From e50e2049e4d0c9ef84ffd15658c52e001a288133 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Sat, 18 Mar 2023 13:55:46 -0700 Subject: [PATCH 001/413] Fix minor warning (#289) --- src/expr/literal.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/expr/literal.rs b/src/expr/literal.rs index b29497e64..f99216957 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::errors::{py_runtime_err, DataFusionError}; +use crate::errors::DataFusionError; use datafusion_common::ScalarValue; use pyo3::prelude::*; From 024ba3aed68fbbe8558d0d4918eb6a559b4e430c Mon Sep 17 00:00:00 2001 From: Dejan Simic <10134699+simicd@users.noreply.github.com> Date: Tue, 21 Mar 2023 18:18:11 +0100 Subject: [PATCH 002/413] Implement describe method (#293) --- datafusion/tests/test_dataframe.py | 24 ++++++++++++++++++++++++ src/dataframe.rs | 7 +++++++ 2 files changed, 31 insertions(+) diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index 611bcabe4..db59e3c52 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -607,3 +607,27 @@ def test_to_pydict(df): pydict = df.to_pydict() assert type(pydict) == dict assert pydict == {"a": [1, 2, 3], "b": [4, 5, 6], "c": [8, 5, 8]} + + +def test_describe(df): + + # Calculate statistics + df = df.describe() + + # Collect the result + result = df.to_pydict() + + assert result == { + "describe": [ + "count", + "null_count", + "mean", + "std", + "min", + "max", + "median", + ], + "a": [3.0, 3.0, 2.0, 1.0, 1.0, 3.0, 2.0], + "b": [3.0, 3.0, 5.0, 1.0, 4.0, 6.0, 5.0], + "c": [3.0, 3.0, 7.0, 1.7320508075688772, 5.0, 8.0, 8.0], + } diff --git a/src/dataframe.rs b/src/dataframe.rs index b21f56104..605e11611 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -76,6 +76,13 @@ impl PyDataFrame { } } + /// Calculate summary statistics for a DataFrame + fn describe(&self, py: Python) -> PyResult { + let df = self.df.as_ref().clone(); + let stat_df = wait_for_future(py, df.describe())?; + Ok(Self::new(stat_df)) + } + /// Returns the schema from the logical plan fn schema(&self) -> PyArrowType { PyArrowType(self.df.schema().into()) From b70f1c0383ffb68f4e95999df7a819a5c43bf1ff Mon Sep 17 00:00:00 2001 From: Dejan Simic <10134699+simicd@users.noreply.github.com> Date: Wed, 22 Mar 2023 15:35:22 +0100 Subject: [PATCH 003/413] fix: Printed results not visible in debugger & notebooks (#296) * Use builtin print function for show * Refactor --- src/dataframe.rs | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/dataframe.rs b/src/dataframe.rs index 605e11611..fcce90d90 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -172,8 +172,7 @@ impl PyDataFrame { #[pyo3(signature = (num=20))] fn show(&self, py: Python, num: usize) -> PyResult<()> { let df = self.df.as_ref().clone().limit(0, Some(num))?; - let batches = wait_for_future(py, df.collect())?; - pretty::print_batches(&batches).map_err(|err| PyArrowException::new_err(err.to_string())) + print_dataframe(py, df) } /// Filter out duplicate rows @@ -217,8 +216,7 @@ impl PyDataFrame { #[pyo3(signature = (verbose=false, analyze=false))] fn explain(&self, py: Python, verbose: bool, analyze: bool) -> PyResult<()> { let df = self.df.as_ref().clone().explain(verbose, analyze)?; - let batches = wait_for_future(py, df.collect())?; - pretty::print_batches(&batches).map_err(|err| PyArrowException::new_err(err.to_string())) + print_dataframe(py, df) } /// Get the logical plan for this `DataFrame` @@ -389,3 +387,20 @@ impl PyDataFrame { Ok(wait_for_future(py, self.df.as_ref().clone().count())?) } } + +/// Print DataFrame +fn print_dataframe(py: Python, df: DataFrame) -> PyResult<()> { + // Get string representation of record batches + let batches = wait_for_future(py, df.collect())?; + let batches_as_string = pretty::pretty_format_batches(&batches); + let result = match batches_as_string { + Ok(batch) => format!("DataFrame()\n{batch}"), + Err(err) => format!("Error: {:?}", err.to_string()), + }; + + // Import the Python 'builtins' module to access the print function + // Note that println! does not print to the Python debug console and is not visible in notebooks for instance + let print = py.import("builtins")?.getattr("print")?; + print.call1((result,))?; + Ok(()) +} From a3880400fb290e9c0428503b3fa2d5dacc7dceed Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 22 Mar 2023 08:35:54 -0600 Subject: [PATCH 004/413] add package.include and remove wildcard dependency (#295) --- Cargo.toml | 3 ++- dev/release/README.md | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 03dafefe8..03cd68422 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ readme = "README.md" license = "Apache-2.0" edition = "2021" rust-version = "1.64" +include = ["/src", "/LICENSE.txt"] [features] default = ["mimalloc"] @@ -41,7 +42,7 @@ datafusion-optimizer = { version = "20.0.0" } datafusion-sql = { version = "20.0.0" } datafusion-substrait = { version = "20.0.0" } uuid = { version = "1.2", features = ["v4"] } -mimalloc = { version = "*", optional = true, default-features = false } +mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" futures = "0.3" object_store = { version = "0.5.3", features = ["aws", "gcp", "azure"] } diff --git a/dev/release/README.md b/dev/release/README.md index cec0eef5e..84058a606 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -198,6 +198,14 @@ Create the source release tarball: ./dev/release/release-tarball.sh 0.8.0 1 ``` +### Publishing Rust Crate to crates.io + +Some projects depend on the Rust crate directly, so we publish this to crates.io + +```shell +cargo publish +``` + ### Publishing Python Artifacts to PyPi Go to the Test PyPI page of Datafusion, and download From 5269719dca2a82d9f8b40305307c02fe7bc44ddf Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 29 Mar 2023 08:39:07 -0600 Subject: [PATCH 005/413] update main branch name in docs workflow (#303) --- .github/workflows/docs.yaml | 2 +- docs/build.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 docs/build.sh diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b1c9ffc12..170f77a19 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -1,7 +1,7 @@ on: push: branches: - - master + - main tags-ignore: - "**-rc**" diff --git a/docs/build.sh b/docs/build.sh old mode 100644 new mode 100755 From 65620f42d35c6c7af0214ef76f502e80339dcfb6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 30 Mar 2023 10:17:40 -0600 Subject: [PATCH 006/413] Upgrade to DF 21 (#301) --- Cargo.lock | 1309 +++++++++++++++++++++++----- Cargo.toml | 15 +- datafusion/tests/test_dataframe.py | 7 +- datafusion/tests/test_indexing.py | 2 +- datafusion/tests/test_store.py | 2 +- src/common/df_field.rs | 15 +- src/context.rs | 18 +- src/dataframe.rs | 2 +- src/expr/column.rs | 2 +- src/expr/table_scan.rs | 4 +- 10 files changed, 1134 insertions(+), 242 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23f1486b4..91a703ee5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" [[package]] name = "apache-avro" @@ -93,11 +93,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "arrayref" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" @@ -113,16 +119,16 @@ checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921" dependencies = [ "ahash", "arrow-arith", - "arrow-array", - "arrow-buffer", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", "arrow-cast", "arrow-csv", - "arrow-data", + "arrow-data 34.0.0", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema", + "arrow-schema 34.0.0", "arrow-select", "arrow-string", "comfy-table", @@ -135,10 +141,10 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "chrono", "half", "num", @@ -151,10 +157,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", + "chrono", + "half", + "hashbrown 0.13.2", + "num", +] + +[[package]] +name = "arrow-array" +version = "35.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43489bbff475545b78b0e20bde1d22abd6c99e54499839f9e815a2fa5134a51b" +dependencies = [ + "ahash", + "arrow-buffer 35.0.0", + "arrow-data 35.0.0", + "arrow-schema 35.0.0", "chrono", + "chrono-tz", "half", "hashbrown 0.13.2", "num", @@ -170,16 +193,26 @@ dependencies = [ "num", ] +[[package]] +name = "arrow-buffer" +version = "35.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3759e4a52c593281184787af5435671dc8b1e78333e5a30242b2e2d6e3c9d1f" +dependencies = [ + "half", + "num", +] + [[package]] name = "arrow-cast" version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "arrow-select", "chrono", "lexical-core", @@ -192,11 +225,11 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7" dependencies = [ - "arrow-array", - "arrow-buffer", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "chrono", "csv", "csv-core", @@ -211,8 +244,20 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 34.0.0", + "arrow-schema 34.0.0", + "half", + "num", +] + +[[package]] +name = "arrow-data" +version = "35.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19c7787c6cdbf9539b1ffb860bfc18c5848926ec3d62cbd52dc3b1ea35c874fd" +dependencies = [ + "arrow-buffer 35.0.0", + "arrow-schema 35.0.0", "half", "num", ] @@ -223,11 +268,11 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a" dependencies = [ - "arrow-array", - "arrow-buffer", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "flatbuffers", ] @@ -237,11 +282,11 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266" dependencies = [ - "arrow-array", - "arrow-buffer", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "chrono", "half", "indexmap", @@ -256,10 +301,10 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "arrow-select", "num", ] @@ -271,10 +316,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "half", "hashbrown 0.13.2", ] @@ -288,16 +333,22 @@ dependencies = [ "bitflags", ] +[[package]] +name = "arrow-schema" +version = "35.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6b26f6a6f8410e3b9531cbd1886399b99842701da77d4b4cf2013f7708f20f" + [[package]] name = "arrow-select" version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "num", ] @@ -307,10 +358,10 @@ version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", + "arrow-data 34.0.0", + "arrow-schema 34.0.0", "arrow-select", "regex", "regex-syntax", @@ -336,24 +387,24 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b015a331cc64ebd1774ba119538573603427eaace0a1950c423ab971f903796" +checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.10", ] [[package]] name = "async-trait" -version = "0.1.66" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.10", ] [[package]] @@ -427,6 +478,27 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bstr" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" +dependencies = [ + "memchr", + "once_cell", + "regex-automata", + "serde", +] + +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" version = "3.12.0" @@ -492,11 +564,39 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time", + "time 0.1.45", "wasm-bindgen", "winapi", ] +[[package]] +name = "chrono-tz" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa48fa079165080f11d7753fd0bc175b7d391f276b965fe4b55bfad67856e463" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "clru" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8191fa7302e03607ff0e237d4246cc043ff5b3cb9409d995172ba3bea16b807" + [[package]] name = "codespan-reporting" version = "0.11.1" @@ -554,9 +654,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "cpufeatures" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d997bd5e24a5928dd43e46dc529867e207907fe0b239c3477d924f7f2ca320" +checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" dependencies = [ "libc", ] @@ -609,9 +709,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" +checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" dependencies = [ "cc", "cxxbridge-flags", @@ -621,9 +721,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" +checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" dependencies = [ "cc", "codespan-reporting", @@ -631,24 +731,24 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn", + "syn 2.0.10", ] [[package]] name = "cxxbridge-flags" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" +checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" [[package]] name = "cxxbridge-macro" -version = "1.0.92" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" +checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.10", ] [[package]] @@ -666,9 +766,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c467c5802cb75ecb0acffa2121d8361a8903fef05b21fd1ca12a55797df8a75" +checksum = "c187589ce9ddf0bbc90e2e3dc0a89b90cc3d4bfdeefc7cf2aaa8ac15f7725811" dependencies = [ "ahash", "apache-avro", @@ -699,7 +799,6 @@ dependencies = [ "object_store", "parking_lot", "parquet", - "paste", "percent-encoding", "pin-project-lite", "rand", @@ -717,12 +816,13 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c60e0a92bae06c6a55c4618947ae0837f833929b8d52ade1d50cf8b5f3b381" +checksum = "ecbbfb88a799beca6a361c1282795f0f185b96201dab496d733a49bdf4684f7f" dependencies = [ "apache-avro", "arrow", + "arrow-array 35.0.0", "chrono", "num_cpus", "object_store", @@ -733,9 +833,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96eab4ef369c3972a4f99418ba8673aa0cf818aa4b892d44c35b473edce8e021" +checksum = "73a38825b879024a87937b3b5ea8e43287ab3432db8786a2839dcbf141b6d938" dependencies = [ "dashmap", "datafusion-common", @@ -751,22 +851,21 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0533db37a619a045a4fd37dabc2d99a85f10127d7f100d75914c194e9e36ed7" +checksum = "05454741d8496faf9f433a666e97ce693807e8374e0fd513eda5a8218ba8456d" dependencies = [ "ahash", "arrow", "datafusion-common", - "log", "sqlparser", ] [[package]] name = "datafusion-optimizer" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dbb8f467d31c3efebd372854e35cf1ca1572f163d0f031e2c6f8d78b317a43b" +checksum = "d5d551c428b8557790cceecb59615f624b24dddf60b4d843c5994f8120b48c7f" dependencies = [ "arrow", "async-trait", @@ -782,14 +881,14 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb003f6c0cdc25d2e9e92f8ee0322faca2a8f94f03d070b5ca15224c5503a3c" +checksum = "08aa1047edf92d59f97b18dfbb1cade176a970b1a98b0a27f909409ceb05906e" dependencies = [ "ahash", "arrow", - "arrow-buffer", - "arrow-schema", + "arrow-buffer 34.0.0", + "arrow-schema 34.0.0", "blake2", "blake3", "chrono", @@ -802,7 +901,6 @@ dependencies = [ "itertools", "lazy_static", "md-5", - "num-traits", "paste", "petgraph", "rand", @@ -814,7 +912,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "20.0.0" +version = "21.0.0" dependencies = [ "async-trait", "datafusion", @@ -831,14 +929,15 @@ dependencies = [ "rand", "regex-syntax", "tokio", + "url", "uuid", ] [[package]] name = "datafusion-row" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba0b4478a205b767f6d98e26730a604562155a8b3cf64fe3600367e741b0f129" +checksum = "2fc83ac8761c251617c1b7e1122adf79ebbf215ecabc4e2346cda1c4307d5152" dependencies = [ "arrow", "datafusion-common", @@ -848,11 +947,11 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07af4200a9353ebe7bf0d6024894275cdebb303bb534b3a38eb0f532583083d" +checksum = "46d6cbfa8c6ac06202badbac6e4675c33b91d299f711a4fee23327b83906e2ee" dependencies = [ - "arrow-schema", + "arrow-schema 34.0.0", "datafusion-common", "datafusion-expr", "log", @@ -861,9 +960,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "20.0.0" +version = "21.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71a9cc3e51c77ada10990e996c15c5bda293b3e4a508f55c2f3993ea6c6a39a7" +checksum = "92afff2fc078214072992424e56e081fbe4b80c639f037f3f1916a8d6d28c25d" dependencies = [ "async-recursion", "chrono", @@ -872,7 +971,6 @@ dependencies = [ "object_store", "prost 0.11.8", "prost-build 0.9.0", - "prost-types 0.11.8", "substrait", "tokio", ] @@ -888,12 +986,38 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "doc-comment" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +[[package]] +name = "dunce" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" + [[package]] name = "dyn-clone" version = "1.0.11" @@ -945,6 +1069,18 @@ dependencies = [ "instant", ] +[[package]] +name = "filetime" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.45.0", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -1012,88 +1148,588 @@ dependencies = [ ] [[package]] -name = "futures-core" -version = "0.3.27" +name = "futures-core" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" + +[[package]] +name = "futures-executor" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" + +[[package]] +name = "futures-macro" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "futures-sink" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" + +[[package]] +name = "futures-task" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" + +[[package]] +name = "futures-util" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "gix" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd5e0d9c5df90c9b4d325ec716762beb7d6c1465a4049fec5c4f6b72e7824656" +dependencies = [ + "gix-actor", + "gix-attributes", + "gix-config", + "gix-credentials", + "gix-date", + "gix-diff", + "gix-discover", + "gix-features", + "gix-glob", + "gix-hash", + "gix-hashtable", + "gix-index", + "gix-lock", + "gix-mailmap", + "gix-object", + "gix-odb", + "gix-pack", + "gix-path", + "gix-prompt", + "gix-ref", + "gix-refspec", + "gix-revision", + "gix-sec", + "gix-tempfile", + "gix-traverse", + "gix-url", + "gix-validate", + "gix-worktree", + "log", + "once_cell", + "signal-hook", + "smallvec", + "thiserror", + "unicode-normalization", +] + +[[package]] +name = "gix-actor" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc22b0cdc52237667c301dd7cdc6ead8f8f73c9f824e9942c8ebd6b764f6c0bf" +dependencies = [ + "bstr", + "btoi", + "gix-date", + "itoa", + "nom", + "thiserror", +] + +[[package]] +name = "gix-attributes" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2231a25934a240d0a4b6f4478401c73ee81d8be52de0293eedbc172334abf3e1" +dependencies = [ + "bstr", + "gix-features", + "gix-glob", + "gix-path", + "gix-quote", + "thiserror", + "unicode-bom", +] + +[[package]] +name = "gix-bitmap" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "024bca0c7187517bda5ea24ab148c9ca8208dd0c3e2bea88cdb2008f91791a6d" +dependencies = [ + "thiserror", +] + +[[package]] +name = "gix-chunk" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0d39583cab06464b8bf73b3f1707458270f0e7383cb24c3c9c1a16e6f792978" +dependencies = [ + "thiserror", +] + +[[package]] +name = "gix-command" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2c6f75c1e0f924de39e750880a6e21307194bb1ab773efe3c7d2d787277f8ab" +dependencies = [ + "bstr", +] + +[[package]] +name = "gix-config" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa7d7dd60256b7a0c0506a1d708ec92767c2662ee57b3301b538eaa3e064f8a" +dependencies = [ + "bstr", + "gix-config-value", + "gix-features", + "gix-glob", + "gix-path", + "gix-ref", + "gix-sec", + "memchr", + "nom", + "once_cell", + "smallvec", + "thiserror", + "unicode-bom", +] + +[[package]] +name = "gix-config-value" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d4a4ba0531e46fe558459557a5b29fb86c3e4b2666c1c0861d93c7c678331" +dependencies = [ + "bitflags", + "bstr", + "gix-path", + "libc", + "thiserror", +] + +[[package]] +name = "gix-credentials" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "750b684197374518ea057e0a0594713e07683faa0a3f43c0f93d97f64130ad8d" +dependencies = [ + "bstr", + "gix-command", + "gix-config-value", + "gix-path", + "gix-prompt", + "gix-sec", + "gix-url", + "thiserror", +] + +[[package]] +name = "gix-date" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96271912ce39822501616f177dea7218784e6c63be90d5f36322ff3a722aae2" +dependencies = [ + "bstr", + "itoa", + "thiserror", + "time 0.3.20", +] + +[[package]] +name = "gix-diff" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "103a0fa79b0d438f5ecb662502f052e530ace4fe1fe8e1c83c0c6da76d728e67" +dependencies = [ + "gix-hash", + "gix-object", + "imara-diff", + "thiserror", +] + +[[package]] +name = "gix-discover" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630ac89fa142bd4dff7337104670a0486c39fc3849f7fe0847f6af6648b2cbca" +dependencies = [ + "bstr", + "dunce", + "gix-hash", + "gix-path", + "gix-ref", + "gix-sec", + "thiserror", +] + +[[package]] +name = "gix-features" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b76f9a80f6dd7be66442ae86e1f534effad9546676a392acc95e269d0c21c22" +dependencies = [ + "crc32fast", + "flate2", + "gix-hash", + "libc", + "once_cell", + "prodash", + "sha1_smol", + "thiserror", + "walkdir", +] + +[[package]] +name = "gix-glob" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e43efd776bc543f46f0fd0ca3d920c37af71a764a16f2aebd89765e9ff2993" +dependencies = [ + "bitflags", + "bstr", +] + +[[package]] +name = "gix-hash" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0c5a9f4d621d4f4ea046bb331df5c746ca735b8cae5b234cc2be70ee4dbef0" +dependencies = [ + "hex", + "thiserror", +] + +[[package]] +name = "gix-hashtable" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9609c1b8f36f12968e6a6098f7cdb52004f7d42d570f47a2d6d7c16612f19acb" +dependencies = [ + "gix-hash", + "hashbrown 0.13.2", + "parking_lot", +] + +[[package]] +name = "gix-index" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "546ee7855d5d8731288f05a63c07ab41b59cb406660a825ed3fe89d7223823df" +dependencies = [ + "bitflags", + "bstr", + "btoi", + "filetime", + "gix-bitmap", + "gix-features", + "gix-hash", + "gix-lock", + "gix-object", + "gix-traverse", + "itoa", + "memmap2", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-lock" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b80172055c5d8017a48ddac5cc7a95421c00211047db0165c97853c4f05194" +dependencies = [ + "fastrand", + "gix-tempfile", + "thiserror", +] + +[[package]] +name = "gix-mailmap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b66aea5e52875cd4915f4957a6f4b75831a36981e2ec3f5fad9e370e444fe1a" +dependencies = [ + "bstr", + "gix-actor", + "thiserror", +] + +[[package]] +name = "gix-object" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df068db9180ee935fbb70504848369e270bdcb576b05c0faa8b9fd3b86fc017" +dependencies = [ + "bstr", + "btoi", + "gix-actor", + "gix-features", + "gix-hash", + "gix-validate", + "hex", + "itoa", + "nom", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-odb" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa63fce01e5bce663bb24ad01fa2b77266e91b1d1982aab3f67cb0aed8af8169" +dependencies = [ + "arc-swap", + "gix-features", + "gix-hash", + "gix-object", + "gix-pack", + "gix-path", + "gix-quote", + "parking_lot", + "tempfile", + "thiserror", +] + +[[package]] +name = "gix-pack" +version = "0.33.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9401911c7fe032ad7b31c6a6b5be59cb283d1d6c999417a8215056efe6d635f3" +dependencies = [ + "clru", + "gix-chunk", + "gix-diff", + "gix-features", + "gix-hash", + "gix-hashtable", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-traverse", + "memmap2", + "parking_lot", + "smallvec", + "thiserror", +] + +[[package]] +name = "gix-path" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6c104a66dec149cb8f7aaafc6ab797654cf82d67f050fd0cb7e7294e328354b" +dependencies = [ + "bstr", + "thiserror", +] + +[[package]] +name = "gix-prompt" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3034d4d935aef2c7bf719aaa54b88c520e82413118d886ae880a31d5bdee57" +dependencies = [ + "gix-command", + "gix-config-value", + "nix", + "parking_lot", + "thiserror", +] + +[[package]] +name = "gix-quote" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a282f5a8d9ee0b09ec47390ac727350c48f2f5c76d803cd8da6b3e7ad56e0bcb" +dependencies = [ + "bstr", + "btoi", + "thiserror", +] + +[[package]] +name = "gix-ref" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" +checksum = "0949e07aa4ed00a5936c2f4529013540708f367906f542cf19db814957e80449" +dependencies = [ + "gix-actor", + "gix-features", + "gix-hash", + "gix-lock", + "gix-object", + "gix-path", + "gix-tempfile", + "gix-validate", + "memmap2", + "nom", + "thiserror", +] [[package]] -name = "futures-executor" -version = "0.3.27" +name = "gix-refspec" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +checksum = "aba332462bda2e8efeae4302b39a6ed01ad56ef772fd5b7ef197cf2798294d65" dependencies = [ - "futures-core", - "futures-task", - "futures-util", + "bstr", + "gix-hash", + "gix-revision", + "gix-validate", + "smallvec", + "thiserror", ] [[package]] -name = "futures-io" -version = "0.3.27" +name = "gix-revision" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" +checksum = "b12fc4bbc3161a5b2d68079fce93432cef8771ff88ca017abb01187fddfc41a1" +dependencies = [ + "bstr", + "gix-date", + "gix-hash", + "gix-hashtable", + "gix-object", + "thiserror", +] [[package]] -name = "futures-macro" -version = "0.3.27" +name = "gix-sec" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +checksum = "e8ffa5bf0772f9b01de501c035b6b084cf9b8bb07dec41e3afc6a17336a65f47" dependencies = [ - "proc-macro2", - "quote", - "syn", + "bitflags", + "dirs", + "gix-path", + "libc", + "windows 0.43.0", ] [[package]] -name = "futures-sink" -version = "0.3.27" +name = "gix-tempfile" +version = "5.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" +checksum = "c2ceb30a610e3f5f2d5f9a5114689fde507ba9417705a8cf3429604275b2153c" +dependencies = [ + "libc", + "once_cell", + "parking_lot", + "signal-hook", + "signal-hook-registry", + "tempfile", +] [[package]] -name = "futures-task" -version = "0.3.27" +name = "gix-traverse" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" +checksum = "dd9a4a07bb22168dc79c60e1a6a41919d198187ca83d8a5940ad8d7122a45df3" +dependencies = [ + "gix-hash", + "gix-hashtable", + "gix-object", + "thiserror", +] [[package]] -name = "futures-util" -version = "0.3.27" +name = "gix-url" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +checksum = "b6a22b4b32ad14d68f7b7fb6458fa58d44b01797d94c1b8f4db2d9c7b3c366b5" dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", + "bstr", + "gix-features", + "gix-path", + "home", + "thiserror", + "url", ] [[package]] -name = "generic-array" -version = "0.14.6" +name = "gix-validate" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "7bd629d3680773e1785e585d76fd4295b740b559cad9141517300d99a0c8c049" dependencies = [ - "typenum", - "version_check", + "bstr", + "thiserror", ] [[package]] -name = "getrandom" -version = "0.2.8" +name = "gix-worktree" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "27248bdec2c90cd2d40fc43bb7d782b26bacc53d9248e78c32ac8268044aebbe" dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "bstr", + "gix-attributes", + "gix-features", + "gix-glob", + "gix-hash", + "gix-index", + "gix-object", + "gix-path", + "io-close", + "thiserror", ] [[package]] @@ -1170,6 +1806,18 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "home" version = "0.5.4" @@ -1252,16 +1900,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows 0.46.0", ] [[package]] @@ -1284,11 +1932,21 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "imara-diff" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8" +dependencies = [ + "ahash", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", "hashbrown 0.12.3", @@ -1315,12 +1973,23 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "io-close" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "io-lifetimes" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfa919a82ea574332e2de6e74b4c36e74d41982b335080fa59d4ef31be20fdf3" +checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ + "hermit-abi 0.3.1", "libc", "windows-sys 0.45.0", ] @@ -1556,6 +2225,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.8.0" @@ -1576,9 +2254,15 @@ dependencies = [ [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" @@ -1607,6 +2291,28 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +[[package]] +name = "nix" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" +dependencies = [ + "bitflags", + "cfg-if", + "libc", + "static_assertions", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num" version = "0.4.0" @@ -1690,7 +2396,16 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", + "libc", +] + +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ "libc", ] @@ -1767,12 +2482,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", + "arrow-array 34.0.0", + "arrow-buffer 34.0.0", "arrow-cast", - "arrow-data", + "arrow-data 34.0.0", "arrow-ipc", - "arrow-schema", + "arrow-schema 34.0.0", "arrow-select", "base64", "brotli", @@ -1793,6 +2508,15 @@ dependencies = [ "zstd 0.12.3+zstd.1.5.2", ] +[[package]] +name = "parse-zoneinfo" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +dependencies = [ + "regex", +] + [[package]] name = "paste" version = "1.0.12" @@ -1825,6 +2549,44 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phf" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.9" @@ -1857,13 +2619,19 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.52" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" +checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" dependencies = [ "unicode-ident", ] +[[package]] +name = "prodash" +version = "23.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9516b775656bc3e8985e19cd4b8c0c0de045095074e453d2c0a513b5f978392d" + [[package]] name = "prost" version = "0.9.0" @@ -1934,7 +2702,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1947,7 +2715,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -1971,9 +2739,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06a3d8e8a46ab2738109347433cb7b96dffda2e4a218b03ef27090238886b147" +checksum = "cfb848f80438f926a9ebddf0a539ed6065434fd7aae03a89312a9821f81b8501" dependencies = [ "cfg-if", "indoc", @@ -1988,9 +2756,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75439f995d07ddfad42b192dfcf3bc66a7ecfd8b4a1f5f6f046aa5c2c5d7677d" +checksum = "98a42e7f42e917ce6664c832d5eee481ad514c98250c49e0b03b20593e2c7ed0" dependencies = [ "once_cell", "target-lexicon", @@ -1998,9 +2766,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839526a5c07a17ff44823679b68add4a58004de00512a95b6c1c98a6dcac0ee5" +checksum = "a0707f0ab26826fe4ccd59b69106e9df5e12d097457c7b8f9c0fd1d2743eec4d" dependencies = [ "libc", "pyo3-build-config", @@ -2008,25 +2776,25 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd44cf207476c6a9760c4653559be4f206efafb924d3e4cbf2721475fc0d6cc5" +checksum = "978d18e61465ecd389e1f235ff5a467146dc4e3c3968b90d274fe73a5dd4a438" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "pyo3-macros-backend" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1f43d8e30460f36350d18631ccf85ded64c059829208fe680904c65bcd0a4c" +checksum = "8e0e1128f85ce3fca66e435e08aa2089a2689c1c48ce97803e13f63124058462" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2093,22 +2861,39 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +dependencies = [ + "getrandom", + "redox_syscall", + "thiserror", +] + [[package]] name = "regex" -version = "1.7.1" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" + [[package]] name = "regex-syntax" -version = "0.6.28" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regress" @@ -2121,9 +2906,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.14" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21eed90ec8570952d53b772ecf8f206aa1ec9a3d76b2521c56c42973f2d91ee9" +checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" dependencies = [ "base64", "bytes", @@ -2205,9 +2990,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.9" +version = "0.36.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" +checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" dependencies = [ "bitflags", "errno", @@ -2280,7 +3065,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 1.0.109", ] [[package]] @@ -2337,22 +3122,22 @@ checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" [[package]] name = "serde" -version = "1.0.156" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4" +checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.156" +version = "1.0.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" +checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.10", ] [[package]] @@ -2363,7 +3148,7 @@ checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2385,7 +3170,7 @@ checksum = "797ba1d80299b264f3aac68ab5d12e5825a561749db4df7cd7c8083900c5d4e9" dependencies = [ "proc-macro2", "serde", - "syn", + "syn 1.0.109", ] [[package]] @@ -2413,6 +3198,12 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.6" @@ -2424,6 +3215,31 @@ dependencies = [ "digest", ] +[[package]] +name = "signal-hook" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.8" @@ -2458,7 +3274,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2501,7 +3317,7 @@ checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2526,20 +3342,22 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 1.0.109", ] [[package]] name = "substrait" -version = "0.4.2" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e977fc98d1e03cf99220bb6bb96f8838ffa5c1306a8c83c1b25aa20817eb6d0" +checksum = "522c38065c57c4e5b03354157bbe8272b42fbf3af6abff304fad10669436b1fb" dependencies = [ + "gix", "heck 0.4.1", "prost 0.11.8", "prost-build 0.11.8", "prost-types 0.11.8", "schemars", + "semver 1.0.17", "serde", "serde_json", "serde_yaml", @@ -2564,6 +3382,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "2.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "target-lexicon" version = "0.12.6" @@ -2594,22 +3423,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.39" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.10", ] [[package]] @@ -2634,6 +3463,35 @@ dependencies = [ "winapi", ] +[[package]] +name = "time" +version = "0.3.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" +dependencies = [ + "itoa", + "libc", + "num_threads", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" + +[[package]] +name = "time-macros" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" +dependencies = [ + "time-core", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -2685,7 +3543,7 @@ checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2772,7 +3630,7 @@ checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2808,7 +3666,7 @@ checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -2841,7 +3699,7 @@ dependencies = [ "rustfmt-wrapper", "schemars", "serde_json", - "syn", + "syn 1.0.109", "thiserror", "unicode-ident", ] @@ -2858,7 +3716,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 1.0.109", "typify-impl", ] @@ -2870,9 +3728,15 @@ checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" [[package]] name = "unicode-bidi" -version = "0.3.11" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-bom" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524b68aca1d05e03fdf03fcdce2c6c94b6daf6d16861ddaa7e4f2b6638a9052c" +checksum = "63ec69f541d875b783ca40184d655f2927c95f0bffd486faa83cd3ac3529ec32" [[package]] name = "unicode-ident" @@ -2948,12 +3812,11 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" dependencies = [ "same-file", - "winapi", "winapi-util", ] @@ -3000,7 +3863,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-shared", ] @@ -3034,7 +3897,7 @@ checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3129,6 +3992,30 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.43.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.42.0" @@ -3246,7 +4133,7 @@ checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 03cd68422..75f458d7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "20.0.0" +version = "21.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -35,12 +35,12 @@ default = ["mimalloc"] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "20.0.0", features = ["pyarrow", "avro"]} -datafusion-common = { version = "20.0.0", features = ["pyarrow"]} -datafusion-expr = { version = "20.0.0" } -datafusion-optimizer = { version = "20.0.0" } -datafusion-sql = { version = "20.0.0" } -datafusion-substrait = { version = "20.0.0" } +datafusion = { version = "21.0.0", features = ["pyarrow", "avro"]} +datafusion-common = { version = "21.0.0", features = ["pyarrow"]} +datafusion-expr = { version = "21.0.0" } +datafusion-optimizer = { version = "21.0.0" } +datafusion-sql = { version = "21.0.0" } +datafusion-substrait = { version = "21.0.0" } uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" @@ -48,6 +48,7 @@ futures = "0.3" object_store = { version = "0.5.3", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.6.28" +url = "2.2" [lib] name = "datafusion_python" diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index db59e3c52..c6ef95772 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -164,16 +164,17 @@ def test_join(): [pa.array([1, 2, 3]), pa.array([4, 5, 6])], names=["a", "b"], ) - df = ctx.create_dataframe([[batch]]) + df = ctx.create_dataframe([[batch]], "l") batch = pa.RecordBatch.from_arrays( [pa.array([1, 2]), pa.array([8, 10])], names=["a", "c"], ) - df1 = ctx.create_dataframe([[batch]]) + df1 = ctx.create_dataframe([[batch]], "r") df = df.join(df1, join_keys=(["a"], ["a"]), how="inner") - df = df.sort(column("a").sort(ascending=True)) + df.show() + df = df.sort(column("l.a").sort(ascending=True)) table = pa.Table.from_batches(df.collect()) expected = {"a": [1, 2], "c": [8, 10], "b": [4, 5]} diff --git a/datafusion/tests/test_indexing.py b/datafusion/tests/test_indexing.py index 9e65c0efd..1c7f8627d 100644 --- a/datafusion/tests/test_indexing.py +++ b/datafusion/tests/test_indexing.py @@ -44,7 +44,7 @@ def test_err(df): with pytest.raises(Exception) as e_info: df["c"] - assert "Schema error: No field named 'c'" in e_info.value.args[0] + assert 'Schema error: No field named "c"' in e_info.value.args[0] with pytest.raises(Exception) as e_info: df[1] diff --git a/datafusion/tests/test_store.py b/datafusion/tests/test_store.py index d6f0db583..9174c246f 100644 --- a/datafusion/tests/test_store.py +++ b/datafusion/tests/test_store.py @@ -30,7 +30,7 @@ def local(): @pytest.fixture def ctx(local): ctx = SessionContext() - ctx.register_object_store("local", local, None) + ctx.register_object_store("file://local", local, None) return ctx diff --git a/src/common/df_field.rs b/src/common/df_field.rs index fa65e0495..d7745bfb5 100644 --- a/src/common/df_field.rs +++ b/src/common/df_field.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::arrow::datatypes::DataType; -use datafusion_common::DFField; +use datafusion_common::{DFField, OwnedTableReference}; use pyo3::prelude::*; use super::data_type::PyDataType; @@ -46,9 +46,14 @@ impl From for PyDFField { impl PyDFField { #[new] #[pyo3(signature = (qualifier=None, name="", data_type=DataType::Int64.into(), nullable=false))] - fn new(qualifier: Option<&str>, name: &str, data_type: PyDataType, nullable: bool) -> Self { + fn new(qualifier: Option, name: &str, data_type: PyDataType, nullable: bool) -> Self { PyDFField { - field: DFField::new(qualifier, name, data_type.into(), nullable), + field: DFField::new( + qualifier.map(|q| OwnedTableReference::from(q)), + name, + data_type.into(), + nullable, + ), } } @@ -91,8 +96,8 @@ impl PyDFField { // fn py_unqualified_column(&self) -> PyResult {} #[pyo3(name = "qualifier")] - fn py_qualifier(&self) -> PyResult> { - Ok(self.field.qualifier()) + fn py_qualifier(&self) -> PyResult> { + Ok(self.field.qualifier().map(|q| format!("{}", q))) } // TODO: Need bindings for Arrow `Field` first diff --git a/src/context.rs b/src/context.rs index e2c509abe..448fa7742 100644 --- a/src/context.rs +++ b/src/context.rs @@ -20,6 +20,7 @@ use std::path::PathBuf; use std::sync::Arc; use object_store::ObjectStore; +use url::Url; use uuid::Uuid; use pyo3::exceptions::{PyKeyError, PyValueError}; @@ -49,7 +50,6 @@ use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; -use datafusion_common::config::Extensions; use datafusion_common::ScalarValue; use pyo3::types::PyTuple; use tokio::runtime::Runtime; @@ -260,10 +260,9 @@ impl PySessionContext { } else { &upstream_host }; - - self.ctx - .runtime_env() - .register_object_store(scheme, derived_host, store); + let url_string = format!("{}{}", scheme, derived_host); + let url = Url::parse(&url_string).unwrap(); + self.ctx.runtime_env().register_object_store(&url, store); Ok(()) } @@ -699,20 +698,19 @@ impl PySessionContext { part: usize, py: Python, ) -> PyResult { - let ctx = TaskContext::try_new( - "task_id".to_string(), + let ctx = TaskContext::new( + None, "session_id".to_string(), - HashMap::new(), + SessionConfig::new(), HashMap::new(), HashMap::new(), Arc::new(RuntimeEnv::default()), - Extensions::default(), ); // create a Tokio runtime to run the async code let rt = Runtime::new().unwrap(); let plan = plan.plan.clone(); let fut: JoinHandle> = - rt.spawn(async move { plan.execute(part, Arc::new(ctx?)) }); + rt.spawn(async move { plan.execute(part, Arc::new(ctx)) }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; Ok(PyRecordBatchStream::new(stream?)) } diff --git a/src/dataframe.rs b/src/dataframe.rs index fcce90d90..7e1ce03b5 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -20,7 +20,7 @@ use crate::sql::logical::PyLogicalPlan; use crate::utils::wait_for_future; use crate::{errors::DataFusionError, expr::PyExpr}; use datafusion::arrow::datatypes::Schema; -use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowException, PyArrowType}; +use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType}; use datafusion::arrow::util::pretty; use datafusion::dataframe::DataFrame; use datafusion::prelude::*; diff --git a/src/expr/column.rs b/src/expr/column.rs index 16b8bce3c..68123fb04 100644 --- a/src/expr/column.rs +++ b/src/expr/column.rs @@ -45,7 +45,7 @@ impl PyColumn { /// Get the column relation fn relation(&self) -> Option { - self.col.relation.clone() + self.col.relation.as_ref().map(|r| format!("{}", r)) } /// Get the fully-qualified column name diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index 63684fe7f..ac848d9eb 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -67,8 +67,8 @@ impl Display for PyTableScan { impl PyTableScan { /// Retrieves the name of the table represented by this `TableScan` instance #[pyo3(name = "table_name")] - fn py_table_name(&self) -> PyResult<&str> { - Ok(&self.table_scan.table_name) + fn py_table_name(&self) -> PyResult { + Ok(format!("{}", self.table_scan.table_name)) } /// TODO: Bindings for `TableSource` need to exist first. Left as a From a7e9161e99f2e8231f5dbf1c556755d58971aa45 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 31 Mar 2023 08:57:37 -0600 Subject: [PATCH 007/413] Prepare 21.0.0 release (#304) * cargo update * changelog * fix --- CHANGELOG.md | 13 +++ Cargo.lock | 256 ++++++++++++++++++++++++++++++++------------------- 2 files changed, 172 insertions(+), 97 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cf05413a..35982d9a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,19 @@ # Changelog +## [21.0.0](https://github.com/apache/arrow-datafusion-python/tree/21.0.0) (2023-03-30) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/20.0.0...21.0.0) + +**Merged pull requests:** + +- minor: Fix minor warning on unused import [#289](https://github.com/apache/arrow-datafusion-python/pull/289) (viirya) +- feature: Implement `describe()` method [#293](https://github.com/apache/arrow-datafusion-python/pull/293) (simicd) +- fix: Printed results not visible in debugger & notebooks [#296](https://github.com/apache/arrow-datafusion-python/pull/296) (simicd) +- add package.include and remove wildcard dependency [#295](https://github.com/apache/arrow-datafusion-python/pull/295) (andygrove) +- Update main branch name in docs workflow [#303](https://github.com/apache/arrow-datafusion-python/pull/303) (andygrove) +- Upgrade to DF 21 [#301](https://github.com/apache/arrow-datafusion-python/pull/301) (andygrove) + ## [20.0.0](https://github.com/apache/arrow-datafusion-python/tree/20.0.0) (2023-03-17) [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/0.8.0...20.0.0) diff --git a/Cargo.lock b/Cargo.lock index 91a703ee5..dab3530b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,7 +393,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.10", + "syn 2.0.11", ] [[package]] @@ -404,7 +404,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.10", + "syn 2.0.11", ] [[package]] @@ -731,7 +731,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.10", + "syn 2.0.11", ] [[package]] @@ -748,7 +748,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.10", + "syn 2.0.11", ] [[package]] @@ -1041,13 +1041,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys", ] [[package]] @@ -1077,8 +1077,8 @@ checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.45.0", + "redox_syscall 0.2.16", + "windows-sys", ] [[package]] @@ -1213,9 +1213,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -1353,9 +1353,9 @@ dependencies = [ [[package]] name = "gix-config-value" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d4a4ba0531e46fe558459557a5b29fb86c3e4b2666c1c0861d93c7c678331" +checksum = "d09154c0c8677e4da0ec35e896f56ee3e338e741b9599fae06075edd83a4081c" dependencies = [ "bitflags", "bstr", @@ -1406,9 +1406,9 @@ dependencies = [ [[package]] name = "gix-discover" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630ac89fa142bd4dff7337104670a0486c39fc3849f7fe0847f6af6648b2cbca" +checksum = "6eba8ba458cb8f4a6c33409b0fe650b1258655175a7ffd1d24fafd3ed31d880b" dependencies = [ "bstr", "dunce", @@ -1469,9 +1469,9 @@ dependencies = [ [[package]] name = "gix-index" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "546ee7855d5d8731288f05a63c07ab41b59cb406660a825ed3fe89d7223823df" +checksum = "717ab601ece7921f59fe86849dbe27d44a46ebb883b5885732c4f30df4996177" dependencies = [ "bitflags", "bstr", @@ -1532,9 +1532,9 @@ dependencies = [ [[package]] name = "gix-odb" -version = "0.43.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa63fce01e5bce663bb24ad01fa2b77266e91b1d1982aab3f67cb0aed8af8169" +checksum = "e83af2e3e36005bfe010927f0dff41fb5acc3e3d89c6f1174135b3a34086bda2" dependencies = [ "arc-swap", "gix-features", @@ -1572,9 +1572,9 @@ dependencies = [ [[package]] name = "gix-path" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c104a66dec149cb8f7aaafc6ab797654cf82d67f050fd0cb7e7294e328354b" +checksum = "32370dce200bb951df013e03dff35b4233fc7a89458642b047629b91734a7e19" dependencies = [ "bstr", "thiserror", @@ -1606,9 +1606,9 @@ dependencies = [ [[package]] name = "gix-ref" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0949e07aa4ed00a5936c2f4529013540708f367906f542cf19db814957e80449" +checksum = "e4e909396ed3b176823991ccc391c276ae2a015e54edaafa3566d35123cfac9d" dependencies = [ "gix-actor", "gix-features", @@ -1716,9 +1716,9 @@ dependencies = [ [[package]] name = "gix-worktree" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27248bdec2c90cd2d40fc43bb7d782b26bacc53d9248e78c32ac8268044aebbe" +checksum = "54ec9a000b4f24af706c3cc680c7cda235656cbe3216336522f5692773b8a301" dependencies = [ "bstr", "gix-attributes", @@ -1900,16 +1900,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.54" +version = "0.1.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +checksum = "716f12fbcfac6ffab0a5e9ec51d0a0ff70503742bb2dc7b99396394c9dc323f0" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows 0.46.0", + "windows 0.47.0", ] [[package]] @@ -1991,14 +1991,14 @@ checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" +checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" [[package]] name = "itertools" @@ -2156,9 +2156,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.1.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" +checksum = "cd550e73688e6d578f0ac2119e32b797a327631a42f9433e59d02e139c8df60d" [[package]] name = "lock_api" @@ -2282,7 +2282,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -2470,9 +2470,9 @@ checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -2611,6 +2611,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +dependencies = [ + "proc-macro2", + "syn 1.0.109", +] + [[package]] name = "proc-macro-hack" version = "0.5.20+deprecated" @@ -2861,6 +2871,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -2868,7 +2887,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] @@ -2906,9 +2925,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.15" +version = "0.11.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba30cc2c0cd02af1222ed216ba659cdb2f879dfe3181852fe7c50b1d0005949" +checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" dependencies = [ "base64", "bytes", @@ -2990,16 +3009,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.11" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db4165c9963ab29e422d6c26fbc1d37f15bace6b2810221f9d925023480fcf0e" +checksum = "0e78cc525325c06b4a7ff02db283472f3c042b7ff0c391f96c6d5ac6f4f91b75" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -3122,22 +3141,22 @@ checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" [[package]] name = "serde" -version = "1.0.158" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.158" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn 2.0.10", + "syn 2.0.11", ] [[package]] @@ -3153,9 +3172,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" +checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" dependencies = [ "itoa", "ryu", @@ -3347,12 +3366,13 @@ dependencies = [ [[package]] name = "substrait" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522c38065c57c4e5b03354157bbe8272b42fbf3af6abff304fad10669436b1fb" +checksum = "58dbeec746a1da14b7af982eabbd18031563f418fe4217612bc1a4cab5961259" dependencies = [ "gix", "heck 0.4.1", + "prettyplease", "prost 0.11.8", "prost-build 0.11.8", "prost-types 0.11.8", @@ -3361,6 +3381,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "syn 1.0.109", "typify", "walkdir", ] @@ -3384,9 +3405,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.10" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40" +checksum = "21e3787bb71465627110e7d87ed4faaa36c1f61042ee67badb9e2ef173accc40" dependencies = [ "proc-macro2", "quote", @@ -3401,15 +3422,15 @@ checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" [[package]] name = "tempfile" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af18f7ae1acd354b992402e9ec5864359d693cd8a79dcbef59f76891701c1e95" +checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] @@ -3438,7 +3459,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.10", + "syn 2.0.11", ] [[package]] @@ -3518,32 +3539,31 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.26.0" +version = "1.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" +checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" dependencies = [ "autocfg", "bytes", "libc", - "memchr", "mio", "num_cpus", "parking_lot", "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.11", ] [[package]] @@ -3998,61 +4018,61 @@ version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows" -version = "0.46.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "2649ff315bee4c98757f15dac226efe3d81927adbb6e882084bb1ee3e0c330a7" dependencies = [ - "windows-targets", + "windows-targets 0.47.0", ] [[package]] name = "windows-sys" -version = "0.42.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-targets 0.42.2", ] [[package]] -name = "windows-sys" -version = "0.45.0" +name = "windows-targets" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows-targets", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "2f8996d3f43b4b2d44327cd71b7b0efd1284ab60e6e9d0e8b630e18555d87d3e" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.47.0", + "windows_aarch64_msvc 0.47.0", + "windows_i686_gnu 0.47.0", + "windows_i686_msvc 0.47.0", + "windows_x86_64_gnu 0.47.0", + "windows_x86_64_gnullvm 0.47.0", + "windows_x86_64_msvc 0.47.0", ] [[package]] @@ -4061,42 +4081,84 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831d567d53d4f3cb1db332b68e6e2b6260228eb4d99a777d8b2e8ed794027c90" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a42d54a417c60ce4f0e31661eed628f0fa5aca73448c093ec4d45fab4c51cdf" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1925beafdbb22201a53a483db861a5644123157c1c3cee83323a2ed565d71e3" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8ef8f2f1711b223947d9b69b596cf5a4e452c930fb58b6fc3fdae7d0ec6b31" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acaa0c2cf0d2ef99b61c308a0c3dbae430a51b7345dedec470bd8f53f5a3642" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a0628f71be1d11e17ca4a0e9e15b3a5180f6fbf1c2d55e3ba3f850378052c1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6e62c256dc6d40b8c8707df17df8d774e60e39db723675241e7c15e910bce7" + [[package]] name = "winreg" version = "0.10.1" From 48f759b5355d4f7ceeb46b615252e62fb55d0d3a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 31 Mar 2023 09:27:23 -0600 Subject: [PATCH 008/413] DataFusion bump to use valid typify instance for build with RustFMT (#307) * DataFusion bump to use valid typify instance for build with RustFMT * specify syn version --------- Co-authored-by: Jeremy Dyer --- Cargo.lock | 619 +++++++++++++++++------------------------------------ Cargo.toml | 13 +- 2 files changed, 200 insertions(+), 432 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dab3530b4..86e323375 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,69 +113,52 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f410d3907b6b3647b9e7bca4551274b2e3d716aa940afb67b7287257401da921" +checksum = "990dfa1a9328504aa135820da1c95066537b69ad94c04881b785f64328e0fa6b" dependencies = [ "ahash", "arrow-arith", - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", "arrow-csv", - "arrow-data 34.0.0", + "arrow-data", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema 34.0.0", + "arrow-schema", "arrow-select", "arrow-string", - "comfy-table", "pyo3", ] [[package]] name = "arrow-arith" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87391cf46473c9bc53dab68cb8872c3a81d4dfd1703f1c8aa397dba9880a043" -dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", - "chrono", - "half", - "num", -] - -[[package]] -name = "arrow-array" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d35d5475e65c57cffba06d0022e3006b677515f99b54af33a7cd54f6cdd4a5b5" +checksum = "f2b2e52de0ab54173f9b08232b7184c26af82ee7ab4ac77c83396633c90199fa" dependencies = [ - "ahash", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "half", - "hashbrown 0.13.2", "num", ] [[package]] name = "arrow-array" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43489bbff475545b78b0e20bde1d22abd6c99e54499839f9e815a2fa5134a51b" +checksum = "e10849b60c17dbabb334be1f4ef7550701aa58082b71335ce1ed586601b2f423" dependencies = [ "ahash", - "arrow-buffer 35.0.0", - "arrow-data 35.0.0", - "arrow-schema 35.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "chrono-tz", "half", @@ -185,19 +168,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b4ec72eda7c0207727df96cf200f539749d736b21f3e782ece113e18c1a0a7" -dependencies = [ - "half", - "num", -] - -[[package]] -name = "arrow-buffer" -version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3759e4a52c593281184787af5435671dc8b1e78333e5a30242b2e2d6e3c9d1f" +checksum = "b0746ae991b186be39933147117f8339eb1c4bbbea1c8ad37e7bf5851a1a06ba" dependencies = [ "half", "num", @@ -205,31 +178,32 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a7285272c9897321dfdba59de29f5b05aeafd3cdedf104a941256d155f6d304" +checksum = "b88897802515d7b193e38b27ddd9d9e43923d410a9e46307582d756959ee9595" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "chrono", + "comfy-table", "lexical-core", "num", ] [[package]] name = "arrow-csv" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981ee4e7f6a120da04e00d0b39182e1eeacccb59c8da74511de753c56b7fddf7" +checksum = "1c8220d9741fc37961262710ceebd8451a5b393de57c464f0267ffdda1775c0a" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-data", + "arrow-schema", "chrono", "csv", "csv-core", @@ -240,53 +214,41 @@ dependencies = [ [[package]] name = "arrow-data" -version = "34.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27cc673ee6989ea6e4b4e8c7d461f7e06026a096c8f0b1a7288885ff71ae1e56" -dependencies = [ - "arrow-buffer 34.0.0", - "arrow-schema 34.0.0", - "half", - "num", -] - -[[package]] -name = "arrow-data" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19c7787c6cdbf9539b1ffb860bfc18c5848926ec3d62cbd52dc3b1ea35c874fd" +checksum = "533f937efa1aaad9dc86f6a0e382c2fa736a4943e2090c946138079bdf060cef" dependencies = [ - "arrow-buffer 35.0.0", - "arrow-schema 35.0.0", + "arrow-buffer", + "arrow-schema", "half", "num", ] [[package]] name = "arrow-ipc" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e37b8b69d9e59116b6b538e8514e0ec63a30f08b617ce800d31cb44e3ef64c1a" +checksum = "18b75296ff01833f602552dff26a423fc213db8e5049b540ca4a00b1c957e41c" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-data", + "arrow-schema", "flatbuffers", ] [[package]] name = "arrow-json" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c3fa0bed7cfebf6d18e46b733f9cb8a1cb43ce8e6539055ca3e1e48a426266" +checksum = "e501d3de4d612c90677594896ca6c0fa075665a7ff980dc4189bb531c17e19f6" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -297,71 +259,66 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d247dce7bed6a8d6a3c6debfa707a3a2f694383f0c692a39d736a593eae5ef94" +checksum = "33d2671eb3793f9410230ac3efb0e6d36307be8a2dac5fad58ac9abde8e9f01e" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", + "half", "num", ] [[package]] name = "arrow-row" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d609c0181f963cea5c70fddf9a388595b5be441f3aa1d1cdbf728ca834bbd3a" +checksum = "fc11fa039338cebbf4e29cf709c8ac1d6a65c7540063d4a25f991ab255ca85c8" dependencies = [ "ahash", - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", "hashbrown 0.13.2", ] [[package]] name = "arrow-schema" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64951898473bfb8e22293e83a44f02874d2257514d49cd95f9aa4afcff183fbc" +checksum = "d04f17f7b86ded0b5baf98fe6123391c4343e031acc3ccc5fa604cc180bff220" dependencies = [ - "bitflags", + "bitflags 2.0.2", ] -[[package]] -name = "arrow-schema" -version = "35.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf6b26f6a6f8410e3b9531cbd1886399b99842701da77d4b4cf2013f7708f20f" - [[package]] name = "arrow-select" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a513d89c2e1ac22b28380900036cf1f3992c6443efc5e079de631dcf83c6888" +checksum = "163e35de698098ff5f5f672ada9dc1f82533f10407c7a11e2cd09f3bcf31d18a" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num", ] [[package]] name = "arrow-string" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5288979b2705dae1114c864d73150629add9153b9b8f1d7ee3963db94c372ba5" +checksum = "bfdfbed1b10209f0dc68e6aa4c43dc76079af65880965c7c3b73f641f23d4aba" dependencies = [ - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", - "arrow-data 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "arrow-select", "regex", "regex-syntax", @@ -393,7 +350,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -404,7 +361,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -425,6 +382,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" + [[package]] name = "blake2" version = "0.10.6" @@ -731,7 +694,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -748,7 +711,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -767,8 +730,7 @@ dependencies = [ [[package]] name = "datafusion" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c187589ce9ddf0bbc90e2e3dc0a89b90cc3d4bfdeefc7cf2aaa8ac15f7725811" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "ahash", "apache-avro", @@ -817,12 +779,11 @@ dependencies = [ [[package]] name = "datafusion-common" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecbbfb88a799beca6a361c1282795f0f185b96201dab496d733a49bdf4684f7f" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "apache-avro", "arrow", - "arrow-array 35.0.0", + "arrow-array", "chrono", "num_cpus", "object_store", @@ -834,8 +795,7 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73a38825b879024a87937b3b5ea8e43287ab3432db8786a2839dcbf141b6d938" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "dashmap", "datafusion-common", @@ -852,8 +812,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05454741d8496faf9f433a666e97ce693807e8374e0fd513eda5a8218ba8456d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "ahash", "arrow", @@ -864,8 +823,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d551c428b8557790cceecb59615f624b24dddf60b4d843c5994f8120b48c7f" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "arrow", "async-trait", @@ -882,13 +840,13 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08aa1047edf92d59f97b18dfbb1cade176a970b1a98b0a27f909409ceb05906e" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "ahash", "arrow", - "arrow-buffer 34.0.0", - "arrow-schema 34.0.0", + "arrow-array", + "arrow-buffer", + "arrow-schema", "blake2", "blake3", "chrono", @@ -928,6 +886,7 @@ dependencies = [ "pyo3", "rand", "regex-syntax", + "syn 2.0.12", "tokio", "url", "uuid", @@ -936,8 +895,7 @@ dependencies = [ [[package]] name = "datafusion-row" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fc83ac8761c251617c1b7e1122adf79ebbf215ecabc4e2346cda1c4307d5152" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "arrow", "datafusion-common", @@ -948,10 +906,9 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46d6cbfa8c6ac06202badbac6e4675c33b91d299f711a4fee23327b83906e2ee" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ - "arrow-schema 34.0.0", + "arrow-schema", "datafusion-common", "datafusion-expr", "log", @@ -961,16 +918,14 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "21.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92afff2fc078214072992424e56e081fbe4b80c639f037f3f1916a8d6d28c25d" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" dependencies = [ "async-recursion", "chrono", "datafusion", "itertools", "object_store", - "prost 0.11.8", - "prost-build 0.9.0", + "prost", "substrait", "tokio", ] @@ -1093,7 +1048,7 @@ version = "23.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619" dependencies = [ - "bitflags", + "bitflags 1.3.2", "rustc_version", ] @@ -1124,9 +1079,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" dependencies = [ "futures-channel", "futures-core", @@ -1139,9 +1094,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", "futures-sink", @@ -1149,15 +1104,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-executor" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" dependencies = [ "futures-core", "futures-task", @@ -1166,38 +1121,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.12", ] [[package]] name = "futures-sink" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-channel", "futures-core", @@ -1357,7 +1312,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d09154c0c8677e4da0ec35e896f56ee3e338e741b9599fae06075edd83a4081c" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bstr", "gix-path", "libc", @@ -1442,7 +1397,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93e43efd776bc543f46f0fd0ca3d920c37af71a764a16f2aebd89765e9ff2993" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bstr", ] @@ -1473,7 +1428,7 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "717ab601ece7921f59fe86849dbe27d44a46ebb883b5885732c4f30df4996177" dependencies = [ - "bitflags", + "bitflags 1.3.2", "bstr", "btoi", "filetime", @@ -1657,7 +1612,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8ffa5bf0772f9b01de501c035b6b084cf9b8bb07dec41e3afc6a17336a65f47" dependencies = [ - "bitflags", + "bitflags 1.3.2", "dirs", "gix-path", "libc", @@ -1782,15 +1737,6 @@ dependencies = [ "ahash", ] -[[package]] -name = "heck" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" -dependencies = [ - "unicode-segmentation", -] - [[package]] name = "heck" version = "0.4.1" @@ -1900,16 +1846,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.55" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "716f12fbcfac6ffab0a5e9ec51d0a0ff70503742bb2dc7b99396394c9dc323f0" +checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows 0.47.0", + "windows 0.46.0", ] [[package]] @@ -2156,9 +2102,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd550e73688e6d578f0ac2119e32b797a327631a42f9433e59d02e139c8df60d" +checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" [[package]] name = "lock_api" @@ -2297,7 +2243,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" dependencies = [ - "bitflags", + "bitflags 1.3.2", "cfg-if", "libc", "static_assertions", @@ -2477,17 +2423,17 @@ dependencies = [ [[package]] name = "parquet" -version = "34.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ac135ecf63ebb5f53dda0921b0b76d6048b3ef631a5f4760b9e8f863ff00cfa" +checksum = "321a15f8332645759f29875b07f8233d16ed8ec1b3582223de81625a9f8506b7" dependencies = [ "ahash", - "arrow-array 34.0.0", - "arrow-buffer 34.0.0", + "arrow-array", + "arrow-buffer", "arrow-cast", - "arrow-data 34.0.0", + "arrow-data", "arrow-ipc", - "arrow-schema 34.0.0", + "arrow-schema", "arrow-select", "base64", "brotli", @@ -2529,16 +2475,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" -[[package]] -name = "pest" -version = "2.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7" -dependencies = [ - "thiserror", - "ucd-trie", -] - [[package]] name = "petgraph" version = "0.6.3" @@ -2613,12 +2549,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.1.25" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8646e95016a7a6c4adea95bafa8a16baab64b583356217f2c85db4a39d9a86" +checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 1.0.109", + "syn 2.0.12", ] [[package]] @@ -2642,16 +2578,6 @@ version = "23.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9516b775656bc3e8985e19cd4b8c0c0de045095074e453d2c0a513b5f978392d" -[[package]] -name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive 0.9.0", -] - [[package]] name = "prost" version = "0.11.8" @@ -2659,27 +2585,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" dependencies = [ "bytes", - "prost-derive 0.11.8", -] - -[[package]] -name = "prost-build" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" -dependencies = [ - "bytes", - "heck 0.3.3", - "itertools", - "lazy_static", - "log", - "multimap", - "petgraph", - "prost 0.9.0", - "prost-types 0.9.0", - "regex", - "tempfile", - "which", + "prost-derive", ] [[package]] @@ -2689,32 +2595,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" dependencies = [ "bytes", - "heck 0.4.1", + "heck", "itertools", "lazy_static", "log", "multimap", "petgraph", - "prost 0.11.8", - "prost-types 0.11.8", + "prost", + "prost-types", "regex", "tempfile", "which", ] -[[package]] -name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn 1.0.109", -] - [[package]] name = "prost-derive" version = "0.11.8" @@ -2728,23 +2621,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost 0.9.0", -] - [[package]] name = "prost-types" version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" dependencies = [ - "prost 0.11.8", + "prost", ] [[package]] @@ -2868,7 +2751,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2877,7 +2760,7 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -2916,10 +2799,11 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regress" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a92ff21fe8026ce3f2627faaf43606f0b67b014dbc9ccf027181a804f75d92e" +checksum = "d995d590bd8ec096d1893f414bf3f5e8b0ee4c9eed9a5642b9766ef2c8e2e8e9" dependencies = [ + "hashbrown 0.13.2", "memchr", ] @@ -2991,20 +2875,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.17", -] - -[[package]] -name = "rustfmt-wrapper" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed729e3bee08ec2befd593c27e90ca9fdd25efdc83c94c3b82eaef16e4f7406e" -dependencies = [ - "serde", - "tempfile", - "thiserror", - "toml", - "toolchain_find", + "semver", ] [[package]] @@ -3013,7 +2884,7 @@ version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e78cc525325c06b4a7ff02db283472f3c042b7ff0c391f96c6d5ac6f4f91b75" dependencies = [ - "bitflags", + "bitflags 1.3.2", "errno", "io-lifetimes", "libc", @@ -3109,30 +2980,12 @@ dependencies = [ "untrusted", ] -[[package]] -name = "semver" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" -[[package]] -name = "semver-parser" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" -dependencies = [ - "pest", -] - [[package]] name = "seq-macro" version = "0.3.3" @@ -3156,7 +3009,7 @@ checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -3290,7 +3143,7 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "syn 1.0.109", @@ -3357,7 +3210,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", "rustversion", @@ -3366,22 +3219,22 @@ dependencies = [ [[package]] name = "substrait" -version = "0.5.4" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbeec746a1da14b7af982eabbd18031563f418fe4217612bc1a4cab5961259" +checksum = "a94bab2309478886b270dbace5adb3927d8e397aa804b67c5c37da8591dff989" dependencies = [ "gix", - "heck 0.4.1", + "heck", "prettyplease", - "prost 0.11.8", - "prost-build 0.11.8", - "prost-types 0.11.8", + "prost", + "prost-build", + "prost-types", "schemars", - "semver 1.0.17", + "semver", "serde", "serde_json", "serde_yaml", - "syn 1.0.109", + "syn 2.0.12", "typify", "walkdir", ] @@ -3405,9 +3258,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21e3787bb71465627110e7d87ed4faaa36c1f61042ee67badb9e2ef173accc40" +checksum = "79d9531f94112cfc3e4c8f5f02cb2b58f72c97b7efd85f70203cc6d8efda5927" dependencies = [ "proc-macro2", "quote", @@ -3459,7 +3312,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -3563,7 +3416,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.11", + "syn 2.0.12", ] [[package]] @@ -3602,28 +3455,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "toolchain_find" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e85654a10e7a07a47c6f19d93818f3f343e22927f2fa280c84f7c8042743413" -dependencies = [ - "home", - "lazy_static", - "regex", - "semver 0.11.0", - "walkdir", -] - [[package]] name = "tower-service" version = "0.3.2" @@ -3697,9 +3528,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "typify" -version = "0.0.10" +version = "0.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8486352f3c946e69f983558cfc09b295250b01e01b381ec67a05a812d01d63" +checksum = "30bfde96849e25d7feef1bbf652e9cfc51deb63203fdc07b115b8bc3bcfe20b9" dependencies = [ "typify-impl", "typify-macro", @@ -3707,16 +3538,15 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.10" +version = "0.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7624d0b911df6e2bbf34a236f76281f93b294cdde1d4df1dbdb748e5a7fefa5" +checksum = "95d27d749378ceab6ec22188ed7ad102205c89ddb92ab662371c850ffc71aa1a" dependencies = [ - "heck 0.4.1", + "heck", "log", "proc-macro2", "quote", "regress", - "rustfmt-wrapper", "schemars", "serde_json", "syn 1.0.109", @@ -3726,9 +3556,9 @@ dependencies = [ [[package]] name = "typify-macro" -version = "0.0.10" +version = "0.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c42802aa033cee7650a4e1509ba7d5848a56f84be7c4b31e4385ee12445e942" +checksum = "35db6fc2bd9220ecdac6eeb88158824b83610de3dda0c6d0f2142b49efd858b0" dependencies = [ "proc-macro2", "quote", @@ -3740,12 +3570,6 @@ dependencies = [ "typify-impl", ] -[[package]] -name = "ucd-trie" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e79c4d996edb816c91e4308506774452e55e95c3c9de07b6729e17e15a5ef81" - [[package]] name = "unicode-bidi" version = "0.3.13" @@ -4018,22 +3842,22 @@ version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows" -version = "0.47.0" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2649ff315bee4c98757f15dac226efe3d81927adbb6e882084bb1ee3e0c330a7" +checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" dependencies = [ - "windows-targets 0.47.0", + "windows-targets", ] [[package]] @@ -4042,7 +3866,7 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets 0.42.2", + "windows-targets", ] [[package]] @@ -4051,28 +3875,13 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8996d3f43b4b2d44327cd71b7b0efd1284ab60e6e9d0e8b630e18555d87d3e" -dependencies = [ - "windows_aarch64_gnullvm 0.47.0", - "windows_aarch64_msvc 0.47.0", - "windows_i686_gnu 0.47.0", - "windows_i686_msvc 0.47.0", - "windows_x86_64_gnu 0.47.0", - "windows_x86_64_gnullvm 0.47.0", - "windows_x86_64_msvc 0.47.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -4081,84 +3890,42 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831d567d53d4f3cb1db332b68e6e2b6260228eb4d99a777d8b2e8ed794027c90" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" -[[package]] -name = "windows_aarch64_msvc" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a42d54a417c60ce4f0e31661eed628f0fa5aca73448c093ec4d45fab4c51cdf" - [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" -[[package]] -name = "windows_i686_gnu" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1925beafdbb22201a53a483db861a5644123157c1c3cee83323a2ed565d71e3" - [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" -[[package]] -name = "windows_i686_msvc" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8ef8f2f1711b223947d9b69b596cf5a4e452c930fb58b6fc3fdae7d0ec6b31" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" -[[package]] -name = "windows_x86_64_gnu" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acaa0c2cf0d2ef99b61c308a0c3dbae430a51b7345dedec470bd8f53f5a3642" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a0628f71be1d11e17ca4a0e9e15b3a5180f6fbf1c2d55e3ba3f850378052c1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" -[[package]] -name = "windows_x86_64_msvc" -version = "0.47.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6e62c256dc6d40b8c8707df17df8d774e60e39db723675241e7c15e910bce7" - [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index 75f458d7e..ddabb2b8c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,12 +35,12 @@ default = ["mimalloc"] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "21.0.0", features = ["pyarrow", "avro"]} -datafusion-common = { version = "21.0.0", features = ["pyarrow"]} -datafusion-expr = { version = "21.0.0" } -datafusion-optimizer = { version = "21.0.0" } -datafusion-sql = { version = "21.0.0" } -datafusion-substrait = { version = "21.0.0" } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad", features = ["pyarrow", "avro"] } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad", features = ["pyarrow"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" @@ -48,6 +48,7 @@ futures = "0.3" object_store = { version = "0.5.3", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.6.28" +syn = "2.0.11" url = "2.2" [lib] From 5b023942c68b6e2d08ab86975fba7c2b9bf69dda Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 31 Mar 2023 11:12:17 -0600 Subject: [PATCH 009/413] fix invalid build yaml (#308) --- .github/workflows/build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fc8910766..e57696f8b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -105,8 +105,6 @@ jobs: - name: Build wheels uses: PyO3/maturin-action@v1 with: - env: - RUST_BACKTRACE: 1 rust-toolchain: nightly target: x86_64 manylinux: auto From 9688d57417f7dd17535cac84cc20b6cb4c6c47e3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 31 Mar 2023 13:16:00 -0600 Subject: [PATCH 010/413] Try fix release build (#309) * Fix broken use of environment variables in GitHub actions * Fix broken workflow file and add actionlint pre-commit check to prevent future errors * Install protoc * Add rustup-components * fix maturin-action bugs * add explanation * add protoc to sdist and manylinux * Update .github/workflows/build.yml Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> * install protobuf compiler * add protoc * fix invalid build yaml * try set protoc path * try suggestion from ChatGPT * experiment * revert change --------- Co-authored-by: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Co-authored-by: Ian Alexander Joiner <14581281+iajoiner@users.noreply.github.com> --- .github/workflows/build.yml | 31 +++++++++++++++++++++++++++++++ .github/workflows/docs.yaml | 4 ++-- .pre-commit-config.yaml | 4 ++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e57696f8b..a6541f775 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,6 +60,12 @@ jobs: with: toolchain: stable + - name: Upgrade pip + run: python -m pip install --upgrade pip + + - name: Install maturin + run: pip install maturin==0.14.2 + - run: rm LICENSE.txt - name: Download LICENSE.txt uses: actions/download-artifact@v3 @@ -67,12 +73,21 @@ jobs: name: python-wheel-license path: . + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build Python package run: maturin build --release --strip - name: List Windows wheels if: matrix.os == 'windows-latest' run: dir target\wheels\ + # since the runner is dynamic shellcheck (from actionlint) can't infer this is powershell + # so we specify it explicitly + shell: powershell - name: List Mac wheels if: matrix.os != 'windows-latest' @@ -97,17 +112,25 @@ jobs: name: python-wheel-license path: . - run: cat LICENSE.txt + - run: sudo apt-get install protobuf-compiler - name: Install Protoc uses: arduino/setup-protoc@v1 with: version: '3.x' repo-token: ${{ secrets.GITHUB_TOKEN }} + - run: echo "PROTOC=$(which protoc)" >> "$GITHUB_ENV" + - name: Confirm PROTOC is set + - run: echo $PROTOC - name: Build wheels uses: PyO3/maturin-action@v1 + env: + RUST_BACKTRACE: 1 + PROTOC: /opt/hostedtoolcache/protoc/3.20.3/x64/bin/protoc with: rust-toolchain: nightly target: x86_64 manylinux: auto + rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 args: --release --manylinux 2014 - name: Archive wheels uses: actions/upload-artifact@v3 @@ -128,11 +151,19 @@ jobs: name: python-wheel-license path: . - run: cat LICENSE.txt + - run: sudo apt-get install protobuf-compiler + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + - run: echo "PROTOC=$(which protoc)" >> "$GITHUB_ENV" - name: Build sdist uses: PyO3/maturin-action@v1 with: rust-toolchain: stable manylinux: auto + rustup-components: rust-std rustfmt args: --release --sdist --out dist - name: Archive wheels uses: actions/upload-artifact@v3 diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 170f77a19..bef42d538 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -17,9 +17,9 @@ jobs: run: | set -x if test '${{ github.ref }}' = 'refs/heads/main'; then - echo "value=asf-staging" >> $GITHUB_OUTPUT + echo "value=asf-staging" >> "$GITHUB_OUTPUT" elif test '${{ github.ref_type }}' = 'tag'; then - echo "value=asf-site" >> $GITHUB_OUTPUT + echo "value=asf-site" >> "$GITHUB_OUTPUT" else echo "Unsupported input: ${{ github.ref }} / ${{ github.ref_type }}" exit 1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c6805322..39049bf49 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,10 @@ # under the License. repos: + - repo: https://github.com/rhysd/actionlint + rev: v1.6.23 + hooks: + - id: actionlint-docker - repo: https://github.com/psf/black rev: 22.3.0 hooks: From 503647c4be6caa96d2ec05e63aee12616bdf8ed8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 31 Mar 2023 14:34:26 -0600 Subject: [PATCH 011/413] Fix release build (#310) --- .github/workflows/build.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a6541f775..115648b07 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,19 +113,20 @@ jobs: path: . - run: cat LICENSE.txt - run: sudo apt-get install protobuf-compiler - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - version: '3.x' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: echo "PROTOC=$(which protoc)" >> "$GITHUB_ENV" - - name: Confirm PROTOC is set - - run: echo $PROTOC + - name: Install protoc + run: | + PROTOC_VERSION=3.12.4 + curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_OS_ARCH}.zip" + unzip -d "${GITHUB_WORKSPACE}" "protoc-${PROTOC_VERSION}-${PROTOC_OS_ARCH}.zip" bin/protoc + chmod +x "${GITHUB_WORKSPACE}/bin/protoc" + ${GITHUB_WORKSPACE}/bin/protoc --version + export PATH=${GITHUB_WORKSPACE}/bin/:$PATH + which protoc + echo "::set-env name=PATH::$PATH" - name: Build wheels uses: PyO3/maturin-action@v1 env: RUST_BACKTRACE: 1 - PROTOC: /opt/hostedtoolcache/protoc/3.20.3/x64/bin/protoc with: rust-toolchain: nightly target: x86_64 From 5eb0161d581ad1e6d274206e26d58848057c54e2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 3 Apr 2023 12:42:48 -0600 Subject: [PATCH 012/413] Enable datafusion-substrait protoc feature, to remove compile-time dependency on protoc (#312) --- .github/workflows/build.yml | 35 ++---- Cargo.lock | 212 ++++++++++++++++++++++++------------ Cargo.toml | 12 +- 3 files changed, 161 insertions(+), 98 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 115648b07..5d3199432 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,6 +52,17 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup Cpp + uses: aminya/setup-cpp@v1 + with: + compiler: gcc + vcvarsall: ${{ contains(matrix.os, 'windows') }} + cmake: true + ninja: true + vcpkg: true + cppcheck: true + clangtidy: true + - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -73,12 +84,6 @@ jobs: name: python-wheel-license path: . - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - version: '3.x' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Build Python package run: maturin build --release --strip @@ -112,17 +117,6 @@ jobs: name: python-wheel-license path: . - run: cat LICENSE.txt - - run: sudo apt-get install protobuf-compiler - - name: Install protoc - run: | - PROTOC_VERSION=3.12.4 - curl -LO "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-${PROTOC_OS_ARCH}.zip" - unzip -d "${GITHUB_WORKSPACE}" "protoc-${PROTOC_VERSION}-${PROTOC_OS_ARCH}.zip" bin/protoc - chmod +x "${GITHUB_WORKSPACE}/bin/protoc" - ${GITHUB_WORKSPACE}/bin/protoc --version - export PATH=${GITHUB_WORKSPACE}/bin/:$PATH - which protoc - echo "::set-env name=PATH::$PATH" - name: Build wheels uses: PyO3/maturin-action@v1 env: @@ -152,13 +146,6 @@ jobs: name: python-wheel-license path: . - run: cat LICENSE.txt - - run: sudo apt-get install protobuf-compiler - - name: Install Protoc - uses: arduino/setup-protoc@v1 - with: - version: '3.x' - repo-token: ${{ secrets.GITHUB_TOKEN }} - - run: echo "PROTOC=$(which protoc)" >> "$GITHUB_ENV" - name: Build sdist uses: PyO3/maturin-action@v1 with: diff --git a/Cargo.lock b/Cargo.lock index 86e323375..5d5196067 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,7 +350,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -361,7 +361,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -370,6 +370,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "autotools" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +dependencies = [ + "cc", +] + [[package]] name = "base64" version = "0.21.0" @@ -611,9 +620,9 @@ checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" @@ -694,7 +703,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -711,7 +720,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -729,8 +738,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "ahash", "apache-avro", @@ -778,8 +787,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "apache-avro", "arrow", @@ -794,8 +803,8 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "dashmap", "datafusion-common", @@ -811,8 +820,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "ahash", "arrow", @@ -822,8 +831,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "arrow", "async-trait", @@ -839,8 +848,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "ahash", "arrow", @@ -886,7 +895,7 @@ dependencies = [ "pyo3", "rand", "regex-syntax", - "syn 2.0.12", + "syn 2.0.13", "tokio", "url", "uuid", @@ -894,8 +903,8 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "arrow", "datafusion-common", @@ -905,8 +914,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "arrow-schema", "datafusion-common", @@ -917,8 +926,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "21.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=c09edad#c09edade14d456f1d2161c5ebb8c1e51e592a8ef" +version = "21.1.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" dependencies = [ "async-recursion", "chrono", @@ -1133,7 +1142,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -1594,9 +1603,9 @@ dependencies = [ [[package]] name = "gix-revision" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b12fc4bbc3161a5b2d68079fce93432cef8771ff88ca017abb01187fddfc41a1" +checksum = "3c6f6ff53f888858afc24bf12628446a14279ceec148df6194481f306f553ad2" dependencies = [ "bstr", "gix-date", @@ -1846,16 +1855,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.54" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c17cc76786e99f8d2f055c11159e7f0091c42474dcc3189fbab96072e873e6d" +checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows 0.46.0", + "windows 0.48.0", ] [[package]] @@ -2083,9 +2092,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libmimalloc-sys" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8c7cbf8b89019683667e347572e6d55a7df7ea36b0c4ce69961b0cde67b174" +checksum = "ef2c45001fb108f37d41bed8efd715769acb14674c1ce3e266ef0e317ef5f877" dependencies = [ "cc", "libc", @@ -2357,9 +2366,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ea8f683b4f89a64181393742c041520a1a87e9775e6b4c0dd5a3281af05fc6" +checksum = "ec9cd6ca25e796a49fa242876d1c4de36a24a6da5258e9f0bc062dbf5e81c53b" dependencies = [ "async-trait", "base64", @@ -2554,7 +2563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -2565,9 +2574,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.54" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" +checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" dependencies = [ "unicode-ident", ] @@ -2630,6 +2639,15 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf-src" +version = "1.1.0+21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +dependencies = [ + "autotools", +] + [[package]] name = "pyo3" version = "0.18.2" @@ -2698,9 +2716,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quick-xml" -version = "0.27.1" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc053f057dd768a56f62cd7e434c42c831d296968997e9ac1f76ea7c2d14c41" +checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" dependencies = [ "memchr", "serde", @@ -2880,9 +2898,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.5" +version = "0.37.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e78cc525325c06b4a7ff02db283472f3c042b7ff0c391f96c6d5ac6f4f91b75" +checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" dependencies = [ "bitflags 1.3.2", "errno", @@ -3009,7 +3027,7 @@ checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -3219,9 +3237,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a94bab2309478886b270dbace5adb3927d8e397aa804b67c5c37da8591dff989" +checksum = "2b121815df0d0ce3f2c3b68d38fb0c5a56e6076ba1c9a713b6ad1fec41674906" dependencies = [ "gix", "heck", @@ -3229,12 +3247,13 @@ dependencies = [ "prost", "prost-build", "prost-types", + "protobuf-src", "schemars", "semver", "serde", "serde_json", "serde_yaml", - "syn 2.0.12", + "syn 2.0.13", "typify", "walkdir", ] @@ -3258,9 +3277,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.12" +version = "2.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79d9531f94112cfc3e4c8f5f02cb2b58f72c97b7efd85f70203cc6d8efda5927" +checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" dependencies = [ "proc-macro2", "quote", @@ -3312,7 +3331,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -3416,7 +3435,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.12", + "syn 2.0.13", ] [[package]] @@ -3842,22 +3861,22 @@ version = "0.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] name = "windows" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets", + "windows-targets 0.48.0", ] [[package]] @@ -3866,7 +3885,7 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.2", ] [[package]] @@ -3875,13 +3894,28 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -3890,42 +3924,84 @@ version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "winreg" version = "0.10.1" diff --git a/Cargo.toml b/Cargo.toml index ddabb2b8c..1a51e61ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,12 +35,12 @@ default = ["mimalloc"] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad", features = ["pyarrow", "avro"] } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad", features = ["pyarrow"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "c09edad" } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69", features = ["pyarrow", "avro"] } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69", features = ["pyarrow"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69", features = ["protoc"] } uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" From 47dd8e573956f6dcfb6d378b08e17ced25264700 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 4 Apr 2023 08:54:38 -0600 Subject: [PATCH 013/413] Fix Mac/Win release builds in CI (#313) --- .github/workflows/build.yml | 21 ++++++++------------- Cargo.toml | 3 ++- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5d3199432..6296bf49e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -52,17 +52,6 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Setup Cpp - uses: aminya/setup-cpp@v1 - with: - compiler: gcc - vcvarsall: ${{ contains(matrix.os, 'windows') }} - cmake: true - ninja: true - vcpkg: true - cppcheck: true - clangtidy: true - - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} @@ -84,6 +73,12 @@ jobs: name: python-wheel-license path: . + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Build Python package run: maturin build --release --strip @@ -126,7 +121,7 @@ jobs: target: x86_64 manylinux: auto rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 - args: --release --manylinux 2014 + args: --release --manylinux 2014 --features protoc - name: Archive wheels uses: actions/upload-artifact@v3 with: @@ -152,7 +147,7 @@ jobs: rust-toolchain: stable manylinux: auto rustup-components: rust-std rustfmt - args: --release --sdist --out dist + args: --release --sdist --out dist --features protoc - name: Archive wheels uses: actions/upload-artifact@v3 with: diff --git a/Cargo.toml b/Cargo.toml index 1a51e61ee..bf6cfe9ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ include = ["/src", "/LICENSE.txt"] [features] default = ["mimalloc"] +protoc = [ "datafusion-substrait/protoc" ] [dependencies] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } @@ -40,7 +41,7 @@ datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", re datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69", features = ["protoc"] } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" From 59f90f65010eddd511dbb4460417f098d7fe20f7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 5 Apr 2023 20:21:59 -0600 Subject: [PATCH 014/413] install protoc in docs workflow (#314) --- .github/workflows/docs.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index bef42d538..d9e7ad4ad 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -37,6 +37,12 @@ jobs: with: python-version: "3.10" + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + - name: Install dependencies run: | set -x From 31418b5f5cdc876142c34a3b15bbd89fe3a34a04 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 6 Apr 2023 07:56:33 -0600 Subject: [PATCH 015/413] Fix documentation generation in CI (#315) * install protoc in docs workflow * fix docs gen --- docs/README.md | 6 +++--- docs/build.sh | 2 -- docs/requirements.txt | 2 +- docs/source/api.rst | 2 +- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/README.md b/docs/README.md index a6f4998c8..6d8b2d57d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,8 +20,8 @@ # DataFusion Documentation This folder contains the source content of the [python api](./source/api). -These are both published to https://arrow.apache.org/datafusion/ -as part of the release process. +This is published to https://arrow.apache.org/datafusion-python/ by a GitHub action +when changes are merged to the main branch. ## Dependencies @@ -29,7 +29,7 @@ It's recommended to install build dependencies and build the documentation inside a Python virtualenv. - Python -- `pip install -r requirements.txt` +- `pip3 install -r requirements.txt` ## Build & Preview diff --git a/docs/build.sh b/docs/build.sh index 3f24f8eec..7e8bb0b54 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -23,6 +23,4 @@ rm -rf build 2> /dev/null rm -rf temp 2> /dev/null mkdir temp cp -rf source/* temp/ -# replace relative URLs with absolute URLs -#sed -i 's/\.\.\/\.\.\/\.\.\//https:\/\/github.com\/apache\/arrow-datafusion\/blob\/master\//g' temp/contributor-guide/index.md make SOURCEDIR=`pwd`/temp html \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index 2af563587..8eb744968 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -sphinx +sphinx==5.3.0 pydata-sphinx-theme==0.8.0 myst-parser maturin diff --git a/docs/source/api.rst b/docs/source/api.rst index a3e7e24df..a5d65433d 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -27,6 +27,6 @@ API Reference api/config api/dataframe api/execution_context - api/expr + api/expression api/functions api/object_store From bb54b10a2532915fc24b49664fa7db0c5dd14b5b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 10 Apr 2023 14:12:06 -0600 Subject: [PATCH 016/413] Source wheel fix (#319) --- Cargo.toml | 2 +- pyproject.toml | 2 +- requirements.in | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bf6cfe9ac..361a06e04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ readme = "README.md" license = "Apache-2.0" edition = "2021" rust-version = "1.64" -include = ["/src", "/LICENSE.txt"] +include = ["/src", "/datafusion", "/LICENSE.txt", "pyproject.toml", "Cargo.toml", "Cargo.lock"] [features] default = ["mimalloc"] diff --git a/pyproject.toml b/pyproject.toml index 6d8a9d213..87d0be6cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [build-system] -requires = ["maturin>=0.11,<0.15"] +requires = ["maturin>=0.14,<0.15"] build-backend = "maturin" [project] diff --git a/requirements.in b/requirements.in index 7ee6a48dc..098e9f809 100644 --- a/requirements.in +++ b/requirements.in @@ -18,7 +18,7 @@ black flake8 isort -maturin +maturin>=0.14 mypy numpy pyarrow From 5c90187207e218393297ff3cbe4f08b69049d224 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 10 Apr 2023 17:37:22 -0600 Subject: [PATCH 017/413] Prepare for 22.0.0 release (#320) * Prepare for 22.0.0 release * fix tests * lint --- CHANGELOG.md | 15 +++ Cargo.lock | 150 ++++++++++++++++------------- Cargo.toml | 14 +-- datafusion/tests/test_context.py | 4 +- datafusion/tests/test_dataframe.py | 4 +- datafusion/tests/test_indexing.py | 2 +- 6 files changed, 112 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35982d9a9..7f4a35c70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,21 @@ # Changelog +## [22.0.0](https://github.com/apache/arrow-datafusion-python/tree/22.0.0) (2023-04-10) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/21.0.0...22.0.0) + +**Merged pull requests:** + +- Fix invalid build yaml [#308](https://github.com/apache/arrow-datafusion-python/pull/308) (andygrove) +- Try fix release build [#309](https://github.com/apache/arrow-datafusion-python/pull/309) (andygrove) +- Fix release build [#310](https://github.com/apache/arrow-datafusion-python/pull/310) (andygrove) +- Enable datafusion-substrait protoc feature, to remove compile-time dependency on protoc [#312](https://github.com/apache/arrow-datafusion-python/pull/312) (andygrove) +- Fix Mac/Win release builds in CI [#313](https://github.com/apache/arrow-datafusion-python/pull/313) (andygrove) +- install protoc in docs workflow [#314](https://github.com/apache/arrow-datafusion-python/pull/314) (andygrove) +- Fix documentation generation in CI [#315](https://github.com/apache/arrow-datafusion-python/pull/315) (andygrove) +- Source wheel fix [#319](https://github.com/apache/arrow-datafusion-python/pull/319) (andygrove) + ## [21.0.0](https://github.com/apache/arrow-datafusion-python/tree/21.0.0) (2023-03-30) [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/20.0.0...21.0.0) diff --git a/Cargo.lock b/Cargo.lock index 5d5196067..3451163dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -293,7 +293,7 @@ version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d04f17f7b86ded0b5baf98fe6123391c4343e031acc3ccc5fa604cc180bff220" dependencies = [ - "bitflags 2.0.2", + "bitflags 2.1.0", ] [[package]] @@ -393,9 +393,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" +checksum = "c70beb79cbb5ce9c4f8e20849978f34225931f665bb49efa6982875a4d5facb3" [[package]] name = "blake2" @@ -543,9 +543,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa48fa079165080f11d7753fd0bc175b7d391f276b965fe4b55bfad67856e463" +checksum = "cf9cc2b23599e6d7479755f3594285efb3f74a1bdca7a7374948bc831e23a552" dependencies = [ "chrono", "chrono-tz-build", @@ -738,8 +738,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bdb93fee4f30368f1f71bfd5cd28882ec9fab0183db7924827b76129d33227c" dependencies = [ "ahash", "apache-avro", @@ -787,8 +788,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82401ce129e601d406012b6d718f8978ba84c386e1c342fa155877120d68824" dependencies = [ "apache-avro", "arrow", @@ -803,8 +805,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b08b2078aed21a27239cd93f3015e492a58b0d50ebeeaf8d2236cf108ef583ce" dependencies = [ "dashmap", "datafusion-common", @@ -820,8 +823,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b5b977ce9695fb4c67614266ec57f384fc11e9a9f9b3e6d0e62b9c5a9f2c1f" dependencies = [ "ahash", "arrow", @@ -831,8 +835,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b2bb9e73ed778d1bc5af63a270f0154bf6eab5099c77668a6362296888e46b" dependencies = [ "arrow", "async-trait", @@ -848,8 +853,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80cd8ea5ab0a07b1b2a3e17d5909f1b1035bd129ffeeb5c66842a32e682f8f79" dependencies = [ "ahash", "arrow", @@ -879,7 +885,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "21.0.0" +version = "22.0.0" dependencies = [ "async-trait", "datafusion", @@ -903,8 +909,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a95d6badab19fd6e9195fdc5209ac0a7e5ce9bcdedc67767b9ffc1b4e645760" dependencies = [ "arrow", "datafusion-common", @@ -914,9 +921,11 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a78f8fc67123c4357e63bc0c87622a2a663d26f074958d749a633d0ecde90f" dependencies = [ + "arrow", "arrow-schema", "datafusion-common", "datafusion-expr", @@ -926,8 +935,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "21.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=2191a69#2191a69b2d48b0b7230c9da5a75f86a4e659361b" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae6ed64a2005f0d78f2b1b3ec3f8148183f4523d5d364e5367115f8d8a82b7df" dependencies = [ "async-recursion", "chrono", @@ -1005,13 +1015,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1035,14 +1045,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ "cfg-if", "libc", "redox_syscall 0.2.16", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1187,9 +1197,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -1198,9 +1208,9 @@ dependencies = [ [[package]] name = "gix" -version = "0.42.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd5e0d9c5df90c9b4d325ec716762beb7d6c1465a4049fec5c4f6b72e7824656" +checksum = "c256ea71cc1967faaefdaad15f334146b7c806f12460dcafd3afed845c8c78dd" dependencies = [ "gix-actor", "gix-attributes", @@ -1296,9 +1306,9 @@ dependencies = [ [[package]] name = "gix-config" -version = "0.19.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aa7d7dd60256b7a0c0506a1d708ec92767c2662ee57b3301b538eaa3e064f8a" +checksum = "7fbad5ce54a8fc997acc50febd89ec80fa6e97cb7f8d0654cb229936407489d8" dependencies = [ "bstr", "gix-config-value", @@ -1307,6 +1317,7 @@ dependencies = [ "gix-path", "gix-ref", "gix-sec", + "log", "memchr", "nom", "once_cell", @@ -1940,13 +1951,13 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" +checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2060,9 +2071,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.140" +version = "0.2.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" [[package]] name = "libflate" @@ -2092,9 +2103,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libmimalloc-sys" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2c45001fb108f37d41bed8efd715769acb14674c1ce3e266ef0e317ef5f877" +checksum = "43a558e3d911bc3c7bfc8c78bc580b404d6e51c1cefbf656e176a94b49b0df40" dependencies = [ "cc", "libc", @@ -2200,9 +2211,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dcb174b18635f7561a0c6c9fc2ce57218ac7523cf72c50af80e2d79ab8f3ba1" +checksum = "3d88dad3f985ec267a3fcb7a1726f5cb1a7e8cad8b646e70a84f967210df23da" dependencies = [ "libmimalloc-sys", ] @@ -2237,7 +2248,7 @@ dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -2427,7 +2438,7 @@ dependencies = [ "libc", "redox_syscall 0.2.16", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -2898,16 +2909,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.6" +version = "0.37.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3077,9 +3088,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.19" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82e6c8c047aa50a7328632d067bcae6ef38772a79e28daf32f735e0e4f3dd10" +checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" dependencies = [ "indexmap", "itoa", @@ -3237,9 +3248,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.6.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b121815df0d0ce3f2c3b68d38fb0c5a56e6076ba1c9a713b6ad1fec41674906" +checksum = "fcd7c95895a69f92b0491cb0764d49140f57ad918a8abb3b7ec7f8e507d2a240" dependencies = [ "gix", "heck", @@ -3302,7 +3313,7 @@ dependencies = [ "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -3424,7 +3435,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -3636,9 +3647,9 @@ checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" [[package]] name = "unsafe-libyaml" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad2024452afd3874bf539695e04af6732ba06517424dbf958fdb16a01f3bef6c" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" [[package]] name = "untrusted" @@ -3659,9 +3670,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" +checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" dependencies = [ "getrandom", "serde", @@ -3888,6 +3899,15 @@ dependencies = [ "windows-targets 0.42.2", ] +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -4056,7 +4076,7 @@ version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe 6.0.4+zstd.1.5.4", + "zstd-safe 6.0.5+zstd.1.5.4", ] [[package]] @@ -4071,9 +4091,9 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "6.0.4+zstd.1.5.4" +version = "6.0.5+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7afb4b54b8910cf5447638cb54bf4e8a65cbedd783af98b98c62ffe91f185543" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" dependencies = [ "libc", "zstd-sys", @@ -4081,9 +4101,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 361a06e04..5ca3eee50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "21.0.0" +version = "22.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69", features = ["pyarrow", "avro"] } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69", features = ["pyarrow"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "2191a69" } +datafusion = { version = "22.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "22.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "22.0.0" } +datafusion-optimizer = { version = "22.0.0" } +datafusion-sql = { version = "22.0.0" } +datafusion-substrait = { version = "22.0.0" } uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" diff --git a/datafusion/tests/test_context.py b/datafusion/tests/test_context.py index 1aea21c21..6b1223a16 100644 --- a/datafusion/tests/test_context.py +++ b/datafusion/tests/test_context.py @@ -282,10 +282,10 @@ def test_dataset_filter_nested_data(ctx): # This filter will not be pushed down to DatasetExec since it # isn't supported - df = df.select( + df = df.filter(column("nested_data")["b"] > literal(5)).select( column("nested_data")["a"] + column("nested_data")["b"], column("nested_data")["a"] - column("nested_data")["b"], - ).filter(column("nested_data")["b"] > literal(5)) + ) result = df.collect() diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index c6ef95772..cd78f3c88 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -84,10 +84,10 @@ def test_select_columns(df): def test_filter(df): - df = df.select( + df = df.filter(column("a") > literal(2)).select( column("a") + column("b"), column("a") - column("b"), - ).filter(column("a") > literal(2)) + ) # execute and collect the first (and only) batch result = df.collect()[0] diff --git a/datafusion/tests/test_indexing.py b/datafusion/tests/test_indexing.py index 1c7f8627d..8ca3eab19 100644 --- a/datafusion/tests/test_indexing.py +++ b/datafusion/tests/test_indexing.py @@ -44,7 +44,7 @@ def test_err(df): with pytest.raises(Exception) as e_info: df["c"] - assert 'Schema error: No field named "c"' in e_info.value.args[0] + assert "Schema error: No field named c." in e_info.value.args[0] with pytest.raises(Exception) as e_info: df[1] From 6ef1c5b554c452bbae2d25b44308722ccfa8fbf7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 12 Apr 2023 04:26:52 -0600 Subject: [PATCH 018/413] Improve API docs, README, and examples for configuring context (#321) --- README.md | 33 +++++++++++++++++++++++ dev/release/rat_exclude_files.txt | 2 +- docs/README.md | 4 +-- docs/source/api/config.rst | 27 ------------------- docs/source/api/execution_context.rst | 4 ++- examples/README.md | 4 +++ examples/create-context.py | 39 +++++++++++++++++++++++++++ src/context.rs | 29 +++++++++++++++----- 8 files changed, 104 insertions(+), 38 deletions(-) delete mode 100644 docs/source/api/config.rst create mode 100644 examples/create-context.py diff --git a/README.md b/README.md index 7c29defdc..506a38227 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,39 @@ This produces the following chart: ![Chart](examples/chart.png) +## Configuration + +It is possible to configure runtime (memory and disk settings) and configuration settings when creating a context. + +```python +runtime = ( + RuntimeConfig() + .with_disk_manager_os() + .with_fair_spill_pool(10000000) +) +config = ( + SessionConfig() + .with_create_default_catalog_and_schema(True) + .with_default_catalog_and_schema("foo", "bar") + .with_target_partitions(8) + .with_information_schema(True) + .with_repartition_joins(False) + .with_repartition_aggregations(False) + .with_repartition_windows(False) + .with_parquet_pruning(False) + .set("datafusion.execution.parquet.pushdown_filters", "true") +) +ctx = SessionContext(config, runtime) +``` + +Refer to the [API documentation](https://arrow.apache.org/datafusion-python/#api-reference) for more information. + +Printing the context will show the current configuration settings. + +```python +print(ctx) +``` + ## More Examples See [examples](examples/README.md) for more information. diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index db5379d89..c7754f350 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -42,4 +42,4 @@ Cargo.lock .history *rat.txt */.git -docs.yaml \ No newline at end of file +.github/* \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index 6d8b2d57d..04f46a907 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ # DataFusion Documentation This folder contains the source content of the [python api](./source/api). -This is published to https://arrow.apache.org/datafusion-python/ by a GitHub action +This is published to https://arrow.apache.org/datafusion-python/ by a GitHub action when changes are merged to the main branch. ## Dependencies @@ -61,4 +61,4 @@ version of the docs, follow these steps: - `cp -rT ./build/html/ ../../arrow-site/datafusion/` (doesn't work on mac) - `rsync -avzr ./build/html/ ../../arrow-site/datafusion/` -5. Commit changes in `arrow-site` and send a PR. \ No newline at end of file +5. Commit changes in `arrow-site` and send a PR. diff --git a/docs/source/api/config.rst b/docs/source/api/config.rst deleted file mode 100644 index df244aec5..000000000 --- a/docs/source/api/config.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api.config: -.. currentmodule:: datafusion - -Config -========= - -.. autosummary:: - :toctree: ../generated/ - - Config diff --git a/docs/source/api/execution_context.rst b/docs/source/api/execution_context.rst index 5b7e0f82f..a3bda76d7 100644 --- a/docs/source/api/execution_context.rst +++ b/docs/source/api/execution_context.rst @@ -19,9 +19,11 @@ .. currentmodule:: datafusion SessionContext -================ +============== .. autosummary:: :toctree: ../generated/ + SessionConfig + RuntimeConfig SessionContext diff --git a/examples/README.md b/examples/README.md index 2c4775ea4..82405955b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -27,6 +27,10 @@ Here is a direct link to the file used in the examples: - https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet +### Creating a SessionContext + +- [Creating a SessionContext](./create-context.py) + ### Executing Queries with DataFusion - [Query a Parquet file using SQL](./sql-parquet.py) diff --git a/examples/create-context.py b/examples/create-context.py new file mode 100644 index 000000000..3184d4085 --- /dev/null +++ b/examples/create-context.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datafusion import RuntimeConfig, SessionConfig, SessionContext + +# create a session context with default settings +ctx = SessionContext() +print(ctx) + +# create a session context with explicit runtime and config settings +runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) +config = ( + SessionConfig() + .with_create_default_catalog_and_schema(True) + .with_default_catalog_and_schema("foo", "bar") + .with_target_partitions(8) + .with_information_schema(True) + .with_repartition_joins(False) + .with_repartition_aggregations(False) + .with_repartition_windows(False) + .with_parquet_pruning(False) + .set("datafusion.execution.parquet.pushdown_filters", "true") +) +ctx = SessionContext(config, runtime) +print(ctx) diff --git a/src/context.rs b/src/context.rs index 448fa7742..0ba120022 100644 --- a/src/context.rs +++ b/src/context.rs @@ -55,6 +55,7 @@ use pyo3::types::PyTuple; use tokio::runtime::Runtime; use tokio::task::JoinHandle; +/// Configuration options for a SessionContext #[pyclass(name = "SessionConfig", module = "datafusion", subclass, unsendable)] #[derive(Clone, Default)] pub(crate) struct PySessionConfig { @@ -141,8 +142,13 @@ impl PySessionConfig { fn with_parquet_pruning(&self, enabled: bool) -> Self { Self::from(self.config.clone().with_parquet_pruning(enabled)) } + + fn set(&self, key: &str, value: &str) -> Self { + Self::from(self.config.clone().set_str(key, value)) + } } +/// Runtime options for a SessionContext #[pyclass(name = "RuntimeConfig", module = "datafusion", subclass, unsendable)] #[derive(Clone)] pub(crate) struct PyRuntimeConfig { @@ -549,8 +555,8 @@ impl PySessionContext { Ok(PyDataFrame::new(self.ctx.read_empty()?)) } - fn session_id(&self) -> PyResult { - Ok(self.ctx.session_id()) + fn session_id(&self) -> String { + self.ctx.session_id() } #[allow(clippy::too_many_arguments)] @@ -684,11 +690,20 @@ impl PySessionContext { } fn __repr__(&self) -> PyResult { - let id = self.session_id(); - match id { - Ok(value) => Ok(format!("SessionContext(session_id={value})")), - Err(err) => Ok(format!("Error: {:?}", err.to_string())), - } + let config = self.ctx.copied_config(); + let mut config_entries = config + .options() + .entries() + .iter() + .filter(|e| e.value.is_some()) + .map(|e| format!("{} = {}", e.key, e.value.as_ref().unwrap())) + .collect::>(); + config_entries.sort(); + Ok(format!( + "SessionContext: id={}; configs=[\n\t{}]", + self.session_id(), + config_entries.join("\n\t") + )) } /// Execute a partition of an execution plan and return a stream of record batches From 94936380e58a266f5dd5de6b70a06d3aa36fbe22 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 19 Apr 2023 12:08:04 -0400 Subject: [PATCH 019/413] Osx build linker args (#330) * Add linker flags for macOS to ensure proper linking with macOS libpython library * Include custom build.rs script * Add apache license to build.rs * Add apache license v2 --- .cargo/config.toml | 12 +++++ Cargo.lock | 111 +++++++++++++++++++++++---------------------- Cargo.toml | 3 ++ build.rs | 20 ++++++++ 4 files changed, 91 insertions(+), 55 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 build.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..91a099a61 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,12 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + diff --git a/Cargo.lock b/Cargo.lock index 3451163dc..69505061f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,7 +350,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -361,7 +361,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -703,7 +703,7 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -720,7 +720,7 @@ checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -899,9 +899,10 @@ dependencies = [ "object_store", "parking_lot", "pyo3", + "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.13", + "syn 2.0.15", "tokio", "url", "uuid", @@ -988,9 +989,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dunce" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd4b30a6560bbd9b4620f4de34c3f14f60848e58a9b7216801afcb4c7b31c3c" +checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "dyn-clone" @@ -1152,7 +1153,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -1423,9 +1424,9 @@ dependencies = [ [[package]] name = "gix-hash" -version = "0.10.3" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0c5a9f4d621d4f4ea046bb331df5c746ca735b8cae5b234cc2be70ee4dbef0" +checksum = "2a258595457bc192d1f1c59d0d168a1e34e2be9b97a614e14995416185de41a7" dependencies = [ "hex", "thiserror", @@ -1433,9 +1434,9 @@ dependencies = [ [[package]] name = "gix-hashtable" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9609c1b8f36f12968e6a6098f7cdb52004f7d42d570f47a2d6d7c16612f19acb" +checksum = "e4e55e40dfd694884f0eb78796c5bddcf2f8b295dace47039099dd7e76534973" dependencies = [ "gix-hash", "hashbrown 0.13.2", @@ -1715,9 +1716,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.16" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" +checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" dependencies = [ "bytes", "fnv", @@ -1829,9 +1830,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.25" +version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899" +checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ "bytes", "futures-channel", @@ -2122,9 +2123,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59d8c75012853d2e872fb56bc8a2e53718e2cafe1a4c823143141c6d90c322f" +checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" [[package]] name = "lock_api" @@ -2574,7 +2575,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" dependencies = [ "proc-macro2", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -2600,9 +2601,9 @@ checksum = "9516b775656bc3e8985e19cd4b8c0c0de045095074e453d2c0a513b5f978392d" [[package]] name = "prost" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48e50df39172a3e7eb17e14642445da64996989bc212b583015435d39a58537" +checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", "prost-derive", @@ -2610,9 +2611,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c828f93f5ca4826f97fedcbd3f9a536c16b12cff3dbbb4a007f932bbad95b12" +checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", @@ -2630,9 +2631,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea9b0f8cbe5e15a8a042d030bd96668db28ecb567ec37d691971ff5731d2b1b" +checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools", @@ -2643,9 +2644,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379119666929a1afd7a043aa6cf96fa67a6dce9af60c88095a4686dbce4c9c88" +checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ "prost", ] @@ -2661,9 +2662,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfb848f80438f926a9ebddf0a539ed6065434fd7aae03a89312a9821f81b8501" +checksum = "e3b1ac5b3731ba34fdaa9785f8d74d17448cd18f30cf19e0c7e7b1fdb5272109" dependencies = [ "cfg-if", "indoc", @@ -2678,9 +2679,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a42e7f42e917ce6664c832d5eee481ad514c98250c49e0b03b20593e2c7ed0" +checksum = "9cb946f5ac61bb61a5014924910d936ebd2b23b705f7a4a3c40b05c720b079a3" dependencies = [ "once_cell", "target-lexicon", @@ -2688,9 +2689,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0707f0ab26826fe4ccd59b69106e9df5e12d097457c7b8f9c0fd1d2743eec4d" +checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" dependencies = [ "libc", "pyo3-build-config", @@ -2698,9 +2699,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978d18e61465ecd389e1f235ff5a467146dc4e3c3968b90d274fe73a5dd4a438" +checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2710,9 +2711,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.18.2" +version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e0e1128f85ce3fca66e435e08aa2089a2689c1c48ce97803e13f63124058462" +checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" dependencies = [ "proc-macro2", "quote", @@ -2727,9 +2728,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quick-xml" -version = "0.28.1" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5c1a97b1bc42b1d550bfb48d4262153fe400a12bab1511821736f7eac76d7e2" +checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" dependencies = [ "memchr", "serde", @@ -2909,9 +2910,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.11" +version = "0.37.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85597d61f83914ddeba6a47b3b8ffe7365107221c2e557ed94426489fefb5f77" +checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" dependencies = [ "bitflags 1.3.2", "errno", @@ -3023,22 +3024,22 @@ checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" [[package]] name = "serde" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" +checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.159" +version = "1.0.160" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" +checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -3054,9 +3055,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -3248,9 +3249,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcd7c95895a69f92b0491cb0764d49140f57ad918a8abb3b7ec7f8e507d2a240" +checksum = "e3ae64fb7ad0670c7d6d53d57b1b91beb2212afc30e164cc8edb02d6b2cff32a" dependencies = [ "gix", "heck", @@ -3264,7 +3265,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.13", + "syn 2.0.15", "typify", "walkdir", ] @@ -3288,9 +3289,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.13" +version = "2.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec" +checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" dependencies = [ "proc-macro2", "quote", @@ -3342,7 +3343,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] @@ -3446,7 +3447,7 @@ checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.13", + "syn 2.0.15", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5ca3eee50..22a2989ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,9 @@ regex-syntax = "0.6.28" syn = "2.0.11" url = "2.2" +[build-dependencies] +pyo3-build-config = "0.18.3" + [lib] name = "datafusion_python" crate-type = ["cdylib", "rlib"] diff --git a/build.rs b/build.rs new file mode 100644 index 000000000..4878d8b0e --- /dev/null +++ b/build.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +fn main() { + pyo3_build_config::add_extension_module_link_args(); +} From 16298058cb10139a96e1b10fb7424a36c3c1c686 Mon Sep 17 00:00:00 2001 From: "r.4ntix" Date: Sun, 23 Apr 2023 23:11:07 +0800 Subject: [PATCH 020/413] Add requirements file for python 3.11 (#332) --- requirements-311.txt | 199 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 requirements-311.txt diff --git a/requirements-311.txt b/requirements-311.txt new file mode 100644 index 000000000..9161e3f5b --- /dev/null +++ b/requirements-311.txt @@ -0,0 +1,199 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes --output-file=requirements-311.txt +# +black==23.3.0 \ + --hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \ + --hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \ + --hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \ + --hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \ + --hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \ + --hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \ + --hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \ + --hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \ + --hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \ + --hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \ + --hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \ + --hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \ + --hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \ + --hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \ + --hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \ + --hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \ + --hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \ + --hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \ + --hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \ + --hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \ + --hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \ + --hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \ + --hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \ + --hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \ + --hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3 + # via -r requirements.in +click==8.1.3 \ + --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ + --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 + # via black +flake8==6.0.0 \ + --hash=sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7 \ + --hash=sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181 + # via -r requirements.in +iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + # via pytest +isort==5.12.0 \ + --hash=sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504 \ + --hash=sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6 + # via -r requirements.in +maturin==0.14.17 \ + --hash=sha256:0308d4124311c92ec5c08600f93385a5ff16f7764410585261d1ca0aed739764 \ + --hash=sha256:2684302ded1559a26635ec45e91ae7913aeb386a8b658d5c2eb13ff410df7930 \ + --hash=sha256:3bf173adac3e9f84a84123ec0b5b4b28b2de79446a49da2b4549016614025b83 \ + --hash=sha256:64dd1b017eab2f9c938d833522bb505506908d824388927011dbdfff26ccea2c \ + --hash=sha256:683e1127b54b9fbdbd661ce3b27b073bd5d7c51d7e980870f4caffdb40f364b0 \ + --hash=sha256:7fef3950a34c0b5c1806637169fca12f89b7440d7608832bf7765b34f394b06a \ + --hash=sha256:8076993118089abb19e7d2312b138afea418dccea60be4198ab9f348618b47df \ + --hash=sha256:a7f06984bd3ffa4ab7be67f7352353a486f262d7a0dbd4dc305944e0b09c8f35 \ + --hash=sha256:abfd98529cfab59deb59639962d93df72121013d5151a5f01c66cda19d9c88a9 \ + --hash=sha256:bb8f8ac375d022c5de5da3efed79e712129de13ebdb9f668b96bf5df0673bc03 \ + --hash=sha256:cf1429a7854ecd4830bb42407d30ff0625c3f6480fa7b1e90da2896ba7a1b2fc \ + --hash=sha256:dcd44c2fa66d79497e2f4361654ed0d1c2ef1c58139469e8e0083409f948eea4 \ + --hash=sha256:fb4e3311e8ce707843235fbe8748a05a3ae166c3efd6d2aa335b53dfc2bd3b88 + # via -r requirements.in +mccabe==0.7.0 \ + --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ + --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e + # via flake8 +mypy==1.2.0 \ + --hash=sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521 \ + --hash=sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140 \ + --hash=sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48 \ + --hash=sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128 \ + --hash=sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336 \ + --hash=sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a \ + --hash=sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41 \ + --hash=sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f \ + --hash=sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e \ + --hash=sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8 \ + --hash=sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238 \ + --hash=sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119 \ + --hash=sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb \ + --hash=sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d \ + --hash=sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed \ + --hash=sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9 \ + --hash=sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e \ + --hash=sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a \ + --hash=sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5 \ + --hash=sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950 \ + --hash=sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937 \ + --hash=sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394 \ + --hash=sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6 \ + --hash=sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602 \ + --hash=sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1 \ + --hash=sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba + # via -r requirements.in +mypy-extensions==1.0.0 \ + --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ + --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 + # via + # black + # mypy +numpy==1.24.2 \ + --hash=sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22 \ + --hash=sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f \ + --hash=sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9 \ + --hash=sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96 \ + --hash=sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0 \ + --hash=sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a \ + --hash=sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281 \ + --hash=sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04 \ + --hash=sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468 \ + --hash=sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253 \ + --hash=sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756 \ + --hash=sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a \ + --hash=sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb \ + --hash=sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d \ + --hash=sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0 \ + --hash=sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910 \ + --hash=sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978 \ + --hash=sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5 \ + --hash=sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f \ + --hash=sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a \ + --hash=sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5 \ + --hash=sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2 \ + --hash=sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d \ + --hash=sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95 \ + --hash=sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5 \ + --hash=sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d \ + --hash=sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780 \ + --hash=sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa + # via + # -r requirements.in + # pyarrow +packaging==23.1 \ + --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ + --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f + # via + # black + # pytest +pathspec==0.11.1 \ + --hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \ + --hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293 + # via black +platformdirs==3.2.0 \ + --hash=sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08 \ + --hash=sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e + # via black +pluggy==1.0.0 \ + --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ + --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 + # via pytest +pyarrow==11.0.0 \ + --hash=sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc \ + --hash=sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c \ + --hash=sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06 \ + --hash=sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841 \ + --hash=sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2 \ + --hash=sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835 \ + --hash=sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854 \ + --hash=sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2 \ + --hash=sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e \ + --hash=sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97 \ + --hash=sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d \ + --hash=sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418 \ + --hash=sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba \ + --hash=sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70 \ + --hash=sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85 \ + --hash=sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00 \ + --hash=sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56 \ + --hash=sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689 \ + --hash=sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c \ + --hash=sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4 \ + --hash=sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841 \ + --hash=sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b \ + --hash=sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca \ + --hash=sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5 \ + --hash=sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34 + # via -r requirements.in +pycodestyle==2.10.0 \ + --hash=sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053 \ + --hash=sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610 + # via flake8 +pyflakes==3.0.1 \ + --hash=sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf \ + --hash=sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd + # via flake8 +pytest==7.3.1 \ + --hash=sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362 \ + --hash=sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3 + # via -r requirements.in +toml==0.10.2 \ + --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ + --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f + # via -r requirements.in +typing-extensions==4.5.0 \ + --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ + --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 + # via mypy From fb3c6e22789a2ddf33505c0b29a4464105c6c1ba Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 23 Apr 2023 13:52:08 -0600 Subject: [PATCH 021/413] mac arm64 build (#338) --- .github/workflows/build.yml | 52 +++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6296bf49e..c667dab80 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -99,6 +99,58 @@ jobs: name: dist path: target/wheels/* + build-macos-aarch64: + needs: [generate-license] + name: Mac arm64 + runs-on: macos-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Set up Rust targets + run: rustup target add aarch64-apple-darwin + + - name: Upgrade pip + run: python -m pip install --upgrade pip + + - name: Install maturin + run: pip install maturin==0.14.2 + + - run: rm LICENSE.txt + - name: Download LICENSE.txt + uses: actions/download-artifact@v3 + with: + name: python-wheel-license + path: . + + - name: Install Protoc + uses: arduino/setup-protoc@v1 + with: + version: '3.x' + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Build Python package + run: maturin build --release --strip --target aarch64-apple-darwin + - name: List Mac wheels + run: find target/wheels/ + + - name: Archive wheels + uses: actions/upload-artifact@v3 + with: + name: dist + path: target/wheels/* + build-manylinux: needs: [generate-license] name: Manylinux From b6c600d301ea56f9f56440f2a6e3670a0e917ce5 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sun, 23 Apr 2023 15:52:28 -0400 Subject: [PATCH 022/413] Add conda.yaml baseline workflow file (#281) * point to custom upstream arrow-datafusion with updated build script * Add conda.yaml baseline workflow file * Change conda recipe path * Add some debug output * fix syntax issues * update rustfmt path * test invoking rustfmt * Allow for RUSTFMT environment to pass through to the conda build * conda build updates * github workflow syntax issue * workflow syntax * more syntax * updates * use minimal rustup profile * use minimal rustup profile * use minimal rustup profile * use minimal rustup profile * use minimal rustup profile * changes * changes * trigger workflow rebuild * Add container * Ouput Rust env varialbes * full rustup command path * Point to typify fork for testing * Adjust conda CI workflow * Adjust conda build instead of mambabuild * Adjust command * Install conda-build * Some initial changes to get noarch builds working * Remove beta rust toolchain from CI * Add libprotobuf to host/build deps * Bump libprotobuf version * Revert back to 3 for libprotobuf --------- Co-authored-by: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> --- .github/workflows/conda.yml | 61 +++++++++++++++++++++++++++++++++++++ .github/workflows/test.yaml | 1 - Cargo.lock | 12 ++++---- conda/recipes/build.sh | 20 ------------ conda/recipes/meta.yaml | 25 ++++++++++----- pyproject.toml | 2 +- 6 files changed, 85 insertions(+), 36 deletions(-) create mode 100644 .github/workflows/conda.yml delete mode 100644 conda/recipes/build.sh diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 000000000..9853230de --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,61 @@ +name: Build conda nightly +on: [push, pull_request] + +# Cancel any already running instances of this build +concurrency: + group: conda-${{ github.head_ref }} + cancel-in-progress: true + +# Required shell entrypoint to have properly activated conda environments +defaults: + run: + shell: bash -l {0} + +jobs: + conda: + name: Build (and optionally upload) the conda nightly + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python + uses: conda-incubator/setup-miniconda@v2.2.0 + with: + miniforge-variant: Mambaforge + use-mamba: true + python-version: "3.10" + channel-priority: strict + - name: Install dependencies + run: | + mamba install boa conda-verify + + which python + pip list + mamba list + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + components: clippy, rustfmt + - name: Build conda package + run: | + # suffix for nightly package versions + export VERSION_SUFFIX=a`date +%y%m%d` + + conda mambabuild conda/recipes \ + --no-anaconda-upload \ + --output-folder . + # - name: Upload conda package + # if: | + # github.event_name == 'push' + # && github.repository == 'apache/arrow-datafusion-python' + # env: + # ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }} + # LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }} + # run: | + # # install anaconda for upload + # mamba install anaconda-client + + # anaconda upload --label $LABEL linux-64/*.tar.bz2 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 164b09e15..c69205be3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -36,7 +36,6 @@ jobs: - "3.10" toolchain: - "stable" - - "beta" # we are not that much eager in walking on the edge yet # - nightly # build stable for only 3.7 diff --git a/Cargo.lock b/Cargo.lock index 69505061f..f4d8e0f7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2072,9 +2072,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.141" +version = "0.2.142" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5" +checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" [[package]] name = "libflate" @@ -2123,9 +2123,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" +checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" [[package]] name = "lock_api" @@ -2910,9 +2910,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.12" +version = "0.37.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659" +checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" dependencies = [ "bitflags 1.3.2", "errno", diff --git a/conda/recipes/build.sh b/conda/recipes/build.sh deleted file mode 100644 index f10be3581..000000000 --- a/conda/recipes/build.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -maturin build -r diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index cce614b2f..48e95eb08 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -15,11 +15,14 @@ # specific language governing permissions and limitations # under the License. -{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %} -{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %} +{% set name = "datafusion" %} +{% set major_minor_patch = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').split('.') %} +{% set new_patch = major_minor_patch[2] | int + 1 %} +{% set version = (major_minor_patch[:2] + [new_patch]) | join('.') + environ.get('VERSION_SUFFIX', '') %} + package: - name: datafusion + name: {{ name|lower }} version: {{ version }} source: @@ -27,20 +30,26 @@ source: build: noarch: python + script: {{ PYTHON }} -m pip install . -vv number: 0 requirements: host: - - {{ compiler('rust') }} - - python - - setuptools - - maturin + - python >=3.6 + - maturin >=0.14,<0.15 + - libprotobuf =3 + - pip run: - - python + - python >=3.6 + - pyarrow >=11.0.0 test: imports: - datafusion + commands: + - pip check + requires: + - pip about: home: https://arrow.apache.org/datafusion diff --git a/pyproject.toml b/pyproject.toml index 87d0be6cf..cf68a33c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ classifier = [ "Programming Language :: Rust", ] dependencies = [ - "pyarrow>=6.0.1", + "pyarrow>=11.0.0", ] [project.urls] From e6e8e128b95ceb91b5baaec6d04389b1c642c9d6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 24 Apr 2023 16:14:29 -0600 Subject: [PATCH 023/413] Prepare for 23.0.0 release (#335) --- CHANGELOG.md | 25 +++-- Cargo.lock | 182 ++++++++++++++++++------------------- Cargo.toml | 14 +-- dev/release/README.md | 12 ++- src/common/data_type.rs | 2 +- src/expr/literal.rs | 4 +- src/expr/subquery_alias.rs | 2 +- src/udaf.rs | 4 +- src/udf.rs | 2 +- 9 files changed, 127 insertions(+), 120 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f4a35c70..b8942f944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,15 @@ # Changelog +## [23.0.0](https://github.com/apache/arrow-datafusion-python/tree/23.0.0) (2023-04-23) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/22.0.0...23.0.0) + +**Merged pull requests:** + +- Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/arrow-datafusion-python/pull/321) (andygrove) +- Osx build linker args [#330](https://github.com/apache/arrow-datafusion-python/pull/330) (jdye64) + ## [22.0.0](https://github.com/apache/arrow-datafusion-python/tree/22.0.0) (2023-04-10) [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/21.0.0...22.0.0) @@ -163,7 +172,7 @@ - Arrow type bindings [\#173](https://github.com/apache/arrow-datafusion-python/pull/173) ([jdye64](https://github.com/jdye64)) - Pyo3 bump [\#171](https://github.com/apache/arrow-datafusion-python/pull/171) ([jdye64](https://github.com/jdye64)) - feature: Add additional aggregation functions [\#170](https://github.com/apache/arrow-datafusion-python/pull/170) ([simicd](https://github.com/simicd)) -- Make from\_substrait\_plan return DataFrame instead of LogicalPlan [\#164](https://github.com/apache/arrow-datafusion-python/pull/164) ([andygrove](https://github.com/andygrove)) +- Make from_substrait_plan return DataFrame instead of LogicalPlan [\#164](https://github.com/apache/arrow-datafusion-python/pull/164) ([andygrove](https://github.com/andygrove)) - feature: Implement count method [\#163](https://github.com/apache/arrow-datafusion-python/pull/163) ([simicd](https://github.com/simicd)) - CI Fixes [\#162](https://github.com/apache/arrow-datafusion-python/pull/162) ([jdye64](https://github.com/jdye64)) - Upgrade to DataFusion 17 [\#160](https://github.com/apache/arrow-datafusion-python/pull/160) ([andygrove](https://github.com/andygrove)) @@ -180,7 +189,7 @@ - Prepare for 0.8.0 release [\#141](https://github.com/apache/arrow-datafusion-python/pull/141) ([andygrove](https://github.com/andygrove)) - Improve README and add more examples [\#137](https://github.com/apache/arrow-datafusion-python/pull/137) ([andygrove](https://github.com/andygrove)) - test: Expand tests for built-in functions [\#129](https://github.com/apache/arrow-datafusion-python/pull/129) ([simicd](https://github.com/simicd)) -- build\(deps\): bump object\_store from 0.5.2 to 0.5.3 [\#126](https://github.com/apache/arrow-datafusion-python/pull/126) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump object_store from 0.5.2 to 0.5.3 [\#126](https://github.com/apache/arrow-datafusion-python/pull/126) ([dependabot[bot]](https://github.com/apps/dependabot)) - build\(deps\): bump mimalloc from 0.1.32 to 0.1.34 [\#125](https://github.com/apache/arrow-datafusion-python/pull/125) ([dependabot[bot]](https://github.com/apps/dependabot)) - Introduce conda directory containing datafusion-dev.yaml conda enviro… [\#124](https://github.com/apache/arrow-datafusion-python/pull/124) ([jdye64](https://github.com/jdye64)) - build\(deps\): bump bzip2 from 0.4.3 to 0.4.4 [\#121](https://github.com/apache/arrow-datafusion-python/pull/121) ([dependabot[bot]](https://github.com/apps/dependabot)) @@ -188,7 +197,7 @@ - build\(deps\): bump async-trait from 0.1.60 to 0.1.61 [\#118](https://github.com/apache/arrow-datafusion-python/pull/118) ([dependabot[bot]](https://github.com/apps/dependabot)) - Upgrade to DataFusion 16.0.0 [\#115](https://github.com/apache/arrow-datafusion-python/pull/115) ([andygrove](https://github.com/andygrove)) - Bump async-trait from 0.1.57 to 0.1.60 [\#114](https://github.com/apache/arrow-datafusion-python/pull/114) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump object\_store from 0.5.1 to 0.5.2 [\#112](https://github.com/apache/arrow-datafusion-python/pull/112) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump object_store from 0.5.1 to 0.5.2 [\#112](https://github.com/apache/arrow-datafusion-python/pull/112) ([dependabot[bot]](https://github.com/apps/dependabot)) - Bump tokio from 1.21.2 to 1.23.0 [\#109](https://github.com/apache/arrow-datafusion-python/pull/109) ([dependabot[bot]](https://github.com/apps/dependabot)) - Add entries for publishing production \(asf-site\) and staging docs [\#107](https://github.com/apache/arrow-datafusion-python/pull/107) ([martin-g](https://github.com/martin-g)) - Add a workflow that builds the docs and deploys them at staged or production [\#104](https://github.com/apache/arrow-datafusion-python/pull/104) ([martin-g](https://github.com/martin-g)) @@ -212,9 +221,9 @@ **Implemented enhancements:** -- Add bindings for datafusion\_common::DFField [\#184](https://github.com/apache/arrow-datafusion-python/issues/184) +- Add bindings for datafusion_common::DFField [\#184](https://github.com/apache/arrow-datafusion-python/issues/184) - Add bindings for DFSchema/DFSchemaRef [\#181](https://github.com/apache/arrow-datafusion-python/issues/181) -- Add bindings for datafusion\_expr Projection [\#179](https://github.com/apache/arrow-datafusion-python/issues/179) +- Add bindings for datafusion_expr Projection [\#179](https://github.com/apache/arrow-datafusion-python/issues/179) - Add bindings for `TableScan` struct from `datafusion_expr::TableScan` [\#177](https://github.com/apache/arrow-datafusion-python/issues/177) - Add a "mapping" struct for types [\#172](https://github.com/apache/arrow-datafusion-python/issues/172) - Improve string representation of datafusion classes \(dataframe, context, expression, ...\) [\#158](https://github.com/apache/arrow-datafusion-python/issues/158) @@ -233,7 +242,7 @@ - Build is broken [\#161](https://github.com/apache/arrow-datafusion-python/issues/161) - Out of memory when sorting [\#157](https://github.com/apache/arrow-datafusion-python/issues/157) -- window\_lead test appears to be non-deterministic [\#135](https://github.com/apache/arrow-datafusion-python/issues/135) +- window_lead test appears to be non-deterministic [\#135](https://github.com/apache/arrow-datafusion-python/issues/135) - Reading csv does not work [\#130](https://github.com/apache/arrow-datafusion-python/issues/130) - Github actions produce a lot of warnings [\#94](https://github.com/apache/arrow-datafusion-python/issues/94) - ASF source release tarball has wrong directory name [\#90](https://github.com/apache/arrow-datafusion-python/issues/90) @@ -252,7 +261,6 @@ [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/0.7.0...0.7.0-rc2) - ## [Unreleased](https://github.com/datafusion-contrib/datafusion-python/tree/HEAD) [Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.1...HEAD) @@ -319,5 +327,4 @@ \* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ - -\* *This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)* +\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ diff --git a/Cargo.lock b/Cargo.lock index f4d8e0f7b..2622625c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" dependencies = [ "memchr", ] @@ -113,9 +113,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990dfa1a9328504aa135820da1c95066537b69ad94c04881b785f64328e0fa6b" +checksum = "1aea9fcb25bbb70f7f922f95b99ca29c1013dab47f6df61a6f24861842dd7f2e" dependencies = [ "ahash", "arrow-arith", @@ -136,9 +136,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b2e52de0ab54173f9b08232b7184c26af82ee7ab4ac77c83396633c90199fa" +checksum = "8d967b42f7b12c91fd78acd396b20c2973b184c8866846674abbb00c963e93ab" dependencies = [ "arrow-array", "arrow-buffer", @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10849b60c17dbabb334be1f4ef7550701aa58082b71335ce1ed586601b2f423" +checksum = "3190f208ee7aa0f3596fa0098d42911dec5e123ca88c002a08b24877ad14c71e" dependencies = [ "ahash", "arrow-buffer", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0746ae991b186be39933147117f8339eb1c4bbbea1c8ad37e7bf5851a1a06ba" +checksum = "5d33c733c5b6c44a0fc526f29c09546e04eb56772a7a21e48e602f368be381f6" dependencies = [ "half", "num", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88897802515d7b193e38b27ddd9d9e43923d410a9e46307582d756959ee9595" +checksum = "abd349520b6a1ed4924ae2afc9d23330a3044319e4ec3d5b124c09e4d440ae87" dependencies = [ "arrow-array", "arrow-buffer", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c8220d9741fc37961262710ceebd8451a5b393de57c464f0267ffdda1775c0a" +checksum = "c80af3c3e290a2a7e1cc518f1471dff331878cb4af9a5b088bf030b89debf649" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f937efa1aaad9dc86f6a0e382c2fa736a4943e2090c946138079bdf060cef" +checksum = "b1c8361947aaa96d331da9df3f7a08bdd8ab805a449994c97f5c4d24c4b7e2cf" dependencies = [ "arrow-buffer", "arrow-schema", @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18b75296ff01833f602552dff26a423fc213db8e5049b540ca4a00b1c957e41c" +checksum = "9a46ee000b9fbd1e8db6e8b26acb8c760838512b39d8c9f9d73892cb55351d50" dependencies = [ "arrow-array", "arrow-buffer", @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e501d3de4d612c90677594896ca6c0fa075665a7ff980dc4189bb531c17e19f6" +checksum = "4bf2366607be867ced681ad7f272371a5cf1fc2941328eef7b4fee14565166fb" dependencies = [ "arrow-array", "arrow-buffer", @@ -254,14 +254,15 @@ dependencies = [ "indexmap", "lexical-core", "num", + "serde", "serde_json", ] [[package]] name = "arrow-ord" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d2671eb3793f9410230ac3efb0e6d36307be8a2dac5fad58ac9abde8e9f01e" +checksum = "304069901c867200e21ec868ae7521165875470ef2f1f6d58f979a443d63997e" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc11fa039338cebbf4e29cf709c8ac1d6a65c7540063d4a25f991ab255ca85c8" +checksum = "0d57fe8ceef3392fdd493269d8a2d589de17bafce151aacbffbddac7a57f441a" dependencies = [ "ahash", "arrow-array", @@ -289,18 +290,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04f17f7b86ded0b5baf98fe6123391c4343e031acc3ccc5fa604cc180bff220" +checksum = "a16b88a93ac8350f0200b1cd336a1f887315925b8dd7aa145a37b8bdbd8497a4" dependencies = [ - "bitflags 2.1.0", + "bitflags 2.2.1", ] [[package]] name = "arrow-select" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "163e35de698098ff5f5f672ada9dc1f82533f10407c7a11e2cd09f3bcf31d18a" +checksum = "98e8a4d6ca37d5212439b24caad4d80743fcbb706706200dd174bb98e68fe9d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -311,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdfbed1b10209f0dc68e6aa4c43dc76079af65880965c7c3b73f641f23d4aba" +checksum = "cbb594efa397eb6a546f42b1f8df3d242ea84dbfda5232e06035dc2b2e2c8459" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,7 +322,7 @@ dependencies = [ "arrow-schema", "arrow-select", "regex", - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] @@ -393,9 +394,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c70beb79cbb5ce9c4f8e20849978f34225931f665bb49efa6982875a4d5facb3" +checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813" [[package]] name = "blake2" @@ -473,9 +474,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" [[package]] name = "byteorder" @@ -532,12 +533,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", - "js-sys", "num-integer", "num-traits", "serde", - "time 0.1.45", - "wasm-bindgen", "winapi", ] @@ -626,9 +624,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280a9f2d8b3a38871a3c8a46fb80db65e5e5ed97da80c4d08bf27fb63e35e181" +checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" dependencies = [ "libc", ] @@ -738,13 +736,15 @@ dependencies = [ [[package]] name = "datafusion" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bdb93fee4f30368f1f71bfd5cd28882ec9fab0183db7924827b76129d33227c" +checksum = "a8a7d4b334f4512ff2fdbce87f511f570ae895af1ac7c729e77c12583253b22a" dependencies = [ "ahash", "apache-avro", "arrow", + "arrow-array", + "arrow-schema", "async-compression", "async-trait", "bytes", @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82401ce129e601d406012b6d718f8978ba84c386e1c342fa155877120d68824" +checksum = "80abfcb1dbc6390f952f21de9069e6177ad6318fcae5fbceabb50666d96533dd" dependencies = [ "apache-avro", "arrow", @@ -805,9 +805,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b08b2078aed21a27239cd93f3015e492a58b0d50ebeeaf8d2236cf108ef583ce" +checksum = "df2524f1b4b58319895b112809d2a59e54fa662d0e46330a455f22882c2cb7b9" dependencies = [ "dashmap", "datafusion-common", @@ -823,9 +823,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b5b977ce9695fb4c67614266ec57f384fc11e9a9f9b3e6d0e62b9c5a9f2c1f" +checksum = "af8040b7a75b04685f4db0a1b11ffa93cd163c1bc13751df3f5cf76baabaf5a1" dependencies = [ "ahash", "arrow", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b2bb9e73ed778d1bc5af63a270f0154bf6eab5099c77668a6362296888e46b" +checksum = "74ceae25accc0f640a4238283f55f3a9fd181d55398703a4330fb2c46261e6a2" dependencies = [ "arrow", "async-trait", @@ -848,14 +848,14 @@ dependencies = [ "hashbrown 0.13.2", "itertools", "log", - "regex-syntax", + "regex-syntax 0.6.29", ] [[package]] name = "datafusion-physical-expr" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80cd8ea5ab0a07b1b2a3e17d5909f1b1035bd129ffeeb5c66842a32e682f8f79" +checksum = "df4cf228b312f2758cb78e93fe3d2dc602345028efdf7cfa5b338cb370d0a347" dependencies = [ "ahash", "arrow", @@ -873,6 +873,7 @@ dependencies = [ "indexmap", "itertools", "lazy_static", + "libc", "md-5", "paste", "petgraph", @@ -885,7 +886,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "22.0.0" +version = "23.0.0" dependencies = [ "async-trait", "datafusion", @@ -901,7 +902,7 @@ dependencies = [ "pyo3", "pyo3-build-config", "rand", - "regex-syntax", + "regex-syntax 0.6.29", "syn 2.0.15", "tokio", "url", @@ -910,9 +911,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a95d6badab19fd6e9195fdc5209ac0a7e5ce9bcdedc67767b9ffc1b4e645760" +checksum = "b52b486fb3d81bb132e400304be01af5aba0ad6737e3518045bb98944991fe32" dependencies = [ "arrow", "datafusion-common", @@ -922,9 +923,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a78f8fc67123c4357e63bc0c87622a2a663d26f074958d749a633d0ecde90f" +checksum = "773e985c182e41cfd68f7a7b483ab6bfb68beaac241c348cd4b1bf9f9d61b762" dependencies = [ "arrow", "arrow-schema", @@ -936,9 +937,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "22.0.0" +version = "23.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae6ed64a2005f0d78f2b1b3ec3f8148183f4523d5d364e5367115f8d8a82b7df" +checksum = "836e9b1c0ea430199c9bd4b88024cb8d617e3768ffdb412064169e2504a850ed" dependencies = [ "async-recursion", "chrono", @@ -1204,7 +1205,7 @@ checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -1365,7 +1366,7 @@ dependencies = [ "bstr", "itoa", "thiserror", - "time 0.3.20", + "time", ] [[package]] @@ -2123,9 +2124,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b085a4f2cde5781fc4b1717f2e86c62f5cda49de7ba99a7c2eae02b61c9064c" +checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" [[package]] name = "lock_api" @@ -2248,7 +2249,7 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.45.0", ] @@ -2444,9 +2445,9 @@ dependencies = [ [[package]] name = "parquet" -version = "36.0.0" +version = "37.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "321a15f8332645759f29875b07f8233d16ed8ec1b3582223de81625a9f8506b7" +checksum = "b5022d98333271f4ca3e87bab760498e61726bf5a6ca919123c80517e20ded29" dependencies = [ "ahash", "arrow-array", @@ -2806,13 +2807,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.3" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" +checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.7.1", ] [[package]] @@ -2827,6 +2828,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" + [[package]] name = "regress" version = "0.5.0" @@ -2910,9 +2917,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.13" +version = "0.37.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0" +checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f" dependencies = [ "bitflags 1.3.2", "errno", @@ -3203,9 +3210,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.32.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0366f270dbabb5cc2e4c88427dc4c08bba144f81e32fbd459a013f26a4d16aa0" +checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a" dependencies = [ "log", "sqlparser_derive", @@ -3357,17 +3364,6 @@ dependencies = [ "ordered-float", ] -[[package]] -name = "time" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.3.20" @@ -3506,13 +3502,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" +checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.15", ] [[package]] @@ -3705,12 +3701,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 22a2989ae..8c205a4cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "22.0.0" +version = "23.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "22.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "22.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "22.0.0" } -datafusion-optimizer = { version = "22.0.0" } -datafusion-sql = { version = "22.0.0" } -datafusion-substrait = { version = "22.0.0" } +datafusion = { version = "23.0.0" , features = ["pyarrow", "avro"] } +datafusion-common = { version = "23.0.0", features = ["pyarrow"] } +datafusion-expr = "23.0.0" +datafusion-optimizer = "23.0.0" +datafusion-sql = "23.0.0" +datafusion-substrait = "23.0.0" uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" diff --git a/dev/release/README.md b/dev/release/README.md index 84058a606..1ba44fb50 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -145,14 +145,22 @@ Pushing an `rc` tag to the release branch will cause a GitHub Workflow to run th Go to https://github.com/apache/arrow-datafusion-python/actions and look for an action named "Python Release Build" that has run against the pushed tag. -Click on the action and scroll down to the bottom of the page titled "Artifacts". Download `dist.zip`. +Click on the action and scroll down to the bottom of the page titled "Artifacts". Download `dist.zip`. It should +contain files such as: + +```text +datafusion-22.0.0-cp37-abi3-macosx_10_7_x86_64.whl +datafusion-22.0.0-cp37-abi3-macosx_11_0_arm64.whl +datafusion-22.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl +datafusion-22.0.0-cp37-abi3-win_amd64.whl +``` Upload the wheels to testpypi. ```bash unzip dist.zip python3 -m pip install --upgrade setuptools twine build -python3 -m twine upload --repository testpypi datafusion-0.7.0-cp37-abi3-*.whl +python3 -m twine upload --repository testpypi datafusion-22.0.0-cp37-abi3-*.whl ``` When prompted for username, enter `__token__`. When prompted for a password, enter a valid GitHub Personal Access Token diff --git a/src/common/data_type.rs b/src/common/data_type.rs index e07805c52..a7b79f49f 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -180,7 +180,7 @@ impl DataTypeMap { DataType::Struct(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( format!("{:?}", arrow_type), ))), - DataType::Union(_, _, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( + DataType::Union(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( format!("{:?}", arrow_type), ))), DataType::Dictionary(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( diff --git a/src/expr/literal.rs b/src/expr/literal.rs index f99216957..076f89a66 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -117,7 +117,9 @@ impl PyLiteral { ScalarValue::TimestampNanosecond(iv, tz) | ScalarValue::TimestampMicrosecond(iv, tz) | ScalarValue::TimestampMillisecond(iv, tz) - | ScalarValue::TimestampSecond(iv, tz) => Ok((*iv, tz.clone())), + | ScalarValue::TimestampSecond(iv, tz) => { + Ok((*iv, tz.as_ref().map(|s| s.as_ref().to_string()))) + } other => Err(unexpected_literal_value(other)), } } diff --git a/src/expr/subquery_alias.rs b/src/expr/subquery_alias.rs index 5360bbbc4..d3abd2e8c 100644 --- a/src/expr/subquery_alias.rs +++ b/src/expr/subquery_alias.rs @@ -68,7 +68,7 @@ impl PySubqueryAlias { } fn alias(&self) -> PyResult { - Ok(self.subquery_alias.alias.clone()) + Ok(self.subquery_alias.alias.to_string()) } fn __repr__(&self) -> PyResult { diff --git a/src/udaf.rs b/src/udaf.rs index d5866f840..756fa5659 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -56,7 +56,7 @@ impl Accumulator for RustAccumulator { // 1. cast args to Pyarrow array let py_args = values .iter() - .map(|arg| arg.data().to_owned().to_pyarrow(py).unwrap()) + .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); let py_args = PyTuple::new(py, py_args); @@ -76,7 +76,7 @@ impl Accumulator for RustAccumulator { // 1. cast states to Pyarrow array let state = state - .data() + .into_data() .to_pyarrow(py) .map_err(|e| DataFusionError::Execution(format!("{e}")))?; diff --git a/src/udf.rs b/src/udf.rs index f3e6cfb58..dcb9cd572 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -41,7 +41,7 @@ fn to_rust_function(func: PyObject) -> ScalarFunctionImplementation { // 1. cast args to Pyarrow arrays let py_args = args .iter() - .map(|arg| arg.data().to_owned().to_pyarrow(py).unwrap()) + .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); let py_args = PyTuple::new(py, py_args); From 545e93e1739a71ef77cee1afe9196c318aabb667 Mon Sep 17 00:00:00 2001 From: Kyle Brooks <84413234+kylebrooks-8451@users.noreply.github.com> Date: Mon, 24 Apr 2023 19:48:04 -0400 Subject: [PATCH 024/413] Store the Tokio Runtime in an _internal module level attribute and reuse it. (#341) --- src/context.rs | 5 ++--- src/lib.rs | 9 +++++++++ src/utils.rs | 11 +++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/context.rs b/src/context.rs index 0ba120022..3dc8a8fba 100644 --- a/src/context.rs +++ b/src/context.rs @@ -36,7 +36,7 @@ use crate::sql::logical::PyLogicalPlan; use crate::store::StorageContexts; use crate::udaf::PyAggregateUDF; use crate::udf::PyScalarUDF; -use crate::utils::wait_for_future; +use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; @@ -52,7 +52,6 @@ use datafusion::prelude::{ }; use datafusion_common::ScalarValue; use pyo3::types::PyTuple; -use tokio::runtime::Runtime; use tokio::task::JoinHandle; /// Configuration options for a SessionContext @@ -722,7 +721,7 @@ impl PySessionContext { Arc::new(RuntimeEnv::default()), ); // create a Tokio runtime to run the async code - let rt = Runtime::new().unwrap(); + let rt = &get_tokio_runtime(py).0; let plan = plan.plan.clone(); let fut: JoinHandle> = rt.spawn(async move { plan.execute(part, Arc::new(ctx)) }); diff --git a/src/lib.rs b/src/lib.rs index 4a6574c16..0bb4d9a12 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,12 +59,21 @@ pub mod utils; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; +// Used to define Tokio Runtime as a Python module attribute +#[pyclass] +pub(crate) struct TokioRuntime(tokio::runtime::Runtime); + /// Low-level DataFusion internal package. /// /// The higher-level public API is defined in pure python files under the /// datafusion directory. #[pymodule] fn _internal(py: Python, m: &PyModule) -> PyResult<()> { + // Register the Tokio Runtime as a module attribute so we can reuse it + m.add( + "runtime", + TokioRuntime(tokio::runtime::Runtime::new().unwrap()), + )?; // Register the python classes m.add_class::()?; m.add_class::()?; diff --git a/src/utils.rs b/src/utils.rs index 4158b7485..427a8a064 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -16,19 +16,26 @@ // under the License. use crate::errors::DataFusionError; +use crate::TokioRuntime; use datafusion_expr::Volatility; use pyo3::prelude::*; use std::future::Future; use tokio::runtime::Runtime; +/// Utility to get the Tokio Runtime from Python +pub(crate) fn get_tokio_runtime(py: Python) -> PyRef { + let datafusion = py.import("datafusion._internal").unwrap(); + datafusion.getattr("runtime").unwrap().extract().unwrap() +} + /// Utility to collect rust futures with GIL released pub fn wait_for_future(py: Python, f: F) -> F::Output where F: Send, F::Output: Send, { - let rt = Runtime::new().unwrap(); - py.allow_threads(|| rt.block_on(f)) + let runtime: &Runtime = &get_tokio_runtime(py).0; + py.allow_threads(|| runtime.block_on(f)) } pub(crate) fn parse_volatility(value: &str) -> Result { From 14f48408ee56694ca17e6fa67408be626bb56fe0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 24 Apr 2023 18:27:41 -0600 Subject: [PATCH 025/413] update changelog (#342) --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8942f944..26c4eb18e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,15 @@ **Merged pull requests:** +**Merged pull requests:** + - Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/arrow-datafusion-python/pull/321) (andygrove) - Osx build linker args [#330](https://github.com/apache/arrow-datafusion-python/pull/330) (jdye64) +- Add requirements file for python 3.11 [#332](https://github.com/apache/arrow-datafusion-python/pull/332) (r4ntix) +- mac arm64 build [#338](https://github.com/apache/arrow-datafusion-python/pull/338) (andygrove) +- Add conda.yaml baseline workflow file [#281](https://github.com/apache/arrow-datafusion-python/pull/281) (jdye64) +- Prepare for 23.0.0 release [#335](https://github.com/apache/arrow-datafusion-python/pull/335) (andygrove) +- Reuse the Tokio Runtime [#341](https://github.com/apache/arrow-datafusion-python/pull/341) (kylebrooks-8451) ## [22.0.0](https://github.com/apache/arrow-datafusion-python/tree/22.0.0) (2023-04-10) From b6c21150d84d390418bf7514983771e483375828 Mon Sep 17 00:00:00 2001 From: Kyle Brooks <84413234+kylebrooks-8451@users.noreply.github.com> Date: Tue, 25 Apr 2023 19:53:55 -0400 Subject: [PATCH 026/413] Add interface to directly serialize Substrait plans to Python Bytes. (#344) Clean up existing substrait bindings to return bytes instead of List[int]. --- Cargo.lock | 2 ++ Cargo.toml | 2 ++ datafusion/tests/test_substrait.py | 2 ++ examples/substrait.py | 7 ++++++- src/errors.rs | 3 +++ src/substrait.rs | 24 ++++++++++++++++++++---- 6 files changed, 35 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2622625c7..db7c9a768 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -899,6 +899,8 @@ dependencies = [ "mimalloc", "object_store", "parking_lot", + "prost", + "prost-types", "pyo3", "pyo3-build-config", "rand", diff --git a/Cargo.toml b/Cargo.toml index 8c205a4cd..4c5203d69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,8 @@ datafusion-expr = "23.0.0" datafusion-optimizer = "23.0.0" datafusion-sql = "23.0.0" datafusion-substrait = "23.0.0" +prost = "0.11" +prost-types = "0.11" uuid = { version = "1.2", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" diff --git a/datafusion/tests/test_substrait.py b/datafusion/tests/test_substrait.py index 9a08b760e..01df2d746 100644 --- a/datafusion/tests/test_substrait.py +++ b/datafusion/tests/test_substrait.py @@ -41,6 +41,8 @@ def test_substrait_serialization(ctx): substrait_plan = ss.substrait.serde.serialize_to_plan( "SELECT * FROM t", ctx ) + substrait_bytes = substrait_plan.encode() + assert type(substrait_bytes) is bytes substrait_bytes = ss.substrait.serde.serialize_bytes( "SELECT * FROM t", ctx ) diff --git a/examples/substrait.py b/examples/substrait.py index c167f7d90..515311d85 100644 --- a/examples/substrait.py +++ b/examples/substrait.py @@ -32,8 +32,13 @@ ) # type(substrait_plan) -> +# Encode it to bytes +substrait_bytes = substrait_plan.encode() +# type(substrait_bytes) -> , at this point the bytes can be distributed to file, network, etc safely +# where they could subsequently be deserialized on the receiving end. + # Alternative serialization approaches -# type(substrait_bytes) -> , at this point the bytes can be distributed to file, network, etc safely +# type(substrait_bytes) -> , at this point the bytes can be distributed to file, network, etc safely # where they could subsequently be deserialized on the receiving end. substrait_bytes = ss.substrait.serde.serialize_bytes( "SELECT * FROM aggregate_test_data", ctx diff --git a/src/errors.rs b/src/errors.rs index ce6b3c28a..d12b6ade1 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -21,6 +21,7 @@ use std::fmt::Debug; use datafusion::arrow::error::ArrowError; use datafusion::error::DataFusionError as InnerDataFusionError; +use prost::EncodeError; use pyo3::{exceptions::PyException, PyErr}; pub type Result = std::result::Result; @@ -31,6 +32,7 @@ pub enum DataFusionError { ArrowError(ArrowError), Common(String), PythonError(PyErr), + EncodeError(EncodeError), } impl fmt::Display for DataFusionError { @@ -40,6 +42,7 @@ impl fmt::Display for DataFusionError { DataFusionError::ArrowError(e) => write!(f, "Arrow error: {e:?}"), DataFusionError::PythonError(e) => write!(f, "Python error {e:?}"), DataFusionError::Common(e) => write!(f, "{e}"), + DataFusionError::EncodeError(e) => write!(f, "Failed to encode substrait plan: {e}"), } } } diff --git a/src/substrait.rs b/src/substrait.rs index 2bde01123..361e9ac4b 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use pyo3::prelude::*; +use pyo3::{prelude::*, types::PyBytes}; use crate::context::PySessionContext; use crate::errors::{py_datafusion_err, DataFusionError}; @@ -25,6 +25,7 @@ use crate::utils::wait_for_future; use datafusion_substrait::logical_plan::{consumer, producer}; use datafusion_substrait::serializer; use datafusion_substrait::substrait::proto::Plan; +use prost::Message; #[pyclass(name = "plan", module = "datafusion.substrait", subclass, unsendable)] #[derive(Debug, Clone)] @@ -32,6 +33,17 @@ pub(crate) struct PyPlan { pub(crate) plan: Plan, } +#[pymethods] +impl PyPlan { + fn encode(&self, py: Python) -> PyResult { + let mut proto_bytes = Vec::::new(); + self.plan + .encode(&mut proto_bytes) + .map_err(|e| DataFusionError::EncodeError(e))?; + Ok(PyBytes::new(py, &proto_bytes).into()) + } +} + impl From for Plan { fn from(plan: PyPlan) -> Plan { plan.plan @@ -63,16 +75,19 @@ impl PySubstraitSerializer { #[staticmethod] pub fn serialize_to_plan(sql: &str, ctx: PySessionContext, py: Python) -> PyResult { match PySubstraitSerializer::serialize_bytes(sql, ctx, py) { - Ok(proto_bytes) => PySubstraitSerializer::deserialize_bytes(proto_bytes, py), + Ok(proto_bytes) => { + let proto_bytes: &PyBytes = proto_bytes.as_ref(py).downcast().unwrap(); + PySubstraitSerializer::deserialize_bytes(proto_bytes.as_bytes().to_vec(), py) + } Err(e) => Err(py_datafusion_err(e)), } } #[staticmethod] - pub fn serialize_bytes(sql: &str, ctx: PySessionContext, py: Python) -> PyResult> { + pub fn serialize_bytes(sql: &str, ctx: PySessionContext, py: Python) -> PyResult { let proto_bytes: Vec = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx)) .map_err(DataFusionError::from)?; - Ok(proto_bytes) + Ok(PyBytes::new(py, &proto_bytes).into()) } #[staticmethod] @@ -136,6 +151,7 @@ impl PySubstraitConsumer { } pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From af1a8ebade3d537ce24f6f01a037a56177a917fa Mon Sep 17 00:00:00 2001 From: Kyle Brooks <84413234+kylebrooks-8451@users.noreply.github.com> Date: Tue, 25 Apr 2023 19:54:40 -0400 Subject: [PATCH 027/413] Add partition_count property to ExecutionPlan. (#346) --- datafusion/tests/test_dataframe.py | 3 +++ examples/substrait.py | 2 +- src/physical_plan.rs | 5 +++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index cd78f3c88..221b0cc09 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -370,6 +370,9 @@ def test_execution_plan(aggregate_df): assert expected == plan.display() + # Check the number of partitions is as expected. + assert type(plan.partition_count) is int + expected = ( "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n" " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" diff --git a/examples/substrait.py b/examples/substrait.py index 515311d85..c579751d2 100644 --- a/examples/substrait.py +++ b/examples/substrait.py @@ -23,7 +23,7 @@ ctx = SessionContext() # Register table with context -ctx.register_parquet( +ctx.register_csv( "aggregate_test_data", "./testing/data/csv/aggregate_test_100.csv" ) diff --git a/src/physical_plan.rs b/src/physical_plan.rs index 340d527fa..4c35f3e60 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -53,6 +53,11 @@ impl PyExecutionPlan { let d = displayable(self.plan.as_ref()); format!("{}", d.indent()) } + + #[getter] + pub fn partition_count(&self) -> usize { + self.plan.output_partitioning().partition_count() + } } impl From for Arc { From a1e2b8ba8a233c7ea2edde3918334200a5727226 Mon Sep 17 00:00:00 2001 From: Kyle Brooks <84413234+kylebrooks-8451@users.noreply.github.com> Date: Thu, 27 Apr 2023 17:59:03 -0400 Subject: [PATCH 028/413] Remove unsendable from all Rust pyclass types. (#348) --- src/context.rs | 6 +++--- src/store.rs | 23 ++++------------------- src/substrait.rs | 18 ++++-------------- 3 files changed, 11 insertions(+), 36 deletions(-) diff --git a/src/context.rs b/src/context.rs index 3dc8a8fba..0733a58f4 100644 --- a/src/context.rs +++ b/src/context.rs @@ -55,7 +55,7 @@ use pyo3::types::PyTuple; use tokio::task::JoinHandle; /// Configuration options for a SessionContext -#[pyclass(name = "SessionConfig", module = "datafusion", subclass, unsendable)] +#[pyclass(name = "SessionConfig", module = "datafusion", subclass)] #[derive(Clone, Default)] pub(crate) struct PySessionConfig { pub(crate) config: SessionConfig, @@ -148,7 +148,7 @@ impl PySessionConfig { } /// Runtime options for a SessionContext -#[pyclass(name = "RuntimeConfig", module = "datafusion", subclass, unsendable)] +#[pyclass(name = "RuntimeConfig", module = "datafusion", subclass)] #[derive(Clone)] pub(crate) struct PyRuntimeConfig { pub(crate) config: RuntimeConfig, @@ -210,7 +210,7 @@ impl PyRuntimeConfig { /// `PySessionContext` is able to plan and execute DataFusion plans. /// It has a powerful optimizer, a physical planner for local execution, and a /// multi-threaded execution engine to perform the execution. -#[pyclass(name = "SessionContext", module = "datafusion", subclass, unsendable)] +#[pyclass(name = "SessionContext", module = "datafusion", subclass)] #[derive(Clone)] pub(crate) struct PySessionContext { pub(crate) ctx: SessionContext, diff --git a/src/store.rs b/src/store.rs index 7d9bb7518..542cfa925 100644 --- a/src/store.rs +++ b/src/store.rs @@ -32,12 +32,7 @@ pub enum StorageContexts { LocalFileSystem(PyLocalFileSystemContext), } -#[pyclass( - name = "LocalFileSystem", - module = "datafusion.store", - subclass, - unsendable -)] +#[pyclass(name = "LocalFileSystem", module = "datafusion.store", subclass)] #[derive(Debug, Clone)] pub struct PyLocalFileSystemContext { pub inner: Arc, @@ -63,12 +58,7 @@ impl PyLocalFileSystemContext { } } -#[pyclass( - name = "MicrosoftAzure", - module = "datafusion.store", - subclass, - unsendable -)] +#[pyclass(name = "MicrosoftAzure", module = "datafusion.store", subclass)] #[derive(Debug, Clone)] pub struct PyMicrosoftAzureContext { pub inner: Arc, @@ -140,12 +130,7 @@ impl PyMicrosoftAzureContext { } } -#[pyclass( - name = "GoogleCloud", - module = "datafusion.store", - subclass, - unsendable -)] +#[pyclass(name = "GoogleCloud", module = "datafusion.store", subclass)] #[derive(Debug, Clone)] pub struct PyGoogleCloudContext { pub inner: Arc, @@ -175,7 +160,7 @@ impl PyGoogleCloudContext { } } -#[pyclass(name = "AmazonS3", module = "datafusion.store", subclass, unsendable)] +#[pyclass(name = "AmazonS3", module = "datafusion.store", subclass)] #[derive(Debug, Clone)] pub struct PyAmazonS3Context { pub inner: Arc, diff --git a/src/substrait.rs b/src/substrait.rs index 361e9ac4b..5d2e7a485 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -27,7 +27,7 @@ use datafusion_substrait::serializer; use datafusion_substrait::substrait::proto::Plan; use prost::Message; -#[pyclass(name = "plan", module = "datafusion.substrait", subclass, unsendable)] +#[pyclass(name = "plan", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub(crate) struct PyPlan { pub(crate) plan: Plan, @@ -59,7 +59,7 @@ impl From for PyPlan { /// A PySubstraitSerializer is a representation of a Serializer that is capable of both serializing /// a `LogicalPlan` instance to Substrait Protobuf bytes and also deserialize Substrait Protobuf bytes /// to a valid `LogicalPlan` instance. -#[pyclass(name = "serde", module = "datafusion.substrait", subclass, unsendable)] +#[pyclass(name = "serde", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub(crate) struct PySubstraitSerializer; @@ -105,12 +105,7 @@ impl PySubstraitSerializer { } } -#[pyclass( - name = "producer", - module = "datafusion.substrait", - subclass, - unsendable -)] +#[pyclass(name = "producer", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub(crate) struct PySubstraitProducer; @@ -126,12 +121,7 @@ impl PySubstraitProducer { } } -#[pyclass( - name = "consumer", - module = "datafusion.substrait", - subclass, - unsendable -)] +#[pyclass(name = "consumer", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub(crate) struct PySubstraitConsumer; From e1a8a3c623110b5214ce417c12431d2ae4238c60 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 27 Apr 2023 17:30:23 -0600 Subject: [PATCH 029/413] Fix link to user guide (#354) --- .asf.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.asf.yaml b/.asf.yaml index f27975c84..de776871c 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -22,7 +22,7 @@ notifications: jira_options: link label worklog github: description: "Apache Arrow DataFusion Python Bindings" - homepage: https://arrow.apache.org/datafusion + homepage: https://arrow.apache.org/datafusion-python enabled_merge_buttons: squash: true merge: false From 6642660dcfa4ab9e7c743b6537e0e3499f0923d5 Mon Sep 17 00:00:00 2001 From: Kyle Brooks <84413234+kylebrooks-8451@users.noreply.github.com> Date: Thu, 27 Apr 2023 19:31:05 -0400 Subject: [PATCH 030/413] Fix SessionContext execute. (#353) --- src/context.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/context.rs b/src/context.rs index 0733a58f4..b7f82230f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -712,14 +712,7 @@ impl PySessionContext { part: usize, py: Python, ) -> PyResult { - let ctx = TaskContext::new( - None, - "session_id".to_string(), - SessionConfig::new(), - HashMap::new(), - HashMap::new(), - Arc::new(RuntimeEnv::default()), - ); + let ctx: TaskContext = TaskContext::from(&self.ctx.state()); // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let plan = plan.plan.clone(); From befd967558a77ab6048e5130354d29063c843b9c Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Fri, 28 Apr 2023 19:17:31 -0400 Subject: [PATCH 031/413] Make expr module public for library consumption (#357) --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 0bb4d9a12..2512aefa4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -40,7 +40,7 @@ mod dataset; mod dataset_exec; pub mod errors; #[allow(clippy::borrow_deref_ref)] -mod expr; +pub mod expr; #[allow(clippy::borrow_deref_ref)] mod functions; pub mod physical_plan; From 0911bf31b85624e76e12eb2f93c71ad1caa6da68 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 3 May 2023 11:50:32 -0600 Subject: [PATCH 032/413] Add benchmark derived from TPC-H (#355) --- benchmarks/tpch/.gitignore | 2 + benchmarks/tpch/README.md | 78 ++++++++++++++++++++ benchmarks/tpch/create_tables.sql | 119 ++++++++++++++++++++++++++++++ benchmarks/tpch/queries/q1.sql | 23 ++++++ benchmarks/tpch/queries/q10.sql | 33 +++++++++ benchmarks/tpch/queries/q11.sql | 29 ++++++++ benchmarks/tpch/queries/q12.sql | 30 ++++++++ benchmarks/tpch/queries/q13.sql | 22 ++++++ benchmarks/tpch/queries/q14.sql | 15 ++++ benchmarks/tpch/queries/q15.sql | 33 +++++++++ benchmarks/tpch/queries/q16.sql | 32 ++++++++ benchmarks/tpch/queries/q17.sql | 19 +++++ benchmarks/tpch/queries/q18.sql | 34 +++++++++ benchmarks/tpch/queries/q19.sql | 37 ++++++++++ benchmarks/tpch/queries/q2.sql | 45 +++++++++++ benchmarks/tpch/queries/q20.sql | 39 ++++++++++ benchmarks/tpch/queries/q21.sql | 41 ++++++++++ benchmarks/tpch/queries/q22.sql | 39 ++++++++++ benchmarks/tpch/queries/q3.sql | 24 ++++++ benchmarks/tpch/queries/q4.sql | 23 ++++++ benchmarks/tpch/queries/q5.sql | 26 +++++++ benchmarks/tpch/queries/q6.sql | 11 +++ benchmarks/tpch/queries/q7.sql | 41 ++++++++++ benchmarks/tpch/queries/q8.sql | 39 ++++++++++ benchmarks/tpch/queries/q9.sql | 34 +++++++++ benchmarks/tpch/tpch-gen.sh | 53 +++++++++++++ benchmarks/tpch/tpch.py | 102 +++++++++++++++++++++++++ dev/release/rat_exclude_files.txt | 4 +- src/context.rs | 2 +- 29 files changed, 1027 insertions(+), 2 deletions(-) create mode 100644 benchmarks/tpch/.gitignore create mode 100644 benchmarks/tpch/README.md create mode 100644 benchmarks/tpch/create_tables.sql create mode 100644 benchmarks/tpch/queries/q1.sql create mode 100644 benchmarks/tpch/queries/q10.sql create mode 100644 benchmarks/tpch/queries/q11.sql create mode 100644 benchmarks/tpch/queries/q12.sql create mode 100644 benchmarks/tpch/queries/q13.sql create mode 100644 benchmarks/tpch/queries/q14.sql create mode 100644 benchmarks/tpch/queries/q15.sql create mode 100644 benchmarks/tpch/queries/q16.sql create mode 100644 benchmarks/tpch/queries/q17.sql create mode 100644 benchmarks/tpch/queries/q18.sql create mode 100644 benchmarks/tpch/queries/q19.sql create mode 100644 benchmarks/tpch/queries/q2.sql create mode 100644 benchmarks/tpch/queries/q20.sql create mode 100644 benchmarks/tpch/queries/q21.sql create mode 100644 benchmarks/tpch/queries/q22.sql create mode 100644 benchmarks/tpch/queries/q3.sql create mode 100644 benchmarks/tpch/queries/q4.sql create mode 100644 benchmarks/tpch/queries/q5.sql create mode 100644 benchmarks/tpch/queries/q6.sql create mode 100644 benchmarks/tpch/queries/q7.sql create mode 100644 benchmarks/tpch/queries/q8.sql create mode 100644 benchmarks/tpch/queries/q9.sql create mode 100755 benchmarks/tpch/tpch-gen.sh create mode 100644 benchmarks/tpch/tpch.py diff --git a/benchmarks/tpch/.gitignore b/benchmarks/tpch/.gitignore new file mode 100644 index 000000000..4471c6d15 --- /dev/null +++ b/benchmarks/tpch/.gitignore @@ -0,0 +1,2 @@ +data +results.csv \ No newline at end of file diff --git a/benchmarks/tpch/README.md b/benchmarks/tpch/README.md new file mode 100644 index 000000000..a118a7449 --- /dev/null +++ b/benchmarks/tpch/README.md @@ -0,0 +1,78 @@ + + +# DataFusion Python Benchmarks Derived from TPC-H + +## Create Release Build + +From repo root: + +```bash +maturin develop --release +``` + +Note that release builds take a really long time, so you may want to temporarily comment out this section of the +root Cargo.toml when frequently building. + +```toml +[profile.release] +lto = true +codegen-units = 1 +``` + +## Generate Data + +```bash +./tpch-gen.sh 1 +``` + +## Run Benchmarks + +```bash +python tpch.py ./data ./queries +``` + +A summary of the benchmark timings will be written to `results.csv`. For example: + +```csv +setup,1.4 +q1,2978.6 +q2,679.7 +q3,2943.7 +q4,2894.9 +q5,3592.3 +q6,1691.4 +q7,3003.9 +q8,3818.7 +q9,4237.9 +q10,2344.7 +q11,526.1 +q12,2284.6 +q13,1009.2 +q14,1738.4 +q15,1942.1 +q16,499.8 +q17,5178.9 +q18,4127.7 +q19,2056.6 +q20,2162.5 +q21,8046.5 +q22,754.9 +total,58513.2 +``` \ No newline at end of file diff --git a/benchmarks/tpch/create_tables.sql b/benchmarks/tpch/create_tables.sql new file mode 100644 index 000000000..4b2209c4b --- /dev/null +++ b/benchmarks/tpch/create_tables.sql @@ -0,0 +1,119 @@ +-- Schema derived from TPC-H schema under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. + +CREATE EXTERNAL TABLE customer ( + c_custkey INT NOT NULL, + c_name VARCHAR NOT NULL, + c_address VARCHAR NOT NULL, + c_nationkey INT NOT NULL, + c_phone VARCHAR NOT NULL, + c_acctbal DECIMAL(15, 2) NOT NULL, + c_mktsegment VARCHAR NOT NULL, + c_comment VARCHAR NOT NULL, + c_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/customer.csv'; + +CREATE EXTERNAL TABLE lineitem ( + l_orderkey INT NOT NULL, + l_partkey INT NOT NULL, + l_suppkey INT NOT NULL, + l_linenumber INT NOT NULL, + l_quantity DECIMAL(15, 2) NOT NULL, + l_extendedprice DECIMAL(15, 2) NOT NULL, + l_discount DECIMAL(15, 2) NOT NULL, + l_tax DECIMAL(15, 2) NOT NULL, + l_returnflag VARCHAR NOT NULL, + l_linestatus VARCHAR NOT NULL, + l_shipdate DATE NOT NULL, + l_commitdate DATE NOT NULL, + l_receiptdate DATE NOT NULL, + l_shipinstruct VARCHAR NOT NULL, + l_shipmode VARCHAR NOT NULL, + l_comment VARCHAR NOT NULL, + l_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/lineitem.csv'; + +CREATE EXTERNAL TABLE nation ( + n_nationkey INT NOT NULL, + n_name VARCHAR NOT NULL, + n_regionkey INT NOT NULL, + n_comment VARCHAR NOT NULL, + n_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/nation.csv'; + +CREATE EXTERNAL TABLE orders ( + o_orderkey INT NOT NULL, + o_custkey INT NOT NULL, + o_orderstatus VARCHAR NOT NULL, + o_totalprice DECIMAL(15, 2) NOT NULL, + o_orderdate DATE NOT NULL, + o_orderpriority VARCHAR NOT NULL, + o_clerk VARCHAR NOT NULL, + o_shippriority INT NULL, + o_comment VARCHAR NOT NULL, + o_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/orders.csv'; + +CREATE EXTERNAL TABLE part ( + p_partkey INT NOT NULL, + p_name VARCHAR NOT NULL, + p_mfgr VARCHAR NOT NULL, + p_brand VARCHAR NOT NULL, + p_type VARCHAR NOT NULL, + p_size INT NULL, + p_container VARCHAR NOT NULL, + p_retailprice DECIMAL(15, 2) NOT NULL, + p_comment VARCHAR NOT NULL, + p_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/part.csv'; + +CREATE EXTERNAL TABLE partsupp ( + ps_partkey INT NOT NULL, + ps_suppkey INT NOT NULL, + ps_availqty INT NOT NULL, + ps_supplycost DECIMAL(15, 2) NOT NULL, + ps_comment VARCHAR NOT NULL, + ps_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/partsupp.csv'; + +CREATE EXTERNAL TABLE region ( + r_regionkey INT NOT NULL, + r_name VARCHAR NOT NULL, + r_comment VARCHAR NOT NULL, + r_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/region.csv'; + +CREATE EXTERNAL TABLE supplier ( + s_suppkey INT NOT NULL, + s_name VARCHAR NOT NULL, + s_address VARCHAR NOT NULL, + s_nationkey INT NOT NULL, + s_phone VARCHAR NOT NULL, + s_acctbal DECIMAL(15, 2) NOT NULL, + s_comment VARCHAR NOT NULL, + s_extra VARCHAR NOT NULL, +) +STORED AS CSV +WITH HEADER ROW DELIMITER '|' +LOCATION '$PATH/supplier.csv'; \ No newline at end of file diff --git a/benchmarks/tpch/queries/q1.sql b/benchmarks/tpch/queries/q1.sql new file mode 100644 index 000000000..e7e8e32b8 --- /dev/null +++ b/benchmarks/tpch/queries/q1.sql @@ -0,0 +1,23 @@ +-- Benchmark Query 1 derived from TPC-H query 1 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + l_returnflag, + l_linestatus, + sum(l_quantity) as sum_qty, + sum(l_extendedprice) as sum_base_price, + sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + avg(l_quantity) as avg_qty, + avg(l_extendedprice) as avg_price, + avg(l_discount) as avg_disc, + count(*) as count_order +from + lineitem +where + l_shipdate <= date '1998-12-01' - interval '68 days' +group by + l_returnflag, + l_linestatus +order by + l_returnflag, + l_linestatus; diff --git a/benchmarks/tpch/queries/q10.sql b/benchmarks/tpch/queries/q10.sql new file mode 100644 index 000000000..8391f6277 --- /dev/null +++ b/benchmarks/tpch/queries/q10.sql @@ -0,0 +1,33 @@ +-- Benchmark Query 10 derived from TPC-H query 10 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + c_custkey, + c_name, + sum(l_extendedprice * (1 - l_discount)) as revenue, + c_acctbal, + n_name, + c_address, + c_phone, + c_comment +from + customer, + orders, + lineitem, + nation +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate >= date '1993-07-01' + and o_orderdate < date '1993-07-01' + interval '3' month + and l_returnflag = 'R' + and c_nationkey = n_nationkey +group by + c_custkey, + c_name, + c_acctbal, + c_phone, + n_name, + c_address, + c_comment +order by + revenue desc limit 20; diff --git a/benchmarks/tpch/queries/q11.sql b/benchmarks/tpch/queries/q11.sql new file mode 100644 index 000000000..58776d369 --- /dev/null +++ b/benchmarks/tpch/queries/q11.sql @@ -0,0 +1,29 @@ +-- Benchmark Query 11 derived from TPC-H query 11 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + ps_partkey, + sum(ps_supplycost * ps_availqty) as value +from + partsupp, + supplier, + nation +where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'ALGERIA' +group by + ps_partkey having + sum(ps_supplycost * ps_availqty) > ( + select + sum(ps_supplycost * ps_availqty) * 0.0001000000 + from + partsupp, + supplier, + nation + where + ps_suppkey = s_suppkey + and s_nationkey = n_nationkey + and n_name = 'ALGERIA' + ) +order by + value desc; diff --git a/benchmarks/tpch/queries/q12.sql b/benchmarks/tpch/queries/q12.sql new file mode 100644 index 000000000..0b973de98 --- /dev/null +++ b/benchmarks/tpch/queries/q12.sql @@ -0,0 +1,30 @@ +-- Benchmark Query 12 derived from TPC-H query 12 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + l_shipmode, + sum(case + when o_orderpriority = '1-URGENT' + or o_orderpriority = '2-HIGH' + then 1 + else 0 + end) as high_line_count, + sum(case + when o_orderpriority <> '1-URGENT' + and o_orderpriority <> '2-HIGH' + then 1 + else 0 + end) as low_line_count +from + orders, + lineitem +where + o_orderkey = l_orderkey + and l_shipmode in ('FOB', 'SHIP') + and l_commitdate < l_receiptdate + and l_shipdate < l_commitdate + and l_receiptdate >= date '1995-01-01' + and l_receiptdate < date '1995-01-01' + interval '1' year +group by + l_shipmode +order by + l_shipmode; diff --git a/benchmarks/tpch/queries/q13.sql b/benchmarks/tpch/queries/q13.sql new file mode 100644 index 000000000..145dd6f10 --- /dev/null +++ b/benchmarks/tpch/queries/q13.sql @@ -0,0 +1,22 @@ +-- Benchmark Query 13 derived from TPC-H query 13 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + c_count, + count(*) as custdist +from + ( + select + c_custkey, + count(o_orderkey) + from + customer left outer join orders on + c_custkey = o_custkey + and o_comment not like '%express%requests%' + group by + c_custkey + ) as c_orders (c_custkey, c_count) +group by + c_count +order by + custdist desc, + c_count desc; diff --git a/benchmarks/tpch/queries/q14.sql b/benchmarks/tpch/queries/q14.sql new file mode 100644 index 000000000..1a91a04df --- /dev/null +++ b/benchmarks/tpch/queries/q14.sql @@ -0,0 +1,15 @@ +-- Benchmark Query 14 derived from TPC-H query 14 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + 100.00 * sum(case + when p_type like 'PROMO%' + then l_extendedprice * (1 - l_discount) + else 0 + end) / sum(l_extendedprice * (1 - l_discount)) as promo_revenue +from + lineitem, + part +where + l_partkey = p_partkey + and l_shipdate >= date '1995-02-01' + and l_shipdate < date '1995-02-01' + interval '1' month; diff --git a/benchmarks/tpch/queries/q15.sql b/benchmarks/tpch/queries/q15.sql new file mode 100644 index 000000000..68cc32cb7 --- /dev/null +++ b/benchmarks/tpch/queries/q15.sql @@ -0,0 +1,33 @@ +-- Benchmark Query 15 derived from TPC-H query 15 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +create view revenue0 (supplier_no, total_revenue) as + select + l_suppkey, + sum(l_extendedprice * (1 - l_discount)) + from + lineitem + where + l_shipdate >= date '1996-08-01' + and l_shipdate < date '1996-08-01' + interval '3' month + group by + l_suppkey; +select + s_suppkey, + s_name, + s_address, + s_phone, + total_revenue +from + supplier, + revenue0 +where + s_suppkey = supplier_no + and total_revenue = ( + select + max(total_revenue) + from + revenue0 + ) +order by + s_suppkey; +drop view revenue0; diff --git a/benchmarks/tpch/queries/q16.sql b/benchmarks/tpch/queries/q16.sql new file mode 100644 index 000000000..098b4f3b3 --- /dev/null +++ b/benchmarks/tpch/queries/q16.sql @@ -0,0 +1,32 @@ +-- Benchmark Query 16 derived from TPC-H query 16 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + p_brand, + p_type, + p_size, + count(distinct ps_suppkey) as supplier_cnt +from + partsupp, + part +where + p_partkey = ps_partkey + and p_brand <> 'Brand#14' + and p_type not like 'SMALL PLATED%' + and p_size in (14, 6, 5, 31, 49, 15, 41, 47) + and ps_suppkey not in ( + select + s_suppkey + from + supplier + where + s_comment like '%Customer%Complaints%' + ) +group by + p_brand, + p_type, + p_size +order by + supplier_cnt desc, + p_brand, + p_type, + p_size; diff --git a/benchmarks/tpch/queries/q17.sql b/benchmarks/tpch/queries/q17.sql new file mode 100644 index 000000000..ed02d7b77 --- /dev/null +++ b/benchmarks/tpch/queries/q17.sql @@ -0,0 +1,19 @@ +-- Benchmark Query 17 derived from TPC-H query 17 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + sum(l_extendedprice) / 7.0 as avg_yearly +from + lineitem, + part +where + p_partkey = l_partkey + and p_brand = 'Brand#42' + and p_container = 'LG BAG' + and l_quantity < ( + select + 0.2 * avg(l_quantity) + from + lineitem + where + l_partkey = p_partkey + ); diff --git a/benchmarks/tpch/queries/q18.sql b/benchmarks/tpch/queries/q18.sql new file mode 100644 index 000000000..cf1f8c89a --- /dev/null +++ b/benchmarks/tpch/queries/q18.sql @@ -0,0 +1,34 @@ +-- Benchmark Query 18 derived from TPC-H query 18 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice, + sum(l_quantity) +from + customer, + orders, + lineitem +where + o_orderkey in ( + select + l_orderkey + from + lineitem + group by + l_orderkey having + sum(l_quantity) > 313 + ) + and c_custkey = o_custkey + and o_orderkey = l_orderkey +group by + c_name, + c_custkey, + o_orderkey, + o_orderdate, + o_totalprice +order by + o_totalprice desc, + o_orderdate limit 100; diff --git a/benchmarks/tpch/queries/q19.sql b/benchmarks/tpch/queries/q19.sql new file mode 100644 index 000000000..3968f0d24 --- /dev/null +++ b/benchmarks/tpch/queries/q19.sql @@ -0,0 +1,37 @@ +-- Benchmark Query 19 derived from TPC-H query 19 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + sum(l_extendedprice* (1 - l_discount)) as revenue +from + lineitem, + part +where + ( + p_partkey = l_partkey + and p_brand = 'Brand#21' + and p_container in ('SM CASE', 'SM BOX', 'SM PACK', 'SM PKG') + and l_quantity >= 8 and l_quantity <= 8 + 10 + and p_size between 1 and 5 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#13' + and p_container in ('MED BAG', 'MED BOX', 'MED PKG', 'MED PACK') + and l_quantity >= 20 and l_quantity <= 20 + 10 + and p_size between 1 and 10 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ) + or + ( + p_partkey = l_partkey + and p_brand = 'Brand#52' + and p_container in ('LG CASE', 'LG BOX', 'LG PACK', 'LG PKG') + and l_quantity >= 30 and l_quantity <= 30 + 10 + and p_size between 1 and 15 + and l_shipmode in ('AIR', 'AIR REG') + and l_shipinstruct = 'DELIVER IN PERSON' + ); diff --git a/benchmarks/tpch/queries/q2.sql b/benchmarks/tpch/queries/q2.sql new file mode 100644 index 000000000..46ec5d239 --- /dev/null +++ b/benchmarks/tpch/queries/q2.sql @@ -0,0 +1,45 @@ +-- Benchmark Query 2 derived from TPC-H query 2 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + s_acctbal, + s_name, + n_name, + p_partkey, + p_mfgr, + s_address, + s_phone, + s_comment +from + part, + supplier, + partsupp, + nation, + region +where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and p_size = 48 + and p_type like '%TIN' + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'ASIA' + and ps_supplycost = ( + select + min(ps_supplycost) + from + partsupp, + supplier, + nation, + region + where + p_partkey = ps_partkey + and s_suppkey = ps_suppkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'ASIA' + ) +order by + s_acctbal desc, + n_name, + s_name, + p_partkey limit 100; diff --git a/benchmarks/tpch/queries/q20.sql b/benchmarks/tpch/queries/q20.sql new file mode 100644 index 000000000..5bb16563b --- /dev/null +++ b/benchmarks/tpch/queries/q20.sql @@ -0,0 +1,39 @@ +-- Benchmark Query 20 derived from TPC-H query 20 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + s_name, + s_address +from + supplier, + nation +where + s_suppkey in ( + select + ps_suppkey + from + partsupp + where + ps_partkey in ( + select + p_partkey + from + part + where + p_name like 'blanched%' + ) + and ps_availqty > ( + select + 0.5 * sum(l_quantity) + from + lineitem + where + l_partkey = ps_partkey + and l_suppkey = ps_suppkey + and l_shipdate >= date '1993-01-01' + and l_shipdate < date '1993-01-01' + interval '1' year + ) + ) + and s_nationkey = n_nationkey + and n_name = 'KENYA' +order by + s_name; diff --git a/benchmarks/tpch/queries/q21.sql b/benchmarks/tpch/queries/q21.sql new file mode 100644 index 000000000..6f84b876e --- /dev/null +++ b/benchmarks/tpch/queries/q21.sql @@ -0,0 +1,41 @@ +-- Benchmark Query 21 derived from TPC-H query 21 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + s_name, + count(*) as numwait +from + supplier, + lineitem l1, + orders, + nation +where + s_suppkey = l1.l_suppkey + and o_orderkey = l1.l_orderkey + and o_orderstatus = 'F' + and l1.l_receiptdate > l1.l_commitdate + and exists ( + select + * + from + lineitem l2 + where + l2.l_orderkey = l1.l_orderkey + and l2.l_suppkey <> l1.l_suppkey + ) + and not exists ( + select + * + from + lineitem l3 + where + l3.l_orderkey = l1.l_orderkey + and l3.l_suppkey <> l1.l_suppkey + and l3.l_receiptdate > l3.l_commitdate + ) + and s_nationkey = n_nationkey + and n_name = 'ARGENTINA' +group by + s_name +order by + numwait desc, + s_name limit 100; diff --git a/benchmarks/tpch/queries/q22.sql b/benchmarks/tpch/queries/q22.sql new file mode 100644 index 000000000..65ea49b04 --- /dev/null +++ b/benchmarks/tpch/queries/q22.sql @@ -0,0 +1,39 @@ +-- Benchmark Query 22 derived from TPC-H query 22 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + cntrycode, + count(*) as numcust, + sum(c_acctbal) as totacctbal +from + ( + select + substring(c_phone from 1 for 2) as cntrycode, + c_acctbal + from + customer + where + substring(c_phone from 1 for 2) in + ('24', '34', '16', '30', '33', '14', '13') + and c_acctbal > ( + select + avg(c_acctbal) + from + customer + where + c_acctbal > 0.00 + and substring(c_phone from 1 for 2) in + ('24', '34', '16', '30', '33', '14', '13') + ) + and not exists ( + select + * + from + orders + where + o_custkey = c_custkey + ) + ) as custsale +group by + cntrycode +order by + cntrycode; diff --git a/benchmarks/tpch/queries/q3.sql b/benchmarks/tpch/queries/q3.sql new file mode 100644 index 000000000..161f2e1e4 --- /dev/null +++ b/benchmarks/tpch/queries/q3.sql @@ -0,0 +1,24 @@ +-- Benchmark Query 3 derived from TPC-H query 3 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + l_orderkey, + sum(l_extendedprice * (1 - l_discount)) as revenue, + o_orderdate, + o_shippriority +from + customer, + orders, + lineitem +where + c_mktsegment = 'BUILDING' + and c_custkey = o_custkey + and l_orderkey = o_orderkey + and o_orderdate < date '1995-03-15' + and l_shipdate > date '1995-03-15' +group by + l_orderkey, + o_orderdate, + o_shippriority +order by + revenue desc, + o_orderdate limit 10; diff --git a/benchmarks/tpch/queries/q4.sql b/benchmarks/tpch/queries/q4.sql new file mode 100644 index 000000000..e444dbfce --- /dev/null +++ b/benchmarks/tpch/queries/q4.sql @@ -0,0 +1,23 @@ +-- Benchmark Query 4 derived from TPC-H query 4 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + o_orderpriority, + count(*) as order_count +from + orders +where + o_orderdate >= date '1995-04-01' + and o_orderdate < date '1995-04-01' + interval '3' month + and exists ( + select + * + from + lineitem + where + l_orderkey = o_orderkey + and l_commitdate < l_receiptdate + ) +group by + o_orderpriority +order by + o_orderpriority; diff --git a/benchmarks/tpch/queries/q5.sql b/benchmarks/tpch/queries/q5.sql new file mode 100644 index 000000000..4426bd245 --- /dev/null +++ b/benchmarks/tpch/queries/q5.sql @@ -0,0 +1,26 @@ +-- Benchmark Query 5 derived from TPC-H query 5 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + n_name, + sum(l_extendedprice * (1 - l_discount)) as revenue +from + customer, + orders, + lineitem, + supplier, + nation, + region +where + c_custkey = o_custkey + and l_orderkey = o_orderkey + and l_suppkey = s_suppkey + and c_nationkey = s_nationkey + and s_nationkey = n_nationkey + and n_regionkey = r_regionkey + and r_name = 'AFRICA' + and o_orderdate >= date '1994-01-01' + and o_orderdate < date '1994-01-01' + interval '1' year +group by + n_name +order by + revenue desc; diff --git a/benchmarks/tpch/queries/q6.sql b/benchmarks/tpch/queries/q6.sql new file mode 100644 index 000000000..3d6e51cfe --- /dev/null +++ b/benchmarks/tpch/queries/q6.sql @@ -0,0 +1,11 @@ +-- Benchmark Query 6 derived from TPC-H query 6 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + sum(l_extendedprice * l_discount) as revenue +from + lineitem +where + l_shipdate >= date '1994-01-01' + and l_shipdate < date '1994-01-01' + interval '1' year + and l_discount between 0.04 - 0.01 and 0.04 + 0.01 + and l_quantity < 24; diff --git a/benchmarks/tpch/queries/q7.sql b/benchmarks/tpch/queries/q7.sql new file mode 100644 index 000000000..6e36ad616 --- /dev/null +++ b/benchmarks/tpch/queries/q7.sql @@ -0,0 +1,41 @@ +-- Benchmark Query 7 derived from TPC-H query 7 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + supp_nation, + cust_nation, + l_year, + sum(volume) as revenue +from + ( + select + n1.n_name as supp_nation, + n2.n_name as cust_nation, + extract(year from l_shipdate) as l_year, + l_extendedprice * (1 - l_discount) as volume + from + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2 + where + s_suppkey = l_suppkey + and o_orderkey = l_orderkey + and c_custkey = o_custkey + and s_nationkey = n1.n_nationkey + and c_nationkey = n2.n_nationkey + and ( + (n1.n_name = 'GERMANY' and n2.n_name = 'IRAQ') + or (n1.n_name = 'IRAQ' and n2.n_name = 'GERMANY') + ) + and l_shipdate between date '1995-01-01' and date '1996-12-31' + ) as shipping +group by + supp_nation, + cust_nation, + l_year +order by + supp_nation, + cust_nation, + l_year; diff --git a/benchmarks/tpch/queries/q8.sql b/benchmarks/tpch/queries/q8.sql new file mode 100644 index 000000000..e28235ed4 --- /dev/null +++ b/benchmarks/tpch/queries/q8.sql @@ -0,0 +1,39 @@ +-- Benchmark Query 8 derived from TPC-H query 8 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + o_year, + sum(case + when nation = 'IRAQ' then volume + else 0 + end) / sum(volume) as mkt_share +from + ( + select + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) as volume, + n2.n_name as nation + from + part, + supplier, + lineitem, + orders, + customer, + nation n1, + nation n2, + region + where + p_partkey = l_partkey + and s_suppkey = l_suppkey + and l_orderkey = o_orderkey + and o_custkey = c_custkey + and c_nationkey = n1.n_nationkey + and n1.n_regionkey = r_regionkey + and r_name = 'MIDDLE EAST' + and s_nationkey = n2.n_nationkey + and o_orderdate between date '1995-01-01' and date '1996-12-31' + and p_type = 'LARGE PLATED STEEL' + ) as all_nations +group by + o_year +order by + o_year; diff --git a/benchmarks/tpch/queries/q9.sql b/benchmarks/tpch/queries/q9.sql new file mode 100644 index 000000000..86ae02482 --- /dev/null +++ b/benchmarks/tpch/queries/q9.sql @@ -0,0 +1,34 @@ +-- Benchmark Query 9 derived from TPC-H query 9 under the terms of the TPC Fair Use Policy. +-- TPC-H queries are Copyright 1993-2022 Transaction Processing Performance Council. +select + nation, + o_year, + sum(amount) as sum_profit +from + ( + select + n_name as nation, + extract(year from o_orderdate) as o_year, + l_extendedprice * (1 - l_discount) - ps_supplycost * l_quantity as amount + from + part, + supplier, + lineitem, + partsupp, + orders, + nation + where + s_suppkey = l_suppkey + and ps_suppkey = l_suppkey + and ps_partkey = l_partkey + and p_partkey = l_partkey + and o_orderkey = l_orderkey + and s_nationkey = n_nationkey + and p_name like '%moccasin%' + ) as profit +group by + nation, + o_year +order by + nation, + o_year desc; diff --git a/benchmarks/tpch/tpch-gen.sh b/benchmarks/tpch/tpch-gen.sh new file mode 100755 index 000000000..e27472a3d --- /dev/null +++ b/benchmarks/tpch/tpch-gen.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +mkdir -p data/answers 2>/dev/null + +set -e + +#pushd .. +#. ./dev/build-set-env.sh +#popd + +# Generate data into the ./data directory if it does not already exist +FILE=./data/supplier.tbl +if test -f "$FILE"; then + echo "$FILE exists." +else + docker run -v `pwd`/data:/data -it --rm ghcr.io/databloom-ai/tpch-docker:main -vf -s $1 + + # workaround for https://github.com/apache/arrow-datafusion/issues/6147 + mv data/customer.tbl data/customer.csv + mv data/lineitem.tbl data/lineitem.csv + mv data/nation.tbl data/nation.csv + mv data/orders.tbl data/orders.csv + mv data/part.tbl data/part.csv + mv data/partsupp.tbl data/partsupp.csv + mv data/region.tbl data/region.csv + mv data/supplier.tbl data/supplier.csv + + ls -l data +fi + +# Copy expected answers (at SF=1) into the ./data/answers directory if it does not already exist +FILE=./data/answers/q1.out +if test -f "$FILE"; then + echo "$FILE exists." +else + docker run -v `pwd`/data:/data -it --entrypoint /bin/bash --rm ghcr.io/databloom-ai/tpch-docker:main -c "cp /opt/tpch/2.18.0_rc2/dbgen/answers/* /data/answers/" +fi \ No newline at end of file diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py new file mode 100644 index 000000000..ea830a1ff --- /dev/null +++ b/benchmarks/tpch/tpch.py @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +from datafusion import SessionContext +import time + + +def bench(data_path, query_path): + with open("results.csv", "w") as results: + # register tables + start = time.time() + total_time_millis = 0 + + # create context + # runtime = ( + # RuntimeConfig() + # .with_disk_manager_os() + # .with_fair_spill_pool(10000000) + # ) + # config = ( + # SessionConfig() + # .with_create_default_catalog_and_schema(True) + # .with_default_catalog_and_schema("datafusion", "tpch") + # .with_information_schema(True) + # ) + # ctx = SessionContext(config, runtime) + + ctx = SessionContext() + print("Configuration:\n", ctx) + + # register tables + with open("create_tables.sql") as f: + sql = "" + for line in f.readlines(): + if line.startswith("--"): + continue + sql = sql + line + if sql.strip().endswith(";"): + sql = sql.strip().replace("$PATH", data_path) + ctx.sql(sql) + sql = "" + + end = time.time() + time_millis = (end - start) * 1000 + total_time_millis += time_millis + print("setup,{}".format(round(time_millis, 1))) + results.write("setup,{}\n".format(round(time_millis, 1))) + results.flush() + + # run queries + for query in range(1, 23): + with open("{}/q{}.sql".format(query_path, query)) as f: + text = f.read() + tmp = text.split(";") + queries = [] + for str in tmp: + if len(str.strip()) > 0: + queries.append(str.strip()) + + try: + start = time.time() + for sql in queries: + print(sql) + df = ctx.sql(sql) + # result_set = df.collect() + df.show() + end = time.time() + time_millis = (end - start) * 1000 + total_time_millis += time_millis + print("q{},{}".format(query, round(time_millis, 1))) + results.write( + "q{},{}\n".format(query, round(time_millis, 1)) + ) + results.flush() + except Exception as e: + print("query", query, "failed", e) + + print("total,{}".format(round(total_time_millis, 1))) + results.write("total,{}\n".format(round(total_time_millis, 1))) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("data_path") + parser.add_argument("query_path") + args = parser.parse_args() + bench(args.data_path, args.query_path) diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index c7754f350..6d0fee185 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -42,4 +42,6 @@ Cargo.lock .history *rat.txt */.git -.github/* \ No newline at end of file +.github/* +benchmarks/tpch/queries/q*.sql +benchmarks/tpch/create_tables.sql \ No newline at end of file diff --git a/src/context.rs b/src/context.rs index b7f82230f..b603c010e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -224,7 +224,7 @@ impl PySessionContext { let config = if let Some(c) = config { c.config } else { - SessionConfig::default() + SessionConfig::default().with_information_schema(true) }; let runtime_config = if let Some(c) = runtime { c.config From 31a86eefddbb00600c95fc4b0a73cb82704d5d35 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 3 May 2023 19:47:15 -0600 Subject: [PATCH 033/413] Add db-benchmark (#365) * Add db-benchmark * python lint --- benchmarks/db-benchmark/README.md | 31 ++ .../db-benchmark/db-benchmark.dockerfile | 106 +++++++ benchmarks/db-benchmark/groupby-datafusion.py | 288 ++++++++++++++++++ benchmarks/db-benchmark/join-datafusion.py | 238 +++++++++++++++ benchmarks/db-benchmark/run-bench.sh | 21 ++ 5 files changed, 684 insertions(+) create mode 100644 benchmarks/db-benchmark/README.md create mode 100644 benchmarks/db-benchmark/db-benchmark.dockerfile create mode 100644 benchmarks/db-benchmark/groupby-datafusion.py create mode 100755 benchmarks/db-benchmark/join-datafusion.py create mode 100644 benchmarks/db-benchmark/run-bench.sh diff --git a/benchmarks/db-benchmark/README.md b/benchmarks/db-benchmark/README.md new file mode 100644 index 000000000..fe268199f --- /dev/null +++ b/benchmarks/db-benchmark/README.md @@ -0,0 +1,31 @@ + + +# Run db-benchmark + +This directory contains scripts for running DataFusion with the https://github.com/h2oai/db-benchmark + +## Directions + +Run the following from root `arrow-datafusion` directory + +```bash +$ docker buildx build -t db-benchmark -f benchmarks/db-benchmark/db-benchmark.dockerfile . +$ docker run --privileged db-benchmark +``` diff --git a/benchmarks/db-benchmark/db-benchmark.dockerfile b/benchmarks/db-benchmark/db-benchmark.dockerfile new file mode 100644 index 000000000..b21d3a0d1 --- /dev/null +++ b/benchmarks/db-benchmark/db-benchmark.dockerfile @@ -0,0 +1,106 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM ubuntu +ARG DEBIAN_FRONTEND=noninteractive +ARG TARGETPLATFORM + +RUN apt-get update && \ + apt-get install -y git build-essential + +# Install R, curl, and python deps +RUN apt-get -y install --no-install-recommends --no-install-suggests \ + ca-certificates software-properties-common gnupg2 gnupg1 \ + && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \ + && add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' \ + && apt-get -y install r-base \ + && apt-get -y install curl \ + && apt-get -y install python3.8 \ + && apt-get -y install python3-pip + +# Install R libraries +RUN R -e "install.packages('data.table',dependencies=TRUE, repos='http://cran.rstudio.com/')" \ + && R -e "install.packages('dplyr',dependencies=TRUE, repos='http://cran.rstudio.com/')" + +# Install Rust +RUN curl https://sh.rustup.rs -sSf | bash -s -- -y +ENV PATH="/root/.cargo/bin:${PATH}" + +# Clone db-benchmark and download data +RUN git clone https://github.com/h2oai/db-benchmark \ + && cd db-benchmark \ + && Rscript _data/groupby-datagen.R 1e7 1e2 0 0 \ + && Rscript _data/join-datagen.R 1e7 0 0 0 \ + && mkdir data \ + && mv G1_1e7_1e2_0_0.csv data \ + && mv J1_1e7_1e1_0_0.csv data \ + && mv J1_1e7_1e4_0_0.csv data \ + && mv J1_1e7_1e7_0_0.csv data \ + && mv J1_1e7_NA_0_0.csv data \ + && cd .. + +# Clone datafusion-python and build python library +# Not sure if the wheel will be the same on all computers +RUN git clone https://github.com/datafusion-contrib/datafusion-python \ + && cd datafusion-python && git reset --hard 368b50ed9662d5e93c70b539f94cceace685265e \ + && python3 -m pip install pip \ + && python3 -m pip install pandas \ + && python3 -m pip install -r requirements.txt \ + && cd .. + +# Copy local arrow-datafusion +COPY . arrow-datafusion + +# 1. datafusion-python that builds from datafusion version referenced datafusion-python +RUN cd datafusion-python \ + && maturin build --release \ + && case "${TARGETPLATFORM}" in \ + */amd64) CPUARCH=x86_64 ;; \ + */arm64) CPUARCH=aarch64 ;; \ + *) exit 1 ;; \ + esac \ + # Version will need to be updated in conjunction with datafusion-python version + && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \ + && cd .. + +# 2. datafusion-python that builds from local datafusion. use this when making local changes to datafusion. +# Currently, as of March 5th 2022, this done not build (i think) because datafusion is being split into multiple crates +# and datafusion-python has not yet been updated to reflect this. +# RUN cd datafusion-python \ +# && sed -i '/datafusion =/c\datafusion = { path = "../arrow-datafusion/datafusion", features = ["pyarrow"] }' Cargo.toml \ +# && sed -i '/fuzz-utils/d' ../arrow-datafusion/datafusion/Cargo.toml \ +# && maturin build --release \ +# && case "${TARGETPLATFORM}" in \ +# */amd64) CPUARCH=x86_64 ;; \ +# */amd64) CPUARCH=aarch64 ;; \ +# *) exit 1 ;; \ +# esac \ +# && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \ +# && cd .. + +# Make datafusion directory in db-benchmark +RUN mkdir db-benchmark/datafusion \ + && cp ../arrow-datafusion/benchmarks/db-benchmark/groupby-datafusion.py db-benchmark/datafusion \ + && cp ../arrow-datafusion/benchmarks/db-benchmark/join-datafusion.py db-benchmark/datafusion \ + && cp ../arrow-datafusion/benchmarks/db-benchmark/run-bench.sh db-benchmark/ \ + && chmod +x db-benchmark/run-bench.sh + +WORKDIR /db-benchmark + +RUN ls && ls -al data/ + +ENTRYPOINT ./run-bench.sh \ No newline at end of file diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py new file mode 100644 index 000000000..7268cc872 --- /dev/null +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -0,0 +1,288 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import gc +import timeit +import datafusion as df +from datafusion import functions as f +from datafusion import col +from pyarrow import csv as pacsv + + +print("# groupby-datafusion.py", flush=True) + +# exec(open("./_helpers/helpers.py").read()) + + +def ans_shape(batches): + rows, cols = 0, 0 + for batch in batches: + rows += batch.num_rows + if cols == 0: + cols = batch.num_columns + else: + assert cols == batch.num_columns + return rows, cols + + +# ver = df.__version__ +ver = "7.0.0" +git = "" +task = "groupby" +solution = "datafusion" +fun = ".groupby" +cache = "TRUE" +on_disk = "FALSE" + +data_name = os.environ["SRC_DATANAME"] +src_grp = os.path.join("data", data_name + ".csv") +print("loading dataset %s" % src_grp, flush=True) + +data = pacsv.read_csv( + src_grp, convert_options=pacsv.ConvertOptions(auto_dict_encode=True) +) +print("dataset loaded") + +ctx = df.ExecutionContext() +ctx.register_record_batches("x", [data.to_batches()]) +print("registered record batches") +# cols = ctx.sql("SHOW columns from x") +# ans.show() + +in_rows = data.num_rows +# print(in_rows, flush=True) + +task_init = timeit.default_timer() + +question = "sum v1 by id1" # q1 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql("SELECT id1, SUM(v1) AS v1 FROM x GROUP BY id1").collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q1: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = df.aggregate([], [f.sum(col("v1"))]).collect()[0].column(0)[0] +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "sum v1 by id1:id2" # q2 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id1, id2, SUM(v1) AS v1 FROM x GROUP BY id1, id2" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q2: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = df.aggregate([], [f.sum(col("v1"))]).collect()[0].column(0)[0] +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "sum v1 mean v3 by id3" # q3 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id3, SUM(v1) AS v1, AVG(v3) AS v3 FROM x GROUP BY id3" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q3: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v3"))]) + .collect()[0] + .to_pandas() + .to_numpy()[0] +) +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "mean v1:v3 by id4" # q4 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id4, AVG(v1) AS v1, AVG(v2) AS v2, AVG(v3) AS v3 FROM x GROUP BY id4" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q4: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v2")), f.sum(col("v3"))]) + .collect()[0] + .to_pandas() + .to_numpy()[0] +) +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "sum v1:v3 by id6" # q5 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id6, SUM(v1) AS v1, SUM(v2) AS v2, SUM(v3) AS v3 FROM x GROUP BY id6" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q5: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v2")), f.sum(col("v3"))]) + .collect()[0] + .to_pandas() + .to_numpy()[0] +) +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "median v3 sd v3 by id4 id5" # q6 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id4, id5, approx_percentile_cont(v3, .5) AS median_v3, stddev(v3) AS stddev_v3 FROM x GROUP BY id4, id5" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q6: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("median_v3")), f.sum(col("stddev_v3"))]) + .collect()[0] + .to_pandas() + .to_numpy()[0] +) +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "max v1 - min v2 by id3" # q7 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id3, MAX(v1) - MIN(v2) AS range_v1_v2 FROM x GROUP BY id3" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q7: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = df.aggregate([], [f.sum(col("range_v1_v2"))]).collect()[0].column(0)[0] +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "largest two v3 by id6" # q8 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id6, v3 from (SELECT id6, v3, row_number() OVER (PARTITION BY id6 ORDER BY v3 DESC) AS row FROM x) t WHERE row <= 2" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q8: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = df.aggregate([], [f.sum(col("v3"))]).collect()[0].column(0)[0] +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "regression v1 v2 by id2 id4" # q9 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql("SELECT corr(v1, v2) as corr FROM x GROUP BY id2, id4").collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q9: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = df.aggregate([], [f.sum(col("corr"))]).collect()[0].column(0)[0] +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "sum v3 count by id1:id6" # q10 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT id1, id2, id3, id4, id5, id6, SUM(v3) as v3, COUNT(*) AS cnt FROM x GROUP BY id1, id2, id3, id4, id5, id6" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q10: {t}") +# m = memory_usage() +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v3")), f.sum(col("cnt"))]) + .collect()[0] + .to_pandas() + .to_numpy()[0] +) +chkt = timeit.default_timer() - t_start +# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +print( + "grouping finished, took %0.fs" % (timeit.default_timer() - task_init), + flush=True, +) + +exit(0) diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py new file mode 100755 index 000000000..1993a5c83 --- /dev/null +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import gc +import timeit +import datafusion as df +from datafusion import functions as f +from datafusion import col +from pyarrow import csv as pacsv + + +print("# join-datafusion.py", flush=True) + +# exec(open("./_helpers/helpers.py").read()) + + +def join_to_tbls(data_name): + x_n = int(float(data_name.split("_")[1])) + y_n = [ + "{:.0e}".format(x_n / 1e6), + "{:.0e}".format(x_n / 1e3), + "{:.0e}".format(x_n), + ] + y_n = [ + y_n[0].replace("+0", ""), + y_n[1].replace("+0", ""), + y_n[2].replace("+0", ""), + ] + return [ + data_name.replace("NA", y_n[0]), + data_name.replace("NA", y_n[1]), + data_name.replace("NA", y_n[2]), + ] + + +def ans_shape(batches): + rows, cols = 0, 0 + for batch in batches: + rows += batch.num_rows + if cols == 0: + cols = batch.num_columns + else: + assert cols == batch.num_columns + return rows, cols + + +ver = "6.0.0" +task = "join" +git = "" +solution = "datafusion" +fun = ".join" +cache = "TRUE" +on_disk = "FALSE" + +data_name = os.environ["SRC_DATANAME"] +src_jn_x = os.path.join("data", data_name + ".csv") +y_data_name = join_to_tbls(data_name) +src_jn_y = [ + os.path.join("data", y_data_name[0] + ".csv"), + os.path.join("data", y_data_name[1] + ".csv"), + os.path.join("data", y_data_name[2] + ".csv"), +] +if len(src_jn_y) != 3: + raise Exception("Something went wrong in preparing files used for join") + +print( + "loading datasets " + + data_name + + ", " + + y_data_name[0] + + ", " + + y_data_name[2] + + ", " + + y_data_name[2], + flush=True, +) + +ctx = df.ExecutionContext() + +x_data = pacsv.read_csv( + src_jn_x, convert_options=pacsv.ConvertOptions(auto_dict_encode=True) +) +ctx.register_record_batches("x", [x_data.to_batches()]) +small_data = pacsv.read_csv( + src_jn_y[0], convert_options=pacsv.ConvertOptions(auto_dict_encode=True) +) +ctx.register_record_batches("small", [small_data.to_batches()]) +medium_data = pacsv.read_csv( + src_jn_y[1], convert_options=pacsv.ConvertOptions(auto_dict_encode=True) +) +ctx.register_record_batches("medium", [medium_data.to_batches()]) +large_data = pacsv.read_csv( + src_jn_y[2], convert_options=pacsv.ConvertOptions(auto_dict_encode=True) +) +ctx.register_record_batches("large", [large_data.to_batches()]) + +print(x_data.num_rows, flush=True) +print(small_data.num_rows, flush=True) +print(medium_data.num_rows, flush=True) +print(large_data.num_rows, flush=True) + +task_init = timeit.default_timer() +print("joining...", flush=True) + +question = "small inner on int" # q1 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT x.id1, x.id2, x.id3, x.id4 as xid4, small.id4 as smallid4, x.id5, x.id6, x.v1, small.v2 FROM x INNER JOIN small ON x.id1 = small.id1" +).collect() +# ans = ctx.sql("SELECT * FROM x INNER JOIN small ON x.id1 = small.id1").collect() +# print(set([b.schema for b in ans])) +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q1: {t}") +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = df.aggregate([], [f.sum(col("v1"))]).collect()[0].column(0)[0] +chkt = timeit.default_timer() - t_start +# m = memory_usage() +# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "medium inner on int" # q2 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x INNER JOIN medium ON x.id2 = medium.id2" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q2: {t}") +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) + .collect()[0] + .column(0)[0] +) +chkt = timeit.default_timer() - t_start +# m = memory_usage() +# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "medium outer on int" # q3 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x LEFT JOIN medium ON x.id2 = medium.id2" +).collect() +shape = ans_shape(ans) +# print(shape, flush=True) +t = timeit.default_timer() - t_start +print(f"q3: {t}") +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) + .collect()[0] + .column(0)[0] +) +chkt = timeit.default_timer() - t_start +# m = memory_usage() +# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "medium inner on factor" # q4 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT x.id1 as xid1, medium.id1 as mediumid1, x.id2, x.id3, x.id4 as xid4, medium.id4 as mediumid4, x.id5 as xid5, medium.id5 as mediumid5, x.id6, x.v1, medium.v2 FROM x LEFT JOIN medium ON x.id5 = medium.id5" +).collect() +shape = ans_shape(ans) +# print(shape) +t = timeit.default_timer() - t_start +print(f"q4: {t}") +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) + .collect()[0] + .column(0)[0] +) +chkt = timeit.default_timer() - t_start +# m = memory_usage() +# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +question = "big inner on int" # q5 +gc.collect() +t_start = timeit.default_timer() +ans = ctx.sql( + "SELECT x.id1 as xid1, large.id1 as largeid1, x.id2 as xid2, large.id2 as largeid2, x.id3, x.id4 as xid4, large.id4 as largeid4, x.id5 as xid5, large.id5 as largeid5, x.id6 as xid6, large.id6 as largeid6, x.v1, large.v2 FROM x LEFT JOIN large ON x.id3 = large.id3" +).collect() +shape = ans_shape(ans) +# print(shape) +t = timeit.default_timer() - t_start +print(f"q5: {t}") +t_start = timeit.default_timer() +df = ctx.create_dataframe([ans]) +chk = ( + df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) + .collect()[0] + .column(0)[0] +) +chkt = timeit.default_timer() - t_start +# m = memory_usage() +# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +del ans +gc.collect() + +print( + "joining finished, took %0.fs" % (timeit.default_timer() - task_init), + flush=True, +) + +exit(0) diff --git a/benchmarks/db-benchmark/run-bench.sh b/benchmarks/db-benchmark/run-bench.sh new file mode 100644 index 000000000..9ccc26804 --- /dev/null +++ b/benchmarks/db-benchmark/run-bench.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +set -e + +SRC_DATANAME=G1_1e7_1e2_0_0 python3 datafusion/groupby-datafusion.py +SRC_DATANAME=J1_1e7_NA_0_0 python3 datafusion/join-datafusion.py From 4175134cdf0ed5168282eb88feeb35a7540ac312 Mon Sep 17 00:00:00 2001 From: Matthew Powers Date: Thu, 4 May 2023 10:43:29 -0300 Subject: [PATCH 034/413] First pass of documentation in mdBook (#364) * First pass of documentation in mdBook * Address code review comments * Add missing ASF headers --- .gitignore | 4 +- docs/mdbook/README.md | 33 +++++ docs/mdbook/book.toml | 23 ++++ docs/mdbook/src/SUMMARY.md | 24 ++++ .../src/images/datafusion-jupyterlab.png | Bin 0 -> 86143 bytes docs/mdbook/src/index.md | 43 ++++++ docs/mdbook/src/installation.md | 62 +++++++++ docs/mdbook/src/quickstart.md | 77 +++++++++++ docs/mdbook/src/usage/create-table.md | 59 +++++++++ docs/mdbook/src/usage/index.md | 25 ++++ docs/mdbook/src/usage/query-table.md | 125 ++++++++++++++++++ 11 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 docs/mdbook/README.md create mode 100644 docs/mdbook/book.toml create mode 100644 docs/mdbook/src/SUMMARY.md create mode 100644 docs/mdbook/src/images/datafusion-jupyterlab.png create mode 100644 docs/mdbook/src/index.md create mode 100644 docs/mdbook/src/installation.md create mode 100644 docs/mdbook/src/quickstart.md create mode 100644 docs/mdbook/src/usage/create-table.md create mode 100644 docs/mdbook/src/usage/index.md create mode 100644 docs/mdbook/src/usage/query-table.md diff --git a/.gitignore b/.gitignore index 4e4450082..1d0a84a43 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,6 @@ venv apache-rat-*.jar *rat.txt .env -CHANGELOG.md.bak \ No newline at end of file +CHANGELOG.md.bak + +docs/mdbook/book \ No newline at end of file diff --git a/docs/mdbook/README.md b/docs/mdbook/README.md new file mode 100644 index 000000000..664b4b4ce --- /dev/null +++ b/docs/mdbook/README.md @@ -0,0 +1,33 @@ + +# DataFusion Book + +This folder builds a DataFusion user guide using [mdBook](https://github.com/rust-lang/mdBook). + +## Build and run book locally + +Build the latest files with `mdbook build`. + +Open the book locally by running `open book/index.html`. + +## Install mdBook + +Download the `mdbook` binary. + +Then manually open it, so you have permissions to run it on your Mac. + +Add it to your path with a command like this so you can easily run the commands: `mv ~/Downloads/mdbook /Users/matthew.powers/.local/bin`. diff --git a/docs/mdbook/book.toml b/docs/mdbook/book.toml new file mode 100644 index 000000000..089cb9a97 --- /dev/null +++ b/docs/mdbook/book.toml @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[book] +authors = ["Apache Arrow "] +language = "en" +multilingual = false +src = "src" +title = "DataFusion Book" diff --git a/docs/mdbook/src/SUMMARY.md b/docs/mdbook/src/SUMMARY.md new file mode 100644 index 000000000..b4908994d --- /dev/null +++ b/docs/mdbook/src/SUMMARY.md @@ -0,0 +1,24 @@ + +# Summary + +- [Index](./index.md) +- [Installation](./installation.md) +- [Quickstart](./quickstart.md) +- [Usage](./usage/index.md) + - [Create a table](./usage/create-table.md) + - [Query a table](./usage/query-table.md) \ No newline at end of file diff --git a/docs/mdbook/src/images/datafusion-jupyterlab.png b/docs/mdbook/src/images/datafusion-jupyterlab.png new file mode 100644 index 0000000000000000000000000000000000000000..c4d46884e93a077c7ae8e3a13f4077854709b2d6 GIT binary patch literal 86143 zcmeFZXIN89_dXmzKtaR;qI4?)(xgcVMd?T{p%)S9gx;$t2uM*t5Tr=20ci<@fQSl$ z5PIkV=_T|KAe47=p5k--9l!oRy6=|p#sX!nQjk1!Q zHV8!C1_F`Ro~8iK_#uMjK%lcSb`KwFDnES4uIUDSYUgMT0x3l%K`0G$zF*ETeiJYA z;0!zat?k<{`$5_dY)Nt%niZ5tpI>~;UO3bvPmbZeT=AjgdKAlfiN`itjzbL8r0w+v z72Hs58BE};|5l*;eADs!J@h+wn^hE9$UY~`;C+}qD4v@u{!92N{%^Mn_?1jSl*J(w z??Wh)llh*?D=LBvC`Ox-z2Tti4&@SU)knj}=Fzv!0i@ zfMpg}WSkPe|8-bd$?Id)s>Esd)nh|Astc9RS=qSq!!<$F~# zY>@I}!b?6nlB}+fAyHA$I9gY(FCyIvXFop5e76&({tYYC7cNvc_Uc+`N_0cvDXOin zVqe#JJnXUbFCUQ7?HgFu-@LDHXP%lSec#;CJwN_Z(HBap zd(#i;_nK3EO?(erc!%^tUncPv-e_ZE61dOM$j+}M;9Y9WuJTf6Dd0TAsV`jLe$u(B zSNF1tUYdE{<|Fvhbc=dA(Q4xhDT1AT-=`kIE^c;#!z}SDO|<@(Fg~&iAvB~UWJ;E5 zWWMawr;YuW{io8iIj*>bo_YczanZaNcH_+q__|LFnf#>2%Tw&Zwk9DB5D?Y1W6zg* zCM+`b4RB4V>r9ei_ZgmtG5hc(#a*QFRXwF~?Z!cbH*cxr73jIr%A;U3Z6)^VVfGOuU#xvGmbu0KhLMSh=kG;Ej z_w51Q?3Hd3d{|PQ5>bEwLErDuLgvC&crMA+bPRNV64p&iO$f^vXYbJe#?wJH&4Fv` zoQx>=P<=zD`1qbzya|x;b4S z`Lm)*Zy}5M$O7f(SKpDJy?k)}^cCg{XxCFM!{3L=LBj<3jeWmMH;$dz>v^zszQLeL zcQ*@*$pvrTm75?UcydR2Grh|4vql6Lh#_`8_ss+Hhz{OUyKglkB-Gfhkhxos%}<{l z3l&|blx06F7wQv#_V(um!71_N5Kaq<6lxWjs|FPMVR(y6LsVYRz>}at%GOVIF6T@r zAkAtnw6F~Ib<*un?@7UD>^GlBbzcbla_Xbp$_1O}k-OJrl}=rZzImfaiQ@W$+_tk% zShd4TWd?5;$J;-Vb~_~&VOrKMl?%u5)uBI|#!7<|U#0|%_)AOXXP)aD1!>oOJyctbpcoKziSc!mY z&V%Wf3P^>f+otY`PZOWQ<{o#NrZNa4Ym|)ijbx1i%Jz(mYr1T~AFMpw#nZZS&^geY z%sg-VOEqORVk3ql)+7Ga#x+EbY7Z07i=O#Qx(m#h#qO^jWgcSnV)d$?v_Q>tf?~sxL5t=Uhs+leFcc%A=($oG1KH;hVxWg~vIb3O29&VmxCGW2#<3u4eMD z3VcZ0O+Brq|L(nzZ&$Zv#TBtvHLsp3WEOp}D=g~wh_)+Pv=MZuI#ueM10Ddic1n6u^Qm~Ec+?1;YGwMw)G81g zZa-}!IGo=;7tYi3)-KHn*}PNCVn6>h*$!gq{gt)G8_qBSt$;GgUYQbmAe3z{hYU+4 z>-f<9;oH>f9GKN}tB+Qw>5kWzZ|ErSviSzhh;oP)Fc&zf6BO)XsF?I5i(UI2Y$R+l z++S?bFchm+bT}qcBtu|3Fj7nm3_iZ#Y&CWnAzT;eNpML}6Tc zv+%OCVV%zepE*1uJ{WtDAMu{**7NWPURtO5E3Q88?bXdw6o-CP%(>L8Bi8X#aW_`3 z-@FiG)h%FYr5{_zW_Oh{sx*#N>wRI87HxJ}4u>LJ%(ZWFt-4<%qgm)BZtMgwgPpa< zUjMWc+r;xwjn_44r> zLk+?W3<1GMTtRW46e}KdC#@zlUPno^a9CL@Xylte^?h0%&tt+DkVKza!?_&jrMnZm z68b*(J*tkYuBc8?&M4ba%RR$&qQ61pBMKomCNS)2RsHjOy8!pHzJPub%n#|;yA(YOn@tB=zhA#3<3HJ=oqZ#r{ z>Q-JqQ%Sc--=d>R;%xn8_}i8y=Dcy+_)p98&MwX- zjaPlEaQdIiMsd@l63$ytZ|wKFttIEFaWr9WjAM*1Fl!_0;KNi)v9#my(CE~J>w(r} z&f8YkxHa$;*mpzdcrm$a4_%0EM?R`AMy4a<@TUkSfl{l!^VA#BXw`y<1>-nrQi zO$W#Ji%g@KJ(W%SN&HXIeW+K1XW(c*+xYRi=_(SvrYgM3AmCb{ zA#q^GVH&;;B~ZFiQhYHv13BUjX^zS-AY1pN1ob}wEvyG9TrIeN{GB@cepUO0T@3?Q zg+vB#kp0xL|0BKwq#dONR>bGv+5XZr$;*7w^J`}`8InJU`{*ZRa$CPRjV4L6YRsR_ z4-M9#v*}{{c~nP!PY<&0PUL%L9S|q;UEUQ4DWa?ml%J}pfw+M3Y0xPWMi4nrA^|>9 zBuxJ;E0Ej)k^Oa^6a6Udt!Y$JV?)PA(_U0ZICZ14Som4@-6*Cr4*@aUbyYziNmB<&)EV*V+H7 z;^6?kZlI>g{t)VB%`U>r&&z*Zii(|`UDEBzQ*muMg@4};9D%RfdU&{q^YM9md-Hk= z@?D}uS(Gr2o>X? zqj!HG|LO{Vmc{3g%*S2V=UzURk>{trn20?7%xLe2w6S?=W8-Cmt&e-8(Yj<#*q&I^ z#TQYyb46*LmjRKS`p2(A8&aCn@t7*^uEpyiAkx$9|L}F6gmh-?^gmuYsgBnU0r{dD zR^AK!^X)(#61UYqyk7c=S)(qLxhrOx=LIw2r2!XMhk9wLLHW0x`Gjw+bo$WOp)JfNB_$(!#F&Ni+SIftJ?Hs%X;wvAfxTdGK+G zQa=2XbrkXdEp_Nf%3}Rvcjf5;6V-J-7yd_y8J_}-=qbL<;sV58IL3VTj}ogr z4Hz;HXO8ze{5i)N|ru(+i`~Q_|YMO&88HEh+5 za9p~$^VV(EQIw)&`hM4`f7D6d#XEK(2ZY6@5!3RsH3Dps_Q3~-X@^_pE+$u~Vx_%S z9P-?XbXxT+8Ul$sur8DzzOt%IyhMFK8q*$hh{t#h=s|{=pbeJN#|Q5ATdZKep%4-> zZLGue7cE5=Ypd4Sn%TF;T^T+np?o6Ljuz8cj!+I8?XH8@o?Q{am?szt^g z_LKZw^jXfQ6NO}g+IYZ@pEWYSs_^h*pY43vfPUk(dRi~@k#vTt_AHag50-$Evf9d; zK_hO|b@^Qp>vp=)$I(2G<8&EFXMk?jFqru?zcur-BVvz_82njhMG`>1tB~y~Ht9;m zi8{U-%QS0DtrKQM7>~-oO-IKjWE+p#?p2M97;>!YT-jeP*Dx{TrCu6=dX7}vdCk4E z8A;HPy6S)CvZ;p_a`G8BYE9;;zlc* z&Q`3{zOBScMGK(y4mqdY(KqGtrZE4;`5#_?vu>HXGl_R)j@l5!m8gO)RbLTP9{~wC zk$%wrBmG#z9%H9XG*6|Rz1XGiq~g*F&`|7 zp)T66<2b4PHs`w8%l?2)mgGYkUGt!W}wH)_-}qG%j-1Isi3Vb2=ZV$B|oJMsqyp8Hiv?RT9fy7c-G zXCi1F=T_d;Ds+g{&fRku9t#v!H8#hi@L!unY`T~o^tCu}5VGhX;aTd@CwLy*r_R_) zKJLvzhYY{(ueP`M84f=5%8|+dYb=eCV~N;|ZD{MHYi^jhzi=+0*^V z&B5i^K!RzSklh=mF}JVukZR#UgBJV~eV0<%Fp8w&B9GfuHfXOc&jbH86h@7kc0+u* zG@-VUluyO?BF~b*VLNyE&~kZ`b1^)&sb5>waCxxI2|oaVJHf|AK5`)ZO1MB%9l{us z>Dx)-GC*h{53Ur&25j}0VuO#zS$!r+2!Sm$qOWit%K|1dgOBQ{DRx%JC2E}RZ=j`* zhjcW+vj>fPb4-2gKE+KGO~%28c&9!!QBQ1((?RvKy(QFD9>xV1jA>N|Cy(f!-6X0= zr(lGur1bGIA^3QHOg93t`JE#Z(JWK*$WD-s*?+Zeeq{h+E}R}#TJMERO+ka5GwXgJ z*YKk&1=6=$ohv>1-qsTLiLJa$M@u|ATjn}rmAH{L`x_I8mhDp8QG1rr;Di>Z9`dvy zHMFVqTEJmFPYXD>iopeyGfm{)D*3793b97P4B?$`(9^zYi&Q7oMzf8XXk2 zUxc#~-fs}gx}a)AO{m@B9R59V6}jd(_v0Newuz!>>`8)JsZN1|jgDeB!Ir5%>V(J~B@*K*zBdJDrZTACZTNIu#0Bo)lpe*7 z;7bTTZr*8VEIGE1VH8o3h|o(dcfM4hu9Q~bxhm;c%`+|pzMnaw%z9~`Z3~!l5!8-~ zqS8NK-dL$$hPhYK3rQ`PR`u#@X&xr(s4J&29A;Y-nRP%WMeh2UN{?oP68$p5<(r2P zS-yx^Z4wi}H`ey5aKES;uU;zKx6#QC^Wd7yW1Z(JNcT#`p$HR6Uhdzn+6)zH<%}I5 z8rm5{)MU9c8?$=PaXo#)Sk3=kDos}0rZd$VV&)P2o*TcvLxr5LG1l&&mkdT_Pu;DL(eJL3MZdJ& z1J+1RlQ%dZj*vRbmU90m$qA0|SmIu}U8Z-VakYbPEMq@yLnQ6e77lXYQUmqn5HWTpQ@*sdr8B6F zZ%hUHdga#mi@F+)wB!T0*#@4E za()xO4ywcf|`iN}lmGzG9IWBrmQy6CXkLhO0YgBA&;n+6Am@J=+mT7`Jg6sopRjx4FVj zpr+e3@tsY=0jB7J-`UCx+!@w|*-4kzXTCTcZ(|(c0_w00X($dfX~gi@(L8paJ8&SL z8=h!9#vzMEY9`WHwKi&%uqw;)WHU+nr=Rw}1=abDheL(Nf{zbZ;f%U9?aKIxabWV% zUS7PKXOon{wsh-ou^_YF+HYCDdBU~`@NJ>Jyr%WJjG^1_`;2b{g=m2tw*A37Lsq!S zFxptwkw7#7&WQV3Ha|LO48_HkWlSFJ<=MQQkc-^887D%z1czaZ2Qh17$i=;!i49BI z1Hi(D&w;l!A;$=t;FU-UK~+O<;VZiH&}j7=4K_3RS%iBXcdahne#Cz`z9qki*cRo! zyTk%(qUK^;_wCM-E&A$btU^O;@7NQ><~wtNu!85N%U%sK5BLq9NHohTB-x~919u*! zZWKr?qk|6k%OC~Xr>hOUaEWb?qwhL%IC?uTAv0g@uoJchMm>_|98$Ea?i_yamkxGA zJq)DXtYTfU;S1V%Wj-G0toyn*PwT5iR(XJw2#-tS*hTIUTp;U53i@LPns^7p)+CWy zEhw!9P8Buo?0XVx=WV)2+O!VzG?n=wQM7LgF*A+C)wGd7 zlXy84BYkd>F&O^prPI$Z7OO0O!PXI0f= zXDJNTZ)jI;|HDHD`DahGu)g!@yeu)&;f>}VJ@Tt1z0)Rv_<^9;#8(t;{gX+81w-` zl(D;DbT!nxm}smcev!reR#CGGS|!QW{*I{NT&81i3Dx6e9#(o`z;p+z?Q{sH^JVij z&?kK!yA6XXmLVo+o5R|`3>TVMzqV@;LkQRDv7>AnhN(3ZRJz{0i3;a3>!4s}6OaA& zI+U!?6|x9F$f_KHAI>)^k|)O+sl9^*zjyUN&02sqFYi!D^7P)Ni1R zeZrT_gB=Y2{M^MV<898Bz}QPBn90A|J0? z1Wk4#^WWP299xA9J#}!z%?0jED|!t%E$)>bZrX+%yh^7S)~63Ga^hPyc-<5bK5Eu_ z-)cTujf9tcRkxqa6x>r{Oy{Gmp_C~i1BTJ_(?lOdpwKj|4D7U+m^6K~a%RxX^GoV3 zobo>hOCn`Lo?W2gDk{VjXwYBs4(Q~58e+278#rm)EQwWmtqXUYvtjbki8aQvLzI{B z#S?F&@X>Bk+~`U^%gMXE*5Q6i`?6P)phmt04#fcnhyGq)Do{E|w1`Zj8q#!ShNTPM zr@(Ai9wY#74d{EpP^Hz{U=bI?W|<5YfRm_&E(vgS(FA=XZzFn2jC&b+Yj_`wa(%m` zsghqI*}ry6G&XeZMUz7y263v@$-VYA4p==_g*mUTGGfPGMFoGJ4Kf309`b$Gj_X0* zBX44icL&C+f=$Cz#cQ%4* z?@pFx+-kIOMm!#}pr?5ZA;9y;ut%)we#SRTRYm%SkMwCJR`^AwLU)QLy@WvJ3J+%M z^fyqS$CHnb&XCQh#Lb4)2c>gl_k~yVU^ip!2zhy`9(_b zW|997=ad(JN0c-lgo?AV)B0Q77L(dLdML8+QBXS?V&G_9-8N4^Vw$;MP90Ed;#TbvQDn@6DrWNf*Vx`x5>?IsBH+*A$qAK;c zDFN#V4Ui+7S@J!!m0#PWpkcHZP7_#$j&8Km=s^8!C*sLg-N~M+7hTb<#&~Y|2|E~G zk;D-A`ODWB!zS~?di$8alDDy~9#;y(u2=|-YDaOQp6CVSxHC~UD{RJ<@;4h+UL zs96ir`1~Hnn3P}O^^#5c-vFz(RQ-FPz2-DFE23kfZ+x?DcPXS`g1&jJ4!n8b8A;J2 zYU1gRv>-GmkIxHPPc|U_<%{|J<}vj9R0acpDwX^ zgpf#(?-h0kyQyc)Wj5vxp*CBOw1CEWCU&fU%fprUuESo&i~yPaB)*KQKE)qQKh}G! z%st6`6oQvka)ew6b)m-Q-YA}IxI%zfdFw#!_a);IMASBSE7tvGCJWO+EJe>W#idw^ zwk_YZTwM~k;7!IGGR^XNx3nxo#Y4&_4D3d0%gn(%YD)G`=`SsLHUgR|7%T=>QFuMF zY({K=R!9{P5yCRsUrgjG@6gbVD2g-<@M|*R=+~A%0OoSIduA!G)RjE|a1PV0apj2= zGXPwmPTrUU-L-X5$-As223Um?uom4!mTgw`d(v{JEMwoUJ3V)iewQzZ;>-z69k{a? z&FI4Fx9~wFhCax5_npIdi_^Q`auRH=)XJ5$Tm3N(*XFZA{Py8fi|(j`}!l(tWP!QI>uGLA6O<}r7igM zQE>zUt4;#SEY~B;D}ofLz@`_5r9qK>0XHrMwyDW(#Hwl9?g?zFz-nq(8i!L}Nku#X^JG*86vqaDzo@7Q9=p@shEdwOIoVJ5(bD0(M_iZ&NuWXHk+y(L_cBtfqF znKe8zFqLJ|Vr;R>cH>!h!={7Hjv`sE@piEzopEgH1VS?>i^A~LQT~8IfzXxVM8}{K zswHUsV!pHYJ8gQ-!&kkHg>vqe5DppEbGq+1W!BCy|4Ro6fDT+-mL5UDFIwA#%INfZ zLSq$Uncs=!+=+bhwjIsk?C$ck5~ebg>wodK}Ibk6ncH~mOsGGVU!*?{p$6IZH=s_!mcD9CW_ z$;RvtAhmBTUNL&>z|MYltfaNb@3E+Dw+vDx$tAw)0cBfDMsiR8sYX(&A)-lJk436E z$9~aPZCE3^A6@YWAOXTjMNbK4KT;Kaj5*1&D8}MYM*qh=9V=}EpFPi0QoAfd}%)@B7cUd5b$3}R~o;C4+ z>0EJ=%?bttBTwQvj#1XlVpYy%*fjCYGWIfldyt8r%YGfT{+ap^=5}FV}QSa6yO zp+fu2cVV6HHLLG-zTW#A01gW6J>4G-@)a_7%?XcLnF``dqgdB@r?V04Z-x;TUdW%r zF9ww2YNljwt;w>^MXI&;fFFN3t#i&k$u`8~n9Q70Mt3{)Sb@oDCch>4Gv-*3ysagq ziUP0eBBpbOjFL)(#dqd)l0GdZhT9|L0FDh3%#+a)n+zcvn+&xir+beD@D82w0UIWE z0r=!!^g=uw=#GD*6)sv3IqeI9<9b>w(eNTZ1ctXjeq#6&~- zPG~oO=fgK;VeR5C_zcK?EX)*{znK%^Jd!U?;^wrrT(qdlhw4uZ=Ni>ssweu*?Zf3q zPf}g5P;HFjwA|f_PozQKhQaoq?1lI5Ou}Rkt2Dy*c6G)4tbS%k^R^?J0+updBks!m zHj?UPP^aV3DR?*@QCp>!DaJ$I0D$&hgS^Qj&$f7MF;d{R4B}0&`7SuH)gU^bN0pTw z@*>ID=s3qzzOF)8CYO(D)80aatvZULttFH>seFGXL1QPa9t|Mb^(2!48Vip|+-RO- zK3XZlP4hiSk3g1uO^8YO5{wlW#NUqaJZLaa zJRuf}TY7XuS0%hkc^cL~uOg2)twRo4u^hu#!GeON&Ec6i+%rXWCPB?TG;Ly9s_1#Slcsttk*3Sf(Bx5!Ybz z9@+p7E;V3K_{NEr00hPD{;U3l4p~|>6-;mEGyHi6?iXT9d&{2Z)dSPY_{$7##s*Nm$!kokUu|TqJ96srN;Ywev;#e+&pS5?Bt^zOw%HC@y4eQ+kW1M(w@JEP zc~vK55AISO1f-X=q3d@DN6yGwaqe$4#{0NFGyFEV-%taCyDl+@Y~A|J+CUMUrE&I* zbCPy}f!3|hW`k5xoP{5|#FtZ^Y45Y)skZk4hZ}(!rK`~9SCCDWzijwrwol(eHcWQ^ z4Up0E3-y3h0>Z{>lR)eTE#R=&Olpx7`>5e2<&>=V;V_)UEaEqP3q zUWxwH>Lb)~(!A(hsx}7~HB&|W0j+AnZ8dFKXL7qlzG2aT6#xt3P00z>qhHY)Pm+3Y zi}wzs9dKeVB1@joA8#*?HT{f<33Iq9ld;&{vvKpWR>+wogO`6M#%a{~TZ%oV#W7*%LoC zr_+~U)pAS2WddZ?kK{>a3NnMl%RVY^m24T)FqKsEq13YH#}2m8P(IZ58zzYdTYCL{ z69vNWBntMyZ8e$eVg>UDKyqbupEaJdBDYpgZ*fE<08Gh}F9g?s+>EGk8J?y>A zc_pt|J|XIvJe9(oH3v1oHE&LKKctNTH$3a_;6*yxvp3m-zEa{Ii=KPRHSkKiOD@{4 zhs!73No=Vw$K?E2dAUrz-~Iip+)TEhc8$>Q6$HfFOYE5?xvYbYihc|Ei(fAres*5l zdMRvJXg87#AdT>nR+D%UzI}Nj%5BZqDKAeynwLu{#6&_5iYJ<7jy|mImF0TcuNxw{ zYA_`!Tp^FO?RF!7C%oV^WWnIfpk5Nn!7IZGyCj;vZA`7B{MBiGz9>0hzAOC@({JM$ zT;f-yNE7rf%I|(j54+Km4`w*79-H{N7cuX9ntf_a;)A4TN z$^2W+7(|-KaL4ekq_Jn{O^tY`iUO=D3WtJXchZLI#>*=@2ZzM*sNzTtY;YUQXHzmw z%wJ)oxLtL^KcTS}{Vl>MEb4g6dMPuZY<`mWsKoT&Sp&x=Chv;1puLe!2+VEvZqv?U z^hz^)?p$x}qbhwb%#=oMZOQ6$KRT>S3r!I#G$@#gi+dr5)sbsSw#dTI;OUQ)sm(YC zVnt!W2ft9R)1yT?9Pm555N||G{y>=T?)nt(`O6fyh0YiF&B{kVlU8v(4=eHPN1B8Y zi)`wiH7kn^wjAzd__;Rkx}aRd&t4|(x#`fl2s7K(sa%oi@y5R&a5!Mp{dq-C}&58QZ`F8=ehsK&A6$T7Wf?se1l7_0IMf3q}Eb}KZVAWD-fwq3`K z%M3nNE^7kPsUx@i$5K+V_O{|3n5JsWqp0I z<8ybLd0{#`*rjJlbCpTM%6wKSEi@QN$8CG5W(uUyhjq&g8w{m=n9SA^~8%l zK;;b>Ok#OAI<||#X}}ISMGDoW{hkmOUAM#0Y$uPk!VC9{TxEij>php>s)PzVjR+CK zp9kl2fZ1x+Dw2KC8DJbhgBe^I0%k2M?=O#jo3*m30aAw9wqOjiOFLVFnry}Jao0Wa zj<<%6b=9AIp5TqPJO@h+^Gzy&39S2qn%$eC?DoDA3@*t#AW2W(wMhLZDe~RU>$Q92 zleg91FZe~l9fy1Mjh$*5DD*lAyY07N9@LD8DyPxf+L<@1Pf^@7l?AM;0DhFvF;$Xy zL7WoRjs=pSr39UH`oKf_v)UF~$MnlX14Z5+;W5fz8=gNNp&gP!Od9eme^E$}q{)Giqbph*a8F~n8BEzbr0q?JCH)V|O85&^Io z>H?jJ2#*~*e@nbnf$^fK&Iz{jZAGMnm*0l%(A z!`F-N9kR{D2^!Le{iyUDcT5$R-Q5@KyTD9f8R+&-U)>d4fcI7)6m=OxPZxzgd-iuc zGeqWgs2>wM`uS%d4k%0`RVLZ2_Z8{?C#Gmt{s|AWnxCi$3e7$PaP}C5DIffOd3wSc zY<~bS4-HKe)JaKB-2{lC#Q$66ybL=7Kntw1{m4sty5IhUA-KP(DfV}*pCD^lgTQ;r zJ(HhPT?>(EBxUp=970?3M4<4o`oVhZ(SGmS|H$S$4Ky@@@zEdYqWw(3EAeyM?%BUN z&tHch0P3S=*q_Osem4NJ;Dm|2I^Ew9z+atZ2UxSL_WXb3%xXeU7>Q}BosqvK-hMHJ zp98$dhcx-Wr8Rzac_{{HrJs9|nAZOl-M<e4?uN*`2K2&kCbi|4 z9)2tJ^PX7jvY#f?AH{zCWPDr{iuetG`qi64Q=p{|7<`t^#DZYUvQEEs^4DP$FajHtx|u2dWTjbNCnL-h6E^Z&Cx0E%DFZh2I!o;AA0>u6 zX(s6kKX>;gAa?m@bNs)_>DOV6D$r6~J!}7; zyZishKtk=w+c?zU8z8&F|14Pc_&h)=cRv`={82j%r>HKn1C&o+-y)YkTZ%OwAiEVl z0gjh{H;`YX?DHHlA(HAZe1FO6x6=Rrh+hKve=Q3c67ryf4aGEm%V)y&{cYl&OS-ce ztuahC0Luhk*09+=SZUqCg|Q+3hrtti0Qi9U`EU9_^JNA}DVbeDPKZ3U-*2Z4#fVNA z%z7!Z`sT&}e3?O0Z&T!Sti%b^QqQ@C>Q{e%mFlR^v!lmL)u9_x5p4Atr2u)jrZZKz z!>HQUvV#Zyp8CrPp<6xExHbujfu$SQK+4U$1`Tbjk%r$fC#=qY(Yt|mWAJyK$LoQu z#e+t-%9-euF8cr+-Iax$I4|zAY*J$*aA;=3X0WQ%?j%{?VUgS3Z1SkcsN)v)qSPt~vfU0D*To16`h9{fDoB#&uD^s)fcK|jXO#?8=^(+A< zd9hCO!Gr$5ot5F*mU9wA07U7K25t0JZz{Kn75jEXXgVGScqC35QhT|b08KuxA(5Hs z{|?|gs-faQ+RP%1W(A0}-N4*@$d$^yi|yv^KqmVO!A6g*O6{5bN~nCBhMq(Ne@80Gzg3?0&*U zXZ2svGA`D)22*JfDhRtkH0=mbR?9XKK`$WYfdl8Hb}a}=W;13lH2$yoWK{sO5cRSl zH=!1I=+N%QwCQ-igV7>uswi}|5uGB?BDKG80|`+6A2m1ZpJz}0`d-Z4@%ydhYJk4o zsBb2yr$Q#-wb*-0Pg>ZG8OWH`{i@;U%`&iJhlA7&r5-w0jrEQNS@_?+DG!_#CJ&)h$9D2NY+I>f03+p z_y)sin7(Di`K6Xue5U%?kzyDS3J|>SCuKZ2J|w^|X-EoqReS^XB{W|)d;UX+j9!)U_F0=i;c^qy+v{LUVnKkFAaM$Dtft(33o`-Qj=vn} ze@p$>L*J0M4TKSfZ8C*v6bfYezeHXd&k8Fkt&Hm6WMyevjpWF1S@uCX!0NuKaUE}} zOW^%j6>rrz}EMUWxS)kR{`X`T#em3h0EOMyPV_EkBEBEp%WMdpRcEm14in=TO)f; z@6G`U{UVNSaM1zqrkzh@v_r_XMu(G3O6PmAzS*bxwCidj!(RX=^W=n!zcL5pLQ|2g z+0+nYC*e^be9x)k4#b3{P^mWpWLxKuN3%(Tfw}G&6ukQZwcdz=;0b_~%xFEE1gQlu z?-5|*3KZI~5xywp(3=~{InY5u$!r1mjFRd@AS8KW>|AGU-`MLkBlyyua|sp#d!^ei z)Z5J>%x%&=$EuGNY3?aoo!+&14y&7Ri&st)aT*IouGW`0(5(|UGR5DnmH}LY zp$z}ktE0AA{`0XCt1h1XRS)bQ1>|UbSc2^rdLLi4c|TR`HSUS2!$y|d)MoNE?q;(9 zdncUn?pxIGN06Q_Odrk)mwGtHxgjth#+Q>3qs z{Pi*|i{lsj3Xq?AfzdQ8f&_pVQy0Oph-Xby^_=B(z1Ji++inbI%SXE;iHnuV9yR3n zRn4W99|Bt`M9CLsEl@-qpsWi0<8 zlzv+^$FmANmdmpqKHDn>Oav0Mi8>e8^k^=Ym^IoLMqbY3Ft^jL9wVGOhtc8P1ynmq zPHfq;KidtJlNW11ORP1xxatGUQY3mKL+#c8;scy9Ti#9{0Cn#&Nq9MWye9IZfyn=} zzIhIpTBz~qd$R?xgX zrf}x?;Nau11A~x~L8>HjT-M!jkLOLY4N?v@PrpmJ1#|txr@G&;)<|^x6hcz;m8zeU zzZA5R6DPIgcGKYl>4qgbuqpHz=EQ3gi*fx{2B`cbD`E?+boI&hAUtUtbz&DV)6Gup zkZC&udjMb)jFy8%}wZY{q%|xR$W!>>*7IY4XD>_CXhD&aAlt&hzF@sA9e&F>RyECxn+U` zfTyXTzTn5M0Qj=LYm7Kx2gtNR>800Y;mq^{iglwc4U+?9$kC19%mfUbB{Zng*gy&drK0JZpvb;+k1(uPLwqdpC3TyOsFA?HvL z9T!)9*HRCw8hv&mx5{X?(9NDrU>nw`EG5g#Qx3d{KCOF_?fyA{Z^&kTTueHn5}7T^ zpgU;h?}R+sU)E_hAYUD*8lyZCMgjJa&(ev_NiC``X{~a5QS!WQ3U6vB7m!LDh&k%t z&B~tWn&|Fq`%w28bCcxKB%M$5<%#!yve3fB}Q;11j=75fg z{N9%wl3jicgdc(H4$I}>IQtAno5U^2yDDivx7ASnT#{KDbK~Bm6^vxlc1;^^-&LzJ2%^3nTEV^m#BsvxG`Gb ztZXlH8hb^HEL@|{9iYh?93LGXI$x4@FY`y-KwX=pG}Kf10OXg2Oy2i-q;!}-ZD;)W zW<2=y{{f|0dUwKtrRv^i{Q@jyk${i)^K~aqS_!eIb)IYalx!}Y#+n!O|FQR$VO4GI zyEq_?N`s^zAPoxAwG?Ry=|-ibySo&T?pz?fXz4~IC8d#Wq`T{kwcm4od!McQ{q+A| z=gYq2!^CSX=A2`W@r>uWpZmV&bnWFCkfPQJ_DRd6Q})ll%I0T38b5+jA5IIH@h7Fv z)Ox;p)^rVWn|MqkQ|Mr=IUo>@HhQ=@VH;qc!WvySAbjy2FTdzC*r~{8QJR2UKred4 zm>Y3Ffzw#?{auQw>qy1Ax1UgD56^aEO7#IQD>c$QxyEVf-S@~l-@91Q9fRy)baB1I z#;E7ZnAEHyH}&S-s_d%U(L+K?FFl>Jbh@(OTe*8u^z#AQQq4Um8J$gHJ6^4MIlD)GNkX) zJdwkk+CI{Epz%+>&kJ8{F3!9_KTy8V@9;1JO)F4G$t-@_;;PQ?ccGZp8~>T1`~>WS zuT2TP!X9SD0&1Jnv_o9C-8PTvBaMbGIG0m=ENw1M4%&BfLr%<0HE?wjhwmro2WXl!X4iy5>V-0ryh%LN=PlY|id%DOBnQU-#x~X`T8pVvdcBx0{ zXYa^w7+P3s*PIt&BSHE~$&WaXVL2E(mU*lU^`7!T#eN8xr8wmc!~Ggpjrs!Fqzn*oI;^r4B_SY!_PL4xoml=p&7sp#kT>?cO zxe*e6$nRm+k;f@!(-{{pU+FHgoz5yZ`dDyjG71Hg5?rN~4CwDs{94*Us6^nSKp0Q@ z=#_Uq4Cy0jJ=^SpUs!@~oXrK4-ysjnpLs6xON=YU#Z>Faorw;LV(#06Bx=P!>){nC z==QD~F^7mQwuelYyUKg*wGkKbSlQ!r{S>x6D{8(+xT*9=S!hJXc8#$xt+8kf&H>^t zK)R^SR*5l3TAFyTCm({0>DdvpTZo?3jPBNa;r1jI;S8SY%-DW^+@gDB;}Dpn1`jXa zz*NxQxjH|xtIM^_7y-56W};b<&4uDC?UUdxP?;Ezrd8!d!tY<{D#lJv?8H^rs^=2O z-m)(w#QfIIS96k@MgGJG{yZ8X`eU`I-3iZ0uOIaE{_5WM75iB{qlmqI)A~!-@z?Kv zFc;S^5HY6s?dzSak92#e;mE>pN-;{sPf`vWijX@Eci|Yrkb?XcDCBC_K*L&i#Ow&j zkw3?%>GG{9A0!sLe+Ow{!sKxa+-K}f zm$q5<_iIz(8q2OQ%3718b>z+**`yUzm%|V=(&W#E-LVrEs#(#1?s+epm+yb8Z|)MS zC92Yd6nzYuF|GcY{*kFLs`qw-VVdAJlk_M1)fA@*SPnF?HZDe05^?nfU=&K6aJe47 zRl4>d%tBYx*DVqoo94O4klS?7C+1p6I~G(b{OgiYkU$9{r3$-SeNR?u{!Yb=KL)<` ztM2}fp4%WAv#}TN6EFvse?cOYs>^2}WsnydQy~acfv7HnDn}A!BcH6=N#OQ%0#`4f zZu*jF$Z;1*_BVcC&?riN&2<#2mKSvEF7KD<`+`OOjK)s_KMA_K9RDzu^+V2MxK{wt zQ_Mp@`mTb#{>Y_aL|Q#N99v2?KjnGnk$KpQI(C6hciZ)hewlcTTezvr zRVP0pBQFvp0;}t8xy;{x)H&^x>xh8h=C~9gQS0JL_H|avGYgDxMcAY1PQdgs0Qx^e z^TiO?V`ec~K<&T$H0{7-IPD>JsI@)*IpZR+caqZ7{=3x>&nR3tRk@OAc0Qfu(FfoK zf>vD+podoE6FAyXeq1F{V=BYT`D`~SljuwDVim63CC!s^X=V3;NhaZ0m&S&se!f-m zD4(Di1$Ec8tm!8=f8x}C9ubHG>5{FA##1OjY#yq(p}?x9Y-NbmRbI)|WQ~+*pQU;r z2vpcU!bKG)bXHQZfC0Ncp;W>zDi9VH>85$NVy5`qwDo6LtV=*kxt*I4%8~l)`BNC6 zaR^P-_sPKt9WZEqmp-c*N<pvH}#7PN7|0i-qnpuQA+Am=z|Qyu66RQ-GjP zmXV5}NBNxmsAnw4b@O~-NSUE9AhcL}1>hCcjQ7v9tQrjA#XVD7$e@D%Q{64mF)1HY z9X%c65uGtv6TM+O?QUfPUS(0KC(gxl^{)Ew4FODLt5?wkdFe_(>=>pt5D8)kINO=D?sOwukKZ(0MD`Kft0_vOKUW4=x5 zMd@{E!=rUl=e6>D?BhidLIct!zwtEMyi0pwzu>PJ6@t$cN)TYN5ymKr!ninMO5Qt7 zxdQf2DZm%oSZxo!J|*`$U!#wcYd&G10QFE|<%Frpq-g?DwI{`$dp|(Sx!hrW0q}}- zRY2=jwQuEZxW5pI=`V-y3Y!{ zz-FtS^9@b{ki`tBlyg)x0Z-0$OTovGi$TTkeqmK5tHZPkS_~YC0c*CpM(@GR!z@Wm zrJd*8QWtq&fD-sF8jnh1g3m3q?80Jp0Ag}wsVV*dWBtMpjXuVGopM_Srh9n&E{N1_ zm?6qyz`NWuSn(xIYyrcc;r7p?073>SZ|$74a!O41JbLSL2AoXYZXo-Wvwt~p@-2$K zjxr6?=%O6MGvE=VXl5Qbb~t?O6TUFt0Fa{|>lWvxVbM&Mbnl#Ia-&B+FH338y2>cs=gy4KgsZ`{8RE_P%@Sf~k@C-BPTySH4-#l^&ZWj1<2agzpZa+>kSejMVit zVaqArlhFXz+6x}C^o`$d{$6kZ&?APBC5hK2U8d#}LMoD$bj&lxdT+%L2D%_pq9r!t zJBsT{Lk4!wQ7(sR^!OGCt5y1JwnU#EzYu#Wb@Xnu5)L?%?JD}oF=a_}`!s%{Vs z-oyo24robAKh7Y(SQpzyi|JJsopsZ0Sc*9g{Ovb6;qA-zmSA|8_Ji1loNLJBcs#e-jh_ zNXmSWfFBP#!h^a$+p2#%%5?*mxiQ0)(f;QG`r8o-yzz|i-0`2imQLVZ)&?{2kbmrN zV^V?s8$#eFU_AWUv-s;#+BUq)>NXZk`j4Ch0to0`BI5i&`ORScr$Cv;2UmJhM8fr( zBJV#ZWAFz3L?o(Gtbe!+1fN=frPpukaGD1Hrk?%-<5oZb!;S(mApYAp{rRhdJh;;7 zI-}ux|9HCy@V0_?C);1g;7?`*I9i87p2DAQIqdO2-meESxZl|wgueU#aGu;8q!G;U zF(pp9-}GPpGZWHogNGN_EVe}OkC$Q#P6MqY(fZr;`}5amA26yM)q%-K|9Gj};C}6R z^#6I>IpDJll{iA_Zxi&-UpsZcsBRmX|MNUV1t;mAdtm$9&fOo6=RfVcVAg-U*Dm-Z zBhh{?_cx=ZKc5Yi6L|bJNuU3D8j`>lo!`e!#y>~sZ%0dPpkd?0S^4M9{q2b17C4Ea zH7?)pI-37k+EakSvufE+^yjqu+mRuB&D~~ukn=yCnZI2HRd^kEZd1ME|GX;yog9B% zS^rLszpk5qBgdaX0UZ51IsTm-e;t9pO^&oa9qju zE}+h`rr`bOQ2zjHcmn9$=Yh#hi80irz43gVIf>uFlqJM>30U71S^ydAC?JfR4d=ex zbnST#D05Rb9TdlaG_k5215)2{xZw$KfGNp})18n7XwW|2mS0R13J^4tCkcDF?qj;J ztbPCFw*~C=T+F{c{M%0Q$FFS!xekc?HjC{EzJVFyAzeRS4~P+0fS%N}>(RO)oEzfe z#ti`!Sm5fzKqlmBUx7VTq$A*zx>M3}w-U&d)!e*}T+rZ>C}5+gheD^&G7WqbUw-!} z<@`-H=JOS6I1%9t&`0#uv~2=(0H8Y}+zr}@{it`z-2~V3Kb&IjTAt zG(fbk@1XZf2YS1lW^a2R%4?+?6)+(t-r0OU_guKv^a^YKoJggHX zZSUl#+)>+XE5E54mgy8q5zc^zoyl~qgI$CEqU6bh89Nz3qzwglmn80c7{IMB0riOA z(&_kgF-!|s<>|*gwu3^z2=(P#Oef{Zc?$|^P4@$Wm;o=dg{GB`Hh5wt2hw|4|9Lbv z#@2sh;{Wlq9pM&yQ#~9De0||iQX?1OU^%)B_5iIxJHOL5dEIH6-`%e9Zb8S4 zSJ{I)mw?5hfwt)<3*-;iHmidruUo7gn{ynSpy8n$*J)g&`5kHt~~QC%gdQna`o z4uk}gz@wqDdR>*z3i!xvpTuk&lyy;M+-X-0Jg1r&0DXY-(M)=F`kU6urFT(MTnS`AmIHh1KlL(ctXcj1cqr0OHW; ztuT@6pNhk$0f6c9##|0AF`JInFd}J6WCigr+eIY??ymDEv6ry2MZek~#n*$q%i;(px zN?L+RJ9x?8itCEn7Ec0XBDg#Xj(N2uAkyj8!y)%Tx#RAGRpIDXasMBPRu#8_oS zsblOh=m;2rI9^JI_3Qxn#I8X_!XKVFR-M0BodIl?oUDh>9_Pdz1;@EWH z4yu*~7+aiU`)$N}<<7^eTO4G4XzeVnh&rB=(fp5R?*?#L6&=06RJ#OLV0gE7;6^V*U?)(kmRczHHF!_q7%vm0q=Lh4Lr+3PFu~gL ziGz*F-dh`O=v8Ew3?(ba2ts9o>jUD6tn6)iuIvO5zrc!c2K$%43=h+z)dWTiM6eL3V|oE zRK9w)Yu@K&fbKdrZM0G-vxHZTZj67{H|63={z;N)Y@m|mb#u+;p-|b6$fdi!^A5WD zg8-!q%u8n_(g&Eo*2)Tz2ri1Qghz)U%8V7%!XychN%?FIvhwq(r0OUElL>u zSpE^c3 z`*5)a_PjE9BYpd@G%-iFP0GssEn3#?@Qss#Dv`s7lG-1z!t+-vXYO`ZF;u7wYM~_n z(_!`eRp-jKSvOZob3i!#VYxD@ByTIEl+PqDxaUt{(~7H7NK^-JqB;5Oo8Q{|Eb#i_ zFQYN+;u8r(jvNpZymYY!K$|O3rvsW=NQ3-GJT(AT7u7F;wp}-j-gruwa9AzuC^P+> zfmNol*`{e0tif4$ZgVq|MZ;i#ZF{6X`q(`vaM;H-UIAO4%lZk>SG_M>P%C7tI?YfK zs(J{hB&Ld8>6ud(T+50-5zcyfX3cM*-7O&HZXsQlDpr=3F2hxmCZQ@`*TBCMo*!3J zVaURs>XkbU(L_azR-lYrGqFWSPF7A(RXM}y?T9l)~0yH+`vA6dU zdewf3skstJbQt0JJg>D$@laZ#MXKRe zY03+k<&)i0(J|{{weK{n%|=?PrH}EY)r>ZJE>x0{m935GB4S8tKb9Ic=)%y{xPN8L z)MZsz^=v_!@z#>f5yol&qHP0g5NKb6SMz2^@nl$DgC;}QxMbF98=w^HGKwiGdJ7gP zu??Lek6v9@6t>$1Zp;@Nn`nE_TII`UMGwpscF96v&2lE6;e7&YrC)VBX_Oezv=B<- z@W>ih36Oj(h%NR%gcKo8bC18e6<4L^!;7L7jrzQ%htPk}Ui;;a)bhL0S3bgDG7w6A zhcGYOQOrm6ygYa$^`5l7$X;aF&7n#}ojX{Dn<#K`=f=I&{*cWcFYHkZr)(GIO*NCo zKhYf4QW|4rAq`~jG4TZZYlmgN#o_1|U_|oM_tf(~X32&#DKb~l&v>b`rNz;(zI>Ca zpL?Zex=M()8to`&aMHLNFuEMfQypRps7;OZJ;mIH+7C)uLOR+*$m;Bd=BhDI*Q=J% zhGhlTg4hA&)83qYzj`&ow+ygC#qEh*y!s#R!gswo0tK+~Kv{vcb~Y-~onW{I)JglO zeRPt^W*JC*9jq*KBYNUK;M3O*Kaq0jLc2q%4C8U-J{nd>lvc?y4qa!U&$9TI&2UmR zh{)(%C>{dtTVS!$rXPWe;3r9}eaeVJ=JNH-_jA(k>U@)nq(%-RFdM~NWJg2_V*u$NX|}bhQfFLmQ|FhqG5XNY6m82zrxv46_UE&MKnH z_brrK2H9c-SayPrHj4acg#`p${AehL8Db&$PO~0P_Ku^oAJ<|nvP_vE&?mbV+G$ns zY-cm_`@@h%8!HlqxjK+qv`6OSN(+5lRK#V3OysK-OHTqfts}Az7Zl19yhh?+KW1vc zP*j=IFi|=b2;>e8sFNoRuE<-BMr}}T(Jp`c$RMN}CpK?A0odUAY`<7ju)lp3dz2I! z3%a*$(Ynk^MU=qoVQzcLw8Jz8(xY4S1OR(dyP}SC&!!I{p6!?G@1E# zu+1BCh0oyXEf@#UmDLLc9umuIs%%@C>5wK@*Y+diF)CN-t8kd$*;3SKmDQ|t{@dO9 zw^luw&>t#C)oR6>S>u5lT(FtAk1VBEFB5J?vW+-lYW zF#5*g3=sE80TmcL33&6JP40sKjtHimOt+jRjW9=4P4FuJN1=yzjOUrxNs)P=>Uvdsl+PnQbW%L1}K zmIz~K9uK-Z<)!&XLb+}c+u4-7R7aoVHE;#D07e?H^4h_8h8@T9Xnu%p=LWtI&yCJ% z5s%zgyf{yKuq&@dKuHp$2d#LMbhklL9xB^}=kX4nPQ@q|<;vAO$z(J>hs!Y5l-U{% z6*mSlB(lzW{)Tyj7liB+!B+?)0?i5r{zC#4EyPs6yZpI>k7FpQ*{-Cj(_=f1oWroh z;nu+X*PRhGEgF`ZLX86UKNUQsEKHm|`cw84xeL>PpeY&r*OnnNF}y+7-iqIWIM{ghM%~$=Mve#hY|x>j*#* zIZV2aScpggD$ld=`t$4LNMSo*Vg^JjCTb+B`FkuAUt$QHN9Upe;YYXqs*P2-&n4@L z;sm}B+AuT!Sfq@%;G}E2&?H!OdZ4_9#Eg8L#r7r9O!RCSVnPC7|INCdgcoc)Wn=Qxc6O(H*3Nc*Ppkd+!)Hen2{Qa|I zjB3#vaq3%3wU!{{x)VzDSlMu&&E-$nJ)X+T^V0(x^FEkgb(bsGyw^%kXFiF(KL;My zZPqYLZ@4wE4WlpSKsSFJf{XSp?;Z9I_5jR3s^ab14K^kIaCg%7ucjR-`VJc;#7k@j z-r7P<0Ajk{eS*o|aL^~vahE`|pIuS+%tp2soYm}!S~Twh&GKJFk_dXft`Kv*eN zyn-f23@Ty0CA27{Ee^H`;Ll!yim~W3G;ZWhIB_?t)Z3F3 z&!XO(}#SXTd+-!5@;ttaz6}n`LwjKR<U_O>e$w`ZMkP-e zy-8};eT|8AJ35!bRCoD8kR`o;`JuAJ4J7g*|7-`?ASWBNr+pK0&*!bfs7p4doZs2MAB&pWX`yB4l>Go%0=pY?x|c zV$cSZQRyMwT}|wd&gquxvF1 z>GY?|O``B59WI{r(v>3dE$r%RIlYsM1CHS`&Auoq=^E(Gdkbxx`42GkBOZwL7z*J> zp#X9z50jOv2iqv4zomU%43moV{?fM$ehfTX$ybHx{Af5W)25GK{$X;C$*wPi-9vNN ztf3Q?!R&b z4AV$jDZDZ&93S$>Rk&`Gmd$ccg6{flPuERiXVe?oT$1zIRu_~J)dPhUhvTNYSwtds zHl)#NRv5T&g?2ywZs}y<1iOcV@@ZvdgArMaN5w=m@g{SHJmflK&yr5-8DPxy(j+Q|O6Q3pMHP)!;?U3o|O`hRejnGlaKbQH~8* zoPB|;xG0gNJ1vq9}_{34~vb81F^U5BFRwnkvR)5qg1Kx;BkU$~#lWTH* zd&#*E8-16Aop5qmbBzm!ywn4F*+kkHX0_w;1P#xv2`(S!;PD|HKIXP+?#}YL| zVUs8d#nwG^&!v(~+{(EOBsDs&hd<{PSs!99Ic-n9q|Vt)NedJS4SnG5<;v+iI4MCw zZ7CEP1tqo8{afZ z>sOb9l@CH}R$$GCF0z>;b^&vA3X`!S3L~KjP{k>$M%^nH`5dSY@tsOV+r~u*6kzx-Ns#`O0s&bh*nP7#3#opa@|n>VE`qK7k1+O}rpCUqG3fF6JRaxmvVZQIa>oXaFcb@js-G;bZWLW!woya`r%X()U52lqT zYDL=!McwMt)cN}{hV8l*->@{?!2b!Hx3v?)$ zJd3S@0U>#MO^u|~xC2*+J``toQ&*}i>Lrk6+eh;@?|uo8V4+aS;RLGskn7IErDYPh z#A*nihydv@yS}3EE85pbU)=CSw?%-8whAe{(HEayPWf^FQ7IIaDZE8sVyMK^SsA1v zZFD9NL+=#pIoE8@HtF%SC4;xpOiI0d$Kwiil-?s(RLUq$qn_}o8oP}LcCK?@5T8n- zI(?9Kpp!>B1EYyzQ9>rDKHbJKb~J~m+u-U@mCowl3;GkPm;?-I`AJIXjUbHHxPuhN zC;{5c2XvdNk@WIKAKjtv5|fJW9MFdQ6$L2UpT*b9~+o{f@J*D2})@hCFoOxEopC(%HVQ41!|oOjBt(BK_fT`xqN>9W+-| zzdk}1LSFzn(*1j!q!->)$&dBL?VO-^uhubl@3QJO*5fTySxi0z9g(P!e6@`c${Z-N zt$CQ8tuapO_fI~@AZ7wjpwIMBS3UFur8-9oLR7>tYhdlwRX0V+_H^w1JyYy}v@w?? zFnU$TKaaq-8-%|z77YM{HH%P4$qqZ1j&33@(!L)Lo=S8B`|%vXoB;c4Z&()lHLDjL~MO%L)mjPHlN6s7pb! zQ{+d5Ssc}N1HC93rLe%0-|bL7)e=+_D7I7*ibCi=wgts+)zHE-%iF+Tv~5y(XDBLd zgEcvDhEmDOH>0Ac%EkGmD5O#=M=r$|x;YpHxjRqlTgP7Map4$uZw$I1a#1Q`M$szp z-q?IFQ>w7dv-dqnNA7%-_94$8()Bzu3|pwS)Mu!9e(d>-7uTN)r|U>01V;45Q0w?2X?hSgkLRUQ>tPng+6QwBRm41!$OyOp_B>%CK@fM_m(Kf&&j$(AKqbhAF0O|C@ny?I3tAVY&B z@zyQLF6405A@uko8&ulVxU_=0G-l}eoZ!uHQh*_9;Pj~yi;G8(l#CFhD}go*ro*}O zGWgLs4lk-xaa`%s!x^D+L=?>1wg|1@hE{`|7q|Pm@zHcb${V> z120MPbLv;-EQ6ev;P1d26C~cQG$i>=1{n_7q{87w;&tE(yeA2eeccUlGImxz5(G^Ou1M4y}yH`^@s7%-M}iX z=SY0@D)S9>ZYP!(MsnTT$hJC&9bz0pUUw+&?E1>Zv86O6)fV2DwzMRB#e6#(5_&p> zKJRB1qJVd;{>KBWbvr2vqQ3%=)TzDcPX>qtK0js<0pmic>Ica?)VN>?g(YpI!k&6&afwDKdprVBT# zMW;caemhldqbsJ_APZ;#BjIl7y^WZZfhQsA6^?#rrWK|mn{B#rxKZvP9_#~@xeLTP zngZbJ)X@ogWA19m>oI6wf+cAHk-MS+yWH6x(J>&xf4sLF^uouCW8DSByme zX&|!g=B*EOUZCGgHz=`mb@wwSW$#=Bs}{F8sWKKFkJX*`48$*444fwYoE`BMO_B%VMn#;)lvmpj3v(rkd#7b^{TBub=Zl<-?pcs<`xcJ4ugbR3m;z$r8sZy z%?THHhkqO(cWe=w&5*!$v+?i#8QijpHyZ(cR+Kh`3|*8A#}ZxjW=C>K+p>3(uQVAF zN5!TjpXYm>Rc8^Q{~BcCx>aqRH16zj@sFs#)>aq&^!KCpG>E(rueK|q{Y?gn7Z`#3 zRgr3pc4xYdz9W=E>CL0()IFk?zJXC7#s3z~7-i6xtRWrEfU8ho>QVnnu0K_zxW0?p zU{R#iE-)FVz;>x~9EsFJqt{S$fVS!fC{ z{T~MI;s%rqY<;P@FYK;v14*J5&z4WsrPD^i*6vmS^C)1!-Pa-bkI@26-{kd2qHp?u zz9p|;q0#+pxM9aEN4+FIpv4oX5B-Bn^R0Iq=#-KM$+S|D)^Fc*PI}LxUg`MlOat3J zDNrx;>?FSO5_xxt>;n01XiLLKH}etvY1~Z_`Ad2Be@(tG9!Lb8F2A3U|MS6CDe$4i zCBs#z--e_A`J4jPs?e|5(O>@uhaN+LDl954gqwPitWs;-{fq6Ih#!+0?IzG%MX{q|Nc2JP}u)-ufIL`ccA_l)_(`;-x&p8 zfd4MNe`6FpApCz8%pnk@vt~ausqo*GKYu~w-q0dqELb@F? z*}Nt7mp||un1?D8nIZf=0S~tLf8Duo*x(=Qj(H{Uo4@gwGdz>_Msks~Ie7KouWz2*P^F3!{ULhAo`0sOZc|9^g=|Bo*24XY2pDLNZHE@+?nA}cn!yEOG5 z>@Rnhx*nPGd7PI6LaMmsOg*EJ>*21O$kZ1eB*4Im1ZZomG8=&548BFSWhJA~;+P`Q z%R{5LCadbmD*<8(Fi6rkTcTEBB&|yGmrZD+oHCKLACwE!2Q*t<@B+D!;Pp75Q}R)W zfS#OUsex1u$HiBGRTu#A-g+GiX!AybES^y*JJ|JI>DV#{JV`=+HU}CQu^VH>GQcJA zEciaUq@@hfXMcz~#?s05H0S+Ha9XCl#f}p!&c>qK$tA_;UC8(FVW+m^gs)yXlzQY?NgUfFj-8 zT2+<~jQUTl(dS;*0Lyna0W$E~@N)3t^Z$M8IAP$b7j4JEN^I5j8Fm)0x0|M}Y?DuZ zzCc0|YV$)EymMQl#x@Jzmw-_b{kaq=hW{;}i3&3;a^wQlT6>cO-~v(6Y3R88slqKxHoxOjoJA|>l_@h4v(UX4c zUuvUfHav=rilouUOb{xEj&V+x-XsfhT7Sqpk&3e&OnY;;c+y`cKpm=)`h`Ov$4eNeSt?-S)zO_SDiB1 zO@;=$ws8TH3t)uCoRChyo5wJXvg|RV`$ArlY_rvw>&udH5|Q+B^A1Tr3rbFg<$ky< z=<(TJi|~0J6&T1QDcNRqGiTYUL`O?0iWv_*5InBXQZww>^D^v*QP{2z+SI&x)St|s zwDe>`_n{h{R1}Ma|K$52X|ElUV1_e5p&MpQ=68_3dtbO$ zvQ)X)ME9M`k!k$F2>sOqDNaKUK+`qZq12e!CQ(tXck=pn_M9{4Xol$cMo`1i>!czt z_TU43)-LmOuk5vFU2(EznKUc$jsT=|G*7<>Ym6ix5=!iLPaON_#n(8;`@&8TInq=L z_4P~>TCeP{jpjsy{HN}E^^P+ek7bq^m@yXV4>j3qnWh@uw2nz;D<;FNRA)85;nBqv znAl3zOsUXG)J_eIuvB$i?M!(t3XEusJ8JEDzRp(Gn10OHRmy*%%6M(f-xTI@acp5S zoEw2xoTpaI24n&IX7>TA7zz5)`4*41W}BKiw)!ouB~+cZGDqqAV|7fgFY7d%X0h=N zI&$YtNswIjgsl=bd>`NPad$kzTq!cW;cmgH;>ce<(7K6HNfF3(LD#MDN-`P!&r_RC z2gKL&;bE_9EqZ-557=|9hNs1@JJQo%<$t!7Bwja~lwQ`qPEKyz)J2y{rutx?J1CnN z=Q_HZi%~qlH(PY?VbkSNg9H>~fj&H^yOFA2#aZgn{UV}ZHzUw^*$4A5PwBRkkGsW5 zk?l(HQC?vdY9srP^I51_>P!-UC z-~O#{fJUW4yZ4&D%~g2!i{4FpzOCt&X+_bJcXQ;`9_oxE@zzEKBvek&zV+NJ&PqF4 zb#^o&$%#A^700Bb@@i;y`nk&$@xv$h=#?|(Uk}P?W!5)m^2EB&W#5m<+|256YfO?2 zz%;R4@?HqI*_5+lC-2o3Ol({TsnjWmt|{HLpif@3;>&l=G`M--bruRd3ydY@GG&ke zz`iYdeYeTyFo^kuXXqv&f2DP!_7l$yJX$HHvq?T^3QZlmUZucW^d+rZIGyq=dQ4tN zTWiM(FImIU4xr7P1kkKqx1gQ)=PwyH>jK)IfiPJ@jgx_5dw2{Qy|T@l!>}k!J@0YH zN39=ZL3cKBH5Cnc#(DA1QCryS>fOdL@UrjKLJr3jwvo+uF}Bivn0&Mn*-NH;vO%qp zk1Yn{9i?qb8y-KUnzcJkf3nl0pp#lP5fhnXcCo7BbEj}-5xk~8M7dCNq=I|(SZ9E+%@{x-zjHiG89>EWfvr?Tjq?)UHlfw6z(6X;HgY+XI zjYhqsE`YhNT(>4>*_5aK(5G6AsvY#Fg5x##zjXui8Yb|*zA!B{wn$^LA(Z{NcSVsg zH~jS!VDl*~fSfsfOs$cj|y} zbjiWBFko_IwPL*hST^X58-gS#w(?aabv(9fr4ii?;sWZY12u0rbLhkM~wYQ~4 zx|3i@anO+87L;k?M846`%3fEB#4YrU_D%Sh*gYV0aJ7zGdUI$Adzl7}?oxR=NXN4< zNcuRhitDyj+C7@64wh!OUmbi=##iH}Tlu+lACqsD0nS30G$Lgy73tx5?{>Vi~^uBPzCu- z0VFiuaUu!tvvb5NpOq8{0D{Sy`1I~lHnC)33DV-vU`ea=kHy4>^AP{PR|xI`%I_L5yLL!F-5 zmF60Bjx@Sf&mVVE8``Hld;$Jvb)f+>A@+SlP=>Ys|m+ccsj z4=dChb5VD^a(WtjZKpkICBkZ=M90mW8P~yV5Ld5}g1I@tx0{6OKSCgH}`EyJ=Bpt=_c##nxp5 zTS7ga=^gs1+~Vu`?<^zlUd?jas8IFqzUAYqboNQUa?6;ib=MdqWM(7jQV9EYbxizd?Wxtv^1{l-q#fhAZfA##$xHvhs$uv%?r@Ca+a$%;Ib z9OuN##l~iQ**0*K1UrETvSQ`bn!htUhYx$XYg1eo#EiuEY-4_!MJkeZ5SRfH;u$S> z#Q{us7=myj>-rgKgkxAUq)+$Kdo zD=+HIm;$lX{GO+3mNX$;uSu0~0KAn|N|g&FCc_V>Q>Hm}4T#uvS)6C58a?VJDkUQx z_}&1nd(@KnY1k_*s@Z;t)!vBwj!?;UB$VZM^N8;D*V-S5MH;1%Qu1A}_!=xjE_oKQ zYfO2IGFdfWVA{J&;gMm4ymlTH;&6&hlvk_I6jaVGNOU?uyWnf`$k&X?7{HHw;^(%C zxHZY;6f*sgX8nN2Sfr+iS3J*g{V^K60Sak{j!-_kIE8@64&$(N> zD)2j&TTAI|ZBFEB41SV*7|P--GN%5foRv3MTU`d(CULrGF)p$*mY{zqUI-_kZs&at z1{8!?eDoT~U7t!Ty^}o%uEZr^o2ztA)2$UKeDY-scdKLj{8);fG92f!pk+>YWzJ zUFW}k(UjVtC7ejY-#e9)4wM~0{Ec=M+4vSAW1j|mr@s~@Vz{qK5orpH8c^@z`QVUsI|gXJ_a({8bd zbI|Kh&-QefFwWyA(!~+mjAf$kvP00p>BQXXH&FM}dtL6m*K9EOhW9Og;0Dd4zGtrP zHqi*>lEg}1tn5x>Vd(Ojd~9Mx*7H=v$nUYU)bU7X0IyJ%NU6935Q(rzVvdc+1qavY zOQTzd?&i3i#H2`P9VaNZ{v2zw)`7xpC8?T=*W8SI<@Wi@IOi=D^FC9YHZQ6AubN_Q z4UTWucpT+*)Af#JHxRgg{)i`8!0dO%aVMhN6ll#&dhWkK>0p)3+T8^j|IRw^&ZnC>Iy2)BIQVaGLA9ne zpn@(2<0Y@3Iz7W$y7-ZJ`D4Ziz3aU(Kfm1}KYCG&IG_00Qyl2*+~W$&KB|%So7{Xp z`4Bn3tU|5$g`-8RSSjDkZ^;|a-@^S-vQ+!HbS8=# z>H_oGk;?8G(auvx*+=Skik@C1*WM)TuD|(!*R{2LyxfVtBthI0#hO~bb9_R6AyKzq zwZpK7M{CWmO?47H6uQkmH`8H^GMf962Iv*!w~Ocf!5U-3MTbE`p%#a4%v$2)L4#9y zhu?Loa7B*lJ@JZRiwwHs-h!A?bpV5Q_6w4YmBDHo{gI(9<$E6j>Z-(69D?XilI>z$ zrj@uwd6>@zg^G{d3%zID50CZVe&SBnzrH*p{0PkP)Tkab`F>?!0ZhGK$j#&!OQHbcirp!^FvER>~Z-Dj%>NMVr(`i_4Vr@ZI#T)j2 z^t!fq&ZLW@r_BuqY;a88<3sQ})tjnTD?nv;ya3D+SAVsq@iBna+6~an!hCnA#7~B+ zNDg3S6*eP0{yL3Xbr`Gh-}N%&#S1bPKOygOs-2v*&BdSBLMqEXJ)aPq4ms#_V@;{_?);B^A%R zCu*$5y6OL^xP_gGnuBEAoVxcN3w!zFeZEJ0au=(ySt%z!ziRG``dWM(UeB6plSkGQ zw+2{hvz+Y%Psv4`jf$5%8B&8S5AwAeJs-EDFIC59*{O!ptYS+A(K6Cmz4KLkq=ajt zTpvIy6+gVD`M%|=m6v>EaU35%VFVhrM*32zd%wFznH6lR)~&nqY&S5v>!ln{oFH_M zJ$rnUwdq?)fSnk2L#)If&&9tRJ>GN^U}VtaO(NJvYZ6(aslL{qJ3m>xW56w=li|{-`TuEXN{V*X|>1n>U+xBP#BndSMf9 zSj*dv+Lu2H-!TS%>Sfq2twg#^{_@-J>T*RIbUYyu5ha>UmnN<%R@bk>$)U^lFmxO? zX{B^Z!X1>ju;q7DQYc8pw^vv*f09|%4u5#joX~W`Ay^rgkn~15nyBtTcd(@X0fGw(V#9HvPv3pf& z_R_ecw;l{L;=z`+-i>4J9M^1w<}UN3vQk^DFDo6W`qZl?6zm(G1a(r-%bxQtH1bJvxQAv(P;Wx8AV#fm6Sj1M zP5!^wd&{7@x;0%pAtbnl0Kr{@h2ZY)?!kTGZowhA2M-q9-4op1Z3TB0EZo1D-M#ne zx4Yk6->>uQR3%j@n5pS=jPX48ecjotwE--wt=EKT zUcaxR95R)uWQyjpx&8FAp6#2zlE(x9A|a>HK4jjd^+nWks#q|)@Xn{h#tB~WarNFU zt~{T!O}-lf19ss?Z!VYg-~1(}L9-6#{(U6#Fo^L8&o_B+F<5SJ9R?veO>_Af`0O@{ z**psAE3B9!f)oM0GT$WQahZ&lPINn|8YAV!G-0|ED3pd&J(IY4(Z`W^8fC^pF;EA9-!+*bRkmM_a!b$G?y5KP-%jYt>{!~Oj^1dT$upKdX!2>d ztNwiIpmk|T20beE=D}t#V@azj@=AQo?q?-F^-%BO?rVgzxnlzm;-;ov?Ps6;`QN{4 z#hU6C*P@%qV!@xdfeuJhg}i{nk~5}&gVEdIOCE}F7H2|{t#UJ*0Vlpm-LF?i#&fMz z#;s*XdwVziZkopig*?a4WA&wuhf|dGGWXZt>-EB?@{?f2-Gjnp{W=f1S$+k2F9`?? z6cwPxG{ud}#yzAWMzA4EHhINb&t1`WrCiwEgkz2$ALcr}-{2H=#1%Dh=4`2b2vOv} z31^}dk=~v3J~;>x(b^@}E`lAj7#^4tKv;rA0Obs03Tr?BJ8PKb4VChj@Mua?BMBV{ zne>AXL}+!`41qrM(XipJ6&@MNpSFcD(*o>pHA8LH-h02XYyqa<@Cq8Z1mn(4Z%YNA z03$IyH4CTg<-!M+1%Pt0%}_yy>n9$CNBKuWrXA#jHTF8ksE*WT z7{FFkVq;)lj&cx^orOn{Md*8+MwTCXr6+rxXKmRX6uld`uEic$r`0pxfeFv1iL<%A zFy{I+Iw_?1pLKg7++UOogpk_*sI+o&YnP!?p%);zd94=g>x81j1$I7_4Xsnan8y`W z4fK$JpSI1#Y2vM-Pnnb52uuPZVkuq{PjqfxIR~z__g_aScN`in7O<8eNV2P zD7Ux8eyBIt*##lo&S`b?N>Zb`Q8ukZB`u9 z1fG9)6%RVkaY+m*wAGXcO6slfL9+kQ!+)|pxOpe~PzR83xZ;9BKY6yUJ0Ro z9fZ%SeLt@4_I_t*6L0N|0jyOv_k8tato$=9#g|A`260y^!f3k1yJhHNPyfO6)nM$7|8?>uZ~yy#25SPaRT#pW^_Hg^ z7NGv@ns2+z%$t6xe@@(~I_K%0y$ei-*Vk2mo~KJ$8k=<*69EPx!I=%vv7!V1#sd@Y zp+6X0g}ec(+sHllrwX1<+lVsHGmm<$b&VF8HFevE)nJ87+SvOb+9CSheArAm33W%z zUbYO51JwBb2u}SF7p*4w&98z)@P4g=`fgXkQi(&i98UMAhBeRB5}_3OTW7+=@35y~ zq0=b}Lr$9qrKpuH!P)2(Xd%56fX;k#acT?bo5l|*bw@lLy^{#7>gX_DdSV2^eP$yj z`3XO5x@a}K#aT%N+#q1x%NExl#%t?0J4mq;=vul8^2Zl6dEeHF$i`fSNKAYQG{2TA z3?EfP$r#Q@bs{S|ZleQi9a*F3?)B!ZM(*ACcsYl^R|({~3d?|%ev)8NLz-G$AC0cN z#l7*|L@Ih&1P)ULdn4l3DSBvk{N*aX;j?z5t2+g`D3Um+yOt2unt97_a-H&bSa5z( zIC9RSl^e1-f;;5!@O#W#Wd9XU0>9mQ!Kc``l-w>H^4T4A!}$wD4t-xUH7DM599G_+ z7}>h1N)_FOoT68^Ap=j}&#$5&_qjGs6T zoo0oyX*SFqsrJ>VktpcYHK{(CefXMbqm`cQc>B{=P2FqhLpMU7(}1KOEHu-=kJ5|U zr}FA>|N8E9WWR53?pMhexn9d)_c%BAFEt^ohhOHe3=cm8-K_Xe*T%CFzjB&nfbIY@ zCHhBy0)d{JDSy(lmqehWD?m(Jgu6_GhU5RRrWeQ8xvX>G!)?x8Nn@`w5s?}fjC7lt*9l!Bl=$D7}5SI#q zu9N3mMRcJ`o%ow%DEA&!qyhNh^!2uE!ee!KO`;6BtS>VWm*nvV>508Y&y<5EXtP~P zziSj**7_ouK;mQ$cuBK#_!^a7u4@gs+^Nh)A^JZAhNqiA1AMHWJ3BOVF2Y`@{qD~e zgoP%Ee-M0373-ROj8fv@Z7M*Z%UZFDrll z*jk&q{v;pR%*KbyX*-{megBGt|D;teG&Iw^7?kZI?Y|=3{n(~Pa=kU^dIBD;X&FYc zf_u&5s_r1O3Sp?40!mPvl>4I51YRG&Kcj#9So;~pUZ>7!u@hmvcal~oK z5~-29xOjW&e<{h}^RBRzynnMx8+6xZcA-Y-d%EzW)HBoU37BOq__k)EOCEpcZy&JI8>!9F}&sHBl>bp^x?eIyU9vn;>p7n0`kWqumt=g9g zp3GQ&r^lUc7l=&B%o8t)|4QD$($$n$UFMQ?DM4bed`u}6_Fgl1-9X0M_~P)DQyrm) z_+01uL=5^}!Nu<|z6D|%;$|-ECmL_xT2Dz3UURz>J@8_F6C^Q#;qp~H+eGe;aR=&= zdN+T{KnlamV6KO(#o}U=OPc!YuXg8Pp#iK z3YZ8dCr+27zo)UUNxtB|@P$#d9&ud-W$t0FtNFUOhA_IO9l&NNk;Zm%niQm}UP7wn zC=CM!5WIlZ*xV`29`95&YbMD8Haa^Z7hYBQ)tJz`AP!;NVp(zC)1romR#!SZRIelP zdk13BV+QZk?AcE%104M1%^E?F+z5id$AwcTo_@M`j2&{7mw3!ob%cb;fx)I}UuWkx z2BN16G{%jz=8v*wG_hWt=ztZc>kTYws~!gdA~u_w^QRp^h4J;;0iNQ@v)XV`!{7V% z)A*+P=SH?b^8?IuMG!+3a-HH{*@bd`%+qdF7Lq zIHN-kxYq(F(i1_W)n_+3E~EDAN}H1in}gMELBJ_{hxT_EF30BItLXZZOR;VUFdm$g z)4)7nsav}dXBx0aq>iD(`vEy;dT^wQ$&7k*f8y@m)9Y5&Dqx_B@hDpO>-KX?=#^?T zvDMKM%%W!#9HFPZ4IxlDb-(V9#lm7q&s(k&%z8WvQFi>Y$?9x$i+NjW$242F)QPV@ zF22C{P2M84a}`OsYFefQ*$;i})dm;xy7Z9mLuy)oHcGoeN2DE z^ww$VI%2!%YOSB;kAX~FiguoAdxphj0b>?Gyg0P^Qc^Gwc(+2Q_Tc52S6hkUd z)`nm(2CD7&V2P!ISX5_;TTOZfd)-JdIe-1m|sYDx|#>i}PP{lX( z@Rg1|c0nFrCyUhk0tA3&h=Jed;EU&OQf@Me7eS^yw9(-qO6%db`u)MxiVj~1H20`o?l!dgn`qi+g1NQ*fYgLI#6r(>(_g}Hs1J8BBa!ZsN zNk7jbuxzo@E8hqrXG)nR@jP@Zj}Yg7&lfO{m)n~u==&?|EK>0kv$bB2eZLcJmO`Pd z!%CiJW&2=xe{DlIe~gnDnNnm!2iVT25_8=#phvs~FBBpdblFxu1LF}L5>+7D>89%9 zE(@4OL(?i*eJWCQ;8+IEJ{*E5^05o6=C6hgO_3Foq?(&wu?Ap0| zd9j9Va{Be)HEJt&7gGtv=a<0WhW`#nDP=XzB6$vF2fpsK4cuL|B(g{&I^W(JymqXE zgK1``(6}_ieflWyTq)VYgFz*JI@z#YswYq}ct*&o}ISQ7BS!WMCw`+$BnNr+zTzc%5x6=h&i(F2V zL*|{E5Vl(MuHIHrEj|21ho-OKKwa7+G(b{G6{yv%o`WBtYBAC^z3}bOk)M<$Loz_a zA|;(C!hFsth9<5AwHxytO!MryPZY)#RSk9#;1i;SPY=L-l$1Y$h+xNaCDERLXN6l+ zakmUd6u}&Cb}313WxVrb8Z@KWuckV$>!461;ZSPML=Zi4ZtX_L0`c9&v^~l2oJ*n$ zkOcIYSTnapzIQd%zm_xfp#v&KVJXET3%@XOjC#Ks`c%Z(W&msBU1HuB$F7G{VyQqC zU5()D`oh&_sN*8=EMIKDo$`!46Z8Z#)AfLs$Z@wlA9>_Lq?%Vy4M+6_R+7uJGf|Ta zlGL|#ArX5PwQQA=5>8|fgG~2AGxLO+A$`fPSPLS3HUpQ4`xeea{%~&aftSNUiT^zhe8gKBS8w1 znGp9(Zuwfqb9NL!(dK%CNw<|*jlbPk0H;=q<2xz%u|N*b73MajQY}%tptj|^SO%!) zFeGYUMQsRYY)=_(kP+kS>xR?wJ3w~?x;5-Dc0U|ZIFPCsq1A1eBwA5d@bD`5nprrU zbl8~J6WF$^acFYF9a?z)H7<$*&?Q}U-`5co38JDA9;XI|e&DsawRWFO0{~)6$lw^U zuUKbpIQ+8$)kdigv;AjX+k-r?YqX`@Zi@GLfWpp;qBvL5*o}d!uZ3l?*mn5boz0RWV(XDzxI$ zWpA~iIP`{E_lLo`-^AG77s_PX({O@umIG)4l#lQu+YoH`Sk7q-taa6v-)EsgG)v4HR2W; zNW6!FQKzAdZXkoJRysPK(_u?g!2P>OQHbqYn-;)MsN89Ptfo=~x=7{D$4hEy?6y{Y z(&BsXO!#63oCgr}s6XaV3)9A0Gnv=9Z4dOta|b((WB{p8Qtcc&L*cdaGqkpN^CQ3C zGabP~T~QDv$JNIM%O^gU#`10^w8Zl`NxKlu3E;bB#<@b6RK!lPCHAo4m%}WhV?Umh zt-=kPY>u&7%_@_V(g|m3+JYx#rs^OtK=p-aYi=tKDCTMdMO|aQw(;{&dEGei_TA#U z%Ek|Lgm_}!e#>sW{}5aclm7l~`bH||fvqPj2o3|8+)g@$@fgDBUE*i4q`C;*?=PvK zMoaytKryojAWI}JrPa{^x`dBF|3E5jhDCLz@GWAMvU9NJFQjTzu@IFaKmrVA5qjji z?A}EkkP)c+v^jKD#Htu8J7y3O-5@a|UTRKt@HpvGFSy`fzCLlahdwZAxC-?33Ztzf zuK%mj0L zsWWdf^k#g$xniyDL+A5(o)ukNzYgyKu9$MA{oBFhIhO%1&NWshSqrM?%;n0lm!#6@ zB|VwJEywCg%_h-~AGiF#&7Z9A&qk*n3wKpLg4<@dSE)l z4-H+v_fcZ(6dHE#Atv)~;G$Uzw59FAC!z_BXa?6dH1+3o9H~^En8FfLS_x(ORjHuK zvzxKpIKWTL_h)`U3MGph05}>mfhb+Yn#A_81d*o{vh-QyWgT6|8dsg2kfA58Hoy7i6OW?!Qjj0A-UmrTFmW%97=n%6+ z3@@fG*(Bu%o*4y!QEZ`Pnj$uCb(ad27qjU3*6krw(@g@~Gi-(x#h+lKV-?-2Q;Jw( z#Nn_-p(Qcl&AU}VPqT9rN1E`gHVbABy1fb!w5wN$?D_F__e4nym>28tb&`&edsrEi@TBJDLpt*8kM%dmf#5xOKYom1ZRRa;5AXUP=|9Kd518DZo1O?&tI<(Z&)R zG>EcaIfdPN=CQ!IQudCmS-*L+AL!*Rs75Tj=1BPbz?@0b+Z29HQkQ%{ z_kRF-Y}kZ;D|Ehx4cbmFIV0NCifE3JSmHNnw#%ne^Z`l{!ALxzN~-jc0`M=xG!O0w zF&q0{_y1Pb>n*gD-ee;yKIH+x4bLiuDT7bQy{`?C9_*4RzKcIJDoVx>31tOgY7j99c9+128A1isCgy&qD)~LtP z7Jw#!aZsQR5-K7H(wJlivbBWFoHYR*y{hB;_A&nYPQ{Q`xZ8cC`GUDpRE+n#Q=PU_ zmYOVu6wQ7!>vi9LF32<|XMjnI;9_v)Gc|KuLPx^%)wN8HC&=PE1V#GpbcqCE;18l%CSRRX~`d5jddz%R{dx8Oh~ z;~J}0eF{~j{uMW^F?;Fd*G2VChsgHa#Ipxxt~;>9@k}DRV&bJnA&+$Pi#q%s$$BPQqG!# zZKdOL{Vk*2QJd}MTHbgf#KJau+MAo4SE}2HA!DcvDaYpi7pqsnbUrnuSbXBt_q~Jw zii}g&C_8WFm<%48P8$Lyo@jxaF(|exX`=1?NPx@uXTLa1 z$BN7KNg%b^v?Mj-Ezgl+L2Dh|@~_Q~yiMOf0$}fk&)==yidS7&pH?RhY3Wa}oK1%~ zK>L3TS`o&Rks~J-Mt|9}XnoUc1vYxm`N&oeZ%6>+VFJ9M1mye7xu`pkT2)!h0U&ci zVSm0dn_J;AIwS*=!)^`By|Lr5LBw74U8Dd&o2niM7M+6t$-LC9GD~2+EqM~1YW?>c zC4x>4`vpPh4S*br+S_;-bA{(qROSrj~ zh^A&0UA_q?;(XL>Z~fDV^FXFfsS}GBMCq0?|D#ZPrtXI)^(R{|rRpD1AU7V7+)`6cqGDZJY6Fvr94VsvGP@>afC zi9i)7UIv@eXb2&?_-ZNd3|eOxlQp|SgLGG#MNl8l(lhis)k`8XX)F=>b2(5Bu)x*! zR$V^P%=GJyc5s&z&8z@WAsaqOl|*9a7_hX_3$5rJ(!{o6)=-%_=pEAdQ@a5o8$xp0 zHz5;55N^F6%i#uz^(op={h#lZ+P?Fl3R@YgDMZoTA^)s3;VB+!Nm;nKmJ9 zS#+|=H$Hk_Kv`Xgl6}YdaD}m>VVXguKH5JfM4+~ilK2JmN4~|y2#$Plj?ld?+zRt! z^mg_1+r=$(&px+8?@w0)Nz1e7yZ}2#S6!iP`L*+MRe?eQ@ z%JxinNPYs`8y%AV>au8sD`TA$(u~lEFkBm8MgsKYZ42OhNVJF}T6XPx2xQ~h96egi z6FF@e=FgMFqgX_DFg=wjh}PkwC>pa$HVT;-BEAuHk~yEgzFTsM zXqp_)Tj(}pVQ5nhA&+a~p8!MvsP}V|;4BVfeEbRh3K+Si!!e_Y*8msD0Cl!^E_>7f z&jXGj z<<1e!%@$PNVp1%QiX-oeFgbmwM`-A1Xy$O*Vd!JqEc{}POxYlPd5(O?v7Q(m_GrEv z@dQ?NmjEDOJ>Cp$v>qxHc(m?&JJajL!B}%mD?^?Fl zOgrrJIkxXhMSFf`pFuC69$hYNYx*-cVQok~>)LuS@}3Vv1Co73qU@Q%(m@;i z=0S}xTc{`Kg3azrW&gzG39g9QwfuuVy1;~$`{%LQwqPua=YWxN9g*^(<&r2Y--6b9 z?TH;X$lCD~MyO`SV7!=jX29GFV~M8_&D6Rp{M0J5d<8l8>u?NZQaiRrmot2LG)7yc zRoaah!7P5UH>pz&IiR%RPl>jj?vHJenh!ZPymjKtkTH@yxALo;O<@*7clFQ898HAh^boEm_*qS^5#_UPUrcRFYN@oMirTXIS^ zHFGGi{WI{AMfdpsg0D7`!`zQ>voC_AYU%hUK{R4w&o#zd+JjDqe0$Mt5Aj&0+qHmY zlkNDoB!hvvPCsTRzCD-Xgda&Sqe2jk5+zI~XI&2;{jVM;O_L~Oj=Uc14!OkCNhF$D z0KaU}btqtSEWO4#^d(R0_d?(=RgXBBK;WK#D=cpXM}ByAMpOv4WASVOz{Yd40oOV~ z+LL5W`l&!1(f)E{WxiaxIuK=&{}sSik;3=B>2^cUdXJ6`(}}?BT&7xapPLK<0W6*o-}0>ywR%c7q>Du(C4C+d)Oi^X*SxF~2ymgXnti zImPv|bnAuLFMRdba>hn(<2mKD**Y&$p9hnkQKg=)2n#;)4NfMd%O+spQjwo-~pM0P-?d$LGQX&bt^`ovVs$!Ic zN#n3Cu`qnYbn~j8aX6TsBZ(wy+9HCDs`W7=u-WFF4q*1ME><6H=R7$1h6|mqG8AEb z&&a=ddeVcWtZJ&!>9U_oGDo}-O>QXfNFGwzNFj8xM3gXGHb3G?b&I=R)UJ*lkr(LulBf*DLXMCB^ocsXbhLdu~KqwhD){S9sFH~rgOt;%9TgS zt25ZKm#!l58pwm!y(HgAUN*9aF#w>gsR$OMuhH!hAz$qK_ihyQ_Apdn!!e@Js^YXs z8c*JIVz?TPw(w8f?zNc9>v1zuGZLv08l`}AKXs{^Cg|=HV;czUazgg3T9ddi-N@RM zYkCH^*O6ZXLM|}3V^_}c-1(`Ce#FMyT?c3X&*B%TpLWNyaY{wdjY7#onkqvk%WpC) z0&#&1E4y~wITiSAc)qW@KY@w&vAqkBYH8(X9&fLZxQ@8;ev|G6g7`h z*h(LYI$*oW!Dkr~DZX+&5};(LWI2Z_8M#r%A0e^r&};{Ej87@hZDBmzH$In2lDMMV zQ^!hNjreM{8NBg(j#+B6OUImQ0d)+V(2arMNRe{IOu7))f1xlrbEAC;frmJ1&!C@D z#CGSEh#`%4|9)jT3h-|8op#~YWF_aqd*)bvEnH0)6aeU7q!8%xC}4;K5QXe~u;i>= zy}mY#p~+u({0Y`FCCiA^uoEEf8~9325XbL>3t1d#w16(uiXI$E{BIyRSyfr5&7l0= zg##_-06;s=>DD{%D-Xv0Y8+E>A|LDFg7NEI1?nf--^W@H7Np`^KId@ulQ3phT@m$P z&z`QIh;;W*67gzah8+SlkN!)ZFT2F*u;?MCI|t8Xt3RrJN@(rQKErRpzpM4_x7>f~ z#bd1-_|H0V9LPX-6`kCSSr%91>bK8krD4xNG$9Y{dgLkbFg%h_1Rl(9FF@g&lDXN7 zNA8;21=yH>gh(2LJr|s0-(&PNNc0k7obhAOp-9yoCsWE`7m6-lU3%V570AeE?L;;g z5B6lx`XW{6w;#-Wc6TLP^i;zZk3%OgI6W_l0AG;~dS4g@ig16X1ew1*CBWm- ztwdgaM~%?NXO<_c%YPi$PvLhpgBb=GCXdVCFeW>tfDnZfEWFxHmG&o!zUyx# z0)Bxo*s}1Rhcg&dqqk)3^bLGdd%3nEK@9Hu1t~LUK4*K84I$8oC4*+OBdEUcxBYBUp6hZEy_ONh(qc=vqh_l+?AiG`D>p+;lw8xPO2*qeNp3v(>j- zQ%1AalKlrK1YHlfuM#*OL@BT<=-(Qj!faL{{QDW{2aICJ_**H?#qqEdu?8KkI&|%y zRxQDqX;|?ZI#>>Ln=n9c;}rgAsa}SXRAj`#J+`01c-eSr*0Js#QN3hXgc2g|?8ayw z>rN)A4uK5}%Wan7a-a`K(DTy?9!sjrly#7)5T+3xLKC^Y)hL}7c-q5RXikGLF_8cv+xk6ngM z6Skz@-g*D#aXT3tVS65IZzV%3kRqdm0#}aK$W53id%cS6uY@$gG7|g83tGV9RUQ}1 zpG7gAr80$!kiAK5#yP0z;NXGz2TAjr8rYvNjIqoA#+UmSY|no|2;kEkHAV9x{oCzc z3IN=<>$kY2{~jRMg9?D=jQd@-wEkDT+P{8yBlQKQIs2z(=D&yHxT1d-B;u(c9}oDq z+x=7k_WB6hckt)D|Nc*PAb^Qma3J)+{x=%|zw*WWE_g!CtWN!}tMtG9BaHCRmnHge zp!R=*XaApzd71!Vv;_RCx&L|({}12pA3q0pnZVoUsw3?Ee zz1V0;l{875g6lQI{q6MK4{xb#ITlSg(np_lQ_@X#WbY|k5BR9ZrQ9a1;(HnZDg^~- z(CmNYi2W;ySX7;a@TG?d_(gQ|-{owhzxR)YfevJRlo0z%mgavAt08fI2?L!#S$y?B z;-mgEenk)q5j|F7J-v_ie@O`ZkHZUM6ci+CA@%wnaTNcID+m(?c0%Jl&1w2SsQ>@@ zfBYw2!w-mXn4tab2`kY6I|;VKEBxD7arBlKFFp1`j{ZJ)8WGq@-l^PX;@`%s3*(TG zpsmL8{g3RKe_mtJ9AGC{9hMm%|Mr6I!N6|i;Mc(Y?YYumk&wi6aYU*7ZOS^l-j^@D z&LUU;y4(N#c+i7^ohaS>a2WdA3zma~9vcc8Ao{O&`!A0{t|G7#MDQp1_y4?4|NRF3 zUwSzHJf#de4WjFmPJe$o@L}k&=d}O*2>*3LVN(88uv>taJxYp~5$|tL$n(z=aNWK+ z^7l^wI`9O%jFJ1>4RSSr%jOB@86)`Hy}$yGJNoGY*57XM{|@87+(?rDzgL54jzh1_ zy*h)(O&(aV(gK|17#GwE;(uIa|8M9Ejds~(2A%pJO$JEVtY*T(t#`=&>s=%{2EJs4 ztJibKL$1WU`&Uc84x?(RTVIj)k5gU>FlyIH0zC)wJ2ZTDSvrjhg42e{nX{3cK*#kX ztxA;+GB4C-GHdn9?GKWVFuMwwn9c6LGA!<7Qz`ZCehWMoKhtq_Ct^0aKeg#{IxP_} ziX`Vh%5m1a#f(pnCi2AOw{9S2A1V)bS@2)a7~w|kR$I-|SVeL}j*=+4yao%{^_71+@W>I;rLXRFOK_>Y= z-rXM3rHen19Cm~o802dq(SnO3N+DAm1_?e$Pt28e?&YmmHVZxVjP<;!kEf5Ajkcxp z`-#5L_q3}O?emV<`SSj+WUz-@gkEj!wfJqXgukswXsf>GakQQ0(XI3d5UhgH>5ql&CrjUr9Ph;eJ+UKi7V||?FdwtsHm52(Ljx6Vs6)m%|3|au9 z7GG`LpEkMGaaWD}m$y&y7)A;IACVREnnD_YHGM0Z2fhhl`xA1zY9F{D8VK_HKDc`T zv5_OBf8rF1HLG*M-FCRp@`vST`r0+tQ_Y7dry9F;?^?$CH%U%JK8n$HYg;ZplUT9V zBsJ!1Hclv%MDCIw^)r_bJ*S{B?Fh9u+ru1Y8S`n{0&ZhYTORf>t-hIz5*%d-$a9Ua zD!d-Tf3wRALm1QrJGg_sJdCW1eB3-)_9o)0hua!-IB^A1`;Jlcz^D81x=0+p0;*59 z2QszyA>{os$pNq5@+IQ!MSZrtwJT<7uBeRNW6yC0p0-Mt+R9Fw`)%{i*2(lyQy$mk zve!r+my?PeYy)H|I z&f`C`MW}cQW)bWX*}lB3j>0zqrH3Mm^`<=0i$SdS2Qla75mrx~h$QWT=7{-~D@IbM zn-?Ov(KG}rcziD=B~16H3Y7n_3OF5d^JP-Tj@{e-%a;ilU2=thuRR{%IL$Vuc?pz* zhYNt0Z6~tD4Z?`VheeEh!y^Xc==YJfkvI93V>udM^V8@Xp6jTj?VaaK1upk!EDT^Q z2&+mD3?W|;X!9&!6ZmnK;Gd93qbt-06@*1 zc!-$(GTQrF0E1afh#T$5F0QoT6A)WGv^SYI`hGN%N5fHvR9raf3omJAqE-qX476Mz z27}>Jd8@)}>p+^LU?f;aV)Cc8**VYXS)Ey1^h05REpR3S?VaoJ1LSU3Qap~*a9h-_ zHaL&FX$Ne*cooMkQ`(raGJ%W7(OgKYRAi`bg1$hFWga#Yh?a}-zA=1gOgh>;|7+s4 zXrN#Z3=}A`4FUBADj;$&8MwTIKs7lZpxON4GpeYx0Sp_EFA!YI;&T+-?$tj(_(8v| zY416aO9$WwY!?@9Slb-@(Jc)F6bY-ALe0BNh~U_7N+b#y?9Hrqo& z-tmH)u=v;#1-={6Tc516uNU#xR;B7E^;`vOQLnc`fYs_-J&$LF%B-1v)?L$~E&wI6 zDWui8S(M0>CR3vI)`q=ATajuM%orMz9p^pQ0V?$^-z{ivN2Mg0m*PUxP8ibjd%XMz zB$GEj-g#nFe0j)FlQ&-ua!?;=8IqNDT_4dJ%;2beZ}_coXEeT0nr$$aE22_`d075r zk-=+c(Mlw+Aa^8esc8@Nouxyta`TwaW74;iuA({!Yi4yWNdo zBuBMo=uT5G9{7ND%brLhJA0*lxP+WvB3j2_SD?kPy*}nV_ifpBg>&8>%`f^Ib2nSZ zyPv*|cXgra74~u=B`&{bM;D>0#h}cHdq}#I4`2;ag2#n?7ep=(aAhn8sVf+NKs}xiBSAQ{wVE9DVFsesHMenxMH9?;E8-tsiCW&4*_X#R)rZOZ7j` z@&{H&DMXUXqA63dLAW5Fwed_=B>7{ooiE3w*jl=^*CWHIr!5BUOpjaC?XVXzq_#Px zkv-dZ)jE*4UvjQXOcxH`4fr5}fayq?cD-~BP~Jwu(f0S|20x>0awBB@b@fp!OG5VY z7SNoIYB=tW$7Fw>ObLIpqSvCErCDj*AN>)4BO2)WFit;f)h+{4r)?mGEp{}WBV7#d zD4EG%P2ZNt?Rbe2_z8#0=%pW&O7q9m=R+&(RGoV@i_th#BH{ zzo_GV-WJwsIr%bwg~rB6d#(XVVrY$#*`D$+8Nk16yW_;6U?TyyxRhE8t1{G1mFsZq zf_(VVB!B@FvXR2Ale3GE-9Hh$&CO3usUxuC^f>l@+C;7L`T-ofp%-3Q`pJLqY-)B05`-`4=myVi~H=Cgzje;WLDXjLmMVfB3G ztYfh1%}BqTlaT1S<&b^YRnJWM3ToLndws9(r61EN#XX+QOR8d5Zr74xF{|weR@F*w zyZXAjv*U89(82ucTRU_>z}jT+kSRj`$8PC6n;B7%dwu${9Ev$DuE8=^$2n+`e;=n& ztsvW7T3X<&Zn-AELvx`Z&q8s()be|q&xU)Wnx{&!I@nHk_{n_-!ReI{&Zku7S%_#A zcWS|#p(W>gK|A?Wl!u3fi%n)={)Hr9-HtPVz%xPLqkC(p(3md0rW2m%Q4&aN8OP}3 z**H7)MfW+m?Eb1IqSiR++B5b0O7qD4d4ABy`W^o6p?82*a-x%g<)iHy`x8_&?&=IP zjl%Kqw@{6%ph0Wta$d+xKYNAS{bMUISuTFw)KqEcUDUGUeVVIDb&5Om5KeGja+%W0 zeCr6{R4#sn6WH4xzES>uLv#vvcwhqgpltp z=({&T+6*xGVtV0ppf- z`I&ti?tq8w6a!brkIQRbb`JxyUBO7{A4&0otaa^+Q!S(ofS5G}Q1dpn4_-rq z>oP|3b5v@@W4(={?hsb9^jHQT)AChcc*q=H0Gdx;$BOC_A4%g!Wy{%34z1eTs>Kfb zHJhm_X(2ApjmQzrHkA$eOs8WGu2)d01gb&$%AU|5fG6CskW~?Q?~Z8&cr5UWd1`xC z9d{;#YOlTP?O5^o#Z!Hc?9V->Eth1RNy0q~YeEdqK0M0SZag+?NaGRf zGs}8PjEp@4B0Gsdi*M$LHNHWmo?C7n>A7kJ8O~T{3ZIjCtN6`+*g9VJr2}FAt;y>5 zmo=b??UqH)NA9~7JXgm4xk@(juk`?ykPBF?SkKf|O#jrC(6yS5i%f;;8fPsF*r?pQ z(8O?eJ%c-Z^1V}1C;*(6#K;}efXk<|AH8352tR(Z-5)!nBC}|^ea;&WJ3wRFRH`$$X@Vm$ZE*F=Tqm0jd zQ!g~Z*G>-oVj{Bl=RfksN_|hmiH1Inf6*1`sO^t#7B)Z5Cp8c{e5~yjl&rpIN^!5c&LIcS ziW6SqUUoJHK$#;y@l`p8vl-n(M^zqW7dfaec0$bQpMHwO7YGV9_6~!6WL#x4O(JLc zbtX6F_O=KdhwgU5*S^=-jaiPGKkBU8jD1EZS{3UWz;#~j`g%g`b2X5&|c-1|LbCLvtO@MqyrSP4-WNv za^q^_Z`-k(2b=EhF0U0n-Y9_ADloS0q6i;X=YDiTfcgGsH`bWNuh*A4bM$r(&z zTJyZeS0Ov-9Z%vE5`tNY&+L-&udbs0S+@Ge7aW#8cH`wO52THpTs)F_*JEDKEqt!7 zA<#Wt-`|0P`9vc-OL0)`rPXQykH8LQ&1PSQ&&{4xl zjGEVWYK$Mpa7p4Qxk0_9e$P)8EqP}C z*~ECCtR@N>0^gnLrSgTA?xvDUY<2Cc>|7>kY{w&~er%pwRJgFdK*~P+tbd0eUPwMt z?~#l!`8e+RJr$wxlW6g+n89T13WC@HInS6M0W`5mOwIH~5F(VT|IvvK7VzP7omdg=X$C*Lz2pTNfg_K^S?0eKrR zRcV=0Gv3o)zH3Jst6(Y54IF(_>zGDk)d!u0TdjGv6!%lZ<}o~_Qd4x^SI|uz_F-_g zAK5())2Duh`J7q{%OHD>cCGImnsu)o-}dl#u#z;UcqE5TDtg`LBT%8&4x8_dzGUe9 zE+?{aSdRbtD$KY%w1VGZxgqV+<&C!Ceu4ek``Felr$g3Htu8Ve&z{{9(pN^kaH`B5m{@O8Hl;c}-N{Y7kG-c8k zK0t|h#<%0vzsnIin>wjX)!63YW1=MT>}lq<89yFR(;v2-HiN4JGvrBVlHWN=`oD>4 zdqUj@V-Ut!gI6rLeHxZW3C6zwrlD1C-?B@T2!D+ zqn+6PmC@Sivy@l+3T5CB8IMjQiPk)+It|oOo_AkQT{ttv)mh)xzrs8TReoo1asa6^ zyX}L74hb0a!tEfu25{>6&M1z%)joXlodS1mlId$Cbl}_9ji9?maSH5&#!}J)O||*J0%VOLFpy?qQ@gO@*9Omdj;$zfUfMT=a^LkL^;1inP*4 zLE>%GibFWV?@k$?&;8C#$Dpe}p3(U5BzrM6T5WkuhCFX?LX*t29N%pHiOWOAW&TMZ zVdtn?^qXIK>^Q&c!e}?LxTZ|vfdk#2DF+?TH_Pz(9W+TvT zWTQuT=`#$}XHIgtm#IGme(JmG_-d^spBai^{Vn3NM%#6RP4+{CmK|K;j##XK@<)=G zTA2I#$4*SIi|UZXclvjg#e-!pi{Ip#M-wEYh;%yX1t&Mso+(;4rc+lWk@6#G2cN!qHk6a$`{0?}uGyBV91R83-~L)W6z~RxXkA zUZ^XR;F#Z=;n`*UNL~?B_9)@&N7j*{*aUK4c-b87T6rt>Fe+?b@Qf{^e8N|G z6>kTRUK>(6N8OlS>;08bd+G2($&dKrT383zVcBQK7!RToVkv}yw>+Uy0?4)Ecz%X0 zvnPOqfyNN{YOu{(`|%A}yMZW~`m`yTcz0gfgV6uuc^f;I96J9zwhDdYj5YyX+p z?_00mW?^2l4bp0pI!1Od?fFwuN&G@0G)FOufiz| zl{59kh{}(Bc>q$^6MzZB8i7GM5E8KM!nm$)Z-1+-b(&o`3UW!TNUoKn(RCyObEP$# zL44!a@EF{h^*$-oJ9llQWU-Fde-Pewdx$wfrXn$TUgfB{O0OLC9dghiduzID|hzCou7#z^`l0Q>>`;j14o^J2V=cgqd+qP zZp4w$qfDqiL{EEJC;hzdYo+JX50%*euf4a9in8tch6OQDQ4ldeV3ZOhl$0J72`L4o z!=O`I8pJ{*q#Grrr5R#S5ov}V8ikP>LR$LWb6wB(+aJ*Ja<(N|I=nH}a@|^8 znI9%?jUjhY{bZH4G7}QDI%ujV5_muN*!aWuB&hHwZE_6C+QXOHR&T_3gxqMPH|=m~ zce&nZ1@?I{)qAj3IAe88LSI{8CbK8Ag29J8s(MSP1#j>~hq4K^(eLnqWG!y-WQ^`4 zb4t~qXP~pkD4jU2uYc6~YyS_s;~a*D3G)00a3TxZo;yp!Y<)}85ko(3`i{jWNR7Kr z!ocjCXTu-<87bYTCg>CyM$h>hn-|2mu(yF4!>lh0!Xf?*%CB+gjafTUNARjrtOm=h z8b7Z+7+gtePwdaND$jCyNRqC#LYuC^8^P{x`-5h3up57vt5RjnH=%h!L()O42bo2io6mg6C@=FP$%j{b1&Z-_R9!wv0&4u?sUx4Y#(+ zOf4y}@LObl5|wqkWKo^=^N{f+fVP~53vb?6C>Ela$gBS{8QjJpkTf%bvg1WvbvrnI zwu`PcKuqMuc|?iw{o&7RYMTBzIkYWhI>VA&>WM$D1(!c#LSR%;850!YU!uc1+CHji z_a5D5lPHp*7PWqLWv{KVaJ)dJB;PVlv3ZtJI>ud;?PS)xXJuu>Y^2iF?GJkw%wvX3 z!|=}i7L~@coY*hPEwrnA+5?6jjJd9xa^@oqzwkxJ(vs92PGq9e7aQ&6JSXX{wfOSK z3G~m*d|rzT_0o~IncjLBzh>mq9i#mIE`HfFICtfTr#d{`6FbJa^Ih{v&7#I92xPYD z#+|u8p^uIYY|V&9dJQ;_PW+9&{-UdEu|Cop;B2jzwKl4xo^za(ZU#7%A@C@JA*hLszEydQTV0IWV zqKA%99Otpdf_AHcYLK}Mzxd8Ya)a{TM#n~DCbjzK^tBab=8E&`TEmD{EYEDQ71o>D zw%>1_NoKAT^kC(?3cqG{KPzhFyMmpxi?dg5 zOzOEG%iZzjJ1s-6;@+=cw7Sg?$52geukWJ{+S%?LO-y=UWXNTo%}(;ox@u)XYioZQ z=>m1RqtvQ{GF%0+ei&f?fs;V?`U|HKd^1K6AHiSlu$UyX@}vg|pP(*6kK_(Dy!(QoqLAa^C?sx=v%Z#o=(XxY{%KIK@5t-x z)t{J@Rx~KGrp|ZU@KSAO{i@MG*B6W9W^%7WkJe!rB#oYx4Qqe@C8QUT^)88?e`;}I z$GJt}eFXRQ{kiDx8kC^yu~JUx7E4TpaDFmg-YBeF7fZJX-N%R#JGXM-hv*n5f5>AI z$=tTHK?&kU&qAHPya-)08$RFV;%?-jzWn@+AbnRxl6FYJKDr=AEtJlf(@RFn%!GP6 zJ$J>0uCkiwk3SD#O-0yFD_%fUp_Dx{-EJCU9apI7~$c_Jfd^qm!74fTK4`>}l5 zq;7h%=_%ei7F$n-LbTc`o0O2@$R|s4JFF!pbZA6VzwQWK^LC-glDj9V&`o!1)|hg~ zY`P~ehrD&UYH_xL4;>qvi{`xk$I(?uBJ83_-l*u@7v+~dY08YIt#7XeD+bd-58Ou| z$-W;r#=!4T3u9iH*LAcJ@3wB6hemg+I2A)?fX(!C!(lbcP(%FDhLVu(hOCtNkVk49 zVcO5DoSkt6E2uon&E&+p4P$RU%BH7`aBJXr~{)5u)Z>lg{$;r{VX?^ zV)ju;f7!7Tt!^sT7cu1=#!vfk z=Id@&gRN(*lw{a?I#tfbEAF&xc9fK8#@<5e)f{3w%>TOT>bHSfN4#~{t79WOdQ*yf zE1kdP3SS@la&zM8hIMR}-64DDOXIWF31ql!1F>-J9m@~vi^>L#6I4dex;1iF+ICRx zAq@}x3|!N9bXOr}joMO<6A?(ZKET>9m;dmOv?_vx;I%EAm(KLp26mR?bINf^fg@}_ z^4~T$b6^TfJb5v`Go4A_bE|0BqlM;FR+}0|Z1Htd0;6QUI{w`$PK{d(Za=SfWIvuC z8S`AQA5rqOgdAFXe#%B&Yg&TGUxS6LL!=@YC7qR`#Uy)Q@4>2EJd6%VFlie4IlH42 z0qA8+gx>-Fkft+yJ(*2O&!+>IV=dzk91q%A!+4mke$wfvYTQecyfx?@h_mi1Ot%jLGIHH5V$P|Hp<>V!w$uOHkw?H5IE>Qoz)bIoi=Pt++}Ei^0nPD=K z4W_}PL{PM_kc*#kXr&R8b7@dBpn6FC3WF*y!l@g%qCVFn{3cCl={<`|VM>bEX6q|M z-l|d6u`>I@U#X1Gqv=@Rb+-$N9{Khn;QgS4u*BOEnF+SSsKq&{jx%=d;ahp~trH%{ z57qgVSg>Fvck;2{V}&Thc9fhQB}%7xG+WuX$8Po-XZhY@G{^VWEcZFje*Z-`-e+?> zfy{1p`zRIU`Fv;k4Z5$-4~kQxd3jE;8M7h#oKrKTtURdL zygMt(qUXa^+wm4IyYa6(B{yV64gMhx&-SVCBP0zKnNvQM21s=a!d#RDYH_nFm`nAl zd;cny!FBqwW|@8Z8e{$$J|o$$@*$tr@VSjeR>c82*cIqzjY1z6v!A%7u92Ye2<(d6 z@OM>PU>3NC@9Vlf(p=VZ>D=Sl*eC->>(yMg)pd`QBXMOzv8x9#cS6d|%L6~UjmEGn zEDmHFxg!8p-(pLil%IEKHrGt5-Ox^|+iMAZaBait<$T{f(8@n|J+;Y0rlxn<5M zSz%6*dM!vSPTv@_>HlSuNZn-GsGgQ{$9?Bvf_|Lj0*1EU%DcPzqvt0u1lwuygvS{`9+A(+ZMR`+EtJ_M+>w|LQ5^@JwuxIX-Qg5&BvcGJ1*&) zSjj2OfTh^f>UQ7o?{ygbB&i|PoFo79Qj4|g6?64|x*7dK%vbW5^d=e%x?a^-A z_a~b`mz1402%mz11_|-5v%(Rd>8vZxQP2s7x~V<-;JM?WS!5PQo@osbvbxvI)b!)T zAL7%wOH=QSC@OMMW3+M!5|v@zsUe9s2H6aCVE(iD@pMRYMp^P1w^*#aXKEC6m}0gY zYr33Ugf1SxV6)LGM^UH0K74-72~Xwx-bOCn^`QrQC|jmZp<|iZ=(#gOPJXZB3Tn!u#!<%#`%I8Sf zTT6WnN?7|9toM!83R&_q)z0J}pcNCKwe+Vg8gezZ3%G>+AP0TECdQW3_1M`~XWu04 zY%^({lwwoN;xl=#b2l~C&MtS{MXWr_FTCzTS87}jwU-h5tyV6aN4Sa?$=qU z4NOqsqcKAD0qoR@!mse<6$Y$y4h*-hcSy#Y;ScH94QpsNIJxNMM{pWSjM5^l68NtQ zZY@>v=d(&Qxhd+6kL8TF1ZnIQ6IW|#H)Qcy*3tf|gUYHuXji(n?M^-Y#+{M!ysxkhYKci47Pr9k~$%5f zVprr#WL!tIbl(q0*J&pix#{DpWbDsB@brusbUgm;@Hh@t=Ic?7up4soC~48o3@|8? z*s1S=@+Rg4kM;E4t@pQ+q~u%L{=r+ImYw&~@m}&wMMIK@EIXE|!rcLiq0$B%`z_IG zQHHdI{gx0n+wN?wmYewp(g`U+Cf5DVyY6$=Llvk@4A2}>EPC@-xOD=-1@AaH^Ju0P zOU;$bn_()Cdi_CG4*euuYL(gqww(ix-W)f2Wxg$9M z38AnWzf)|^eBUuj*GN#a#>>Wx@YP6QEO~MY8wdM5yquU8Bn+jN(Hk%G8*vsf%JaepR!^t`xaqc&}@T zm~Qhamy(+5_fOQ_E7wtldk!7^fS+@n9iO7J3Z+fZG`z$;7`d!7=jQ&XS0%Hlp`+zS z&bWZYPJ)hV_H(>L>CRV_W9%0;P>{O8?5)=a2gc3Kk9Tt1C$EbW!jKB1!~HUPMYs1y z81D>LZE<)y%uS(sDjdUA7JARsJ=4PM%<+uuokR*QZ&vKjSG?qIf2{?WoU%2qXmou} zKIx4|{|U01QNDzgKD7;Xv(a)44pRBX9c3f*BKAQsR%P{eVg2|@*VR`68ts?*x%(eD zJpxZ$#0?7ljk>h0D_M?-{q!1ao@ouWnZCJ?IVH0G3uqZ8bGGcu8Q3gEd==hg2WR$T zM~?+xMQ7sQ6gR#|Nh=c8MwJj@rn@nfOw#lGzK?EEkod;L`*~7-TKT*&u0d%dmz4tj z1Yz4Rul2+}802Z6A*UC9e-35baZgdEBR*duH(pT&eagZ<^UU4oW(@8b5DY_$wl*3d z#(ivMB|R4Yc!G+3*+d(AP%~Glo0PscJ>@*VPFvYn3uW$rXPIpM3&!Ol&7SZ{vhcK( zxsX_F7**w1sF{m`tCSh)Xj}@8$&ys0u)yM)?+b<^>DdF9YrSZKtr?a>FqHCDmXA8L z#eZbj&L(j#Z$;_krV5wPl~*xg1uLt?c1o-ndoQ_tSnKxbeuo{aV8wjsw3q!Hc!S5D zcQ~h>R6x3RtOKNuoTjL>2&5! zt~%1aC}ezG;2Xu>iT8`Bp2Y??m{J0xIVwE&#I*mzZMaKHb-2#**%kc7Zjp6ayvA64 zhzx&Zw+g#y5xuC5TJfA;$X*$x#@@wK$sfBklWz@W+1@Q~)L&Dc=qt)<=(ChD#K(`S zls!^H&GyBrX-3&UP{y56K*qP)9M?|UQO(-1%S!s{u{kfJy1cWF=38vd*rB3S($8&!+2%7z1FctB{6SrSs&`$xo_Wa_R5!}i2;)JjT9kS+i><6T>I(ua*tng zU1RPT%`ydK(%0)^%fFhLHifu3u^*KZrIF?VJo;#z47F^tG1j&PNB%#9INE*Z8G>Y( zw9LfD0lbJ~nsiv`D0AQme6Qdy5lc;aFge!C?oLtn5YJHl-3IztrgL7%{KHpd0s~>% z3k*K>A2m~TN+TW}}kzDneiYPiKyGW*WW zPbuH2=P771RZIm?oO4mWdPQ?tGn>UQVro(G$ogxsPr(-2*qQzDc1-z}S>hHcw)QPHg(hMuK`n3FS zw?x{*BFkVmYI@KA`_um3j;C_KxKs$5-v7rc{l>~fl#v06AHOwX@jJWw4;be&GZc%k z$~yY|k^lY;oTt=WcOslKI<< zcwd6AeL)@!JNo;be|w}<1Z@7_8}-{yf4>|V313@HR&Vv&lYd(bcv_-7ELhnPPs_<& zS4;uE_WlzCp+j6 z_ANoI@F^W%`s;4|_QT(90c%8d5jHOC^wzFB+x`?5Ec!7W>fo-6eh+Fqd}xv+cYV`n zV8tp^TQi;V-ep%1lv(hU4-b}Pe%s7H7FU)UR@8LF5_f*rZFxci@}u^8d)GBLy$1WH zSmCcjw(FaIfTw(V)%JIZ!yhm3w+J8<;ee+3mb~%%QzU+Sse@|p6phR4yH;-{lYOD&vp1DXOPSCRK3%%d&IZpxGOf-%(}DKZES4X(^c7==W9+;Ph)Np7lfA--X{*% z9fJ=2F+$dks?7Txvk0ODCFqSx-7>f_pg!IbXE^f3R|4zIodrYTk1$FM*j>)$ z@&?`$5cCuc2<=OtiNcvHkM1G4*_7k?lbT`XEn;=hDZvd!Bav9juo3@$*>ci!NRa;3 z9)i(*>CJ%ic-%^&fph0ilVx{5sS~O_u3l*j$p{!WheVKW9twLd$+Nm{8ATRtUlI+lEC#xs+A;jNs^dhq@vL zVG81>_W?|dBUvMaV1bg`rSe;wYm2-v&UUi5AWc)P(B$?S#ke7~OA}D9WaR{jHp_`W1pi-3$W#?^ zmkcL3!xSiXesLTl#N)a=#RAb~p?Xy9;Zxjo!=5`_1;&_gNU+H`&JM6^X_qNP@g_Pn zLF-bZ1Zu(A!E$x!*QdYtVFAi#DKKj0&0UTZ(R^Rfm7}YjaVLF5#~|K(&^+$jZ5i&l zp1kXY@ea^wusQGIo%aanp_WOg2Qe4b{l=LW49UX1BS6^coz`z`J#kU3sZ(VKSfmCp zKP(t}4NzAQ8U^Sbl@nfto)doE0oc@Kv?sXXeC<_cT^Xs4v#4=~wwUw20><>N+2T3lCoDF8jRvMz52K9=lF`RL zF@)@5OoBxAsoX`*X7S9JlYY(7EicD9LHyQr2G1j(ov2RZRo zfI;M8K#ALIm6Y%C)aNDc*Q;mw0rrON4uD8HUffm9Wp$QIOY5t8hb=p;PNx2~nJ{&| zH^7XD;U;Jm=!C@5IXGSh6-20^{?%bm{UsyP-9!%dejo0|QTpTbaaoFcHMj~!ENDFf z=|#Ic6RcX_dZ&-NcimNwdV`bPS_=76Qxi3-HL6p6r+fxl4A_6k8|1%KVjhp5_)+1! z*q7P#5cVlWEnCZ~Q99r(`_9&!XEQIwoWqixH|ajy&+pZ|HX~BKg{C2z>gg&hZOFcY zzqz_B8V^{2ha4XhgKhDufY(lBdRsI(;Y@!ccRUfBauisn>Ep9!g77ar=t**z8fQ}H zI^ZZ+AF=NRlH>(KTl~sCF_B!|>PPgZdeF0rKuexL^(rRx&s9}6t-YT^1w6&JO}SXf znXYI1Xj`FbEHbc3!*|~Po8SDG)kInxO3nb9!+Ra4L5CctYkTh)*9V`w{#+IsBsR;A zHbwE3uTN4_U>!)Mxm*(!@e0Px(OqJzn{PsCaUgc(Svn{chR5GXs9Qx=(*-(39j8qi z;wEhMujcYGJ6hrKsBroDu2>RO4%SOg#!$5J6AZkU0*;C(-%SyyzcCV$6L&I34pq83 zsjA(mw&A*(r08QtU!U!b9ujNCLlo<{Mmv*)2P=}Te8ZYyX>NGf5?6v&);gsVvww@4 zRpAlcmbIGB`h8^!Env`LWo^gTk@0ww6;O)NIY9j6Q}ecvVcbQzx{{`IAk>GT&vp4V zE3R|4%>r1mEQxy`_pE*MIi}!AZ|X7XFSI@&$WDU7&D?EMRr$?fGRO7{`jNjX_Q553pgWegSIR zg!!9x6c#Y;b_OP zLPGksVwvjPyG%+pSe8yVp+tc-5-f15JC&)&{2cIpT4^>Ds*46O?}1GXeYH^|lp(Bw(uOJJ+W%Fb;4 z=5GGxha&vRlgqpY7~-;8gVN@vWtfMCw-}F+dmZ`<@~n;h1xD0>y{${Qm5e!URL>bG0-mpy5L}4P~{y z83khH`c z)om#%O%=Th+_!Sjmfk%#>8I(xtj_QMqKqNP9QMsOk=|s%xTZGEjO!7T6Av)=?%?m4 z>)Kk%1Tr{;8Mh&{&4V}0fSs785H&)3&T38-b6FqlHkK05)J@I|bWG=J`XDGOO|yle zAyK0&D@>p^j?@+9ly;xOIzrp}2FabxQOCHd%>_R(D3&v-L|dCBvD_t;BZmuTH<3z8 z7Bc&3t(iG~n6W;8N5B6uH{^#xqwd^^U~@Qu9+z<5CFE*j56U35H9kXuS#O1rKbNTu zI0~MR%vG+&pE>7IL!(}bD>@>ZrWDtN2dEYLuRh9L&Q1azPLYqZ_}j$+)$<7DiFF-B zcgU?4@lLNzI7+r*e)9quL(cKuVkM1pCOv2vwOvB4v$t1o!(ynSce(iqbXoe+sd&s@ zdwmxrMoay{>}r~9f!Q5>&fAGk*T%u!TXb%Ecl|}#`sTRgjt%~yKZ8?t!4vfohn2$@ z`5l+@8X|lADj8l-(2Y$Q`#)XLnnT@#(!ZQ-mm~U$n0mB?dH&5@ZnZ~qT2+oSTDi;K zoRNIS0p^V4x(Rmr?_$dZsWu($DE0O!+o zFrf9#cadv4G2I<4!7P5}aa&R{WS9iW;R*$lr^}G+z;&tX3v4YphjHAU*8SyXA8wF$ z&5d9~)vLgby(zJVqb?8=@JAf`F>sy!YR+j#Mdvt5)yBdJ=43fbwSzpmW%ie2Rwp}Z zLtV%n^;>wwTkx_ZSwdAY$P)12Giv@q*1F^!*EXv&UI z*Wj4RV->b|=~G5dhlF+Ia`>5nIgIA4`yhbm){pkRsY+K@u%CcB-*Y2k=;AJgn8S}J$8|1DI?m1{q zA^YOUnFrQWEAM}(6Y?A-hI18*0UAXegW0ihj9D`aKi zH18tjxm{tRd6J?s0s@nhL+F)0^9AVYin6qu>SS+g=Y~{|qc&t;AX2*BMm6H-90Yx1 z8}YeSv)ywL!a41m;4^9p0{6Ns`xx$70l&fn?D&Ab3P$%j4(fxWbv3D=05&;~XVeE!H34 z*Mxu%5Q%y1k#l^#sR!iNUX}nh>#di0-(MZ2+9ei}LDY~gJPI}wrsk@B_h}CNJsv1P zJ2pN`J%D|~B%^SKGW&%t47z8BB4t8+^T|@Bv7z(Ptjoh|jTsXgX;v;FQjOU?KP*Y} z+UaN=gCtB^5!5+s#e>EP?nYKt6E^(QP+XHq-T!oXmjm*3Um`C;d326E;*^(HfExBT ze!bZThH0{t)_kvi0X+~nQk7R{2UDQl;lCC27saCjMleC=d@`5(s{A@{dNsjxMnLk*!c4Su!?@KZO3JT~M9e(0o5BwpCa02n=g-3Wt658X`m&Ez~NwECa z(*eF4=lhq&_{YB}LjW|pyjE=2MtuF?ax$FT{@-r*o6iDR)dLj(%vgsVckd$OSofW$4N8C(D{5I9_rEoUvNQD-7zaiP-3L!AADAt6%`P)}ooe8( zr70(bUwd{4rW@>gFxifRPJhHcO7;dR6-8Z%^)NTV!9QF5oFWALnzZxLIPowDRU2MX zF(pHc3^nt0IB^Y*{&F-1=7PY4wX#T0mEK7k5#nD;$ppJd<&s2RM_$LY)HeEo_+^g> z$%Jc+`$%m`7Y@2FUEw4yk~)C(2e!(1d&Ls27C-7oe0w7Rzh+vxjXp45>Mf^^%j}%LQE4Hu>3^K6#1{1CCZ0ARAlq~yJUVs5B zow@q@W0(XUXoIbgRgPT;@{#A(~1%%~JuAcbFxP)aNf3BUp#Z_4I6#L2cg?{U*dI%CL%Z>r$#n~YN zCiSVx~Pa0KI+* zgTE_)ssXsrx3?t;0_Nc-xU}woCE$z^v^-?CA?CagiP!yb&)cMx!2%D!Q7?D!xCAug zob1Va_#uru!EM2}(4_4edUHX34ZL`tA&;rfbk=*vT&@celPbc`o=_5A3y?Lvq*6cz z#ty{`yle@(7}1|T-V~+!{K%Q}BJDadk+n57sBGow8}K5@;IOkY&tY@*9}HG@V`7Aq z57BTKNI%Zi$op*We(kWmec}E}LK6xo$8dHSglHY_0#*-KgoXit?S-ZtFXPJtwylQI zl~@!@G16xqTvu%Zq_ARQGcH)1GPH;NZbqnGnDY=2huL*XKm1pVI)Ie}bZ3xkxXh~O z!+n)p+ZC;ySe5rY1 z5`!HFU`^qG7S=Y=6_~W0*R1lWL>X1CXqW*O5Cp9Pl~ zb_~wl4^TeAqE1%zFj%Llf(dd8`NkLq0v8e>RR!qO8exIWK?6L`S6c^uUp_rsD?MR= zVE}_Thh~9M80o%4yQ2+zygwY05<}$bRhFH-@(4-rlbx5l)T>==(w&WA9*C{(9KcM> z0CUtkmO;C%V8Ht2iYjh;Y?tS$2W^xB0?F&K`3otO#7JBw5tM2maUX2= zqxlGS<)b9mk`z&c>Ww308jp7ES&<_g!jH92xrxhQ&}%q^^)F*ua|ref8k2rFO58sR z)`z3N=I_oxFjWKN2Yhymq8^Z5rU_z{aMKtAl$j6!Knwdw&m7>MvJ>3-i|R_V6^lys zFn|2jHyI9ue%^y;3b9w(PIGGJOTB`g*RVz5fS<|6i#e;{`l6EgZbS+nB)%NX=PCK1 zy;HV!3XH2Gwnk%CS>(eNKImUjljcBJLt~L6wET7WRuc2YogRvH;U z@OgTd)Fvs_%P2252(H?b<)|MfGNlqg`5S&G!yt#!@MPa%gYC^V?p%PR*)~k3WXW(V z`-~dH6=5_ZH?#l@1CON<#HpD>7Y%9$Iso|%3TQHr*M&R_iz9BckwK)Fse_<^PJMlK zjz{E4ANIl|$=>~S2N>P%{nCS350kC$t_OvFatOSG%sKZ9Y_Ou-8Szm_OEhNYqB%g9 z37jG{&}x8W#qog}vfdakG1*IH#9csS{=qLm+KG(cU*q^_=s!|T4`QAcN!L+%WOtM&}zKVZdUdmZKva#c!(e#QmD?8Tb{@|ANI`9dkS8 zhAPVm`87`LTLNeAzjNm(FCDhY3YZDP0;RLfnUHJ9!Gh`L#T0cmz_1_P#*k6DWKvs) zpIwlpWLfd|f&7A>l0mc%^s+)3^zsmwQ8FfwQD?Z1!BBn$LcUCbC#)EoN73r_>%5zIOqJgUh1b)&_EP^?TkGN$3|_MB%_mNnn*P2)vv3 z;KmtA54ZfYKD$Wg9#UWOFH*>y5TzlTTKcaesPXs^z(R%^4IyFGo^khLff&~ToWA>V z*nr2zj1@Ll=)hEWwmBQEpAY}<8d82{rKBcV^WJ+P8J2W^v-+=W5*zR- zZ(O&vM6XBE+~1^1r^Nxd6$}{NO#t3>)%idkokx7xn(z>czv^aEZh_MrTQ*HcTwq9P zUiy0JTn2*WTOI8PZe1L3=HGytjcRKbcUcO8_&LU5oe4g{#r>GXueWDzK>T54HwaqV zR#8vHdBK1kTw_Q1Vt9x};q;&MssHp2pHPCz&|!<8hRmpbG(^Gm5DgQ2V^a|XCWI{z zFYV3tHAGp0eXz`7QRnOm7AP@nhA(ze$YkD;oMt(x*_1*duUa2@`5`VnEuL41H1<>~H>9NOxNjDZ zgMycC3?S|yAP#y*9;)m50Ro=yyl6y7>7IZxqXvk+pycGp6HYN*sJ>Zs_zF3jAaJ{7 zzkhz#{BrHc8Qx!0;hc>J{|n7_D(Jvuh{YfieGps!vwa^Pk}m8Cns`E7i_}9i-}vb- z=wnEvgc^rh)&)RGS1G=z?BZJ?qJV7^a;9f^mSDNI+yymufxFn5;J&;qdNSuAdRkq- z^?|6xBq39|1G?u1iihzvSh&rFG;d?#J48wD{qg_~y#Fvz!tw!Ug_&Jfrr%t8S$7#! z8~cl6jNuUS5MtBwwnyKHOaL^41GhAoP?Z@c>XfYHGrEft=rk}vUPjB4-z7TgYmn>u^=B(1oBY+_E~!3=lXwWDUeco7l3?G4;5}r zDNr{nO>hg(Jr9+f2B*SsXM2kf>OM@*)~nQllo!{tLo~tQ3Vs3gnoV3bf?kmUhyjt6 zE@TUhufV7PRssX$27kRP2cTkMR7}#cpr{RTb#LljKinyHa!-J{Gh75{eJ44;W7K{Z zB`^I~{4Y)YKNKp#{=Ew%{hx~B*iQxlP34xL_0aTwPw)W1AO~xvhB%yLSNUS;4(t#d z*s}=LG^KiQWqv@8&k9fNdb`*{GhM$5-IO4y4^})%!zp(e2JnSL(!Y_AtoZiHM-NY} z6~rWS)cM!<7tI3rBG+O-(i>oBSRkd+Ug2zq>1xl?_HDmoB;_p026mL zuKal|t;Jk_i87Fp&U6;{p`br{d!{;%^Ttex;m>cMu38gZBzCRB_xx4^8juh?(YfNh zfV+$>hpO$ix8gnBkjX62WyZLJPHl%CvGsR!WLSD` z@efS#KmU{L9O1P9U9FV1VH9<2q7gHvi~}^iL=_Xomj?P+%03wta4FEw&rUm`$L~Kf zlM1p!!14J~=9--#WF3Ek<4)b{pL*bNW7nUaoR~17a3Q2W31(@k4ALFgW$@MN&D|mU zLg=!zmmC{G||Eons z<%SHQ-OI_*1c0MD89EZ6%TnqMI}vI7oufDJ+1)03rlj5w6xI;u|GPjMD6oU>Cl3Bs z7#|`LYC@^m48U5C{RKLKM*)zf5#&_zJ+@r_hAatv2BAG%2zUl^;GjPIuOLg+H+$1| zv5}x#AA$+Eyz<-*MRc_YNZLGH;dcBVV*RiG{J#=r-WeVmtaTCI@!aDXcHjPHT-ud`)@!qNs#aROc~+b+*l&D4 z(cFx6*b*_0n-g}hT6fFAj63Dj8Yf{KDu=NfWAh%Xn1a=uVzaT}OpbQ$f?NyBa?dsqD9)d>Vzc`3f0avBbP{{PRtN(hJ zzpuaGjMP(CKLs4efQE*K2?#c=)RNI?3BbhZ!MgZENOS|7TwA_XFgjmh;Nx+C30?2p zz5Byxi1j9sk3c$;hl(K>#U4M((k|iV1{y%*rTf=XMSv}!%Y4;g0j7Cb)Akq%c!44n zY;gjqhL|TXtITTJcx!z*4;<+w5%P(^D8s>ev2 z+k72}EjE&l%$6nyvfPqT82{-PBz$Aqa9jw27yp`0D96{wtAhN+VPdN(fO z)~{mzD;cEL42CSz-eH10*GVhR?sGLj6!ID5+VS(l5#IGHcz}_2oW*zzJm}VahAu& zm3X-=jy2m%rKf#u2)j7?Co z!gVXoX~-8$EP0*imc9)&Gm&k$r%g(_5KD#w&TAmVIl_YnoQ9l@U(*V3Y|WCb6>poB zDieE{G2vkd#13@38m(Atfoa31zCu!e6f9z3oMRvRU_a?X%eMN!=5I#;&gI`9-4hND z;sKI1JB;`#pjo~k5JnBIiB&71|8nH*Vkdfiy#1Q%gMv6!1Mfj)i_zqPZ zcnYo|T(Byizds?iqG!nu)`>bJJ%IQ11>BPk#xXZA{z>`y`SmL&gM_|6x=QR(0e?SA zSt&4A2gN%G!Q84nx34@;W#PQXyDH$&N(8G{-ceVVv9z>&7lUOW zCLv^f%m@VLHgYJHe%?9-vaU67cKW-gW^&jN)Li4rSEC>SraFzlW4#CnV1ypFvoo13jqZ@L$Yk;akc6uZKkdW{+Zi#l6^CGe z@tFvzk$rPXToi}u`V*6n6213r!o42`ik~bQbkvUNwgD*MvO!&tVAqT+3kiyGa=H#Z ze}vdyKBKO?i`<6fe+@*;eaWlS1(>gSwG92KW%TKHxJlL^2lxQoyvY-2$b9zzYg&{y z&QW}n9yc?h67`Q|AS{gcd2$>Ff+zU?i)gM0uZrtF!=GM|;Y1Hk*bq|^ht4AqJO>|c z&`|}s*4*51@@FHuNy)IhLD4fN@<@=uTF2XAL?3+cV4a^=UpVVN9O**wG_?cKTgAhz zVD_h_CfsUxcK>bJf9@s0uC;+(yTN9K+Dn4+s`wH29jb}`&A5?j5-bDZoNR(pY6IbY z|NecQ*$~T|G3c^9urw2}*I-)0+{84G-h)J69*AU*16CK#1#e@{@(m(S-s3Ip9ee=i z&ouvx$R`rs!r%hPIHk7wzrPRRC$}U(&n##jW;*w8GWyr%9Dxrz^8o*f_{|$+kVriV z^ZU%iZyxjx6v}lAU8;XS0E9(bI|%EMo0;hc?W1K34Got-6Sx_?5({2mU)O-S^!77- z0?-Ti6ohi@wt+u7HX+wM3REHAL8oEyjEsy1<9pIXRZYg>hC!;;Hhf;;uCLuc-LfJ{b`o9=_3vi{r(Hj2?p(y-s5xek4JnppaH&L0PK1I zG^yc+L}OI^lvSPziM7$0AVPZ^AdJVW$w(SDgq{|((8yeNEPUeSCbLhRsGdJZW=kCe zncoV{EAg?XmGvC*m&c(^+-pA`^kcmCjWuqB)02t|bn;2WUaU|P0R|9&t&t`^%xhpb zUvWRM0xkPbf*AU}U~qX)H3NM3fa}M^3+KyD>))gaBC!7RuzIdVS5naK8`3v!-0(Ae zRxwHDd`_TQ? zl^)*4?YDf2oPwOBFnq*b`NU0v2{yk6v^N(Wm&p{wj?O^ ztm)SM3zvu;J5f#qV))vxKJ(lf!wB7QJ%0vq5t#p5SY6t0jwBrFq3v7JM2;469%Ntp zH584+qW#2BvAl*ikI)Hbydya)?{}c7B6W8y? z5c`HgbtG~-w+DjhkR3Nbkc?J{k;&dmrka}B&MdOri?rp zrVct4d&h^^?d*|oJ07j;_hNn-&hL$rlS~2Zj`0HhrF+O}5T)DT)h0mRj5qWv1|nf5 z;rn{i%04BGlbpqcE5uAvPdsvV?*gX5A{Ky_{xR4vn5?Gz1*~UXQ_!C`k)kK&fbM= z&qzLDU0rQ$x{c`qW!7!Y{kZ;^RdW>(5oV{tdCgK z@ivLLUlZAUx-dTZJhp36oTFHb-{d-23DpJC8d$aWC=ffTKU7ej@aP=XF zh1yPP9SwngGwzK)w&vG)1kdvfL1MyVl1or^xa418!Bi8Vs-X6+ zx^9tTCJ@cP0x(gmsrT{;JOuK2H7pT2$Jyc^KM+YH#8$jJgt!Wt#GimN2=!6>U}5Db z-jw+=4Sh0PO>h@cJ32#f>^)kq9w_Qff!@-Won{9Fv2(IDS$h|{zdruveIPa09XKrN z=69V+b&%-lr?f;w5&3oxSiX~#$3&+;0K-`6Fw3*@u)C1>h^~U;_W&^Y{29JX8d_GUuw0WSbN-gY7Szca#6MJF@+sKy+x3MR97qL;S9M_ltO&J!nD z%X*)a6VTV!C-^Fzm^$0z{rS+ry2JS1v2HT(cLc3Z(h%Fp3$MW6xsa7D#DO>!RDzc8 zB)WEy@Yn}aR%w>t@3@KfUx&ds|E{CX5bXOu{hfOaKiX=5Z2JJB-8MMu?=vzu040px6c~BI0&X8YD7B(7$L1)G$2N&8CqQrH^V?9!)&t085^SBas*pOd zM>fd5O(t>H)zu}~4EEZk$&QD0LFWvmR0rPaz*L?20Xt01+p9hgzP>(FGn3V%rQE#+ zf#yrdzl&_;Ha#~cx<)uDSir<*CxZ^x$rmo$%M&r$A5X%E`W8R&(t#_p)3k}bTQT_k z@`W^nQ*hi;eDF9C3IA||@T6=6oPh`v{A4(hHTUKsjoOC`lvp=$TWl6m{05ba;{-+G z!n2mw0(QMhZVjq2PYMk@y1u!+F*m=|pF4^0mRE)Ea*L)l|3gisMJ!ciML9z!sdOQXo_u06h+3yLAL8I12SGcxkN2-iTpD1SSYUfXeqJFjhE1%!G8Xx;Z`ncxm72=RsvLh-XxPn8 z3>?E5kL~P|p;vp4UO_xRN>2-HM+@vC__<2s8xR^xA1^Ffwd=PmNx_8KPUluE+|;qD zp_!p~8Gw(u3`EgVaIHtdB{yGPgiM0s-W5m538Dx8VBvK{ftMRp`!*}t>;UJ||C8Nu zsZ|IX>RVlxCa9Ffm+vsMHHT1-fx=a6&@MkM?qUyQO?HZU=<40AwQTeQycE4@0PrTs z=YVb^L|9>*z5Vee4=9DUfr>N1%rYo!{JM7J?3E9iYo#`$BLJbXPxgd#3Nvwb8j7x{wo}Aue$@!u?^_4|NFJornQHILF>k52(f!0 z)ch6R$yOa&JZd|KmSfBq!)Z7!~ZQG`BVRy;F|PI1CRe6_@CSjC8?Zi_n!PO>7;pG literal 0 HcmV?d00001 diff --git a/docs/mdbook/src/index.md b/docs/mdbook/src/index.md new file mode 100644 index 000000000..3cd0fec1d --- /dev/null +++ b/docs/mdbook/src/index.md @@ -0,0 +1,43 @@ + +# DataFusion Book + +DataFusion is a blazing fast query engine that lets you run data analyses quickly and reliably. + +DataFusion is written in Rust, but also exposes Python and SQL bindings, so you can easily query data in your langauge of choice. You don't need to know any Rust to be a happy and productive user of DataFusion. + +DataFusion lets you run queries faster than pandas. Let's compare query runtimes for a 5GB CSV file with 100 million rows of data. + +Take a look at a few rows of the data: + +``` ++-------+-------+--------------+-----+-----+-------+----+----+-----------+ +| id1 | id2 | id3 | id4 | id5 | id6 | v1 | v2 | v3 | ++-------+-------+--------------+-----+-----+-------+----+----+-----------+ +| id016 | id016 | id0000042202 | 15 | 24 | 5971 | 5 | 11 | 37.211254 | +| id039 | id045 | id0000029558 | 40 | 49 | 39457 | 5 | 4 | 48.951141 | +| id047 | id023 | id0000071286 | 68 | 20 | 74463 | 2 | 14 | 60.469241 | ++-------+-------+--------------+-----+-----+-------+----+----+-----------+ +``` + +Suppose you'd like to run the following query: `SELECT id1, sum(v1) AS v1 from the_table GROUP BY id1`. + +If you use pandas, then this query will take 43.6 seconds to execute. + +It only takes DataFusion 9.8 seconds to execute the same query. + +DataFusion is easy to use, powerful, and fast. Let's learn more! diff --git a/docs/mdbook/src/installation.md b/docs/mdbook/src/installation.md new file mode 100644 index 000000000..ba00c8b80 --- /dev/null +++ b/docs/mdbook/src/installation.md @@ -0,0 +1,62 @@ + +# Installation + +DataFusion is easy to install, just like any other Python library. + +## Using pip + +``` bash +pip install datafusion +``` + +## Conda & JupyterLab setup + +This section explains how to install DataFusion in a conda environment with other libraries that allow for a nice Jupyter workflow. This setup is completely optional. These steps are only needed if you'd like to run DataFusion in a Jupyter notebook and have an interface like this: + +![DataFusion in Jupyter](https://github.com/MrPowers/datafusion-book/raw/main/src/images/datafusion-jupyterlab.png) + +Create a conda environment with DataFusion, Jupyter, and other useful dependencies in the `datafusion-env.yml` file: + +``` +name: datafusion-env +channels: + - conda-forge + - defaults +dependencies: + - python=3.9 + - ipykernel + - nb_conda + - jupyterlab + - jupyterlab_code_formatter + - isort + - black + - pip + - pip: + - datafusion + +``` + +Create the environment with `conda env create -f datafusion-env.yml`. + +Activate the environment with `conda activate datafusion-env`. + +Run `jupyter lab` or open the [JupyterLab Desktop application](https://github.com/jupyterlab/jupyterlab-desktop) to start running DataFusion in a Jupyter notebook. + +## Examples + +See the [DataFusion Python Examples](https://github.com/apache/arrow-datafusion-python/tree/main/examples) for a variety of Python scripts that show DataFusion in action! diff --git a/docs/mdbook/src/quickstart.md b/docs/mdbook/src/quickstart.md new file mode 100644 index 000000000..bba0b36ae --- /dev/null +++ b/docs/mdbook/src/quickstart.md @@ -0,0 +1,77 @@ + +# DataFusion Quickstart + +You can easily query a DataFusion table with the Python API or with pure SQL. + +Let's create a small DataFrame and then run some queries with both APIs. + +Start by creating a DataFrame with four rows of data and two columns: `a` and `b`. + +```python +from datafusion import SessionContext + +ctx = SessionContext() + +df = ctx.from_pydict({"a": [1, 2, 3, 1], "b": [4, 5, 6, 7]}, name="my_table") +``` + +Let's append a column to this DataFrame that adds columns `a` and `b` with the SQL API. + +``` +ctx.sql("select a, b, a + b as sum_a_b from my_table") + ++---+---+---------+ +| a | b | sum_a_b | ++---+---+---------+ +| 1 | 4 | 5 | +| 2 | 5 | 7 | +| 3 | 6 | 9 | +| 1 | 7 | 8 | ++---+---+---------+ +``` + +DataFusion makes it easy to run SQL queries on DataFrames. + +Now let's run the same query with the DataFusion Python API: + +```python +from datafusion import col + +df.select( + col("a"), + col("b"), + col("a") + col("b"), +) +``` + +We get the same result as before: + +``` ++---+---+-------------------------+ +| a | b | my_table.a + my_table.b | ++---+---+-------------------------+ +| 1 | 4 | 5 | +| 2 | 5 | 7 | +| 3 | 6 | 9 | +| 1 | 7 | 8 | ++---+---+-------------------------+ +``` + +DataFusion also allows you to query data with a well-designed Python interface. + +Python users have two great ways to query DataFusion tables. diff --git a/docs/mdbook/src/usage/create-table.md b/docs/mdbook/src/usage/create-table.md new file mode 100644 index 000000000..332863a16 --- /dev/null +++ b/docs/mdbook/src/usage/create-table.md @@ -0,0 +1,59 @@ + +# DataFusion Create Table + +It's easy to create DataFusion tables from a variety of data sources. + +## Create Table from Python Dictionary + +Here's how to create a DataFusion table from a Python dictionary: + +```python +from datafusion import SessionContext + +ctx = SessionContext() + +df = ctx.from_pydict({"a": [1, 2, 3, 1], "b": [4, 5, 6, 7]}, name="my_table") +``` + +Supplying the `name` parameter is optional. You only need to name the table if you'd like to query it with the SQL API. + +You can also create a DataFrame without a name that can be queried with the Python API: + +```python +from datafusion import SessionContext + +ctx = SessionContext() + +df = ctx.from_pydict({"a": [1, 2, 3, 1], "b": [4, 5, 6, 7]}) +``` + +## Create Table from CSV + +You can read a CSV into a DataFusion DataFrame. Here's how to read the `G1_1e8_1e2_0_0.csv` file into a table named `csv_1e8`: + +```python +ctx.register_csv("csv_1e8", "G1_1e8_1e2_0_0.csv") +``` + +## Create Table from Parquet + +You can read a Parquet file into a DataFusion DataFrame. Here's how to read the `yellow_tripdata_2021-01.parquet` file into a table named `taxi`. + +```python +ctx.register_parquet("taxi", "yellow_tripdata_2021-01.parquet") +``` diff --git a/docs/mdbook/src/usage/index.md b/docs/mdbook/src/usage/index.md new file mode 100644 index 000000000..1ef4406f7 --- /dev/null +++ b/docs/mdbook/src/usage/index.md @@ -0,0 +1,25 @@ + +# Usage + +This section shows how to create DataFusion DataFrames from a variety of data sources like CSV files and Parquet files. + +You'll learn more about the SQL statements that are supported by DataFusion. + +You'll also learn about the DataFusion's Python API for querying data. + +The documentation will wrap up with a variety of real-world data processing tasks that are well suited for DataFusion. The lightning-fast speed and reliable execution makes DataFusion the best technology for a variety of data processing tasks. diff --git a/docs/mdbook/src/usage/query-table.md b/docs/mdbook/src/usage/query-table.md new file mode 100644 index 000000000..5e4e38001 --- /dev/null +++ b/docs/mdbook/src/usage/query-table.md @@ -0,0 +1,125 @@ + +# DataFusion Query Table + +DataFusion tables can be queried with SQL or with the Python API. + +Let's create a small table and show the different types of queries that can be run. + +```python +df = ctx.from_pydict( + { + "first_name": ["li", "wang", "ron", "amanda"], + "age": [25, 75, 68, 18], + "country": ["china", "china", "us", "us"], + }, + name="some_people", +) +``` + +Here's the data in the table: + +``` ++------------+-----+---------+ +| first_name | age | country | ++------------+-----+---------+ +| li | 25 | china | +| wang | 75 | china | +| ron | 68 | us | +| amanda | 18 | us | ++------------+-----+---------+ +``` + +## DataFusion Filter DataFrame + +Here's how to find all individuals that are older than 65 years old in the data with SQL: + +``` +ctx.sql("select * from some_people where age > 65") + ++------------+-----+---------+ +| first_name | age | country | ++------------+-----+---------+ +| wang | 75 | china | +| ron | 68 | us | ++------------+-----+---------+ +``` + +Here's how to run the same query with Python: + +```python +df.filter(col("age") > lit(65)) +``` + +``` ++------------+-----+---------+ +| first_name | age | country | ++------------+-----+---------+ +| wang | 75 | china | +| ron | 68 | us | ++------------+-----+---------+ +``` + +## DataFusion Select Columns from DataFrame + +Here's how to select the `first_name` and `country` columns from the DataFrame with SQL: + +``` +ctx.sql("select first_name, country from some_people") + + ++------------+---------+ +| first_name | country | ++------------+---------+ +| li | china | +| wang | china | +| ron | us | +| amanda | us | ++------------+---------+ +``` + +Here's how to run the same query with Python: + +```python +df.select(col("first_name"), col("country")) +``` + +``` ++------------+---------+ +| first_name | country | ++------------+---------+ +| li | china | +| wang | china | +| ron | us | +| amanda | us | ++------------+---------+ +``` + +## DataFusion Aggregation Query + +Here's how to run a group by aggregation query: + +``` +ctx.sql("select country, count(*) as num_people from some_people group by country") + ++---------+------------+ +| country | num_people | ++---------+------------+ +| china | 2 | +| us | 2 | ++---------+------------+ +``` From 0f2cd2e1be1406f0a462f2b106c41a111f90f37c Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 4 May 2023 20:54:10 -0400 Subject: [PATCH 035/413] Add 'pub' and '#[pyo3(get, set)]' to DataTypeMap (#371) --- src/common/data_type.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/common/data_type.rs b/src/common/data_type.rs index a7b79f49f..a9b0d17d5 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -33,12 +33,12 @@ use crate::errors::py_datafusion_err; #[derive(Debug, Clone)] #[pyclass(name = "DataTypeMap", module = "datafusion.common", subclass)] pub struct DataTypeMap { - #[allow(dead_code)] - arrow_type: PyDataType, - #[allow(dead_code)] - python_type: PythonType, - #[allow(dead_code)] - sql_type: SqlType, + #[pyo3(get, set)] + pub arrow_type: PyDataType, + #[pyo3(get, set)] + pub python_type: PythonType, + #[pyo3(get, set)] + pub sql_type: SqlType, } impl DataTypeMap { @@ -421,7 +421,7 @@ impl DataTypeMap { #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[pyclass(name = "DataType", module = "datafusion.common")] pub struct PyDataType { - data_type: DataType, + pub data_type: DataType, } impl From for DataType { From e16af72a3eff4a13f3419e09faf49847debd7ddd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 6 May 2023 10:42:02 -0600 Subject: [PATCH 036/413] Fix db-benchmark (#369) --- .dockerignore | 12 ++ benchmarks/db-benchmark/README.md | 11 +- .../db-benchmark/db-benchmark.dockerfile | 177 ++++++++++-------- benchmarks/db-benchmark/groupby-datafusion.py | 2 +- benchmarks/db-benchmark/join-datafusion.py | 2 +- benchmarks/db-benchmark/run-bench.sh | 4 +- 6 files changed, 117 insertions(+), 91 deletions(-) create mode 100644 .dockerignore mode change 100644 => 100755 benchmarks/db-benchmark/run-bench.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..411e60291 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.cargo +.github +.pytest_cache +ci +conda +dev +docs +examples +parquet +target +testing +venv \ No newline at end of file diff --git a/benchmarks/db-benchmark/README.md b/benchmarks/db-benchmark/README.md index fe268199f..93293b0dc 100644 --- a/benchmarks/db-benchmark/README.md +++ b/benchmarks/db-benchmark/README.md @@ -17,15 +17,16 @@ under the License. --> -# Run db-benchmark +# DataFusion Implementation of db-benchmark -This directory contains scripts for running DataFusion with the https://github.com/h2oai/db-benchmark +This directory contains scripts for running [db-benchmark](https://github.com/duckdblabs/db-benchmark) with +DataFusion's Python bindings. ## Directions -Run the following from root `arrow-datafusion` directory +Run the following from root of this project. ```bash -$ docker buildx build -t db-benchmark -f benchmarks/db-benchmark/db-benchmark.dockerfile . -$ docker run --privileged db-benchmark +$ docker build -t db-benchmark -f benchmarks/db-benchmark/db-benchmark.dockerfile . +$ docker run --privileged -it db-benchmark ``` diff --git a/benchmarks/db-benchmark/db-benchmark.dockerfile b/benchmarks/db-benchmark/db-benchmark.dockerfile index b21d3a0d1..2876b5b63 100644 --- a/benchmarks/db-benchmark/db-benchmark.dockerfile +++ b/benchmarks/db-benchmark/db-benchmark.dockerfile @@ -15,92 +15,105 @@ # specific language governing permissions and limitations # under the License. -FROM ubuntu +FROM ubuntu:22.04 ARG DEBIAN_FRONTEND=noninteractive ARG TARGETPLATFORM -RUN apt-get update && \ - apt-get install -y git build-essential - -# Install R, curl, and python deps -RUN apt-get -y install --no-install-recommends --no-install-suggests \ - ca-certificates software-properties-common gnupg2 gnupg1 \ - && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \ - && add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu bionic-cran35/' \ - && apt-get -y install r-base \ - && apt-get -y install curl \ - && apt-get -y install python3.8 \ - && apt-get -y install python3-pip - -# Install R libraries -RUN R -e "install.packages('data.table',dependencies=TRUE, repos='http://cran.rstudio.com/')" \ - && R -e "install.packages('dplyr',dependencies=TRUE, repos='http://cran.rstudio.com/')" - -# Install Rust -RUN curl https://sh.rustup.rs -sSf | bash -s -- -y -ENV PATH="/root/.cargo/bin:${PATH}" - -# Clone db-benchmark and download data -RUN git clone https://github.com/h2oai/db-benchmark \ - && cd db-benchmark \ - && Rscript _data/groupby-datagen.R 1e7 1e2 0 0 \ - && Rscript _data/join-datagen.R 1e7 0 0 0 \ - && mkdir data \ - && mv G1_1e7_1e2_0_0.csv data \ - && mv J1_1e7_1e1_0_0.csv data \ - && mv J1_1e7_1e4_0_0.csv data \ - && mv J1_1e7_1e7_0_0.csv data \ - && mv J1_1e7_NA_0_0.csv data \ - && cd .. - -# Clone datafusion-python and build python library -# Not sure if the wheel will be the same on all computers -RUN git clone https://github.com/datafusion-contrib/datafusion-python \ - && cd datafusion-python && git reset --hard 368b50ed9662d5e93c70b539f94cceace685265e \ - && python3 -m pip install pip \ - && python3 -m pip install pandas \ - && python3 -m pip install -r requirements.txt \ - && cd .. - -# Copy local arrow-datafusion -COPY . arrow-datafusion - -# 1. datafusion-python that builds from datafusion version referenced datafusion-python -RUN cd datafusion-python \ - && maturin build --release \ - && case "${TARGETPLATFORM}" in \ - */amd64) CPUARCH=x86_64 ;; \ - */arm64) CPUARCH=aarch64 ;; \ - *) exit 1 ;; \ - esac \ - # Version will need to be updated in conjunction with datafusion-python version - && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \ - && cd .. - -# 2. datafusion-python that builds from local datafusion. use this when making local changes to datafusion. -# Currently, as of March 5th 2022, this done not build (i think) because datafusion is being split into multiple crates -# and datafusion-python has not yet been updated to reflect this. -# RUN cd datafusion-python \ -# && sed -i '/datafusion =/c\datafusion = { path = "../arrow-datafusion/datafusion", features = ["pyarrow"] }' Cargo.toml \ -# && sed -i '/fuzz-utils/d' ../arrow-datafusion/datafusion/Cargo.toml \ -# && maturin build --release \ -# && case "${TARGETPLATFORM}" in \ -# */amd64) CPUARCH=x86_64 ;; \ -# */amd64) CPUARCH=aarch64 ;; \ -# *) exit 1 ;; \ -# esac \ -# && python3 -m pip install target/wheels/datafusion-0.4.0-cp36-abi3-linux_${CPUARCH}.whl \ -# && cd .. - -# Make datafusion directory in db-benchmark -RUN mkdir db-benchmark/datafusion \ - && cp ../arrow-datafusion/benchmarks/db-benchmark/groupby-datafusion.py db-benchmark/datafusion \ - && cp ../arrow-datafusion/benchmarks/db-benchmark/join-datafusion.py db-benchmark/datafusion \ - && cp ../arrow-datafusion/benchmarks/db-benchmark/run-bench.sh db-benchmark/ \ - && chmod +x db-benchmark/run-bench.sh +# This section is based on https://github.com/duckdblabs/db-benchmark/blob/master/_utils/repro.sh + +RUN apt-get -qq update +RUN apt-get -qq -y upgrade +RUN apt-get -qq install -y apt-utils + +RUN apt-get -qq install -y lsb-release software-properties-common wget curl vim htop git byobu libcurl4-openssl-dev libssl-dev +RUN apt-get -qq install -y libfreetype6-dev +RUN apt-get -qq install -y libfribidi-dev +RUN apt-get -qq install -y libharfbuzz-dev +RUN apt-get -qq install -y git +RUN apt-get -qq install -y libxml2-dev +RUN apt-get -qq install -y make +RUN apt-get -qq install -y libfontconfig1-dev +RUN apt-get -qq install -y libicu-dev pandoc zlib1g-dev libgit2-dev libcurl4-openssl-dev libssl-dev libjpeg-dev libpng-dev libtiff-dev +# apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 +RUN add-apt-repository "deb [arch=amd64,i386] https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/" + +RUN apt-get -qq install -y r-base-dev virtualenv + +RUN cd /usr/local/lib/R && \ + chmod o+w site-library + +RUN cd / && \ + git clone https://github.com/duckdblabs/db-benchmark.git WORKDIR /db-benchmark -RUN ls && ls -al data/ +RUN mkdir -p .R && \ + echo 'CFLAGS=-O3 -mtune=native' >> .R/Makevars && \ + echo 'CXXFLAGS=-O3 -mtune=native' >> .R/Makevars + +RUN cd pydatatable && \ + virtualenv py-pydatatable --python=/usr/bin/python3.10 +RUN cd pandas && \ + virtualenv py-pandas --python=/usr/bin/python3.10 +RUN cd modin && \ + virtualenv py-modin --python=/usr/bin/python3.10 + +RUN Rscript -e 'install.packages(c("jsonlite","bit64","devtools","rmarkdown"), dependecies=TRUE, repos="https://cloud.r-project.org")' + +SHELL ["/bin/bash", "-c"] + +RUN source ./pandas/py-pandas/bin/activate && \ + python3 -m pip install --upgrade psutil && \ + python3 -m pip install --upgrade pandas && \ + deactivate + +RUN source ./modin/py-modin/bin/activate && \ + python3 -m pip install --upgrade modin && \ + deactivate + +RUN source ./pydatatable/py-pydatatable/bin/activate && \ + python3 -m pip install --upgrade git+https://github.com/h2oai/datatable && \ + deactivate + +## install dplyr +#RUN Rscript -e 'devtools::install_github(c("tidyverse/readr","tidyverse/dplyr"))' + +# install data.table +RUN Rscript -e 'install.packages("data.table", repos="https://rdatatable.gitlab.io/data.table/")' + +## generate data for groupby 0.5GB +RUN Rscript _data/groupby-datagen.R 1e7 1e2 0 0 +RUN #Rscript _data/groupby-datagen.R 1e8 1e2 0 0 +RUN #Rscript _data/groupby-datagen.R 1e9 1e2 0 0 + +RUN mkdir data && \ + mv G1_1e7_1e2_0_0.csv data/ + +# set only groupby task +RUN echo "Changing run.conf and _control/data.csv to run only groupby at 0.5GB" && \ + cp run.conf run.conf.original && \ + sed -i 's/groupby join groupby2014/groupby/g' run.conf && \ + sed -i 's/data.table dplyr pandas pydatatable spark dask clickhouse polars arrow duckdb/data.table dplyr duckdb/g' run.conf && \ + sed -i 's/DO_PUBLISH=true/DO_PUBLISH=false/g' run.conf + +## set sizes +RUN mv _control/data.csv _control/data.csv.original && \ + echo "task,data,nrow,k,na,sort,active" > _control/data.csv && \ + echo "groupby,G1_1e7_1e2_0_0,1e7,1e2,0,0,1" >> _control/data.csv + +RUN #./dplyr/setup-dplyr.sh +RUN #./datatable/setup-datatable.sh +RUN #./duckdb/setup-duckdb.sh + +# END OF SETUP + +RUN python3 -m pip install --upgrade pandas +RUN python3 -m pip install --upgrade datafusion + +# Now add our solution +RUN rm -rf datafusion-python 2>/dev/null && \ + mkdir datafusion-python +ADD benchmarks/db-benchmark/*.py datafusion-python/ +ADD benchmarks/db-benchmark/run-bench.sh . -ENTRYPOINT ./run-bench.sh \ No newline at end of file +ENTRYPOINT [ "/db-benchmark/run-bench.sh" ] \ No newline at end of file diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 7268cc872..76dd38fee 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -58,7 +58,7 @@ def ans_shape(batches): ) print("dataset loaded") -ctx = df.ExecutionContext() +ctx = df.SessionContext() ctx.register_record_batches("x", [data.to_batches()]) print("registered record batches") # cols = ctx.sql("SHOW columns from x") diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py index 1993a5c83..8843b55c0 100755 --- a/benchmarks/db-benchmark/join-datafusion.py +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -90,7 +90,7 @@ def ans_shape(batches): flush=True, ) -ctx = df.ExecutionContext() +ctx = df.SessionContext() x_data = pacsv.read_csv( src_jn_x, convert_options=pacsv.ConvertOptions(auto_dict_encode=True) diff --git a/benchmarks/db-benchmark/run-bench.sh b/benchmarks/db-benchmark/run-bench.sh old mode 100644 new mode 100755 index 9ccc26804..2c3080929 --- a/benchmarks/db-benchmark/run-bench.sh +++ b/benchmarks/db-benchmark/run-bench.sh @@ -17,5 +17,5 @@ # under the License. set -e -SRC_DATANAME=G1_1e7_1e2_0_0 python3 datafusion/groupby-datafusion.py -SRC_DATANAME=J1_1e7_NA_0_0 python3 datafusion/join-datafusion.py +SRC_DATANAME=G1_1e7_1e2_0_0 python3 /db-benchmark/datafusion-python/groupby-datafusion.py +#SRC_DATANAME=J1_1e7_NA_0_0 python3 /db-benchmark/datafusion-python/join-datafusion.py From b8c74a7f50cdc74b163bf4ba071662d93cabfbfd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 8 May 2023 07:49:54 -0600 Subject: [PATCH 037/413] Docs explaining how to view query plans (#373) * Docs explaining how to view query plans * diagram --- docs/mdbook/README.md | 2 +- docs/mdbook/src/SUMMARY.md | 3 +- docs/mdbook/src/images/plan.svg | 111 +++++++++++++++++ docs/mdbook/src/usage/query-plans.md | 170 +++++++++++++++++++++++++++ src/physical_plan.rs | 4 + src/sql/logical.rs | 2 +- 6 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 docs/mdbook/src/images/plan.svg create mode 100644 docs/mdbook/src/usage/query-plans.md diff --git a/docs/mdbook/README.md b/docs/mdbook/README.md index 664b4b4ce..6dae6bc62 100644 --- a/docs/mdbook/README.md +++ b/docs/mdbook/README.md @@ -26,7 +26,7 @@ Open the book locally by running `open book/index.html`. ## Install mdBook -Download the `mdbook` binary. +Download the `mdbook` binary or run `cargo install mdbook`. Then manually open it, so you have permissions to run it on your Mac. diff --git a/docs/mdbook/src/SUMMARY.md b/docs/mdbook/src/SUMMARY.md index b4908994d..23467ed4c 100644 --- a/docs/mdbook/src/SUMMARY.md +++ b/docs/mdbook/src/SUMMARY.md @@ -21,4 +21,5 @@ - [Quickstart](./quickstart.md) - [Usage](./usage/index.md) - [Create a table](./usage/create-table.md) - - [Query a table](./usage/query-table.md) \ No newline at end of file + - [Query a table](./usage/query-table.md) + - [Viewing Query Plans](./usage/query-plans.md) \ No newline at end of file diff --git a/docs/mdbook/src/images/plan.svg b/docs/mdbook/src/images/plan.svg new file mode 100644 index 000000000..927147985 --- /dev/null +++ b/docs/mdbook/src/images/plan.svg @@ -0,0 +1,111 @@ + + + + + + +%3 + + +cluster_1 + +LogicalPlan + + +cluster_6 + +Detailed LogicalPlan + + + +2 + +Projection: my_table.a, SUM(my_table.b) + + + +3 + +Aggregate: groupBy=[[my_table.a]], aggr=[[SUM(my_table.b)]] + + + +2->3 + + + + + +4 + +Filter: my_table.a < Int64(3) + + + +3->4 + + + + + +5 + +TableScan: my_table + + + +4->5 + + + + + +7 + +Projection: my_table.a, SUM(my_table.b) +Schema: [a:Int64;N, SUM(my_table.b):Int64;N] + + + +8 + +Aggregate: groupBy=[[my_table.a]], aggr=[[SUM(my_table.b)]] +Schema: [a:Int64;N, SUM(my_table.b):Int64;N] + + + +7->8 + + + + + +9 + +Filter: my_table.a < Int64(3) +Schema: [a:Int64;N, b:Int64;N] + + + +8->9 + + + + + +10 + +TableScan: my_table +Schema: [a:Int64;N, b:Int64;N] + + + +9->10 + + + + + diff --git a/docs/mdbook/src/usage/query-plans.md b/docs/mdbook/src/usage/query-plans.md new file mode 100644 index 000000000..a39aa9e42 --- /dev/null +++ b/docs/mdbook/src/usage/query-plans.md @@ -0,0 +1,170 @@ + + +# DataFusion Query Plans + +DataFusion's `DataFrame` is a wrapper around a query plan. In this chapter we will learn how to view +logical and physical query plans for DataFrames. + +## Sample Data + +Let's go ahead and create a simple DataFrame. You can do this in the Python shell or in a notebook. + +```python +from datafusion import SessionContext + +ctx = SessionContext() + +df = ctx.from_pydict({"a": [1, 2, 3, 1], "b": [4, 5, 6, 7]}, name="my_table") +``` + +## Logical Plan + +Next, let's look at the logical plan for this dataframe. + +```python +>>> df.logical_plan() +TableScan: my_table +``` + +The logical plan here consists of a single `TableScan` operator. Let's make a more interesting plan by creating a new +`DataFrame` representing an aggregate query with a filter. + +```python +>>> df = ctx.sql("SELECT a, sum(b) FROM my_table WHERE a < 3 GROUP BY a") +``` + +When we view the plan for this `DataFrame` we can see that there are now four operators in the plan, each +representing a logical transformation of the data. We start with a `TableScan` to read the data, followed by +a `Filter` to filter out rows that do not match the filter expression, then an `Aggregate` is performed. Finally, +a `Projection` is applied to ensure that the order of the final columns matches the `SELECT` part of the SQL query. + +```python +>>> df.logical_plan() +Projection: my_table.a, SUM(my_table.b) + Aggregate: groupBy=[[my_table.a]], aggr=[[SUM(my_table.b)]] + Filter: my_table.a < Int64(3) + TableScan: my_table +``` + +## Optimized Logical Plan + +DataFusion has a powerful query optimizer which will rewrite query plans to make them more efficient before they are +executed. We can view the output of the optimized by viewint the optimized logical plan. + +```python +>>> df.optimized_logical_plan() +Aggregate: groupBy=[[my_table.a]], aggr=[[SUM(my_table.b)]] + Filter: my_table.a < Int64(3) + TableScan: my_table projection=[a, b] +``` + +We can see that there are two key differences compared to the unoptimized logical plan: + +- The `Projection` has been removed because it was redundant in this case (the output of the `Aggregatge` plan + already had the columns in the correct order). +- The `TableScan` now has a projection pushed down so that it only reads the columns required to be able to execute + the query. In this case the table only has two columns and we referenced them both in the query, but this optimization + can be very effective in real-world queries against large tables. + +## Physical Plan + +Logical plans provide a representation of "what" the query should do it. Physical plans explain "how" the query +should be executed. + +We can view the physical plan (also known as an execution plan) using the `execution_plan` method. + +```python +>>> df.execution_plan() +AggregateExec: mode=FinalPartitioned, gby=[a@0 as a], aggr=[SUM(my_table.b)] + CoalesceBatchesExec: target_batch_size=8192 + RepartitionExec: partitioning=Hash([Column { name: "a", index: 0 }], 48), input_partitions=48 + AggregateExec: mode=Partial, gby=[a@0 as a], aggr=[SUM(my_table.b)] + CoalesceBatchesExec: target_batch_size=8192 + FilterExec: a@0 < 3 + RepartitionExec: partitioning=RoundRobinBatch(48), input_partitions=1 + MemoryExec: partitions=1, partition_sizes=[1] +``` + +The `TableScan` has now been replaced by a more specific `MemoryExec` for scanning the in-memory data. If we were +querying a CSV file on disk then we would expect to see a `CsvExec` instead. + +This plan has additional operators that were not in the logical plan: + +- `RepartionExec` has been added so that the data can be split into partitions and processed in parallel using + multiple cores. +- `CoalesceBatchesExec` will combine small batches into larger batches to ensure that processing remains efficient. + +The `Aggregate` operator now appears twice. This is because aggregates are performed in a two step process. Data is +aggregated within each partition in parallel and then those results (which could contain duplicate grouping keys) are +combined and the aggregate operations is applied again. + +## Creating Query Plan Diagrams + +DataFusion supports generating query plan diagrams in [DOT format](). + +DOT is a language for describing graphs and there are open source tools such as GraphViz that can render diagrams +from DOT files. + +We can use the following code to generate a DOT file for a logical query plan. + +```python +>>> diagram = df.logical_plan().display_graphviz() +>>> with open('plan.dot', 'w') as f: +>>> f.write(diagram) +``` + +If we view the view, we will see the following content. + +``` +// Begin DataFusion GraphViz Plan (see https://graphviz.org) +digraph { + subgraph cluster_1 + { + graph[label="LogicalPlan"] + 2[shape=box label="Projection: my_table.a, SUM(my_table.b)"] + 3[shape=box label="Aggregate: groupBy=[[my_table.a]], aggr=[[SUM(my_table.b)]]"] + 2 -> 3 [arrowhead=none, arrowtail=normal, dir=back] + 4[shape=box label="Filter: my_table.a < Int64(3)"] + 3 -> 4 [arrowhead=none, arrowtail=normal, dir=back] + 5[shape=box label="TableScan: my_table"] + 4 -> 5 [arrowhead=none, arrowtail=normal, dir=back] + } + subgraph cluster_6 + { + graph[label="Detailed LogicalPlan"] + 7[shape=box label="Projection: my_table.a, SUM(my_table.b)\nSchema: [a:Int64;N, SUM(my_table.b):Int64;N]"] + 8[shape=box label="Aggregate: groupBy=[[my_table.a]], aggr=[[SUM(my_table.b)]]\nSchema: [a:Int64;N, SUM(my_table.b):Int64;N]"] + 7 -> 8 [arrowhead=none, arrowtail=normal, dir=back] + 9[shape=box label="Filter: my_table.a < Int64(3)\nSchema: [a:Int64;N, b:Int64;N]"] + 8 -> 9 [arrowhead=none, arrowtail=normal, dir=back] + 10[shape=box label="TableScan: my_table\nSchema: [a:Int64;N, b:Int64;N]"] + 9 -> 10 [arrowhead=none, arrowtail=normal, dir=back] + } +} +// End DataFusion GraphViz Plan +``` + +We can use GraphViz from the command-line to convert this DOT file into an image. + +```bash +dot -Tsvg plan.dot > plan.svg +``` + +This generates the following diagram: + +![Query Plan Diagram](../images/plan.svg) diff --git a/src/physical_plan.rs b/src/physical_plan.rs index 4c35f3e60..6f02cefaa 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -54,6 +54,10 @@ impl PyExecutionPlan { format!("{}", d.indent()) } + fn __repr__(&self) -> String { + self.display_indent() + } + #[getter] pub fn partition_count(&self) -> usize { self.plan.output_partitioning().partition_count() diff --git a/src/sql/logical.rs b/src/sql/logical.rs index a22f269f1..a75315d36 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -100,7 +100,7 @@ impl PyLogicalPlan { } fn display_graphviz(&self) -> String { - format!("{}", self.plan.display_indent_schema()) + format!("{}", self.plan.display_graphviz()) } } From 6a2df777508ff335f5bad4a97e8af9da6014eaa4 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 8 May 2023 13:06:34 -0600 Subject: [PATCH 038/413] Improve db-benchmark (#372) --- .github/workflows/test.yaml | 2 +- benchmarks/db-benchmark/README.md | 4 +- .../db-benchmark/db-benchmark.dockerfile | 1 + benchmarks/db-benchmark/groupby-datafusion.py | 330 +++++++++++++++--- benchmarks/db-benchmark/join-datafusion.py | 138 ++++++-- benchmarks/db-benchmark/run-bench.sh | 8 +- dev/python_lint.sh | 2 +- 7 files changed, 404 insertions(+), 81 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c69205be3..f672c8129 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -101,7 +101,7 @@ jobs: if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} run: | source venv/bin/activate - flake8 --exclude venv --ignore=E501,W503 + flake8 --exclude venv,benchmarks/db-benchmark --ignore=E501,W503 black --line-length 79 --diff --check . - name: Run tests diff --git a/benchmarks/db-benchmark/README.md b/benchmarks/db-benchmark/README.md index 93293b0dc..8ce45344d 100644 --- a/benchmarks/db-benchmark/README.md +++ b/benchmarks/db-benchmark/README.md @@ -27,6 +27,6 @@ DataFusion's Python bindings. Run the following from root of this project. ```bash -$ docker build -t db-benchmark -f benchmarks/db-benchmark/db-benchmark.dockerfile . -$ docker run --privileged -it db-benchmark +docker build -t db-benchmark -f benchmarks/db-benchmark/db-benchmark.dockerfile . +docker run --privileged -it db-benchmark ``` diff --git a/benchmarks/db-benchmark/db-benchmark.dockerfile b/benchmarks/db-benchmark/db-benchmark.dockerfile index 2876b5b63..d8842b250 100644 --- a/benchmarks/db-benchmark/db-benchmark.dockerfile +++ b/benchmarks/db-benchmark/db-benchmark.dockerfile @@ -108,6 +108,7 @@ RUN #./duckdb/setup-duckdb.sh # END OF SETUP RUN python3 -m pip install --upgrade pandas +RUN python3 -m pip install --upgrade polars psutil RUN python3 -m pip install --upgrade datafusion # Now add our solution diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 76dd38fee..2c35259e8 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -19,14 +19,20 @@ import gc import timeit import datafusion as df -from datafusion import functions as f -from datafusion import col +from datafusion import ( + col, + functions as f, + RuntimeConfig, + SessionConfig, + SessionContext, +) +import pyarrow from pyarrow import csv as pacsv print("# groupby-datafusion.py", flush=True) -# exec(open("./_helpers/helpers.py").read()) +exec(open("./_helpers/helpers.py").read()) def ans_shape(batches): @@ -40,8 +46,12 @@ def ans_shape(batches): return rows, cols -# ver = df.__version__ -ver = "7.0.0" +def execute(df): + print(df.execution_plan().display_indent()) + return df.collect() + + +ver = df.__version__ git = "" task = "groupby" solution = "datafusion" @@ -49,16 +59,47 @@ def ans_shape(batches): cache = "TRUE" on_disk = "FALSE" +# experimental - support running with both DataFrame and SQL APIs +sql = True + data_name = os.environ["SRC_DATANAME"] src_grp = os.path.join("data", data_name + ".csv") print("loading dataset %s" % src_grp, flush=True) +schema = pyarrow.schema( + [ + ("id4", pyarrow.int32()), + ("id5", pyarrow.int32()), + ("id6", pyarrow.int32()), + ("v1", pyarrow.int32()), + ("v2", pyarrow.int32()), + ("v3", pyarrow.float64()), + ] +) + data = pacsv.read_csv( - src_grp, convert_options=pacsv.ConvertOptions(auto_dict_encode=True) + src_grp, + convert_options=pacsv.ConvertOptions( + auto_dict_encode=True, column_types=schema + ), ) print("dataset loaded") -ctx = df.SessionContext() +# create a session context with explicit runtime and config settings +runtime = ( + RuntimeConfig() + .with_disk_manager_os() + .with_fair_spill_pool(64 * 1024 * 1024 * 1024) +) +config = ( + SessionConfig() + .with_repartition_joins(False) + .with_repartition_aggregations(False) + .set("datafusion.execution.coalesce_batches", "false") +) +ctx = SessionContext(config, runtime) +print(ctx) + ctx.register_record_batches("x", [data.to_batches()]) print("registered record batches") # cols = ctx.sql("SHOW columns from x") @@ -72,50 +113,107 @@ def ans_shape(batches): question = "sum v1 by id1" # q1 gc.collect() t_start = timeit.default_timer() -ans = ctx.sql("SELECT id1, SUM(v1) AS v1 FROM x GROUP BY id1").collect() +if sql: + df = ctx.sql("SELECT id1, SUM(v1) AS v1 FROM x GROUP BY id1") +else: + df = ctx.table("x").aggregate( + [f.col("id1")], [f.sum(f.col("v1")).alias("v1")] + ) +ans = execute(df) + shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q1: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = df.aggregate([], [f.sum(col("v1"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() question = "sum v1 by id1:id2" # q2 gc.collect() t_start = timeit.default_timer() -ans = ctx.sql( - "SELECT id1, id2, SUM(v1) AS v1 FROM x GROUP BY id1, id2" -).collect() +if sql: + df = ctx.sql("SELECT id1, id2, SUM(v1) AS v1 FROM x GROUP BY id1, id2") +else: + df = ctx.table("x").aggregate( + [f.col("id1"), f.col("id2")], [f.sum(f.col("v1")).alias("v1")] + ) +ans = execute(df) shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q2: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = df.aggregate([], [f.sum(col("v1"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() question = "sum v1 mean v3 by id3" # q3 gc.collect() t_start = timeit.default_timer() -ans = ctx.sql( - "SELECT id3, SUM(v1) AS v1, AVG(v3) AS v3 FROM x GROUP BY id3" -).collect() +if sql: + df = ctx.sql( + "SELECT id3, SUM(v1) AS v1, AVG(v3) AS v3 FROM x GROUP BY id3" + ) +else: + df = ctx.table("x").aggregate( + [f.col("id3")], + [ + f.sum(f.col("v1")).alias("v1"), + f.avg(f.col("v3")).alias("v3"), + ], + ) +ans = execute(df) shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q3: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = ( @@ -125,7 +223,25 @@ def ans_shape(batches): .to_numpy()[0] ) chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -136,10 +252,10 @@ def ans_shape(batches): "SELECT id4, AVG(v1) AS v1, AVG(v2) AS v2, AVG(v3) AS v3 FROM x GROUP BY id4" ).collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q4: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = ( @@ -149,7 +265,25 @@ def ans_shape(batches): .to_numpy()[0] ) chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -160,10 +294,10 @@ def ans_shape(batches): "SELECT id6, SUM(v1) AS v1, SUM(v2) AS v2, SUM(v3) AS v3 FROM x GROUP BY id6" ).collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q5: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = ( @@ -173,7 +307,25 @@ def ans_shape(batches): .to_numpy()[0] ) chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -184,10 +336,10 @@ def ans_shape(batches): "SELECT id4, id5, approx_percentile_cont(v3, .5) AS median_v3, stddev(v3) AS stddev_v3 FROM x GROUP BY id4, id5" ).collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q6: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = ( @@ -197,7 +349,25 @@ def ans_shape(batches): .to_numpy()[0] ) chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -208,15 +378,33 @@ def ans_shape(batches): "SELECT id3, MAX(v1) - MIN(v2) AS range_v1_v2 FROM x GROUP BY id3" ).collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q7: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = df.aggregate([], [f.sum(col("range_v1_v2"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -227,15 +415,33 @@ def ans_shape(batches): "SELECT id6, v3 from (SELECT id6, v3, row_number() OVER (PARTITION BY id6 ORDER BY v3 DESC) AS row FROM x) t WHERE row <= 2" ).collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q8: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = df.aggregate([], [f.sum(col("v3"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -244,15 +450,33 @@ def ans_shape(batches): t_start = timeit.default_timer() ans = ctx.sql("SELECT corr(v1, v2) as corr FROM x GROUP BY id2, id4").collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q9: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = df.aggregate([], [f.sum(col("corr"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -263,10 +487,10 @@ def ans_shape(batches): "SELECT id1, id2, id3, id4, id5, id6, SUM(v3) as v3, COUNT(*) AS cnt FROM x GROUP BY id1, id2, id3, id4, id5, id6" ).collect() shape = ans_shape(ans) -# print(shape, flush=True) +print(shape, flush=True) t = timeit.default_timer() - t_start print(f"q10: {t}") -# m = memory_usage() +m = memory_usage() t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) chk = ( @@ -276,7 +500,25 @@ def ans_shape(batches): .to_numpy()[0] ) chkt = timeit.default_timer() - t_start -# write_log(task=task, data=data_name, in_rows=in_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +write_log( + task=task, + data=data_name, + in_rows=in_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py index 8843b55c0..602cee697 100755 --- a/benchmarks/db-benchmark/join-datafusion.py +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -26,26 +26,7 @@ print("# join-datafusion.py", flush=True) -# exec(open("./_helpers/helpers.py").read()) - - -def join_to_tbls(data_name): - x_n = int(float(data_name.split("_")[1])) - y_n = [ - "{:.0e}".format(x_n / 1e6), - "{:.0e}".format(x_n / 1e3), - "{:.0e}".format(x_n), - ] - y_n = [ - y_n[0].replace("+0", ""), - y_n[1].replace("+0", ""), - y_n[2].replace("+0", ""), - ] - return [ - data_name.replace("NA", y_n[0]), - data_name.replace("NA", y_n[1]), - data_name.replace("NA", y_n[2]), - ] +exec(open("./_helpers/helpers.py").read()) def ans_shape(batches): @@ -59,7 +40,7 @@ def ans_shape(batches): return rows, cols -ver = "6.0.0" +ver = df.__version__ task = "join" git = "" solution = "datafusion" @@ -84,13 +65,16 @@ def ans_shape(batches): + ", " + y_data_name[0] + ", " - + y_data_name[2] + + y_data_name[1] + ", " + y_data_name[2], flush=True, ) ctx = df.SessionContext() +print(ctx) + +# TODO we should be applying projections to these table reads to crete relations of different sizes x_data = pacsv.read_csv( src_jn_x, convert_options=pacsv.ConvertOptions(auto_dict_encode=True) @@ -133,8 +117,26 @@ def ans_shape(batches): df = ctx.create_dataframe([ans]) chk = df.aggregate([], [f.sum(col("v1"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start -# m = memory_usage() -# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +m = memory_usage() +write_log( + task=task, + data=data_name, + in_rows=x_data.num_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -156,8 +158,26 @@ def ans_shape(batches): .column(0)[0] ) chkt = timeit.default_timer() - t_start -# m = memory_usage() -# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +m = memory_usage() +write_log( + task=task, + data=data_name, + in_rows=x_data.num_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -179,8 +199,26 @@ def ans_shape(batches): .column(0)[0] ) chkt = timeit.default_timer() - t_start -# m = memory_usage() -# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +m = memory_usage() +write_log( + task=task, + data=data_name, + in_rows=x_data.num_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -202,8 +240,26 @@ def ans_shape(batches): .column(0)[0] ) chkt = timeit.default_timer() - t_start -# m = memory_usage() -# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +m = memory_usage() +write_log( + task=task, + data=data_name, + in_rows=x_data.num_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() @@ -225,8 +281,26 @@ def ans_shape(batches): .column(0)[0] ) chkt = timeit.default_timer() - t_start -# m = memory_usage() -# write_log(task=task, data=data_name, in_rows=x_data.num_rows, question=question, out_rows=shape[0], out_cols=shape[1], solution=solution, version=ver, git=git, fun=fun, run=1, time_sec=t, mem_gb=m, cache=cache, chk=make_chk([chk]), chk_time_sec=chkt, on_disk=on_disk) +m = memory_usage() +write_log( + task=task, + data=data_name, + in_rows=x_data.num_rows, + question=question, + out_rows=shape[0], + out_cols=shape[1], + solution=solution, + version=ver, + git=git, + fun=fun, + run=1, + time_sec=t, + mem_gb=m, + cache=cache, + chk=make_chk([chk]), + chk_time_sec=chkt, + on_disk=on_disk, +) del ans gc.collect() diff --git a/benchmarks/db-benchmark/run-bench.sh b/benchmarks/db-benchmark/run-bench.sh index 2c3080929..36a6087d9 100755 --- a/benchmarks/db-benchmark/run-bench.sh +++ b/benchmarks/db-benchmark/run-bench.sh @@ -17,5 +17,11 @@ # under the License. set -e +#SRC_DATANAME=G1_1e7_1e2_0_0 python3 /db-benchmark/polars/groupby-polars.py SRC_DATANAME=G1_1e7_1e2_0_0 python3 /db-benchmark/datafusion-python/groupby-datafusion.py -#SRC_DATANAME=J1_1e7_NA_0_0 python3 /db-benchmark/datafusion-python/join-datafusion.py + +# joins need more work still +#SRC_DATANAME=G1_1e7_1e2_0_0 python3 /db-benchmark/datafusion-python/join-datafusion.py +#SRC_DATANAME=G1_1e7_1e2_0_0 python3 /db-benchmark/polars/join-polars.py + +cat time.csv diff --git a/dev/python_lint.sh b/dev/python_lint.sh index 949346294..3bc67fb12 100755 --- a/dev/python_lint.sh +++ b/dev/python_lint.sh @@ -22,5 +22,5 @@ set -e source venv/bin/activate -flake8 --exclude venv --ignore=E501,W503 +flake8 --exclude venv,benchmarks/db-benchmark --ignore=E501,W503 black --line-length 79 . From bdcbb9f12e59ee863c418fdf21f64182aab66b11 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 9 May 2023 08:39:50 -0400 Subject: [PATCH 039/413] Make expr member of PyExpr public (#375) --- src/common/data_type.rs | 11 ++++++++++ src/expr.rs | 48 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/common/data_type.rs b/src/common/data_type.rs index a9b0d17d5..d55a0e86d 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -21,6 +21,17 @@ use pyo3::prelude::*; use crate::errors::py_datafusion_err; +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(name = "RexType", module = "datafusion.common")] +pub enum RexType { + Alias, + Literal, + Call, + Reference, + ScalarSubquery, + Other, +} + /// These bindings are tying together several disparate systems. /// You have SQL types for the SQL strings and RDBMS systems itself. /// Rust types for the DataFusion code diff --git a/src/expr.rs b/src/expr.rs index 7c80d0d82..4ada4c16d 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -22,6 +22,7 @@ use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::PyArrowType; use datafusion_expr::{col, lit, Cast, Expr, GetIndexedField}; +use crate::common::data_type::RexType; use crate::errors::py_runtime_err; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; @@ -83,7 +84,7 @@ pub mod union; #[pyclass(name = "Expr", module = "datafusion.expr", subclass)] #[derive(Debug, Clone)] pub struct PyExpr { - pub(crate) expr: Expr, + pub expr: Expr, } impl From for Expr { @@ -228,6 +229,51 @@ impl PyExpr { let expr = Expr::Cast(Cast::new(Box::new(self.expr.clone()), to.0)); expr.into() } + + /// A Rex (Row Expression) specifies a single row of data. That specification + /// could include user defined functions or types. RexType identifies the row + /// as one of the possible valid `RexTypes`. + pub fn rex_type(&self) -> PyResult { + Ok(match self.expr { + Expr::Alias(..) => RexType::Alias, + Expr::Column(..) | Expr::QualifiedWildcard { .. } | Expr::GetIndexedField { .. } => { + RexType::Reference + } + Expr::ScalarVariable(..) | Expr::Literal(..) => RexType::Literal, + Expr::BinaryExpr { .. } + | Expr::Not(..) + | Expr::IsNotNull(..) + | Expr::Negative(..) + | Expr::IsNull(..) + | Expr::Like { .. } + | Expr::ILike { .. } + | Expr::SimilarTo { .. } + | Expr::Between { .. } + | Expr::Case { .. } + | Expr::Cast { .. } + | Expr::TryCast { .. } + | Expr::Sort { .. } + | Expr::ScalarFunction { .. } + | Expr::AggregateFunction { .. } + | Expr::WindowFunction { .. } + | Expr::AggregateUDF { .. } + | Expr::InList { .. } + | Expr::Wildcard + | Expr::ScalarUDF { .. } + | Expr::Exists { .. } + | Expr::InSubquery { .. } + | Expr::GroupingSet(..) + | Expr::IsTrue(..) + | Expr::IsFalse(..) + | Expr::IsUnknown(_) + | Expr::IsNotTrue(..) + | Expr::IsNotFalse(..) + | Expr::Placeholder { .. } + | Expr::OuterReferenceColumn(_, _) + | Expr::IsNotUnknown(_) => RexType::Call, + Expr::ScalarSubquery(..) => RexType::ScalarSubquery, + }) + } } /// Initializes the `expr` module to match the pattern of `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/ From 9c75d03d7fa315251e0df2fbc97f081a8257c9bc Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 9 May 2023 08:26:46 -0600 Subject: [PATCH 040/413] Prepare 24.0.0 Release (#376) * bump version * update changelog --- CHANGELOG.md | 27 +++- Cargo.lock | 348 +++++++++++++++++++++++++-------------------------- Cargo.toml | 14 +-- 3 files changed, 203 insertions(+), 186 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 26c4eb18e..22d7c0f61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,12 +19,35 @@ # Changelog -## [23.0.0](https://github.com/apache/arrow-datafusion-python/tree/23.0.0) (2023-04-23) +## [24.0.0](https://github.com/apache/arrow-datafusion-python/tree/24.0.0) (2023-05-09) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/22.0.0...23.0.0) +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/23.0.0...24.0.0) + +**Documentation updates:** + +- Fix link to user guide [#354](https://github.com/apache/arrow-datafusion-python/pull/354) (andygrove) **Merged pull requests:** +- Add interface to serialize Substrait plans to Python Bytes. [#344](https://github.com/apache/arrow-datafusion-python/pull/344) (kylebrooks-8451) +- Add partition_count property to ExecutionPlan. [#346](https://github.com/apache/arrow-datafusion-python/pull/346) (kylebrooks-8451) +- Remove unsendable from all Rust pyclass types. [#348](https://github.com/apache/arrow-datafusion-python/pull/348) (kylebrooks-8451) +- Fix link to user guide [#354](https://github.com/apache/arrow-datafusion-python/pull/354) (andygrove) +- Fix SessionContext execute. [#353](https://github.com/apache/arrow-datafusion-python/pull/353) (kylebrooks-8451) +- Pub mod expr in lib.rs [#357](https://github.com/apache/arrow-datafusion-python/pull/357) (jdye64) +- Add benchmark derived from TPC-H [#355](https://github.com/apache/arrow-datafusion-python/pull/355) (andygrove) +- Add db-benchmark [#365](https://github.com/apache/arrow-datafusion-python/pull/365) (andygrove) +- First pass of documentation in mdBook [#364](https://github.com/apache/arrow-datafusion-python/pull/364) (MrPowers) +- Add 'pub' and '#[pyo3(get, set)]' to DataTypeMap [#371](https://github.com/apache/arrow-datafusion-python/pull/371) (jdye64) +- Fix db-benchmark [#369](https://github.com/apache/arrow-datafusion-python/pull/369) (andygrove) +- Docs explaining how to view query plans [#373](https://github.com/apache/arrow-datafusion-python/pull/373) (andygrove) +- Improve db-benchmark [#372](https://github.com/apache/arrow-datafusion-python/pull/372) (andygrove) +- Make expr member of PyExpr public [#375](https://github.com/apache/arrow-datafusion-python/pull/375) (jdye64) + +## [23.0.0](https://github.com/apache/arrow-datafusion-python/tree/23.0.0) (2023-04-23) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/22.0.0...23.0.0) + **Merged pull requests:** - Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/arrow-datafusion-python/pull/321) (andygrove) diff --git a/Cargo.lock b/Cargo.lock index db7c9a768..17acaf7aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,9 +113,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aea9fcb25bbb70f7f922f95b99ca29c1013dab47f6df61a6f24861842dd7f2e" +checksum = "c107a57b5913d852da9d5a40e280e4695f2258b5b87733c13b770c63a7117287" dependencies = [ "ahash", "arrow-arith", @@ -136,9 +136,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d967b42f7b12c91fd78acd396b20c2973b184c8866846674abbb00c963e93ab" +checksum = "ace6aa3d5617c5d03041a05e01c6819428a8ddf49dd0b055df9b40fef9d96094" dependencies = [ "arrow-array", "arrow-buffer", @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190f208ee7aa0f3596fa0098d42911dec5e123ca88c002a08b24877ad14c71e" +checksum = "104a04520692cc674e6afd7682f213ca41f9b13ff1873f63a5a2857a590b87b3" dependencies = [ "ahash", "arrow-buffer", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d33c733c5b6c44a0fc526f29c09546e04eb56772a7a21e48e602f368be381f6" +checksum = "72c875bcb9530ec403998fb0b2dc6d180a7c64563ca4bc22b90eafb84b113143" dependencies = [ "half", "num", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abd349520b6a1ed4924ae2afc9d23330a3044319e4ec3d5b124c09e4d440ae87" +checksum = "d6d6e18281636c8fc0b93be59834da6bf9a72bb70fd0c98ddfdaf124da466c28" dependencies = [ "arrow-array", "arrow-buffer", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c80af3c3e290a2a7e1cc518f1471dff331878cb4af9a5b088bf030b89debf649" +checksum = "3197dab0963a236ff8e7c82e2272535745955ac1321eb740c29f2f88b353f54e" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c8361947aaa96d331da9df3f7a08bdd8ab805a449994c97f5c4d24c4b7e2cf" +checksum = "eb68113d6ecdbe8bba48b2c4042c151bf9e1c61244e45072a50250a6fc59bafe" dependencies = [ "arrow-buffer", "arrow-schema", @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a46ee000b9fbd1e8db6e8b26acb8c760838512b39d8c9f9d73892cb55351d50" +checksum = "eab4bbf2dd3078facb5ce0a9641316a64f42bfd8cf357e6775c8a5e6708e3a8d" dependencies = [ "arrow-array", "arrow-buffer", @@ -240,9 +240,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf2366607be867ced681ad7f272371a5cf1fc2941328eef7b4fee14565166fb" +checksum = "48c5b650d23746a494665d914a7fa3d21d939153cff9d53bdebe39bffa88f263" dependencies = [ "arrow-array", "arrow-buffer", @@ -260,9 +260,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "304069901c867200e21ec868ae7521165875470ef2f1f6d58f979a443d63997e" +checksum = "68c6fce28e5011e30acc7466b5efcb8ed0197c396240bd2b10e167f275a3c208" dependencies = [ "arrow-array", "arrow-buffer", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d57fe8ceef3392fdd493269d8a2d589de17bafce151aacbffbddac7a57f441a" +checksum = "f20a421f19799d8b93eb8edde5217e910fa1e2d6ceb3c529f000e57b6db144c0" dependencies = [ "ahash", "arrow-array", @@ -290,18 +290,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a16b88a93ac8350f0200b1cd336a1f887315925b8dd7aa145a37b8bdbd8497a4" +checksum = "bc85923d8d6662cc66ac6602c7d1876872e671002d60993dfdf492a6badeae92" dependencies = [ "bitflags 2.2.1", ] [[package]] name = "arrow-select" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98e8a4d6ca37d5212439b24caad4d80743fcbb706706200dd174bb98e68fe9d8" +checksum = "f6ab6613ce65b61d85a3410241744e84e48fbab0fe06e1251b4429d21b3470fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb594efa397eb6a546f42b1f8df3d242ea84dbfda5232e06035dc2b2e2c8459" +checksum = "f3008641239e884aefba66d8b8532da6af40d14296349fcc85935de4ba67b89e" dependencies = [ "arrow-array", "arrow-buffer", @@ -736,9 +736,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a7d4b334f4512ff2fdbce87f511f570ae895af1ac7c729e77c12583253b22a" +checksum = "0404a559d5a6d8320369bb0a290b43bbc4f8622d0ef6f04bd095ace9a663f439" dependencies = [ "ahash", "apache-avro", @@ -788,9 +788,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80abfcb1dbc6390f952f21de9069e6177ad6318fcae5fbceabb50666d96533dd" +checksum = "4653b79a55161852973760db69ea6dcd05c9966a1b588fd83028f625536a1d7f" dependencies = [ "apache-avro", "arrow", @@ -805,9 +805,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2524f1b4b58319895b112809d2a59e54fa662d0e46330a455f22882c2cb7b9" +checksum = "53481c334b73c6759697919d1d05690392381145fa1890849a65b5a71a24a1ec" dependencies = [ "dashmap", "datafusion-common", @@ -823,9 +823,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8040b7a75b04685f4db0a1b11ffa93cd163c1bc13751df3f5cf76baabaf5a1" +checksum = "a8ecd7c6605d0b4269346d03289e2ced1715a303e75e6d313dba0bafb1f823f2" dependencies = [ "ahash", "arrow", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74ceae25accc0f640a4238283f55f3a9fd181d55398703a4330fb2c46261e6a2" +checksum = "70a7c04e94cb4aa9c323993856e18b91f690dda0358a34ab07a3fe0f14bc6600" dependencies = [ "arrow", "async-trait", @@ -848,14 +848,14 @@ dependencies = [ "hashbrown 0.13.2", "itertools", "log", - "regex-syntax 0.6.29", + "regex-syntax 0.7.1", ] [[package]] name = "datafusion-physical-expr" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df4cf228b312f2758cb78e93fe3d2dc602345028efdf7cfa5b338cb370d0a347" +checksum = "9e34eb8668fee1443965fff41ba73b2956d50a07ed8dd929cfa2e839ab91da5a" dependencies = [ "ahash", "arrow", @@ -886,7 +886,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "23.0.0" +version = "24.0.0" dependencies = [ "async-trait", "datafusion", @@ -913,9 +913,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b52b486fb3d81bb132e400304be01af5aba0ad6737e3518045bb98944991fe32" +checksum = "efa800ae88dfd62ea6c58c24a1154d92937c755672f522b84e8ea6539fad369b" dependencies = [ "arrow", "datafusion-common", @@ -925,9 +925,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773e985c182e41cfd68f7a7b483ab6bfb68beaac241c348cd4b1bf9f9d61b762" +checksum = "556642ef90073e39af721362353ccce4e1f418da7a8e31c23510ed9de6eb71f2" dependencies = [ "arrow", "arrow-schema", @@ -939,9 +939,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836e9b1c0ea430199c9bd4b88024cb8d617e3768ffdb412064169e2504a850ed" +checksum = "0d7643a77bb446047095ec21b913adb900b71c7a2ae600f8062906dd2e5642b9" dependencies = [ "async-recursion", "chrono", @@ -964,26 +964,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" -dependencies = [ - "dirs-sys", -] - -[[package]] -name = "dirs-sys" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1d1d91c932ef41c0f2663aa8b0ca0342d444d842c06914aa0a7e352d0bada6" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "doc-comment" version = "0.3.3" @@ -1212,9 +1192,9 @@ dependencies = [ [[package]] name = "gix" -version = "0.43.1" +version = "0.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c256ea71cc1967faaefdaad15f334146b7c806f12460dcafd3afed845c8c78dd" +checksum = "6bf41b61f7df395284f7a579c0fa1a7e012c5aede655174d4e91299ef1cac643" dependencies = [ "gix-actor", "gix-attributes", @@ -1224,9 +1204,11 @@ dependencies = [ "gix-diff", "gix-discover", "gix-features", + "gix-fs", "gix-glob", "gix-hash", "gix-hashtable", + "gix-ignore", "gix-index", "gix-lock", "gix-mailmap", @@ -1242,6 +1224,7 @@ dependencies = [ "gix-tempfile", "gix-traverse", "gix-url", + "gix-utils", "gix-validate", "gix-worktree", "log", @@ -1254,9 +1237,9 @@ dependencies = [ [[package]] name = "gix-actor" -version = "0.19.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc22b0cdc52237667c301dd7cdc6ead8f8f73c9f824e9942c8ebd6b764f6c0bf" +checksum = "848efa0f1210cea8638f95691c82a46f98a74b9e3524f01d4955ebc25a8f84f3" dependencies = [ "bstr", "btoi", @@ -1268,24 +1251,26 @@ dependencies = [ [[package]] name = "gix-attributes" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2231a25934a240d0a4b6f4478401c73ee81d8be52de0293eedbc172334abf3e1" +checksum = "3015baa01ad2122fbcaab7863c857a603eb7b7ec12ac8141207c42c6439805e2" dependencies = [ "bstr", - "gix-features", "gix-glob", "gix-path", "gix-quote", + "kstring", + "log", + "smallvec", "thiserror", "unicode-bom", ] [[package]] name = "gix-bitmap" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "024bca0c7187517bda5ea24ab148c9ca8208dd0c3e2bea88cdb2008f91791a6d" +checksum = "55a95f4942360766c3880bdb2b4b57f1ef73b190fc424755e7fdf480430af618" dependencies = [ "thiserror", ] @@ -1310,9 +1295,9 @@ dependencies = [ [[package]] name = "gix-config" -version = "0.20.1" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbad5ce54a8fc997acc50febd89ec80fa6e97cb7f8d0654cb229936407489d8" +checksum = "1d252a0eddb6df74600d3d8872dc9fe98835a7da43110411d705b682f49d4ac1" dependencies = [ "bstr", "gix-config-value", @@ -1332,11 +1317,11 @@ dependencies = [ [[package]] name = "gix-config-value" -version = "0.10.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09154c0c8677e4da0ec35e896f56ee3e338e741b9599fae06075edd83a4081c" +checksum = "786861e84a5793ad5f863d846de5eb064cd23b87e61ad708c8c402608202e7be" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.2.1", "bstr", "gix-path", "libc", @@ -1345,9 +1330,9 @@ dependencies = [ [[package]] name = "gix-credentials" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "750b684197374518ea057e0a0594713e07683faa0a3f43c0f93d97f64130ad8d" +checksum = "4874a4fc11ffa844a3c2b87a66957bda30a73b577ef1acf15ac34df5745de5ff" dependencies = [ "bstr", "gix-command", @@ -1361,9 +1346,9 @@ dependencies = [ [[package]] name = "gix-date" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96271912ce39822501616f177dea7218784e6c63be90d5f36322ff3a722aae2" +checksum = "99056f37270715f5c7584fd8b46899a2296af9cae92463bf58b8bd1f5a78e553" dependencies = [ "bstr", "itoa", @@ -1373,9 +1358,9 @@ dependencies = [ [[package]] name = "gix-diff" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "103a0fa79b0d438f5ecb662502f052e530ace4fe1fe8e1c83c0c6da76d728e67" +checksum = "644a0f2768bc42d7a69289ada80c9e15c589caefc6a315d2307202df83ed1186" dependencies = [ "gix-hash", "gix-object", @@ -1385,9 +1370,9 @@ dependencies = [ [[package]] name = "gix-discover" -version = "0.16.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eba8ba458cb8f4a6c33409b0fe650b1258655175a7ffd1d24fafd3ed31d880b" +checksum = "1a6b61363e63e7cdaa3e6f96acb0257ebdb3d8883e21eba5930c99f07f0a5fc0" dependencies = [ "bstr", "dunce", @@ -1400,9 +1385,9 @@ dependencies = [ [[package]] name = "gix-features" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b76f9a80f6dd7be66442ae86e1f534effad9546676a392acc95e269d0c21c22" +checksum = "cf69b0f5c701cc3ae22d3204b671907668f6437ca88862d355eaf9bc47a4f897" dependencies = [ "crc32fast", "flate2", @@ -1415,21 +1400,32 @@ dependencies = [ "walkdir", ] +[[package]] +name = "gix-fs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b37a1832f691fdc09910bd267f9a2e413737c1f9ec68c6e31f9e802616278a9" +dependencies = [ + "gix-features", +] + [[package]] name = "gix-glob" -version = "0.5.5" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93e43efd776bc543f46f0fd0ca3d920c37af71a764a16f2aebd89765e9ff2993" +checksum = "c07c98204529ac3f24b34754540a852593d2a4c7349008df389240266627a72a" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.2.1", "bstr", + "gix-features", + "gix-path", ] [[package]] name = "gix-hash" -version = "0.10.4" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a258595457bc192d1f1c59d0d168a1e34e2be9b97a614e14995416185de41a7" +checksum = "078eec3ac2808cc03f0bddd2704cb661da5c5dc33b41a9d7947b141d499c7c42" dependencies = [ "hex", "thiserror", @@ -1437,22 +1433,34 @@ dependencies = [ [[package]] name = "gix-hashtable" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4e55e40dfd694884f0eb78796c5bddcf2f8b295dace47039099dd7e76534973" +checksum = "afebb85691c6a085b114e01a27f4a61364519298c5826cb87a45c304802299bc" dependencies = [ "gix-hash", "hashbrown 0.13.2", "parking_lot", ] +[[package]] +name = "gix-ignore" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba205b6df563e2906768bb22834c82eb46c5fdfcd86ba2c347270bc8309a05b2" +dependencies = [ + "bstr", + "gix-glob", + "gix-path", + "unicode-bom", +] + [[package]] name = "gix-index" -version = "0.15.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "717ab601ece7921f59fe86849dbe27d44a46ebb883b5885732c4f30df4996177" +checksum = "f39c1ccc8f1912cbbd5191efc28dbc5f0d0598042aa56bc09427b7c34efab3ba" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.2.1", "bstr", "btoi", "filetime", @@ -1481,9 +1489,9 @@ dependencies = [ [[package]] name = "gix-mailmap" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b66aea5e52875cd4915f4957a6f4b75831a36981e2ec3f5fad9e370e444fe1a" +checksum = "e8856cec3bdc3610c06970d28b6cb20a0c6621621cf9a8ec48cbd23f2630f362" dependencies = [ "bstr", "gix-actor", @@ -1492,9 +1500,9 @@ dependencies = [ [[package]] name = "gix-object" -version = "0.28.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df068db9180ee935fbb70504848369e270bdcb576b05c0faa8b9fd3b86fc017" +checksum = "c9bb30ce0818d37096daa29efe361a4bc6dd0b51a5726598898be7e9a40a01e1" dependencies = [ "bstr", "btoi", @@ -1511,9 +1519,9 @@ dependencies = [ [[package]] name = "gix-odb" -version = "0.43.1" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83af2e3e36005bfe010927f0dff41fb5acc3e3d89c6f1174135b3a34086bda2" +checksum = "bca2f324aa67672b6d0f2c0fa93f96eb6a7029d260e4c1df5dce3c015f5e5add" dependencies = [ "arc-swap", "gix-features", @@ -1529,9 +1537,9 @@ dependencies = [ [[package]] name = "gix-pack" -version = "0.33.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9401911c7fe032ad7b31c6a6b5be59cb283d1d6c999417a8215056efe6d635f3" +checksum = "164a515900a83257ae4aa80e741655bee7a2e39113fb535d7a5ac623b445ff20" dependencies = [ "clru", "gix-chunk", @@ -1551,24 +1559,26 @@ dependencies = [ [[package]] name = "gix-path" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32370dce200bb951df013e03dff35b4233fc7a89458642b047629b91734a7e19" +checksum = "4fc78f47095a0c15aea0e66103838f0748f4494bf7a9555dfe0f00425400396c" dependencies = [ "bstr", + "home", + "once_cell", "thiserror", ] [[package]] name = "gix-prompt" -version = "0.3.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3034d4d935aef2c7bf719aaa54b88c520e82413118d886ae880a31d5bdee57" +checksum = "330d11fdf88fff3366c2491efde2f3e454958efe7d5ddf60272e8fb1d944bb01" dependencies = [ "gix-command", "gix-config-value", - "nix", "parking_lot", + "rustix", "thiserror", ] @@ -1585,12 +1595,13 @@ dependencies = [ [[package]] name = "gix-ref" -version = "0.27.2" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4e909396ed3b176823991ccc391c276ae2a015e54edaafa3566d35123cfac9d" +checksum = "1e03989e9d49954368e1b526578230fc7189d1634acdfbe79e9ba1de717e15d5" dependencies = [ "gix-actor", "gix-features", + "gix-fs", "gix-hash", "gix-lock", "gix-object", @@ -1604,9 +1615,9 @@ dependencies = [ [[package]] name = "gix-refspec" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aba332462bda2e8efeae4302b39a6ed01ad56ef772fd5b7ef197cf2798294d65" +checksum = "0a6ea733820df67e4cd7797deb12727905824d8f5b7c59d943c456d314475892" dependencies = [ "bstr", "gix-hash", @@ -1618,9 +1629,9 @@ dependencies = [ [[package]] name = "gix-revision" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6f6ff53f888858afc24bf12628446a14279ceec148df6194481f306f553ad2" +checksum = "810f35e9afeccca999d5d348b239f9c162353127d2e13ff3240e31b919e35476" dependencies = [ "bstr", "gix-date", @@ -1632,15 +1643,14 @@ dependencies = [ [[package]] name = "gix-sec" -version = "0.6.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ffa5bf0772f9b01de501c035b6b084cf9b8bb07dec41e3afc6a17336a65f47" +checksum = "794520043d5a024dfeac335c6e520cb616f6963e30dab995892382e998c12897" dependencies = [ - "bitflags 1.3.2", - "dirs", + "bitflags 2.2.1", "gix-path", "libc", - "windows 0.43.0", + "windows", ] [[package]] @@ -1659,9 +1669,9 @@ dependencies = [ [[package]] name = "gix-traverse" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd9a4a07bb22168dc79c60e1a6a41919d198187ca83d8a5940ad8d7122a45df3" +checksum = "a5be1e807f288c33bb005075111886cceb43ed8a167b3182a0f62c186e2a0dd1" dependencies = [ "gix-hash", "gix-hashtable", @@ -1671,9 +1681,9 @@ dependencies = [ [[package]] name = "gix-url" -version = "0.16.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a22b4b32ad14d68f7b7fb6458fa58d44b01797d94c1b8f4db2d9c7b3c366b5" +checksum = "dfc77f89054297cc81491e31f1bab4027e554b5ef742a44bd7035db9a0f78b76" dependencies = [ "bstr", "gix-features", @@ -1683,6 +1693,15 @@ dependencies = [ "url", ] +[[package]] +name = "gix-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c10b69beac219acb8df673187a1f07dde2d74092f974fb3f9eb385aeb667c909" +dependencies = [ + "fastrand", +] + [[package]] name = "gix-validate" version = "0.7.4" @@ -1695,15 +1714,18 @@ dependencies = [ [[package]] name = "gix-worktree" -version = "0.15.2" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ec9a000b4f24af706c3cc680c7cda235656cbe3216336522f5692773b8a301" +checksum = "a69eaff0ae973a9d37c40f02ae5ae50fa726c8fc2fd3ab79d0a19eb61975aafa" dependencies = [ "bstr", + "filetime", "gix-attributes", "gix-features", + "gix-fs", "gix-glob", "gix-hash", + "gix-ignore", "gix-index", "gix-object", "gix-path", @@ -1879,7 +1901,7 @@ dependencies = [ "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows 0.48.0", + "windows", ] [[package]] @@ -2003,6 +2025,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kstring" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3066350882a1cd6d950d055997f379ac37fd39f81cd4d8ed186032eb3c5747" +dependencies = [ + "static_assertions", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2261,18 +2292,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nix" -version = "0.26.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" -dependencies = [ - "bitflags 1.3.2", - "cfg-if", - "libc", - "static_assertions", -] - [[package]] name = "nom" version = "7.1.3" @@ -2447,9 +2466,9 @@ dependencies = [ [[package]] name = "parquet" -version = "37.0.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5022d98333271f4ca3e87bab760498e61726bf5a6ca919123c80517e20ded29" +checksum = "4cbd51311f8d9ff3d2697b1522b18a588782e097d313a1a278b0faf2ccf2d3f6" dependencies = [ "ahash", "arrow-array", @@ -2469,6 +2488,7 @@ dependencies = [ "lz4", "num", "num-bigint", + "object_store", "paste", "seq-macro", "snap", @@ -2796,17 +2816,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_users" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" -dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", -] - [[package]] name = "regex" version = "1.8.1" @@ -3258,9 +3267,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.7.5" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3ae64fb7ad0670c7d6d53d57b1b91beb2212afc30e164cc8edb02d6b2cff32a" +checksum = "54cd43d44620f716d55d46b998b3cf1baab2935aaa8adc14e3d3d9a465ddae15" dependencies = [ "gix", "heck", @@ -3607,9 +3616,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-bom" -version = "1.1.4" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63ec69f541d875b783ca40184d655f2927c95f0bffd486faa83cd3ac3529ec32" +checksum = "98e90c70c9f0d4d1ee6d0a7d04aa06cb9bbd53d8cfbdd62a0269a7c2eb640552" [[package]] name = "unicode-ident" @@ -3859,21 +3868,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.43.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04662ed0e3e5630dfa9b26e4cb823b817f1a9addda855d973a9458c236556244" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - [[package]] name = "windows" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 4c5203d69..4c6248a6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "23.0.0" +version = "24.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "23.0.0" , features = ["pyarrow", "avro"] } -datafusion-common = { version = "23.0.0", features = ["pyarrow"] } -datafusion-expr = "23.0.0" -datafusion-optimizer = "23.0.0" -datafusion-sql = "23.0.0" -datafusion-substrait = "23.0.0" +datafusion = { version = "24.0.0" , features = ["pyarrow", "avro"] } +datafusion-common = { version = "24.0.0", features = ["pyarrow"] } +datafusion-expr = "24.0.0" +datafusion-optimizer = "24.0.0" +datafusion-sql = "24.0.0" +datafusion-substrait = "24.0.0" prost = "0.11" prost-types = "0.11" uuid = { version = "1.2", features = ["v4"] } From 228b6e515b1b3569fbffc7c34d84ba29d3a98e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 09:03:37 +0800 Subject: [PATCH 041/413] build(deps): bump uuid from 1.3.1 to 1.3.2 (#359) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.3.1 to 1.3.2. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/1.3.1...1.3.2) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17acaf7aa..65340664a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3678,9 +3678,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b55a3fef2a1e3b3a00ce878640918820d3c51081576ac657d23af9fc7928fdb" +checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" dependencies = [ "getrandom", "serde", diff --git a/Cargo.toml b/Cargo.toml index 4c6248a6f..4d9ab90d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ datafusion-sql = "24.0.0" datafusion-substrait = "24.0.0" prost = "0.11" prost-types = "0.11" -uuid = { version = "1.2", features = ["v4"] } +uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" futures = "0.3" From 5984bc7db420f688e0989b26662c24d8b2f5768e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 09:03:51 +0800 Subject: [PATCH 042/413] build(deps): bump mimalloc from 0.1.36 to 0.1.37 (#361) Bumps [mimalloc](https://github.com/purpleprotocol/mimalloc_rust) from 0.1.36 to 0.1.37. - [Release notes](https://github.com/purpleprotocol/mimalloc_rust/releases) - [Commits](https://github.com/purpleprotocol/mimalloc_rust/compare/v0.1.36...v0.1.37) --- updated-dependencies: - dependency-name: mimalloc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65340664a..9ef4aaf62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2138,9 +2138,9 @@ checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" [[package]] name = "libmimalloc-sys" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a558e3d911bc3c7bfc8c78bc580b404d6e51c1cefbf656e176a94b49b0df40" +checksum = "f4ac0e912c8ef1b735e92369695618dc5b1819f5a7bf3f167301a3ba1cea515e" dependencies = [ "cc", "libc", @@ -2246,9 +2246,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.36" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d88dad3f985ec267a3fcb7a1726f5cb1a7e8cad8b646e70a84f967210df23da" +checksum = "4e2894987a3459f3ffb755608bd82188f8ed00d0ae077f1edea29c068d639d98" dependencies = [ "libmimalloc-sys", ] From 21ad90f2bebebd4732db9477dd5311721ae6d70f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 May 2023 09:04:08 +0800 Subject: [PATCH 043/413] build(deps): bump regex-syntax from 0.6.29 to 0.7.1 (#334) Bumps [regex-syntax](https://github.com/rust-lang/regex) from 0.6.29 to 0.7.1. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/commits) --- updated-dependencies: - dependency-name: regex-syntax dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ef4aaf62..080f506f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -904,7 +904,7 @@ dependencies = [ "pyo3", "pyo3-build-config", "rand", - "regex-syntax 0.6.29", + "regex-syntax 0.7.1", "syn 2.0.15", "tokio", "url", diff --git a/Cargo.toml b/Cargo.toml index 4d9ab90d2..63481c0eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ async-trait = "0.1" futures = "0.3" object_store = { version = "0.5.3", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" -regex-syntax = "0.6.28" +regex-syntax = "0.7.1" syn = "2.0.11" url = "2.2" From 43b3105cf859e95905b85f3efae69f7c4a4c0bee Mon Sep 17 00:00:00 2001 From: Jiayu Liu Date: Wed, 10 May 2023 20:53:03 +0800 Subject: [PATCH 044/413] upgrade maturin to 0.15.1 (#379) * upgrade maturin to 0.15.1 * migrate maturin meta --- .github/workflows/build.yml | 8 +- Cargo.toml | 3 - conda/environments/datafusion-dev.yaml | 48 ++-- conda/recipes/meta.yaml | 6 +- docs/README.md | 15 +- pyproject.toml | 3 +- requirements-310.txt | 340 ++++++++++++++----------- requirements.in | 4 +- requirements.txt | 284 --------------------- 9 files changed, 232 insertions(+), 479 deletions(-) delete mode 100644 requirements.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c667dab80..fe06b9c86 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: run: python -m pip install --upgrade pip - name: Install maturin - run: pip install maturin==0.14.2 + run: pip install maturin==0.15.1 - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -76,7 +76,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.x' + version: "3.x" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package @@ -125,7 +125,7 @@ jobs: run: python -m pip install --upgrade pip - name: Install maturin - run: pip install maturin==0.14.2 + run: pip install maturin==0.15.1 - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -137,7 +137,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.x' + version: "3.x" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package diff --git a/Cargo.toml b/Cargo.toml index 63481c0eb..cad16cc86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,9 +61,6 @@ pyo3-build-config = "0.18.3" name = "datafusion_python" crate-type = ["cdylib", "rlib"] -[package.metadata.maturin] -name = "datafusion._internal" - [profile.release] lto = true codegen-units = 1 diff --git a/conda/environments/datafusion-dev.yaml b/conda/environments/datafusion-dev.yaml index d9405e4fe..ceab504c8 100644 --- a/conda/environments/datafusion-dev.yaml +++ b/conda/environments/datafusion-dev.yaml @@ -16,29 +16,29 @@ # under the License. channels: -- conda-forge + - conda-forge dependencies: -- black -- flake8 -- isort -- maturin -- mypy -- numpy -- pyarrow -- pytest -- toml -- importlib_metadata -- python>=3.10 -# Packages useful for building distributions and releasing -- mamba -- conda-build -- anaconda-client -# Packages for documentation building -- sphinx -- pydata-sphinx-theme==0.8.0 -- myst-parser -- jinja2 -# GPU packages -- cudf -- cudatoolkit=11.8 + - black + - flake8 + - isort + - maturin>=0.15 + - mypy + - numpy + - pyarrow>=11.0.0 + - pytest + - toml + - importlib_metadata + - python>=3.10 + # Packages useful for building distributions and releasing + - mamba + - conda-build + - anaconda-client + # Packages for documentation building + - sphinx + - pydata-sphinx-theme==0.8.0 + - myst-parser + - jinja2 + # GPU packages + - cudf + - cudatoolkit=11.8 name: datafusion-dev diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index 48e95eb08..e2bb8bee3 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -35,12 +35,12 @@ build: requirements: host: - - python >=3.6 - - maturin >=0.14,<0.15 + - python >=3.7 + - maturin >=0.15,<0.16 - libprotobuf =3 - pip run: - - python >=3.6 + - python >=3.7 - pyarrow >=11.0.0 test: diff --git a/docs/README.md b/docs/README.md index 04f46a907..85278588e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,17 +19,18 @@ # DataFusion Documentation -This folder contains the source content of the [python api](./source/api). +This folder contains the source content of the [Python API](./source/api). This is published to https://arrow.apache.org/datafusion-python/ by a GitHub action when changes are merged to the main branch. ## Dependencies It's recommended to install build dependencies and build the documentation -inside a Python virtualenv. +inside a Python `venv`. -- Python -- `pip3 install -r requirements.txt` +```bash +python -m pip install -r requirements-310.txt +``` ## Build & Preview @@ -57,8 +58,6 @@ version of the docs, follow these steps: 2. Clone the arrow-site repo 3. Checkout to the `asf-site` branch (NOT `master`) 4. Copy build artifacts into `arrow-site` repo's `datafusion` folder with a command such as - -- `cp -rT ./build/html/ ../../arrow-site/datafusion/` (doesn't work on mac) -- `rsync -avzr ./build/html/ ../../arrow-site/datafusion/` - + - `cp -rT ./build/html/ ../../arrow-site/datafusion/` (doesn't work on mac) + - `rsync -avzr ./build/html/ ../../arrow-site/datafusion/` 5. Commit changes in `arrow-site` and send a PR. diff --git a/pyproject.toml b/pyproject.toml index cf68a33c3..4fdc4586f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [build-system] -requires = ["maturin>=0.14,<0.15"] +requires = ["maturin>=0.15,<0.16"] build-backend = "maturin" [project] @@ -55,6 +55,7 @@ repository = "https://github.com/apache/arrow-datafusion-python" profile = "black" [tool.maturin] +module-name = "datafusion._internal" include = [ { path = "Cargo.lock", format = "sdist" } ] diff --git a/requirements-310.txt b/requirements-310.txt index 332abdbcb..517c31905 100644 --- a/requirements-310.txt +++ b/requirements-310.txt @@ -1,41 +1,20 @@ # -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: # -# pip-compile --generate-hashes +# pip-compile --generate-hashes --resolver=backtracking requirements.in # -attrs==21.4.0 \ - --hash=sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4 \ - --hash=sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd +attrs==21.2.0 \ + --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ + --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb # via pytest -black==22.3.0 \ - --hash=sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b \ - --hash=sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176 \ - --hash=sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09 \ - --hash=sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a \ - --hash=sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015 \ - --hash=sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79 \ - --hash=sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb \ - --hash=sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20 \ - --hash=sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464 \ - --hash=sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968 \ - --hash=sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82 \ - --hash=sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21 \ - --hash=sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0 \ - --hash=sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265 \ - --hash=sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b \ - --hash=sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a \ - --hash=sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72 \ - --hash=sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce \ - --hash=sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0 \ - --hash=sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a \ - --hash=sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163 \ - --hash=sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad \ - --hash=sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d +black==21.9b0 \ + --hash=sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115 \ + --hash=sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91 # via -r requirements.in -click==8.1.3 \ - --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ - --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 +click==8.0.3 \ + --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ + --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b # via black flake8==4.0.1 \ --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ @@ -45,53 +24,53 @@ iniconfig==1.1.1 \ --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 # via pytest -isort==5.10.1 \ - --hash=sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7 \ - --hash=sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951 +isort==5.9.3 \ + --hash=sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899 \ + --hash=sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2 # via -r requirements.in -maturin==0.14.2 \ - --hash=sha256:065a0b48259624af1330bbc2d9bd447615ff07603710ad4a839478aa8e9e98cd \ - --hash=sha256:2966ed9b2e66760ffa5e476dab20c9563dd14efa93ed40b706cbe0a03907d824 \ - --hash=sha256:50a7a15d02f0b407eb5cf4cbc072d27e302993d5734af71bb6d6c567da8c6396 \ - --hash=sha256:539545c666506d34474c1133a11e90b4e6b4b6a7e590f5727ae18a49d3b647d2 \ - --hash=sha256:9f843736fc1c065ffba2dcfc581811c2eb7032c7119ff23012ccde68607e549c \ - --hash=sha256:ace60890f19a87e4a421fd9753e42e5fb46b11e770feacd8ff6749d3bf114764 \ - --hash=sha256:ae3a3a754c3b935420eeb1ec34d98d1693251537331feb43277bae7892b7c51a \ - --hash=sha256:bbaa0689a87124c99adaff99e93dcf415e36f7068d5442f311b49ede4bb7b6c3 \ - --hash=sha256:c65b00f4a4773bc56ca690d93abd09c85cc59b161a313fba3d03aca1ffbba7ca \ - --hash=sha256:daefa75e3f03e914924cb5913b40ada47ff646562c3b1874ddaf5b1e4b0f41ef \ - --hash=sha256:e0b808d1a0b78532c1bcbeb2d01d90281585dc460a966f1949fd337a3fb614c0 \ - --hash=sha256:ebc722e9661a40c3d36a06ecc0d1c637ae1af3335ea0ab69230f46a71e344088 \ - --hash=sha256:fa01eeacfd1798cde3ba1c7a0b8d7a889d2ed0e207f4860beb819c5ddd351f43 +maturin==0.15.1 \ + --hash=sha256:0e89a87549d671056f9358832c8aa1bb522abcf13eeca83327a58b091d4f5a98 \ + --hash=sha256:229d7eb6e14455a3c69a10a4546f082c7bd5490b8ec7f50d5d10edcea600dc64 \ + --hash=sha256:247bec13d82021972e5cb4eb38e7a7aea0e7a034beab60f0e0464ffe7423f24b \ + --hash=sha256:37ddbc261db778c73829173f43624b6178dd4244fae8e4b278323e050e899a14 \ + --hash=sha256:3b4c296b12736756d1d084c04b5a60281011e77b763eb9c69c39444d2421ad31 \ + --hash=sha256:616220c8c875526b22605bbb0be189ff859956abad8fe6e49c9ad2caa761ebca \ + --hash=sha256:61eda19c37394660c3b0b9c6dbfa7d411a5dca1a93c9dcf7baa718f3a03fda04 \ + --hash=sha256:7a7e02f265c30bac9f16e65c9d14184b3228c26b03dbe83a15220ab09fceabd7 \ + --hash=sha256:a40c96dc135e1571159155bdbd8c06509dea14a9317ca0e5b7cae4c8d2016b93 \ + --hash=sha256:ab5f8d8858440f35d833bce8bbf8233e595afa23fe00c513f7b671183946ea02 \ + --hash=sha256:b54421ae71e1e3dde92105d300187b9ac3b257a5b33c1d670ff9ee5266a1bd81 \ + --hash=sha256:dabb8ff46461c6fb1d68e8972a172cf1dede3c9825a41e4a6caecc95c26ca3b4 \ + --hash=sha256:f604b65fd9f0b94856e88cf8b345e21a27276297e6df4ad9305937887feda13b # via -r requirements.in mccabe==0.6.1 \ --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f # via flake8 -mypy==0.950 \ - --hash=sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d \ - --hash=sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8 \ - --hash=sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de \ - --hash=sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038 \ - --hash=sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed \ - --hash=sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334 \ - --hash=sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff \ - --hash=sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2 \ - --hash=sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22 \ - --hash=sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2 \ - --hash=sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2 \ - --hash=sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605 \ - --hash=sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb \ - --hash=sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519 \ - --hash=sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0 \ - --hash=sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc \ - --hash=sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b \ - --hash=sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f \ - --hash=sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075 \ - --hash=sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef \ - --hash=sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb \ - --hash=sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a \ - --hash=sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b +mypy==0.910 \ + --hash=sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9 \ + --hash=sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a \ + --hash=sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9 \ + --hash=sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e \ + --hash=sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2 \ + --hash=sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212 \ + --hash=sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b \ + --hash=sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885 \ + --hash=sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150 \ + --hash=sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703 \ + --hash=sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072 \ + --hash=sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457 \ + --hash=sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e \ + --hash=sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0 \ + --hash=sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb \ + --hash=sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97 \ + --hash=sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8 \ + --hash=sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811 \ + --hash=sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6 \ + --hash=sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de \ + --hash=sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504 \ + --hash=sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921 \ + --hash=sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d # via -r requirements.in mypy-extensions==0.4.3 \ --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ @@ -99,81 +78,100 @@ mypy-extensions==0.4.3 \ # via # black # mypy -numpy==1.22.3 \ - --hash=sha256:07a8c89a04997625236c5ecb7afe35a02af3896c8aa01890a849913a2309c676 \ - --hash=sha256:08d9b008d0156c70dc392bb3ab3abb6e7a711383c3247b410b39962263576cd4 \ - --hash=sha256:201b4d0552831f7250a08d3b38de0d989d6f6e4658b709a02a73c524ccc6ffce \ - --hash=sha256:2c10a93606e0b4b95c9b04b77dc349b398fdfbda382d2a39ba5a822f669a0123 \ - --hash=sha256:3ca688e1b9b95d80250bca34b11a05e389b1420d00e87a0d12dc45f131f704a1 \ - --hash=sha256:48a3aecd3b997bf452a2dedb11f4e79bc5bfd21a1d4cc760e703c31d57c84b3e \ - --hash=sha256:568dfd16224abddafb1cbcce2ff14f522abe037268514dd7e42c6776a1c3f8e5 \ - --hash=sha256:5bfb1bb598e8229c2d5d48db1860bcf4311337864ea3efdbe1171fb0c5da515d \ - --hash=sha256:639b54cdf6aa4f82fe37ebf70401bbb74b8508fddcf4797f9fe59615b8c5813a \ - --hash=sha256:8251ed96f38b47b4295b1ae51631de7ffa8260b5b087808ef09a39a9d66c97ab \ - --hash=sha256:92bfa69cfbdf7dfc3040978ad09a48091143cffb778ec3b03fa170c494118d75 \ - --hash=sha256:97098b95aa4e418529099c26558eeb8486e66bd1e53a6b606d684d0c3616b168 \ - --hash=sha256:a3bae1a2ed00e90b3ba5f7bd0a7c7999b55d609e0c54ceb2b076a25e345fa9f4 \ - --hash=sha256:c34ea7e9d13a70bf2ab64a2532fe149a9aced424cd05a2c4ba662fd989e3e45f \ - --hash=sha256:dbc7601a3b7472d559dc7b933b18b4b66f9aa7452c120e87dfb33d02008c8a18 \ - --hash=sha256:e7927a589df200c5e23c57970bafbd0cd322459aa7b1ff73b7c2e84d6e3eae62 \ - --hash=sha256:f8c1f39caad2c896bc0018f699882b345b2a63708008be29b1f355ebf6f933fe \ - --hash=sha256:f950f8845b480cffe522913d35567e29dd381b0dc7e4ce6a4a9f9156417d2430 \ - --hash=sha256:fade0d4f4d292b6f39951b6836d7a3c7ef5b2347f3c420cd9820a1d90d794802 \ - --hash=sha256:fdf3c08bce27132395d3c3ba1503cac12e17282358cb4bddc25cc46b0aca07aa +numpy==1.21.3 \ + --hash=sha256:043e83bfc274649c82a6f09836943e4a4aebe5e33656271c7dbf9621dd58b8ec \ + --hash=sha256:160ccc1bed3a8371bf0d760971f09bfe80a3e18646620e9ded0ad159d9749baa \ + --hash=sha256:188031f833bbb623637e66006cf75e933e00e7231f67e2b45cf8189612bb5dc3 \ + --hash=sha256:28f15209fb535dd4c504a7762d3bc440779b0e37d50ed810ced209e5cea60d96 \ + --hash=sha256:29fb3dcd0468b7715f8ce2c0c2d9bbbaf5ae686334951343a41bd8d155c6ea27 \ + --hash=sha256:2a6ee9620061b2a722749b391c0d80a0e2ae97290f1b32e28d5a362e21941ee4 \ + --hash=sha256:300321e3985c968e3ae7fbda187237b225f3ffe6528395a5b7a5407f73cf093e \ + --hash=sha256:32437f0b275c1d09d9c3add782516413e98cd7c09e6baf4715cbce781fc29912 \ + --hash=sha256:3c09418a14471c7ae69ba682e2428cae5b4420a766659605566c0fa6987f6b7e \ + --hash=sha256:49c6249260890e05b8111ebfc391ed58b3cb4b33e63197b2ec7f776e45330721 \ + --hash=sha256:4cc9b512e9fb590797474f58b7f6d1f1b654b3a94f4fa8558b48ca8b3cfc97cf \ + --hash=sha256:508b0b513fa1266875524ba8a9ecc27b02ad771fe1704a16314dc1a816a68737 \ + --hash=sha256:50cd26b0cf6664cb3b3dd161ba0a09c9c1343db064e7c69f9f8b551f5104d654 \ + --hash=sha256:5c4193f70f8069550a1788bd0cd3268ab7d3a2b70583dfe3b2e7f421e9aace06 \ + --hash=sha256:5dfe9d6a4c39b8b6edd7990091fea4f852888e41919d0e6722fe78dd421db0eb \ + --hash=sha256:63571bb7897a584ca3249c86dd01c10bcb5fe4296e3568b2e9c1a55356b6410e \ + --hash=sha256:75621882d2230ab77fb6a03d4cbccd2038511491076e7964ef87306623aa5272 \ + --hash=sha256:75eb7cadc8da49302f5b659d40ba4f6d94d5045fbd9569c9d058e77b0514c9e4 \ + --hash=sha256:88a5d6b268e9ad18f3533e184744acdaa2e913b13148160b1152300c949bbb5f \ + --hash=sha256:8a10968963640e75cc0193e1847616ab4c718e83b6938ae74dea44953950f6b7 \ + --hash=sha256:90bec6a86b348b4559b6482e2b684db4a9a7eed1fa054b86115a48d58fbbf62a \ + --hash=sha256:98339aa9911853f131de11010f6dd94c8cec254d3d1f7261528c3b3e3219f139 \ + --hash=sha256:a99a6b067e5190ac6d12005a4d85aa6227c5606fa93211f86b1dafb16233e57d \ + --hash=sha256:bffa2eee3b87376cc6b31eee36d05349571c236d1de1175b804b348dc0941e3f \ + --hash=sha256:c6c2d535a7beb1f8790aaa98fd089ceab2e3dd7ca48aca0af7dc60e6ef93ffe1 \ + --hash=sha256:cc14e7519fab2a4ed87d31f99c31a3796e4e1fe63a86ebdd1c5a1ea78ebd5896 \ + --hash=sha256:dd0482f3fc547f1b1b5d6a8b8e08f63fdc250c58ce688dedd8851e6e26cff0f3 \ + --hash=sha256:dde972a1e11bb7b702ed0e447953e7617723760f420decb97305e66fb4afc54f \ + --hash=sha256:e54af82d68ef8255535a6cdb353f55d6b8cf418a83e2be3569243787a4f4866f \ + --hash=sha256:e606e6316911471c8d9b4618e082635cfe98876007556e89ce03d52ff5e8fcf0 \ + --hash=sha256:f41b018f126aac18583956c54544db437f25c7ee4794bcb23eb38bef8e5e192a \ + --hash=sha256:f8f4625536926a155b80ad2bbff44f8cc59e9f2ad14cdda7acf4c135b4dc8ff2 \ + --hash=sha256:fe52dbe47d9deb69b05084abd4b0df7abb39a3c51957c09f635520abd49b29dd # via # -r requirements.in # pyarrow -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 +packaging==21.0 \ + --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ + --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 # via pytest pathspec==0.9.0 \ --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 # via black -platformdirs==2.5.2 \ - --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ - --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 +platformdirs==2.4.0 \ + --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \ + --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d # via black pluggy==1.0.0 \ --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 # via pytest -py==1.11.0 \ - --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ - --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 +py==1.10.0 \ + --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \ + --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a # via pytest -pyarrow==8.0.0 \ - --hash=sha256:03a10daad957970e914920b793f6a49416699e791f4c827927fd4e4d892a5d16 \ - --hash=sha256:15511ce2f50343f3fd5e9f7c30e4d004da9134e9597e93e9c96c3985928cbe82 \ - --hash=sha256:1dd482ccb07c96188947ad94d7536ab696afde23ad172df8e18944ec79f55055 \ - --hash=sha256:25a5f7c7f36df520b0b7363ba9f51c3070799d4b05d587c60c0adaba57763479 \ - --hash=sha256:3bd201af6e01f475f02be88cf1f6ee9856ab98c11d8bbb6f58347c58cd07be00 \ - --hash=sha256:3fee786259d986f8c046100ced54d63b0c8c9f7cdb7d1bbe07dc69e0f928141c \ - --hash=sha256:42b7982301a9ccd06e1dd4fabd2e8e5df74b93ce4c6b87b81eb9e2d86dc79871 \ - --hash=sha256:4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e \ - --hash=sha256:51e58778fcb8829fca37fbfaea7f208d5ce7ea89ea133dd13d8ce745278ee6f0 \ - --hash=sha256:541e7845ce5f27a861eb5b88ee165d931943347eec17b9ff1e308663531c9647 \ - --hash=sha256:65c7f4cc2be195e3db09296d31a654bb6d8786deebcab00f0e2455fd109d7456 \ - --hash=sha256:69b043a3fce064ebd9fbae6abc30e885680296e5bd5e6f7353e6a87966cf2ad7 \ - --hash=sha256:6ea2c54e6b5ecd64e8299d2abb40770fe83a718f5ddc3825ddd5cd28e352cce1 \ - --hash=sha256:78a6ac39cd793582998dac88ab5c1c1dd1e6503df6672f064f33a21937ec1d8d \ - --hash=sha256:81b87b782a1366279411f7b235deab07c8c016e13f9af9f7c7b0ee564fedcc8f \ - --hash=sha256:8392b9a1e837230090fe916415ed4c3433b2ddb1a798e3f6438303c70fbabcfc \ - --hash=sha256:863be6bad6c53797129610930794a3e797cb7d41c0a30e6794a2ac0e42ce41b8 \ - --hash=sha256:8cd86e04a899bef43e25184f4b934584861d787cf7519851a8c031803d45c6d8 \ - --hash=sha256:95c7822eb37663e073da9892f3499fe28e84f3464711a3e555e0c5463fd53a19 \ - --hash=sha256:98c13b2e28a91b0fbf24b483df54a8d7814c074c2623ecef40dce1fa52f6539b \ - --hash=sha256:ba2b7aa7efb59156b87987a06f5241932914e4d5bbb74a465306b00a6c808849 \ - --hash=sha256:c9c97c8e288847e091dfbcdf8ce51160e638346f51919a9e74fe038b2e8aee62 \ - --hash=sha256:cb06cacc19f3b426681f2f6803cc06ff481e7fe5b3a533b406bc5b2138843d4f \ - --hash=sha256:ce64bc1da3109ef5ab9e4c60316945a7239c798098a631358e9ab39f6e5529e9 \ - --hash=sha256:d5ef4372559b191cafe7db8932801eee252bfc35e983304e7d60b6954576a071 \ - --hash=sha256:d6f1e1040413651819074ef5b500835c6c42e6c446532a1ddef8bc5054e8dba5 \ - --hash=sha256:deb400df8f19a90b662babceb6dd12daddda6bb357c216e558b207c0770c7654 \ - --hash=sha256:ea132067ec712d1b1116a841db1c95861508862b21eddbcafefbce8e4b96b867 \ - --hash=sha256:ece333706a94c1221ced8b299042f85fd88b5db802d71be70024433ddf3aecab \ - --hash=sha256:edad25522ad509e534400d6ab98cf1872d30c31bc5e947712bfd57def7af15bb +pyarrow==6.0.0 \ + --hash=sha256:004185e0babc6f3c3fba6ba4f106e406a0113d0f82bb9ad9a8571a1978c45d04 \ + --hash=sha256:0204e80777ab8f4e9abd3a765a8ec07ed1e3c4630bacda50d2ce212ef0f3826f \ + --hash=sha256:072c1a0fca4509eefd7d018b78542fb7e5c63aaf5698f1c0a6e45628ae17ba44 \ + --hash=sha256:15dc0d673d3f865ca63c877bd7a2eced70b0a08969fb733a28247134b8a1f18b \ + --hash=sha256:1c38263ea438a1666b13372e7565450cfeec32dbcd1c2595749476a58465eaec \ + --hash=sha256:281ce5fa03621d786a9beb514abb09846db7f0221b50eabf543caa24037eaacd \ + --hash=sha256:2d2c681659396c745e4f1988d5dd41dcc3ad557bb8d4a8c2e44030edafc08a91 \ + --hash=sha256:376c4b5f248ae63df21fe15c194e9013753164be2d38f4b3fb8bde63ac5a1958 \ + --hash=sha256:465f87fa0be0b2928b2beeba22b5813a0203fb05d90fd8563eea48e08ecc030e \ + --hash=sha256:477c746ef42c039348a288584800e299456c80c5691401bb9b19aa9c02a427b7 \ + --hash=sha256:5144bd9db2920c7cb566c96462d62443cc239104f94771d110f74393f2fb42a2 \ + --hash=sha256:5408fa8d623e66a0445f3fb0e4027fd219bf99bfb57422d543d7b7876e2c5b55 \ + --hash=sha256:5be62679201c441356d3f2a739895dcc8d4d299f2a6eabcd2163bfb6a898abba \ + --hash=sha256:5c666bc6a1cebf01206e2dc1ab05f25f39f35d3a499e0ef5cd635225e07306ca \ + --hash=sha256:6163d82cca7541774b00503c295fe86a1722820eddb958b57f091bb6f5b0a6db \ + --hash=sha256:6a1d9a2f4ee812ed0bd4182cabef99ea914ac297274f0de086f2488093d284ef \ + --hash=sha256:7a683f71b848eb6310b4ec48c0def55dac839e9994c1ac874c9b2d3d5625def1 \ + --hash=sha256:82fe80309e01acf29e3943a1f6d3c98ec109fe1d356bc1ac37d639bcaadcf684 \ + --hash=sha256:8c23f8cdecd3d9e49f9b0f9a651ae5549d1d32fd4901fb1bdc2d327edfba844f \ + --hash=sha256:8d41dfb09ba9236cca6245f33088eb42f3c54023da281139241e0f9f3b4b754e \ + --hash=sha256:a19e58dfb04e451cd8b7bdec3ac8848373b95dfc53492c9a69789aa9074a3c1b \ + --hash=sha256:a50d2f77b86af38ceabf45617208b9105d20e7a5eebc584e7c8c0acededd82ce \ + --hash=sha256:a5bed4f948c032c40597302e9bdfa65f62295240306976ecbe43a54924c6f94f \ + --hash=sha256:ac941a147d14993987cc8b605b721735a34b3e54d167302501fb4db1ad7382c7 \ + --hash=sha256:b86d175262db1eb46afdceb36d459409eb6f8e532d3dec162f8bf572c7f57623 \ + --hash=sha256:bf3400780c4d3c9cb43b1e8a1aaf2e1b7199a0572d0a645529d2784e4d0d8497 \ + --hash=sha256:c7a6e7e0bf8779e9c3428ced85507541f3da9a0675e2f4781d4eb2c7042cbf81 \ + --hash=sha256:cc1d4a70efd583befe92d4ea6f74ed2e0aa31ccdde767cd5cae8e77c65a1c2d4 \ + --hash=sha256:d046dc78a9337baa6415be915c5a16222505233e238a1017f368243c89817eea \ + --hash=sha256:da7860688c33ca88ac05f1a487d32d96d9caa091412496c35f3d1d832145675a \ + --hash=sha256:ddf2e6e3b321adaaf716f2d5af8e92d205a9671e0cb7c0779710a567fd1dd580 \ + --hash=sha256:e81508239a71943759cee272ce625ae208092dd36ef2c6713fccee30bbcf52bb \ + --hash=sha256:ea64a48a85c631eb2a0ea13ccdec5143c85b5897836b16331ee4289d27a57247 \ + --hash=sha256:ed0be080cf595ea15ff1c9ff4097bbf1fcc4b50847d98c0a3c0412fbc6ede7e9 \ + --hash=sha256:fb701ec4a94b92102606d4e88f0b8eba34f09a5ad8e014eaa4af76f42b7f62ae \ + --hash=sha256:fbda7595f24a639bcef3419ecfac17216efacb09f7b0f1b4c4c97f900d65ca0e # via -r requirements.in pycodestyle==2.8.0 \ --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \ @@ -183,27 +181,69 @@ pyflakes==2.4.0 \ --hash=sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c \ --hash=sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e # via flake8 -pyparsing==3.0.9 \ - --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ - --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc +pyparsing==3.0.3 \ + --hash=sha256:9e3511118010f112a4b4b435ae50e1eaa610cda191acb9e421d60cf5fde83455 \ + --hash=sha256:f8d3fe9fc404576c5164f0f0c4e382c96b85265e023c409c43d48f65da9d60d0 # via packaging -pytest==7.1.2 \ - --hash=sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c \ - --hash=sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45 +pytest==6.2.5 \ + --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89 \ + --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134 # via -r requirements.in +regex==2021.10.23 \ + --hash=sha256:0c186691a7995ef1db61205e00545bf161fb7b59cdb8c1201c89b333141c438a \ + --hash=sha256:0dcc0e71118be8c69252c207630faf13ca5e1b8583d57012aae191e7d6d28b84 \ + --hash=sha256:0f7552429dd39f70057ac5d0e897e5bfe211629652399a21671e53f2a9693a4e \ + --hash=sha256:129472cd06062fb13e7b4670a102951a3e655e9b91634432cfbdb7810af9d710 \ + --hash=sha256:13ec99df95003f56edcd307db44f06fbeb708c4ccdcf940478067dd62353181e \ + --hash=sha256:1f2b59c28afc53973d22e7bc18428721ee8ca6079becf1b36571c42627321c65 \ + --hash=sha256:2b20f544cbbeffe171911f6ce90388ad36fe3fad26b7c7a35d4762817e9ea69c \ + --hash=sha256:2fb698037c35109d3c2e30f2beb499e5ebae6e4bb8ff2e60c50b9a805a716f79 \ + --hash=sha256:34d870f9f27f2161709054d73646fc9aca49480617a65533fc2b4611c518e455 \ + --hash=sha256:391703a2abf8013d95bae39145d26b4e21531ab82e22f26cd3a181ee2644c234 \ + --hash=sha256:450dc27483548214314640c89a0f275dbc557968ed088da40bde7ef8fb52829e \ + --hash=sha256:45b65d6a275a478ac2cbd7fdbf7cc93c1982d613de4574b56fd6972ceadb8395 \ + --hash=sha256:5095a411c8479e715784a0c9236568ae72509450ee2226b649083730f3fadfc6 \ + --hash=sha256:530fc2bbb3dc1ebb17f70f7b234f90a1dd43b1b489ea38cea7be95fb21cdb5c7 \ + --hash=sha256:56f0c81c44638dfd0e2367df1a331b4ddf2e771366c4b9c5d9a473de75e3e1c7 \ + --hash=sha256:5e9c9e0ce92f27cef79e28e877c6b6988c48b16942258f3bc55d39b5f911df4f \ + --hash=sha256:6d7722136c6ed75caf84e1788df36397efdc5dbadab95e59c2bba82d4d808a4c \ + --hash=sha256:74d071dbe4b53c602edd87a7476ab23015a991374ddb228d941929ad7c8c922e \ + --hash=sha256:7b568809dca44cb75c8ebb260844ea98252c8c88396f9d203f5094e50a70355f \ + --hash=sha256:80bb5d2e92b2258188e7dcae5b188c7bf868eafdf800ea6edd0fbfc029984a88 \ + --hash=sha256:8d1cdcda6bd16268316d5db1038965acf948f2a6f43acc2e0b1641ceab443623 \ + --hash=sha256:9f665677e46c5a4d288ece12fdedf4f4204a422bb28ff05f0e6b08b7447796d1 \ + --hash=sha256:a30513828180264294953cecd942202dfda64e85195ae36c265daf4052af0464 \ + --hash=sha256:a7a986c45d1099a5de766a15de7bee3840b1e0e1a344430926af08e5297cf666 \ + --hash=sha256:a940ca7e7189d23da2bfbb38973832813eab6bd83f3bf89a977668c2f813deae \ + --hash=sha256:ab7c5684ff3538b67df3f93d66bd3369b749087871ae3786e70ef39e601345b0 \ + --hash=sha256:be04739a27be55631069b348dda0c81d8ea9822b5da10b8019b789e42d1fe452 \ + --hash=sha256:c0938ddd60cc04e8f1faf7a14a166ac939aac703745bfcd8e8f20322a7373019 \ + --hash=sha256:cb46b542133999580ffb691baf67410306833ee1e4f58ed06b6a7aaf4e046952 \ + --hash=sha256:d134757a37d8640f3c0abb41f5e68b7cf66c644f54ef1cb0573b7ea1c63e1509 \ + --hash=sha256:de557502c3bec8e634246588a94e82f1ee1b9dfcfdc453267c4fb652ff531570 \ + --hash=sha256:ded0c4a3eee56b57fcb2315e40812b173cafe79d2f992d50015f4387445737fa \ + --hash=sha256:e1dae12321b31059a1a72aaa0e6ba30156fe7e633355e445451e4021b8e122b6 \ + --hash=sha256:eb672217f7bd640411cfc69756ce721d00ae600814708d35c930930f18e8029f \ + --hash=sha256:ee684f139c91e69fe09b8e83d18b4d63bf87d9440c1eb2eeb52ee851883b1b29 \ + --hash=sha256:f3f9a91d3cc5e5b0ddf1043c0ae5fa4852f18a1c0050318baf5fc7930ecc1f9c + # via black toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via -r requirements.in -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via + # -r requirements.in + # mypy + # pytest +tomli==1.2.2 \ + --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ + --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade # via # black # maturin +typing-extensions==3.10.0.2 \ + --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ + --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ + --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 + # via + # black # mypy - # pytest -typing-extensions==4.2.0 \ - --hash=sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708 \ - --hash=sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376 - # via mypy diff --git a/requirements.in b/requirements.in index 098e9f809..09872b03b 100644 --- a/requirements.in +++ b/requirements.in @@ -18,10 +18,10 @@ black flake8 isort -maturin>=0.14 +maturin>=0.15 mypy numpy -pyarrow +pyarrow>=11.0.0 pytest toml importlib_metadata; python_version < "3.8" diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index ce2404f39..000000000 --- a/requirements.txt +++ /dev/null @@ -1,284 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.10 -# To update, run: -# -# pip-compile --generate-hashes -# -attrs==21.2.0 \ - --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ - --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb - # via pytest -black==21.9b0 \ - --hash=sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115 \ - --hash=sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91 - # via -r requirements.in -click==8.0.3 \ - --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ - --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b - # via black -flake8==4.0.1 \ - --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ - --hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d - # via -r requirements.in -iniconfig==1.1.1 \ - --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ - --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 - # via pytest -isort==5.9.3 \ - --hash=sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899 \ - --hash=sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2 - # via -r requirements.in -maturin==0.14.2 \ - --hash=sha256:065a0b48259624af1330bbc2d9bd447615ff07603710ad4a839478aa8e9e98cd \ - --hash=sha256:2966ed9b2e66760ffa5e476dab20c9563dd14efa93ed40b706cbe0a03907d824 \ - --hash=sha256:50a7a15d02f0b407eb5cf4cbc072d27e302993d5734af71bb6d6c567da8c6396 \ - --hash=sha256:539545c666506d34474c1133a11e90b4e6b4b6a7e590f5727ae18a49d3b647d2 \ - --hash=sha256:9f843736fc1c065ffba2dcfc581811c2eb7032c7119ff23012ccde68607e549c \ - --hash=sha256:ace60890f19a87e4a421fd9753e42e5fb46b11e770feacd8ff6749d3bf114764 \ - --hash=sha256:ae3a3a754c3b935420eeb1ec34d98d1693251537331feb43277bae7892b7c51a \ - --hash=sha256:bbaa0689a87124c99adaff99e93dcf415e36f7068d5442f311b49ede4bb7b6c3 \ - --hash=sha256:c65b00f4a4773bc56ca690d93abd09c85cc59b161a313fba3d03aca1ffbba7ca \ - --hash=sha256:daefa75e3f03e914924cb5913b40ada47ff646562c3b1874ddaf5b1e4b0f41ef \ - --hash=sha256:e0b808d1a0b78532c1bcbeb2d01d90281585dc460a966f1949fd337a3fb614c0 \ - --hash=sha256:ebc722e9661a40c3d36a06ecc0d1c637ae1af3335ea0ab69230f46a71e344088 \ - --hash=sha256:fa01eeacfd1798cde3ba1c7a0b8d7a889d2ed0e207f4860beb819c5ddd351f43 - # via -r requirements.in -mccabe==0.6.1 \ - --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ - --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f - # via flake8 -mypy==0.910 \ - --hash=sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9 \ - --hash=sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a \ - --hash=sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9 \ - --hash=sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e \ - --hash=sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2 \ - --hash=sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212 \ - --hash=sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b \ - --hash=sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885 \ - --hash=sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150 \ - --hash=sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703 \ - --hash=sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072 \ - --hash=sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457 \ - --hash=sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e \ - --hash=sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0 \ - --hash=sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb \ - --hash=sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97 \ - --hash=sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8 \ - --hash=sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811 \ - --hash=sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6 \ - --hash=sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de \ - --hash=sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504 \ - --hash=sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921 \ - --hash=sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d - # via -r requirements.in -mypy-extensions==0.4.3 \ - --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ - --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 - # via - # black - # mypy -numpy==1.21.3 \ - --hash=sha256:043e83bfc274649c82a6f09836943e4a4aebe5e33656271c7dbf9621dd58b8ec \ - --hash=sha256:160ccc1bed3a8371bf0d760971f09bfe80a3e18646620e9ded0ad159d9749baa \ - --hash=sha256:188031f833bbb623637e66006cf75e933e00e7231f67e2b45cf8189612bb5dc3 \ - --hash=sha256:28f15209fb535dd4c504a7762d3bc440779b0e37d50ed810ced209e5cea60d96 \ - --hash=sha256:29fb3dcd0468b7715f8ce2c0c2d9bbbaf5ae686334951343a41bd8d155c6ea27 \ - --hash=sha256:2a6ee9620061b2a722749b391c0d80a0e2ae97290f1b32e28d5a362e21941ee4 \ - --hash=sha256:300321e3985c968e3ae7fbda187237b225f3ffe6528395a5b7a5407f73cf093e \ - --hash=sha256:32437f0b275c1d09d9c3add782516413e98cd7c09e6baf4715cbce781fc29912 \ - --hash=sha256:3c09418a14471c7ae69ba682e2428cae5b4420a766659605566c0fa6987f6b7e \ - --hash=sha256:49c6249260890e05b8111ebfc391ed58b3cb4b33e63197b2ec7f776e45330721 \ - --hash=sha256:4cc9b512e9fb590797474f58b7f6d1f1b654b3a94f4fa8558b48ca8b3cfc97cf \ - --hash=sha256:508b0b513fa1266875524ba8a9ecc27b02ad771fe1704a16314dc1a816a68737 \ - --hash=sha256:50cd26b0cf6664cb3b3dd161ba0a09c9c1343db064e7c69f9f8b551f5104d654 \ - --hash=sha256:5c4193f70f8069550a1788bd0cd3268ab7d3a2b70583dfe3b2e7f421e9aace06 \ - --hash=sha256:5dfe9d6a4c39b8b6edd7990091fea4f852888e41919d0e6722fe78dd421db0eb \ - --hash=sha256:63571bb7897a584ca3249c86dd01c10bcb5fe4296e3568b2e9c1a55356b6410e \ - --hash=sha256:75621882d2230ab77fb6a03d4cbccd2038511491076e7964ef87306623aa5272 \ - --hash=sha256:75eb7cadc8da49302f5b659d40ba4f6d94d5045fbd9569c9d058e77b0514c9e4 \ - --hash=sha256:88a5d6b268e9ad18f3533e184744acdaa2e913b13148160b1152300c949bbb5f \ - --hash=sha256:8a10968963640e75cc0193e1847616ab4c718e83b6938ae74dea44953950f6b7 \ - --hash=sha256:90bec6a86b348b4559b6482e2b684db4a9a7eed1fa054b86115a48d58fbbf62a \ - --hash=sha256:98339aa9911853f131de11010f6dd94c8cec254d3d1f7261528c3b3e3219f139 \ - --hash=sha256:a99a6b067e5190ac6d12005a4d85aa6227c5606fa93211f86b1dafb16233e57d \ - --hash=sha256:bffa2eee3b87376cc6b31eee36d05349571c236d1de1175b804b348dc0941e3f \ - --hash=sha256:c6c2d535a7beb1f8790aaa98fd089ceab2e3dd7ca48aca0af7dc60e6ef93ffe1 \ - --hash=sha256:cc14e7519fab2a4ed87d31f99c31a3796e4e1fe63a86ebdd1c5a1ea78ebd5896 \ - --hash=sha256:dd0482f3fc547f1b1b5d6a8b8e08f63fdc250c58ce688dedd8851e6e26cff0f3 \ - --hash=sha256:dde972a1e11bb7b702ed0e447953e7617723760f420decb97305e66fb4afc54f \ - --hash=sha256:e54af82d68ef8255535a6cdb353f55d6b8cf418a83e2be3569243787a4f4866f \ - --hash=sha256:e606e6316911471c8d9b4618e082635cfe98876007556e89ce03d52ff5e8fcf0 \ - --hash=sha256:f41b018f126aac18583956c54544db437f25c7ee4794bcb23eb38bef8e5e192a \ - --hash=sha256:f8f4625536926a155b80ad2bbff44f8cc59e9f2ad14cdda7acf4c135b4dc8ff2 \ - --hash=sha256:fe52dbe47d9deb69b05084abd4b0df7abb39a3c51957c09f635520abd49b29dd - # via - # -r requirements.in - # pandas - # pyarrow -packaging==21.0 \ - --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ - --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 - # via pytest -pandas==1.3.4 \ - --hash=sha256:003ba92db58b71a5f8add604a17a059f3068ef4e8c0c365b088468d0d64935fd \ - --hash=sha256:10e10a2527db79af6e830c3d5842a4d60383b162885270f8cffc15abca4ba4a9 \ - --hash=sha256:22808afb8f96e2269dcc5b846decacb2f526dd0b47baebc63d913bf847317c8f \ - --hash=sha256:2d1dc09c0013d8faa7474574d61b575f9af6257ab95c93dcf33a14fd8d2c1bab \ - --hash=sha256:35c77609acd2e4d517da41bae0c11c70d31c87aae8dd1aabd2670906c6d2c143 \ - --hash=sha256:372d72a3d8a5f2dbaf566a5fa5fa7f230842ac80f29a931fb4b071502cf86b9a \ - --hash=sha256:42493f8ae67918bf129869abea8204df899902287a7f5eaf596c8e54e0ac7ff4 \ - --hash=sha256:5298a733e5bfbb761181fd4672c36d0c627320eb999c59c65156c6a90c7e1b4f \ - --hash=sha256:5ba0aac1397e1d7b654fccf263a4798a9e84ef749866060d19e577e927d66e1b \ - --hash=sha256:a2aa18d3f0b7d538e21932f637fbfe8518d085238b429e4790a35e1e44a96ffc \ - --hash=sha256:a388960f979665b447f0847626e40f99af8cf191bce9dc571d716433130cb3a7 \ - --hash=sha256:a51528192755f7429c5bcc9e80832c517340317c861318fea9cea081b57c9afd \ - --hash=sha256:b528e126c13816a4374e56b7b18bfe91f7a7f6576d1aadba5dee6a87a7f479ae \ - --hash=sha256:c1aa4de4919358c5ef119f6377bc5964b3a7023c23e845d9db7d9016fa0c5b1c \ - --hash=sha256:c2646458e1dce44df9f71a01dc65f7e8fa4307f29e5c0f2f92c97f47a5bf22f5 \ - --hash=sha256:d47750cf07dee6b55d8423471be70d627314277976ff2edd1381f02d52dbadf9 \ - --hash=sha256:d99d2350adb7b6c3f7f8f0e5dfb7d34ff8dd4bc0a53e62c445b7e43e163fce63 \ - --hash=sha256:dd324f8ee05925ee85de0ea3f0d66e1362e8c80799eb4eb04927d32335a3e44a \ - --hash=sha256:eaca36a80acaacb8183930e2e5ad7f71539a66805d6204ea88736570b2876a7b \ - --hash=sha256:f567e972dce3bbc3a8076e0b675273b4a9e8576ac629149cf8286ee13c259ae5 \ - --hash=sha256:fe48e4925455c964db914b958f6e7032d285848b7538a5e1b19aeb26ffaea3ec - # via -r requirements.in -pathspec==0.9.0 \ - --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ - --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 - # via black -platformdirs==2.4.0 \ - --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \ - --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d - # via black -pluggy==1.0.0 \ - --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ - --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 - # via pytest -py==1.10.0 \ - --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \ - --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a - # via pytest -pyarrow==6.0.0 \ - --hash=sha256:004185e0babc6f3c3fba6ba4f106e406a0113d0f82bb9ad9a8571a1978c45d04 \ - --hash=sha256:0204e80777ab8f4e9abd3a765a8ec07ed1e3c4630bacda50d2ce212ef0f3826f \ - --hash=sha256:072c1a0fca4509eefd7d018b78542fb7e5c63aaf5698f1c0a6e45628ae17ba44 \ - --hash=sha256:15dc0d673d3f865ca63c877bd7a2eced70b0a08969fb733a28247134b8a1f18b \ - --hash=sha256:1c38263ea438a1666b13372e7565450cfeec32dbcd1c2595749476a58465eaec \ - --hash=sha256:281ce5fa03621d786a9beb514abb09846db7f0221b50eabf543caa24037eaacd \ - --hash=sha256:2d2c681659396c745e4f1988d5dd41dcc3ad557bb8d4a8c2e44030edafc08a91 \ - --hash=sha256:376c4b5f248ae63df21fe15c194e9013753164be2d38f4b3fb8bde63ac5a1958 \ - --hash=sha256:465f87fa0be0b2928b2beeba22b5813a0203fb05d90fd8563eea48e08ecc030e \ - --hash=sha256:477c746ef42c039348a288584800e299456c80c5691401bb9b19aa9c02a427b7 \ - --hash=sha256:5144bd9db2920c7cb566c96462d62443cc239104f94771d110f74393f2fb42a2 \ - --hash=sha256:5408fa8d623e66a0445f3fb0e4027fd219bf99bfb57422d543d7b7876e2c5b55 \ - --hash=sha256:5be62679201c441356d3f2a739895dcc8d4d299f2a6eabcd2163bfb6a898abba \ - --hash=sha256:5c666bc6a1cebf01206e2dc1ab05f25f39f35d3a499e0ef5cd635225e07306ca \ - --hash=sha256:6163d82cca7541774b00503c295fe86a1722820eddb958b57f091bb6f5b0a6db \ - --hash=sha256:6a1d9a2f4ee812ed0bd4182cabef99ea914ac297274f0de086f2488093d284ef \ - --hash=sha256:7a683f71b848eb6310b4ec48c0def55dac839e9994c1ac874c9b2d3d5625def1 \ - --hash=sha256:82fe80309e01acf29e3943a1f6d3c98ec109fe1d356bc1ac37d639bcaadcf684 \ - --hash=sha256:8c23f8cdecd3d9e49f9b0f9a651ae5549d1d32fd4901fb1bdc2d327edfba844f \ - --hash=sha256:8d41dfb09ba9236cca6245f33088eb42f3c54023da281139241e0f9f3b4b754e \ - --hash=sha256:a19e58dfb04e451cd8b7bdec3ac8848373b95dfc53492c9a69789aa9074a3c1b \ - --hash=sha256:a50d2f77b86af38ceabf45617208b9105d20e7a5eebc584e7c8c0acededd82ce \ - --hash=sha256:a5bed4f948c032c40597302e9bdfa65f62295240306976ecbe43a54924c6f94f \ - --hash=sha256:ac941a147d14993987cc8b605b721735a34b3e54d167302501fb4db1ad7382c7 \ - --hash=sha256:b86d175262db1eb46afdceb36d459409eb6f8e532d3dec162f8bf572c7f57623 \ - --hash=sha256:bf3400780c4d3c9cb43b1e8a1aaf2e1b7199a0572d0a645529d2784e4d0d8497 \ - --hash=sha256:c7a6e7e0bf8779e9c3428ced85507541f3da9a0675e2f4781d4eb2c7042cbf81 \ - --hash=sha256:cc1d4a70efd583befe92d4ea6f74ed2e0aa31ccdde767cd5cae8e77c65a1c2d4 \ - --hash=sha256:d046dc78a9337baa6415be915c5a16222505233e238a1017f368243c89817eea \ - --hash=sha256:da7860688c33ca88ac05f1a487d32d96d9caa091412496c35f3d1d832145675a \ - --hash=sha256:ddf2e6e3b321adaaf716f2d5af8e92d205a9671e0cb7c0779710a567fd1dd580 \ - --hash=sha256:e81508239a71943759cee272ce625ae208092dd36ef2c6713fccee30bbcf52bb \ - --hash=sha256:ea64a48a85c631eb2a0ea13ccdec5143c85b5897836b16331ee4289d27a57247 \ - --hash=sha256:ed0be080cf595ea15ff1c9ff4097bbf1fcc4b50847d98c0a3c0412fbc6ede7e9 \ - --hash=sha256:fb701ec4a94b92102606d4e88f0b8eba34f09a5ad8e014eaa4af76f42b7f62ae \ - --hash=sha256:fbda7595f24a639bcef3419ecfac17216efacb09f7b0f1b4c4c97f900d65ca0e - # via -r requirements.in -pycodestyle==2.8.0 \ - --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \ - --hash=sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f - # via flake8 -pyflakes==2.4.0 \ - --hash=sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c \ - --hash=sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e - # via flake8 -pyparsing==3.0.3 \ - --hash=sha256:9e3511118010f112a4b4b435ae50e1eaa610cda191acb9e421d60cf5fde83455 \ - --hash=sha256:f8d3fe9fc404576c5164f0f0c4e382c96b85265e023c409c43d48f65da9d60d0 - # via packaging -pytest==6.2.5 \ - --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89 \ - --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134 - # via -r requirements.in -python-dateutil==2.8.2 \ - --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \ - --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9 - # via pandas -pytz==2021.3 \ - --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \ - --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326 - # via pandas -regex==2021.10.23 \ - --hash=sha256:0c186691a7995ef1db61205e00545bf161fb7b59cdb8c1201c89b333141c438a \ - --hash=sha256:0dcc0e71118be8c69252c207630faf13ca5e1b8583d57012aae191e7d6d28b84 \ - --hash=sha256:0f7552429dd39f70057ac5d0e897e5bfe211629652399a21671e53f2a9693a4e \ - --hash=sha256:129472cd06062fb13e7b4670a102951a3e655e9b91634432cfbdb7810af9d710 \ - --hash=sha256:13ec99df95003f56edcd307db44f06fbeb708c4ccdcf940478067dd62353181e \ - --hash=sha256:1f2b59c28afc53973d22e7bc18428721ee8ca6079becf1b36571c42627321c65 \ - --hash=sha256:2b20f544cbbeffe171911f6ce90388ad36fe3fad26b7c7a35d4762817e9ea69c \ - --hash=sha256:2fb698037c35109d3c2e30f2beb499e5ebae6e4bb8ff2e60c50b9a805a716f79 \ - --hash=sha256:34d870f9f27f2161709054d73646fc9aca49480617a65533fc2b4611c518e455 \ - --hash=sha256:391703a2abf8013d95bae39145d26b4e21531ab82e22f26cd3a181ee2644c234 \ - --hash=sha256:450dc27483548214314640c89a0f275dbc557968ed088da40bde7ef8fb52829e \ - --hash=sha256:45b65d6a275a478ac2cbd7fdbf7cc93c1982d613de4574b56fd6972ceadb8395 \ - --hash=sha256:5095a411c8479e715784a0c9236568ae72509450ee2226b649083730f3fadfc6 \ - --hash=sha256:530fc2bbb3dc1ebb17f70f7b234f90a1dd43b1b489ea38cea7be95fb21cdb5c7 \ - --hash=sha256:56f0c81c44638dfd0e2367df1a331b4ddf2e771366c4b9c5d9a473de75e3e1c7 \ - --hash=sha256:5e9c9e0ce92f27cef79e28e877c6b6988c48b16942258f3bc55d39b5f911df4f \ - --hash=sha256:6d7722136c6ed75caf84e1788df36397efdc5dbadab95e59c2bba82d4d808a4c \ - --hash=sha256:74d071dbe4b53c602edd87a7476ab23015a991374ddb228d941929ad7c8c922e \ - --hash=sha256:7b568809dca44cb75c8ebb260844ea98252c8c88396f9d203f5094e50a70355f \ - --hash=sha256:80bb5d2e92b2258188e7dcae5b188c7bf868eafdf800ea6edd0fbfc029984a88 \ - --hash=sha256:8d1cdcda6bd16268316d5db1038965acf948f2a6f43acc2e0b1641ceab443623 \ - --hash=sha256:9f665677e46c5a4d288ece12fdedf4f4204a422bb28ff05f0e6b08b7447796d1 \ - --hash=sha256:a30513828180264294953cecd942202dfda64e85195ae36c265daf4052af0464 \ - --hash=sha256:a7a986c45d1099a5de766a15de7bee3840b1e0e1a344430926af08e5297cf666 \ - --hash=sha256:a940ca7e7189d23da2bfbb38973832813eab6bd83f3bf89a977668c2f813deae \ - --hash=sha256:ab7c5684ff3538b67df3f93d66bd3369b749087871ae3786e70ef39e601345b0 \ - --hash=sha256:be04739a27be55631069b348dda0c81d8ea9822b5da10b8019b789e42d1fe452 \ - --hash=sha256:c0938ddd60cc04e8f1faf7a14a166ac939aac703745bfcd8e8f20322a7373019 \ - --hash=sha256:cb46b542133999580ffb691baf67410306833ee1e4f58ed06b6a7aaf4e046952 \ - --hash=sha256:d134757a37d8640f3c0abb41f5e68b7cf66c644f54ef1cb0573b7ea1c63e1509 \ - --hash=sha256:de557502c3bec8e634246588a94e82f1ee1b9dfcfdc453267c4fb652ff531570 \ - --hash=sha256:ded0c4a3eee56b57fcb2315e40812b173cafe79d2f992d50015f4387445737fa \ - --hash=sha256:e1dae12321b31059a1a72aaa0e6ba30156fe7e633355e445451e4021b8e122b6 \ - --hash=sha256:eb672217f7bd640411cfc69756ce721d00ae600814708d35c930930f18e8029f \ - --hash=sha256:ee684f139c91e69fe09b8e83d18b4d63bf87d9440c1eb2eeb52ee851883b1b29 \ - --hash=sha256:f3f9a91d3cc5e5b0ddf1043c0ae5fa4852f18a1c0050318baf5fc7930ecc1f9c - # via black -six==1.16.0 \ - --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \ - --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 - # via python-dateutil -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via - # -r requirements.in - # maturin - # mypy - # pytest -tomli==1.2.2 \ - --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ - --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade - # via black -typing-extensions==3.10.0.2 \ - --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ - --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ - --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 - # via - # black - # mypy From 433dbca009a25a78e698d2c703ef9f1e4177dae0 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 10 May 2023 08:53:55 -0400 Subject: [PATCH 045/413] Expand Expr to include RexType basic support (#378) * Make expr member of PyExpr public * Add RexType to Expr * Add utility functions for mapping ScalarValue instances to DataTypeMap instances * Add function to get python_value from Expr instance * Fix syntax problems * Add function to get the operands for a Rex::Call * Add function to get operator for RexType::Call * expand types function to include variant support for BinaryExpr * Add variant coverage for Decimal128 and Decimal256 * add function for getting the column name of an Expr from a LogicalPlan * Make PyProjection::projection member public * Add projected_expressions to projection node * Adjust function signature * Add Distinct variant to to_variant function in PyLogicalPlan * Fill in variants for DataType::Timestamp * Address syntax issues * Refactor types() function to extend support for CAST * Update CAST variant handling * Cargo fmt * Cargo clippy * Coverage for INTERVAL in DataType * More cargo fmt changes --- src/common/data_type.rs | 119 +++++++++++++--- src/expr.rs | 300 +++++++++++++++++++++++++++++++++++++++- src/expr/projection.rs | 18 ++- src/sql/logical.rs | 2 + 4 files changed, 414 insertions(+), 25 deletions(-) diff --git a/src/common/data_type.rs b/src/common/data_type.rs index d55a0e86d..622e1aa46 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion::arrow::datatypes::DataType; -use datafusion_common::DataFusionError; +use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; +use datafusion_common::{DataFusionError, ScalarValue}; use pyo3::prelude::*; use crate::errors::py_datafusion_err; @@ -130,9 +130,11 @@ impl DataTypeMap { PythonType::Float, SqlType::FLOAT, )), - DataType::Timestamp(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), - ))), + DataType::Timestamp(unit, tz) => Ok(DataTypeMap::new( + DataType::Timestamp(unit.clone(), tz.clone()), + PythonType::Datetime, + SqlType::DATE, + )), DataType::Date32 => Ok(DataTypeMap::new( DataType::Date32, PythonType::Datetime, @@ -143,18 +145,28 @@ impl DataTypeMap { PythonType::Datetime, SqlType::DATE, )), - DataType::Time32(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), - ))), - DataType::Time64(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), - ))), + DataType::Time32(unit) => Ok(DataTypeMap::new( + DataType::Time32(unit.clone()), + PythonType::Datetime, + SqlType::DATE, + )), + DataType::Time64(unit) => Ok(DataTypeMap::new( + DataType::Time64(unit.clone()), + PythonType::Datetime, + SqlType::DATE, + )), DataType::Duration(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( format!("{:?}", arrow_type), ))), - DataType::Interval(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), - ))), + DataType::Interval(interval_unit) => Ok(DataTypeMap::new( + DataType::Interval(interval_unit.clone()), + PythonType::Datetime, + match interval_unit { + IntervalUnit::DayTime => SqlType::INTERVAL_DAY, + IntervalUnit::MonthDayNano => SqlType::INTERVAL_MONTH, + IntervalUnit::YearMonth => SqlType::INTERVAL_YEAR_MONTH, + }, + )), DataType::Binary => Ok(DataTypeMap::new( DataType::Binary, PythonType::Bytes, @@ -197,12 +209,16 @@ impl DataTypeMap { DataType::Dictionary(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( format!("{:?}", arrow_type), ))), - DataType::Decimal128(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), - ))), - DataType::Decimal256(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), - ))), + DataType::Decimal128(precision, scale) => Ok(DataTypeMap::new( + DataType::Decimal128(*precision, *scale), + PythonType::Float, + SqlType::DECIMAL, + )), + DataType::Decimal256(precision, scale) => Ok(DataTypeMap::new( + DataType::Decimal256(*precision, *scale), + PythonType::Float, + SqlType::DECIMAL, + )), DataType::Map(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( format!("{:?}", arrow_type), ))), @@ -211,6 +227,69 @@ impl DataTypeMap { )), } } + + /// Generate the `DataTypeMap` from a `ScalarValue` instance + pub fn map_from_scalar_value(scalar_val: &ScalarValue) -> Result { + DataTypeMap::map_from_arrow_type(&DataTypeMap::map_from_scalar_to_arrow(scalar_val)?) + } + + /// Maps a `ScalarValue` to an Arrow `DataType` + pub fn map_from_scalar_to_arrow(scalar_val: &ScalarValue) -> Result { + match scalar_val { + ScalarValue::Boolean(_) => Ok(DataType::Boolean), + ScalarValue::Float32(_) => Ok(DataType::Float32), + ScalarValue::Float64(_) => Ok(DataType::Float64), + ScalarValue::Decimal128(_, precision, scale) => { + Ok(DataType::Decimal128(*precision, *scale)) + } + ScalarValue::Dictionary(data_type, scalar_type) => { + // Call this function again to map the dictionary scalar_value to an Arrow type + Ok(DataType::Dictionary( + Box::new(*data_type.clone()), + Box::new(DataTypeMap::map_from_scalar_to_arrow(scalar_type)?), + )) + } + ScalarValue::Int8(_) => Ok(DataType::Int8), + ScalarValue::Int16(_) => Ok(DataType::Int16), + ScalarValue::Int32(_) => Ok(DataType::Int32), + ScalarValue::Int64(_) => Ok(DataType::Int64), + ScalarValue::UInt8(_) => Ok(DataType::UInt8), + ScalarValue::UInt16(_) => Ok(DataType::UInt16), + ScalarValue::UInt32(_) => Ok(DataType::UInt32), + ScalarValue::UInt64(_) => Ok(DataType::UInt64), + ScalarValue::Utf8(_) => Ok(DataType::Utf8), + ScalarValue::LargeUtf8(_) => Ok(DataType::LargeUtf8), + ScalarValue::Binary(_) => Ok(DataType::Binary), + ScalarValue::LargeBinary(_) => Ok(DataType::LargeBinary), + ScalarValue::Date32(_) => Ok(DataType::Date32), + ScalarValue::Date64(_) => Ok(DataType::Date64), + ScalarValue::Time32Second(_) => Ok(DataType::Time32(TimeUnit::Second)), + ScalarValue::Time32Millisecond(_) => Ok(DataType::Time32(TimeUnit::Millisecond)), + ScalarValue::Time64Microsecond(_) => Ok(DataType::Time64(TimeUnit::Microsecond)), + ScalarValue::Time64Nanosecond(_) => Ok(DataType::Time64(TimeUnit::Nanosecond)), + ScalarValue::Null => Ok(DataType::Null), + ScalarValue::TimestampSecond(_, tz) => { + Ok(DataType::Timestamp(TimeUnit::Second, tz.to_owned())) + } + ScalarValue::TimestampMillisecond(_, tz) => { + Ok(DataType::Timestamp(TimeUnit::Millisecond, tz.to_owned())) + } + ScalarValue::TimestampMicrosecond(_, tz) => { + Ok(DataType::Timestamp(TimeUnit::Microsecond, tz.to_owned())) + } + ScalarValue::TimestampNanosecond(_, tz) => { + Ok(DataType::Timestamp(TimeUnit::Nanosecond, tz.to_owned())) + } + ScalarValue::IntervalYearMonth(..) => Ok(DataType::Interval(IntervalUnit::YearMonth)), + ScalarValue::IntervalDayTime(..) => Ok(DataType::Interval(IntervalUnit::DayTime)), + ScalarValue::IntervalMonthDayNano(..) => { + Ok(DataType::Interval(IntervalUnit::MonthDayNano)) + } + ScalarValue::List(_val, field_ref) => Ok(DataType::List(field_ref.to_owned())), + ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())), + ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)), + } + } } #[pymethods] diff --git a/src/expr.rs b/src/expr.rs index 4ada4c16d..c002b3291 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,19 +15,26 @@ // specific language governing permissions and limitations // under the License. +use datafusion_common::DFField; +use datafusion_expr::expr::{AggregateFunction, Sort, WindowFunction}; +use datafusion_expr::utils::exprlist_to_fields; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::PyArrowType; -use datafusion_expr::{col, lit, Cast, Expr, GetIndexedField}; +use datafusion_expr::{ + col, lit, Between, BinaryExpr, Case, Cast, Expr, GetIndexedField, Like, LogicalPlan, Operator, + TryCast, +}; -use crate::common::data_type::RexType; -use crate::errors::py_runtime_err; +use crate::common::data_type::{DataTypeMap, RexType}; +use crate::errors::{py_runtime_err, py_type_err, DataFusionError}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; +use crate::sql::logical::PyLogicalPlan; use datafusion::scalar::ScalarValue; use self::alias::PyAlias; @@ -274,11 +281,296 @@ impl PyExpr { Expr::ScalarSubquery(..) => RexType::ScalarSubquery, }) } + + /// Given the current `Expr` return the DataTypeMap which represents the + /// PythonType, Arrow DataType, and SqlType Enum which represents + pub fn types(&self) -> PyResult { + Self::_types(&self.expr) + } + + /// Extracts the Expr value into a PyObject that can be shared with Python + pub fn python_value(&self, py: Python) -> PyResult { + match &self.expr { + Expr::Literal(scalar_value) => Ok(match scalar_value { + ScalarValue::Null => todo!(), + ScalarValue::Boolean(v) => v.into_py(py), + ScalarValue::Float32(v) => v.into_py(py), + ScalarValue::Float64(v) => v.into_py(py), + ScalarValue::Decimal128(_, _, _) => todo!(), + ScalarValue::Int8(v) => v.into_py(py), + ScalarValue::Int16(v) => v.into_py(py), + ScalarValue::Int32(v) => v.into_py(py), + ScalarValue::Int64(v) => v.into_py(py), + ScalarValue::UInt8(v) => v.into_py(py), + ScalarValue::UInt16(v) => v.into_py(py), + ScalarValue::UInt32(v) => v.into_py(py), + ScalarValue::UInt64(v) => v.into_py(py), + ScalarValue::Utf8(v) => v.clone().into_py(py), + ScalarValue::LargeUtf8(v) => v.clone().into_py(py), + ScalarValue::Binary(v) => v.clone().into_py(py), + ScalarValue::FixedSizeBinary(_, _) => todo!(), + ScalarValue::LargeBinary(v) => v.clone().into_py(py), + ScalarValue::List(_, _) => todo!(), + ScalarValue::Date32(v) => v.into_py(py), + ScalarValue::Date64(v) => v.into_py(py), + ScalarValue::Time32Second(v) => v.into_py(py), + ScalarValue::Time32Millisecond(v) => v.into_py(py), + ScalarValue::Time64Microsecond(v) => v.into_py(py), + ScalarValue::Time64Nanosecond(v) => v.into_py(py), + ScalarValue::TimestampSecond(_, _) => todo!(), + ScalarValue::TimestampMillisecond(_, _) => todo!(), + ScalarValue::TimestampMicrosecond(_, _) => todo!(), + ScalarValue::TimestampNanosecond(_, _) => todo!(), + ScalarValue::IntervalYearMonth(v) => v.into_py(py), + ScalarValue::IntervalDayTime(v) => v.into_py(py), + ScalarValue::IntervalMonthDayNano(v) => v.into_py(py), + ScalarValue::Struct(_, _) => todo!(), + ScalarValue::Dictionary(_, _) => todo!(), + }), + _ => Err(py_type_err(format!( + "Non Expr::Literal encountered in types: {:?}", + &self.expr + ))), + } + } + + /// Row expressions, Rex(s), operate on the concept of operands. Different variants of Expressions, Expr(s), + /// store those operands in different datastructures. This function examines the Expr variant and returns + /// the operands to the calling logic as a Vec of PyExpr instances. + pub fn rex_call_operands(&self) -> PyResult> { + match &self.expr { + // Expr variants that are themselves the operand to return + Expr::Column(..) | Expr::ScalarVariable(..) | Expr::Literal(..) => { + Ok(vec![PyExpr::from(self.expr.clone())]) + } + + // Expr(s) that house the Expr instance to return in their bounded params + Expr::Alias(expr, ..) + | Expr::Not(expr) + | Expr::IsNull(expr) + | Expr::IsNotNull(expr) + | Expr::IsTrue(expr) + | Expr::IsFalse(expr) + | Expr::IsUnknown(expr) + | Expr::IsNotTrue(expr) + | Expr::IsNotFalse(expr) + | Expr::IsNotUnknown(expr) + | Expr::Negative(expr) + | Expr::GetIndexedField(GetIndexedField { expr, .. }) + | Expr::Cast(Cast { expr, .. }) + | Expr::TryCast(TryCast { expr, .. }) + | Expr::Sort(Sort { expr, .. }) + | Expr::InSubquery { expr, .. } => Ok(vec![PyExpr::from(*expr.clone())]), + + // Expr variants containing a collection of Expr(s) for operands + Expr::AggregateFunction(AggregateFunction { args, .. }) + | Expr::AggregateUDF { args, .. } + | Expr::ScalarFunction { args, .. } + | Expr::ScalarUDF { args, .. } + | Expr::WindowFunction(WindowFunction { args, .. }) => { + Ok(args.iter().map(|arg| PyExpr::from(arg.clone())).collect()) + } + + // Expr(s) that require more specific processing + Expr::Case(Case { + expr, + when_then_expr, + else_expr, + }) => { + let mut operands: Vec = Vec::new(); + + if let Some(e) = expr { + operands.push(PyExpr::from(*e.clone())); + }; + + for (when, then) in when_then_expr { + operands.push(PyExpr::from(*when.clone())); + operands.push(PyExpr::from(*then.clone())); + } + + if let Some(e) = else_expr { + operands.push(PyExpr::from(*e.clone())); + }; + + Ok(operands) + } + Expr::InList { expr, list, .. } => { + let mut operands: Vec = vec![PyExpr::from(*expr.clone())]; + for list_elem in list { + operands.push(PyExpr::from(list_elem.clone())); + } + + Ok(operands) + } + Expr::BinaryExpr(BinaryExpr { left, right, .. }) => Ok(vec![ + PyExpr::from(*left.clone()), + PyExpr::from(*right.clone()), + ]), + Expr::Like(Like { expr, pattern, .. }) => Ok(vec![ + PyExpr::from(*expr.clone()), + PyExpr::from(*pattern.clone()), + ]), + Expr::ILike(Like { expr, pattern, .. }) => Ok(vec![ + PyExpr::from(*expr.clone()), + PyExpr::from(*pattern.clone()), + ]), + Expr::SimilarTo(Like { expr, pattern, .. }) => Ok(vec![ + PyExpr::from(*expr.clone()), + PyExpr::from(*pattern.clone()), + ]), + Expr::Between(Between { + expr, + negated: _, + low, + high, + }) => Ok(vec![ + PyExpr::from(*expr.clone()), + PyExpr::from(*low.clone()), + PyExpr::from(*high.clone()), + ]), + + // Currently un-support/implemented Expr types for Rex Call operations + Expr::GroupingSet(..) + | Expr::OuterReferenceColumn(_, _) + | Expr::Wildcard + | Expr::QualifiedWildcard { .. } + | Expr::ScalarSubquery(..) + | Expr::Placeholder { .. } + | Expr::Exists { .. } => Err(py_runtime_err(format!( + "Unimplemented Expr type: {}", + self.expr + ))), + } + } + + /// Extracts the operator associated with a RexType::Call + pub fn rex_call_operator(&self) -> PyResult { + Ok(match &self.expr { + Expr::BinaryExpr(BinaryExpr { + left: _, + op, + right: _, + }) => format!("{op}"), + Expr::ScalarFunction { fun, args: _ } => format!("{fun}"), + Expr::ScalarUDF { fun, .. } => fun.name.clone(), + Expr::Cast { .. } => "cast".to_string(), + Expr::Between { .. } => "between".to_string(), + Expr::Case { .. } => "case".to_string(), + Expr::IsNull(..) => "is null".to_string(), + Expr::IsNotNull(..) => "is not null".to_string(), + Expr::IsTrue(_) => "is true".to_string(), + Expr::IsFalse(_) => "is false".to_string(), + Expr::IsUnknown(_) => "is unknown".to_string(), + Expr::IsNotTrue(_) => "is not true".to_string(), + Expr::IsNotFalse(_) => "is not false".to_string(), + Expr::IsNotUnknown(_) => "is not unknown".to_string(), + Expr::InList { .. } => "in list".to_string(), + Expr::Negative(..) => "negative".to_string(), + Expr::Not(..) => "not".to_string(), + Expr::Like(Like { negated, .. }) => { + if *negated { + "not like".to_string() + } else { + "like".to_string() + } + } + Expr::ILike(Like { negated, .. }) => { + if *negated { + "not ilike".to_string() + } else { + "ilike".to_string() + } + } + Expr::SimilarTo(Like { negated, .. }) => { + if *negated { + "not similar to".to_string() + } else { + "similar to".to_string() + } + } + _ => { + return Err(py_type_err(format!( + "Catch all triggered in get_operator_name: {:?}", + &self.expr + ))) + } + }) + } + + pub fn column_name(&self, plan: PyLogicalPlan) -> PyResult { + self._column_name(&plan.plan()).map_err(py_runtime_err) + } +} + +impl PyExpr { + pub fn _column_name(&self, plan: &LogicalPlan) -> Result { + let field = Self::expr_to_field(&self.expr, plan)?; + Ok(field.qualified_column().flat_name()) + } + + /// Create a [DFField] representing an [Expr], given an input [LogicalPlan] to resolve against + pub fn expr_to_field( + expr: &Expr, + input_plan: &LogicalPlan, + ) -> Result { + match expr { + Expr::Sort(Sort { expr, .. }) => { + // DataFusion does not support create_name for sort expressions (since they never + // appear in projections) so we just delegate to the contained expression instead + Self::expr_to_field(expr, input_plan) + } + _ => { + let fields = + exprlist_to_fields(&[expr.clone()], input_plan).map_err(PyErr::from)?; + Ok(fields[0].clone()) + } + } + } + + fn _types(expr: &Expr) -> PyResult { + match expr { + Expr::BinaryExpr(BinaryExpr { + left: _, + op, + right: _, + }) => match op { + Operator::Eq + | Operator::NotEq + | Operator::Lt + | Operator::LtEq + | Operator::Gt + | Operator::GtEq + | Operator::And + | Operator::Or + | Operator::IsDistinctFrom + | Operator::IsNotDistinctFrom + | Operator::RegexMatch + | Operator::RegexIMatch + | Operator::RegexNotMatch + | Operator::RegexNotIMatch => DataTypeMap::map_from_arrow_type(&DataType::Boolean), + Operator::Plus | Operator::Minus | Operator::Multiply | Operator::Modulo => { + DataTypeMap::map_from_arrow_type(&DataType::Int64) + } + Operator::Divide => DataTypeMap::map_from_arrow_type(&DataType::Float64), + Operator::StringConcat => DataTypeMap::map_from_arrow_type(&DataType::Utf8), + Operator::BitwiseShiftLeft + | Operator::BitwiseShiftRight + | Operator::BitwiseXor + | Operator::BitwiseAnd + | Operator::BitwiseOr => DataTypeMap::map_from_arrow_type(&DataType::Binary), + }, + Expr::Cast(Cast { expr: _, data_type }) => DataTypeMap::map_from_arrow_type(data_type), + Expr::Literal(scalar_value) => DataTypeMap::map_from_scalar_value(scalar_value), + _ => Err(py_type_err(format!( + "Non Expr::Literal encountered in types: {:?}", + expr + ))), + } + } } /// Initializes the `expr` module to match the pattern of `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { - // expressions m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/expr/projection.rs b/src/expr/projection.rs index f5ba12db2..b3296618f 100644 --- a/src/expr/projection.rs +++ b/src/expr/projection.rs @@ -16,6 +16,7 @@ // under the License. use datafusion_expr::logical_plan::Projection; +use datafusion_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; @@ -27,7 +28,7 @@ use crate::sql::logical::PyLogicalPlan; #[pyclass(name = "Projection", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyProjection { - projection: Projection, + pub projection: Projection, } impl PyProjection { @@ -92,6 +93,21 @@ impl PyProjection { } } +impl PyProjection { + /// Projection: Gets the names of the fields that should be projected + pub fn projected_expressions(local_expr: &PyExpr) -> Vec { + let mut projs: Vec = Vec::new(); + match &local_expr.expr { + Expr::Alias(expr, _name) => { + let py_expr: PyExpr = PyExpr::from(*expr.clone()); + projs.extend_from_slice(Self::projected_expressions(&py_expr).as_slice()); + } + _ => projs.push(local_expr.clone()), + } + projs + } +} + impl LogicalNode for PyProjection { fn inputs(&self) -> Vec { vec![PyLogicalPlan::from((*self.projection.input).clone())] diff --git a/src/sql/logical.rs b/src/sql/logical.rs index a75315d36..07a3f65b1 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use crate::errors::py_unsupported_variant_err; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; +use crate::expr::distinct::PyDistinct; use crate::expr::empty_relation::PyEmptyRelation; use crate::expr::explain::PyExplain; use crate::expr::extension::PyExtension; @@ -62,6 +63,7 @@ impl PyLogicalPlan { LogicalPlan::EmptyRelation(plan) => PyEmptyRelation::from(plan.clone()).to_variant(py), LogicalPlan::Explain(plan) => PyExplain::from(plan.clone()).to_variant(py), LogicalPlan::Extension(plan) => PyExtension::from(plan.clone()).to_variant(py), + LogicalPlan::Distinct(plan) => PyDistinct::from(plan.clone()).to_variant(py), LogicalPlan::Filter(plan) => PyFilter::from(plan.clone()).to_variant(py), LogicalPlan::Limit(plan) => PyLimit::from(plan.clone()).to_variant(py), LogicalPlan::Projection(plan) => PyProjection::from(plan.clone()).to_variant(py), From 3711c73f2bb2bf01cd1c6fd50d0264825d2fd821 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 19 May 2023 08:57:53 -0600 Subject: [PATCH 046/413] Add Python script for generating changelog (#383) --- dev/release/README.md | 39 ++++++---- dev/release/generate-changelog.md | 114 ++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 dev/release/generate-changelog.md diff --git a/dev/release/README.md b/dev/release/README.md index 1ba44fb50..e85b288f4 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -56,28 +56,37 @@ Before creating a new release: - a PR should be created and merged to update the major version number of the project - A new release branch should be created, such as `branch-0.8` -### Update CHANGELOG.md +### Change Log -Define release branch (e.g. `branch-0.8`), base version tag (e.g. `0.7.0`) and future version tag (e.g. `0.9.0`). Commits -between the base version tag and the release branch will be used to populate the changelog content. +We maintain a `CHANGELOG.md` so our users know what has been changed between releases. + +The changelog is generated using a Python script: ```bash -# create the changelog -CHANGELOG_GITHUB_TOKEN= ./dev/release/update_change_log-datafusion-python.sh main 0.8.0 0.7.0 -# review change log / edit issues and labels if needed, rerun until you are happy with the result -git commit -a -m 'Create changelog for release' +$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/arrow-datafusion-python 24.0.0 HEAD > dev/changelog/25.0.0.md +``` + +This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for +titles starting with `feat:`, `fix:`, or `docs:` . The script will produce output similar to: + ``` +Fetching list of commits between 24.0.0 and HEAD +Fetching pull requests +Categorizing pull requests +Generating changelog content +``` + +This process is not fully automated, so there are some additional manual steps: -_If you see the error `"You have exceeded a secondary rate limit"` when running this script, try reducing the CPU -allocation to slow the process down and throttle the number of GitHub requests made per minute, by modifying the -value of the `--cpus` argument in the `update_change_log.sh` script._ +- Add the ASF header to the generated file +- Add a link to this changelog from the top-level `/datafusion/CHANGELOG.md` +- Add the following content (copy from the previous version's changelog and update as appropriate: -You can add `invalid` or `development-process` label to exclude items from -release notes. +``` +## [24.0.0](https://github.com/apache/arrow-datafusion-python/tree/24.0.0) (2023-05-06) -Send a PR to get these changes merged into the release branch (e.g. `branch-0.8`). If new commits that could change the -change log content landed in the release branch before you could merge the PR, you need to rerun the changelog update -script to regenerate the changelog and update the PR accordingly. +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/23.0.0...24.0.0) +``` ### Preparing a Release Candidate diff --git a/dev/release/generate-changelog.md b/dev/release/generate-changelog.md new file mode 100644 index 000000000..caa6ae647 --- /dev/null +++ b/dev/release/generate-changelog.md @@ -0,0 +1,114 @@ +#!/usr/bin/env python + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import sys +from github import Github +import os +import re + + +def print_pulls(repo_name, title, pulls): +if len(pulls) > 0: +print("**{}:**".format(title)) +print() +for (pull, commit) in pulls: +url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) +print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) +print() + + +def generate_changelog(repo, repo_name, tag1, tag2): + + # get a list of commits between two tags + print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) + comparison = repo.compare(tag1, tag2) + + # get the pull requests for these commits + print("Fetching pull requests", file=sys.stderr) + unique_pulls = [] + all_pulls = [] + for commit in comparison.commits: + pulls = commit.get_pulls() + for pull in pulls: + # there can be multiple commits per PR if squash merge is not being used and + # in this case we should get all the author names, but for now just pick one + if pull.number not in unique_pulls: + unique_pulls.append(pull.number) + all_pulls.append((pull, commit)) + + # we split the pulls into categories + #TODO: make categories configurable + breaking = [] + bugs = [] + docs = [] + enhancements = [] + + # categorize the pull requests based on GitHub labels + print("Categorizing pull requests", file=sys.stderr) + for (pull, commit) in all_pulls: + + # see if PR title uses Conventional Commits + cc_type = '' + cc_scope = '' + cc_breaking = '' + parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + if len(parts) == 1: + parts_tuple = parts[0] + cc_type = parts_tuple[0] # fix, feat, docs, chore + cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == '!' + + labels = [label.name for label in pull.labels] + #print(pull.number, labels, parts, file=sys.stderr) + if 'api change' in labels or cc_breaking: + breaking.append((pull, commit)) + elif 'bug' in labels or cc_type == 'fix': + bugs.append((pull, commit)) + elif 'enhancement' in labels or cc_type == 'feat': + enhancements.append((pull, commit)) + elif 'documentation' in labels or cc_type == 'docs': + docs.append((pull, commit)) + + # produce the changelog content + print("Generating changelog content", file=sys.stderr) + print_pulls(repo_name, "Breaking changes", breaking) + print_pulls(repo_name, "Implemented enhancements", enhancements) + print_pulls(repo_name, "Fixed bugs", bugs) + print_pulls(repo_name, "Documentation updates", docs) + print_pulls(repo_name, "Merged pull requests", all_pulls) + + +def cli(args=None): +"""Process command line arguments.""" +if not args: +args = sys.argv[1:] + + parser = argparse.ArgumentParser() + parser.add_argument("project", help="The project name e.g. apache/arrow-datafusion-python") + parser.add_argument("tag1", help="The previous release tag") + parser.add_argument("tag2", help="The current release tag") + args = parser.parse_args() + + token = os.getenv("GITHUB_TOKEN") + + g = Github(token) + repo = g.get_repo(args.project) + generate_changelog(repo, args.project, args.tag1, args.tag2) + +if __name__ == "__main__": +cli() \ No newline at end of file From 82b4a9553ed56cae664a7c637c7ad4d88fb21d35 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 May 2023 10:27:44 -0600 Subject: [PATCH 047/413] Update for DataFusion 25.0.0 (#386) * Update for DataFusion 25.0.0 * update changelog * lint --- CHANGELOG.md | 16 +- Cargo.lock | 1200 +++-------------- Cargo.toml | 14 +- ...ate-changelog.md => generate-changelog.py} | 63 +- src/expr.rs | 30 +- src/functions.rs | 17 +- src/pyarrow_filter_expression.rs | 6 +- 7 files changed, 302 insertions(+), 1044 deletions(-) rename dev/release/{generate-changelog.md => generate-changelog.py} (69%) mode change 100644 => 100755 diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d7c0f61..b6d58e4a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,21 @@ under the License. --> -# Changelog +# DataFusion Python Changelog + +## [25.0.0](https://github.com/apache/arrow-datafusion-python/tree/25.0.0) (2023-05-23) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/24.0.0...25.0.0) + +**Merged pull requests:** + +- Prepare 24.0.0 Release [#376](https://github.com/apache/arrow-datafusion-python/pull/376) (andygrove) +- build(deps): bump uuid from 1.3.1 to 1.3.2 [#359](https://github.com/apache/arrow-datafusion-python/pull/359) (dependabot[bot]) +- build(deps): bump mimalloc from 0.1.36 to 0.1.37 [#361](https://github.com/apache/arrow-datafusion-python/pull/361) (dependabot[bot]) +- build(deps): bump regex-syntax from 0.6.29 to 0.7.1 [#334](https://github.com/apache/arrow-datafusion-python/pull/334) (dependabot[bot]) +- upgrade maturin to 0.15.1 [#379](https://github.com/apache/arrow-datafusion-python/pull/379) (Jimexist) +- Expand Expr to include RexType basic support [#378](https://github.com/apache/arrow-datafusion-python/pull/378) (jdye64) +- Add Python script for generating changelog [#383](https://github.com/apache/arrow-datafusion-python/pull/383) (andygrove) ## [24.0.0](https://github.com/apache/arrow-datafusion-python/tree/24.0.0) (2023-05-09) diff --git a/Cargo.lock b/Cargo.lock index 080f506f8..d8fa65e64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.70" +version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4" +checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" [[package]] name = "apache-avro" @@ -93,12 +93,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "arc-swap" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" - [[package]] name = "arrayref" version = "0.3.7" @@ -113,9 +107,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c107a57b5913d852da9d5a40e280e4695f2258b5b87733c13b770c63a7117287" +checksum = "218ca81dd088b102c0fd6687c72e73fad1ba93d2ef7b3cf9a1043b04b2c39dbf" dependencies = [ "ahash", "arrow-arith", @@ -136,9 +130,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace6aa3d5617c5d03041a05e01c6819428a8ddf49dd0b055df9b40fef9d96094" +checksum = "d49309fa2299ec34a709cfc9f487c41ecaead96d1ab70e21857466346bbbd690" dependencies = [ "arrow-array", "arrow-buffer", @@ -151,9 +145,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "104a04520692cc674e6afd7682f213ca41f9b13ff1873f63a5a2857a590b87b3" +checksum = "e7a27466d897d99654357a6d95dc0a26931d9e4306e60c14fc31a894edb86579" dependencies = [ "ahash", "arrow-buffer", @@ -168,9 +162,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c875bcb9530ec403998fb0b2dc6d180a7c64563ca4bc22b90eafb84b113143" +checksum = "9405b78106a9d767c7b97c78a70ee1b23ee51a74f5188a821a716d9a85d1af2b" dependencies = [ "half", "num", @@ -178,9 +172,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d6e18281636c8fc0b93be59834da6bf9a72bb70fd0c98ddfdaf124da466c28" +checksum = "be0ec5a79a87783dc828b7ff8f89f62880b3f553bc5f5b932a82f4a1035024b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -195,9 +189,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3197dab0963a236ff8e7c82e2272535745955ac1321eb740c29f2f88b353f54e" +checksum = "350d8e55c3b2d602a0a04389bcc1da40167657143a9922a7103190603e7b7692" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +208,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb68113d6ecdbe8bba48b2c4042c151bf9e1c61244e45072a50250a6fc59bafe" +checksum = "c6f710d98964d2c069b8baf566130045e79e11baa105623f038a6c942f805681" dependencies = [ "arrow-buffer", "arrow-schema", @@ -226,9 +220,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab4bbf2dd3078facb5ce0a9641316a64f42bfd8cf357e6775c8a5e6708e3a8d" +checksum = "9c99787cb8fabc187285da9e7182d22f2b80ecfac61ca0a42c4299e9eecdf903" dependencies = [ "arrow-array", "arrow-buffer", @@ -240,9 +234,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c5b650d23746a494665d914a7fa3d21d939153cff9d53bdebe39bffa88f263" +checksum = "91c95a58ce63f60d80d7a3a1222d65df0bc060b71d31353c34a8118c2a6eae7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -260,9 +254,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68c6fce28e5011e30acc7466b5efcb8ed0197c396240bd2b10e167f275a3c208" +checksum = "4141e6488610cc144e841da3de5f5371488f3cf5bc6bc7b3e752c64e7639c31b" dependencies = [ "arrow-array", "arrow-buffer", @@ -275,9 +269,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f20a421f19799d8b93eb8edde5217e910fa1e2d6ceb3c529f000e57b6db144c0" +checksum = "940191a3c636c111c41e816325b0941484bf904c46de72cd9553acd1afd24d33" dependencies = [ "ahash", "arrow-array", @@ -290,18 +284,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc85923d8d6662cc66ac6602c7d1876872e671002d60993dfdf492a6badeae92" +checksum = "18c41d058b2895a12f46dfafc306ee3529ad9660406be0ab8a7967d5e27c417e" dependencies = [ - "bitflags 2.2.1", + "bitflags 2.3.1", ] [[package]] name = "arrow-select" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ab6613ce65b61d85a3410241744e84e48fbab0fe06e1251b4429d21b3470fd" +checksum = "9fcbdda2772b7e712e77444f3a71f4ee517095aceb993b35de71de41c70d9b4f" dependencies = [ "arrow-array", "arrow-buffer", @@ -312,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3008641239e884aefba66d8b8532da6af40d14296349fcc85935de4ba67b89e" +checksum = "7081c34f4b534ad320a03db79d58e38972041bb7c65686b98bbcc2f9a67a9cee" dependencies = [ "arrow-array", "arrow-buffer", @@ -322,7 +316,7 @@ dependencies = [ "arrow-schema", "arrow-select", "regex", - "regex-syntax 0.6.29", + "regex-syntax", ] [[package]] @@ -351,7 +345,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -362,7 +356,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -382,9 +376,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" [[package]] name = "bitflags" @@ -394,9 +388,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a6904aef64d73cf10ab17ebace7befb918b82164785cb89907993be7f83813" +checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" [[package]] name = "blake2" @@ -451,32 +445,11 @@ dependencies = [ "alloc-stdlib", ] -[[package]] -name = "bstr" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d4260bcc2e8fc9df1eac4919a720effeb63a3f0952f5bf4944adfa18897f09" -dependencies = [ - "memchr", - "once_cell", - "regex-automata", - "serde", -] - -[[package]] -name = "btoi" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" -dependencies = [ - "num-traits", -] - [[package]] name = "bumpalo" -version = "3.12.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "byteorder" @@ -561,22 +534,6 @@ dependencies = [ "phf_codegen", ] -[[package]] -name = "clru" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8191fa7302e03607ff0e237d4246cc043ff5b3cb9409d995172ba3bea16b807" - -[[package]] -name = "codespan-reporting" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] - [[package]] name = "comfy-table" version = "6.1.4" @@ -677,50 +634,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "cxx" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61f1b6389c3fe1c316bf8a4dccc90a38208354b330925bce1f74a6c4756eb93" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cee708e8962df2aeb38f594aae5d827c022b6460ac71a7a3e2c3c2aae5a07b" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn 2.0.15", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7944172ae7e4068c533afbb984114a56c46e9ccddda550499caa222902c7f7bb" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.94" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2345488264226bf682893e25de0769f3360aac9957980ec49361b083ddaa5bc5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.15", -] - [[package]] name = "dashmap" version = "5.4.0" @@ -736,9 +649,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0404a559d5a6d8320369bb0a290b43bbc4f8622d0ef6f04bd095ace9a663f439" +checksum = "9a99e1dc003c0b310b203e4dc2aadc6531dccb37fa681630b588c5154d1b7637" dependencies = [ "ahash", "apache-avro", @@ -788,9 +701,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4653b79a55161852973760db69ea6dcd05c9966a1b588fd83028f625536a1d7f" +checksum = "362811fd9cb830877f5a538a19e5f9a1e5519486dcd6dc4f39a3d6192d2f4ba6" dependencies = [ "apache-avro", "arrow", @@ -805,9 +718,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53481c334b73c6759697919d1d05690392381145fa1890849a65b5a71a24a1ec" +checksum = "b052014fe38299019dd2e7dec6c2dd44c7ad9a5375ab4a36e5ed800700d7eb7a" dependencies = [ "dashmap", "datafusion-common", @@ -823,9 +736,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8ecd7c6605d0b4269346d03289e2ced1715a303e75e6d313dba0bafb1f823f2" +checksum = "55b590b184c6f59a65ee3ada666475ff8173bfd409c618c3c29b5abf7ef4a92c" dependencies = [ "ahash", "arrow", @@ -835,9 +748,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a7c04e94cb4aa9c323993856e18b91f690dda0358a34ab07a3fe0f14bc6600" +checksum = "543311151a4c068b2b663f07905dd4d4a605279dacb24ab9434f8f29a1917cd3" dependencies = [ "arrow", "async-trait", @@ -848,14 +761,14 @@ dependencies = [ "hashbrown 0.13.2", "itertools", "log", - "regex-syntax 0.7.1", + "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e34eb8668fee1443965fff41ba73b2956d50a07ed8dd929cfa2e839ab91da5a" +checksum = "b49cfd92e225d948a2199650e8e292ef1f0a3d62ab9b540795c3d3ba9edc2b2e" dependencies = [ "ahash", "arrow", @@ -886,7 +799,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "24.0.0" +version = "25.0.0" dependencies = [ "async-trait", "datafusion", @@ -904,8 +817,8 @@ dependencies = [ "pyo3", "pyo3-build-config", "rand", - "regex-syntax 0.7.1", - "syn 2.0.15", + "regex-syntax", + "syn 2.0.16", "tokio", "url", "uuid", @@ -913,9 +826,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efa800ae88dfd62ea6c58c24a1154d92937c755672f522b84e8ea6539fad369b" +checksum = "e98cdd6f0d005bce0035bebe03a35ccb7eca3cf40a9bce27e24b1e2a56941ffc" dependencies = [ "arrow", "datafusion-common", @@ -925,9 +838,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556642ef90073e39af721362353ccce4e1f418da7a8e31c23510ed9de6eb71f2" +checksum = "9585219bbbad5287c762ad9ede044ab6c0b778fde980c9858b15d6dd5bd80f35" dependencies = [ "arrow", "arrow-schema", @@ -939,9 +852,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "24.0.0" +version = "25.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7643a77bb446047095ec21b913adb900b71c7a2ae600f8062906dd2e5642b9" +checksum = "6f0f1a60b63cca4fb2efbf572c3f2875bccbb2dfe448dd562f103ca5d9830216" dependencies = [ "async-recursion", "chrono", @@ -955,9 +868,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -970,12 +883,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" -[[package]] -name = "dunce" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" - [[package]] name = "dyn-clone" version = "1.0.11" @@ -1027,18 +934,6 @@ dependencies = [ "instant", ] -[[package]] -name = "filetime" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.2.16", - "windows-sys 0.48.0", -] - [[package]] name = "fixedbitset" version = "0.4.2" @@ -1057,9 +952,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -1136,7 +1031,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -1191,548 +1086,18 @@ dependencies = [ ] [[package]] -name = "gix" -version = "0.44.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bf41b61f7df395284f7a579c0fa1a7e012c5aede655174d4e91299ef1cac643" -dependencies = [ - "gix-actor", - "gix-attributes", - "gix-config", - "gix-credentials", - "gix-date", - "gix-diff", - "gix-discover", - "gix-features", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-hashtable", - "gix-ignore", - "gix-index", - "gix-lock", - "gix-mailmap", - "gix-object", - "gix-odb", - "gix-pack", - "gix-path", - "gix-prompt", - "gix-ref", - "gix-refspec", - "gix-revision", - "gix-sec", - "gix-tempfile", - "gix-traverse", - "gix-url", - "gix-utils", - "gix-validate", - "gix-worktree", - "log", - "once_cell", - "signal-hook", - "smallvec", - "thiserror", - "unicode-normalization", -] - -[[package]] -name = "gix-actor" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848efa0f1210cea8638f95691c82a46f98a74b9e3524f01d4955ebc25a8f84f3" -dependencies = [ - "bstr", - "btoi", - "gix-date", - "itoa", - "nom", - "thiserror", -] - -[[package]] -name = "gix-attributes" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3015baa01ad2122fbcaab7863c857a603eb7b7ec12ac8141207c42c6439805e2" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "gix-quote", - "kstring", - "log", - "smallvec", - "thiserror", - "unicode-bom", -] - -[[package]] -name = "gix-bitmap" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55a95f4942360766c3880bdb2b4b57f1ef73b190fc424755e7fdf480430af618" -dependencies = [ - "thiserror", -] - -[[package]] -name = "gix-chunk" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d39583cab06464b8bf73b3f1707458270f0e7383cb24c3c9c1a16e6f792978" -dependencies = [ - "thiserror", -] - -[[package]] -name = "gix-command" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2c6f75c1e0f924de39e750880a6e21307194bb1ab773efe3c7d2d787277f8ab" -dependencies = [ - "bstr", -] - -[[package]] -name = "gix-config" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d252a0eddb6df74600d3d8872dc9fe98835a7da43110411d705b682f49d4ac1" -dependencies = [ - "bstr", - "gix-config-value", - "gix-features", - "gix-glob", - "gix-path", - "gix-ref", - "gix-sec", - "log", - "memchr", - "nom", - "once_cell", - "smallvec", - "thiserror", - "unicode-bom", -] - -[[package]] -name = "gix-config-value" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786861e84a5793ad5f863d846de5eb064cd23b87e61ad708c8c402608202e7be" -dependencies = [ - "bitflags 2.2.1", - "bstr", - "gix-path", - "libc", - "thiserror", -] - -[[package]] -name = "gix-credentials" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4874a4fc11ffa844a3c2b87a66957bda30a73b577ef1acf15ac34df5745de5ff" -dependencies = [ - "bstr", - "gix-command", - "gix-config-value", - "gix-path", - "gix-prompt", - "gix-sec", - "gix-url", - "thiserror", -] - -[[package]] -name = "gix-date" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99056f37270715f5c7584fd8b46899a2296af9cae92463bf58b8bd1f5a78e553" -dependencies = [ - "bstr", - "itoa", - "thiserror", - "time", -] - -[[package]] -name = "gix-diff" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644a0f2768bc42d7a69289ada80c9e15c589caefc6a315d2307202df83ed1186" -dependencies = [ - "gix-hash", - "gix-object", - "imara-diff", - "thiserror", -] - -[[package]] -name = "gix-discover" -version = "0.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a6b61363e63e7cdaa3e6f96acb0257ebdb3d8883e21eba5930c99f07f0a5fc0" -dependencies = [ - "bstr", - "dunce", - "gix-hash", - "gix-path", - "gix-ref", - "gix-sec", - "thiserror", -] - -[[package]] -name = "gix-features" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf69b0f5c701cc3ae22d3204b671907668f6437ca88862d355eaf9bc47a4f897" -dependencies = [ - "crc32fast", - "flate2", - "gix-hash", - "libc", - "once_cell", - "prodash", - "sha1_smol", - "thiserror", - "walkdir", -] - -[[package]] -name = "gix-fs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b37a1832f691fdc09910bd267f9a2e413737c1f9ec68c6e31f9e802616278a9" -dependencies = [ - "gix-features", -] - -[[package]] -name = "gix-glob" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07c98204529ac3f24b34754540a852593d2a4c7349008df389240266627a72a" -dependencies = [ - "bitflags 2.2.1", - "bstr", - "gix-features", - "gix-path", -] - -[[package]] -name = "gix-hash" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "078eec3ac2808cc03f0bddd2704cb661da5c5dc33b41a9d7947b141d499c7c42" -dependencies = [ - "hex", - "thiserror", -] - -[[package]] -name = "gix-hashtable" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afebb85691c6a085b114e01a27f4a61364519298c5826cb87a45c304802299bc" -dependencies = [ - "gix-hash", - "hashbrown 0.13.2", - "parking_lot", -] - -[[package]] -name = "gix-ignore" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba205b6df563e2906768bb22834c82eb46c5fdfcd86ba2c347270bc8309a05b2" -dependencies = [ - "bstr", - "gix-glob", - "gix-path", - "unicode-bom", -] - -[[package]] -name = "gix-index" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f39c1ccc8f1912cbbd5191efc28dbc5f0d0598042aa56bc09427b7c34efab3ba" -dependencies = [ - "bitflags 2.2.1", - "bstr", - "btoi", - "filetime", - "gix-bitmap", - "gix-features", - "gix-hash", - "gix-lock", - "gix-object", - "gix-traverse", - "itoa", - "memmap2", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-lock" -version = "5.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b80172055c5d8017a48ddac5cc7a95421c00211047db0165c97853c4f05194" -dependencies = [ - "fastrand", - "gix-tempfile", - "thiserror", -] - -[[package]] -name = "gix-mailmap" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8856cec3bdc3610c06970d28b6cb20a0c6621621cf9a8ec48cbd23f2630f362" -dependencies = [ - "bstr", - "gix-actor", - "thiserror", -] - -[[package]] -name = "gix-object" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9bb30ce0818d37096daa29efe361a4bc6dd0b51a5726598898be7e9a40a01e1" -dependencies = [ - "bstr", - "btoi", - "gix-actor", - "gix-features", - "gix-hash", - "gix-validate", - "hex", - "itoa", - "nom", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-odb" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2f324aa67672b6d0f2c0fa93f96eb6a7029d260e4c1df5dce3c015f5e5add" -dependencies = [ - "arc-swap", - "gix-features", - "gix-hash", - "gix-object", - "gix-pack", - "gix-path", - "gix-quote", - "parking_lot", - "tempfile", - "thiserror", -] - -[[package]] -name = "gix-pack" -version = "0.35.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164a515900a83257ae4aa80e741655bee7a2e39113fb535d7a5ac623b445ff20" -dependencies = [ - "clru", - "gix-chunk", - "gix-diff", - "gix-features", - "gix-hash", - "gix-hashtable", - "gix-object", - "gix-path", - "gix-tempfile", - "gix-traverse", - "memmap2", - "parking_lot", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-path" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fc78f47095a0c15aea0e66103838f0748f4494bf7a9555dfe0f00425400396c" -dependencies = [ - "bstr", - "home", - "once_cell", - "thiserror", -] - -[[package]] -name = "gix-prompt" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "330d11fdf88fff3366c2491efde2f3e454958efe7d5ddf60272e8fb1d944bb01" -dependencies = [ - "gix-command", - "gix-config-value", - "parking_lot", - "rustix", - "thiserror", -] - -[[package]] -name = "gix-quote" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a282f5a8d9ee0b09ec47390ac727350c48f2f5c76d803cd8da6b3e7ad56e0bcb" -dependencies = [ - "bstr", - "btoi", - "thiserror", -] - -[[package]] -name = "gix-ref" -version = "0.29.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e03989e9d49954368e1b526578230fc7189d1634acdfbe79e9ba1de717e15d5" -dependencies = [ - "gix-actor", - "gix-features", - "gix-fs", - "gix-hash", - "gix-lock", - "gix-object", - "gix-path", - "gix-tempfile", - "gix-validate", - "memmap2", - "nom", - "thiserror", -] - -[[package]] -name = "gix-refspec" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6ea733820df67e4cd7797deb12727905824d8f5b7c59d943c456d314475892" -dependencies = [ - "bstr", - "gix-hash", - "gix-revision", - "gix-validate", - "smallvec", - "thiserror", -] - -[[package]] -name = "gix-revision" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "810f35e9afeccca999d5d348b239f9c162353127d2e13ff3240e31b919e35476" -dependencies = [ - "bstr", - "gix-date", - "gix-hash", - "gix-hashtable", - "gix-object", - "thiserror", -] - -[[package]] -name = "gix-sec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794520043d5a024dfeac335c6e520cb616f6963e30dab995892382e998c12897" -dependencies = [ - "bitflags 2.2.1", - "gix-path", - "libc", - "windows", -] - -[[package]] -name = "gix-tempfile" -version = "5.0.2" +name = "git2" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2ceb30a610e3f5f2d5f9a5114689fde507ba9417705a8cf3429604275b2153c" +checksum = "8b7905cdfe33d31a88bb2e8419ddd054451f5432d1da9eaf2ac7804ee1ea12d5" dependencies = [ + "bitflags 1.3.2", "libc", - "once_cell", - "parking_lot", - "signal-hook", - "signal-hook-registry", - "tempfile", -] - -[[package]] -name = "gix-traverse" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5be1e807f288c33bb005075111886cceb43ed8a167b3182a0f62c186e2a0dd1" -dependencies = [ - "gix-hash", - "gix-hashtable", - "gix-object", - "thiserror", -] - -[[package]] -name = "gix-url" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc77f89054297cc81491e31f1bab4027e554b5ef742a44bd7035db9a0f78b76" -dependencies = [ - "bstr", - "gix-features", - "gix-path", - "home", - "thiserror", + "libgit2-sys", + "log", "url", ] -[[package]] -name = "gix-utils" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c10b69beac219acb8df673187a1f07dde2d74092f974fb3f9eb385aeb667c909" -dependencies = [ - "fastrand", -] - -[[package]] -name = "gix-validate" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd629d3680773e1785e585d76fd4295b740b559cad9141517300d99a0c8c049" -dependencies = [ - "bstr", - "thiserror", -] - -[[package]] -name = "gix-worktree" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69eaff0ae973a9d37c40f02ae5ae50fa726c8fc2fd3ab79d0a19eb61975aafa" -dependencies = [ - "bstr", - "filetime", - "gix-attributes", - "gix-features", - "gix-fs", - "gix-glob", - "gix-hash", - "gix-ignore", - "gix-index", - "gix-object", - "gix-path", - "io-close", - "thiserror", -] - [[package]] name = "glob" version = "0.3.1" @@ -1741,9 +1106,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -1804,21 +1169,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "home" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" -dependencies = [ - "winapi", -] - [[package]] name = "http" version = "0.2.9" @@ -1879,9 +1229,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" dependencies = [ "http", "hyper", @@ -1906,12 +1256,11 @@ dependencies = [ [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "cxx", - "cxx-build", + "cc", ] [[package]] @@ -1924,16 +1273,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "imara-diff" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98c1d0ad70fc91b8b9654b1f33db55e59579d3b3de2bffdced0fdb810570cb8" -dependencies = [ - "ahash", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -1965,16 +1304,6 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" -[[package]] -name = "io-close" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cadcf447f06744f8ce713d2d6239bb5bde2c357a452397a9ed90c625da390bc" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "io-lifetimes" version = "1.0.10" @@ -2018,22 +1347,13 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" dependencies = [ "wasm-bindgen", ] -[[package]] -name = "kstring" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3066350882a1cd6d950d055997f379ac37fd39f81cd4d8ed186032eb3c5747" -dependencies = [ - "static_assertions", -] - [[package]] name = "lazy_static" version = "1.4.0" @@ -2106,15 +1426,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.142" +version = "0.2.144" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" [[package]] name = "libflate" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97822bf791bd4d5b403713886a5fbe8bf49520fe78e323b0dc480ca1a03e50b0" +checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" dependencies = [ "adler32", "crc32fast", @@ -2130,11 +1450,23 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libgit2-sys" +version = "0.15.1+1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb4577bde8cdfc7d6a2a4bcb7b049598597de33ffd337276e9c7db6cd4a2cee7" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + [[package]] name = "libm" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libmimalloc-sys" @@ -2147,19 +1479,22 @@ dependencies = [ ] [[package]] -name = "link-cplusplus" -version = "1.0.8" +name = "libz-sys" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" +checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" dependencies = [ "cc", + "libc", + "pkg-config", + "vcpkg", ] [[package]] name = "linux-raw-sys" -version = "0.3.4" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" @@ -2226,15 +1561,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" -[[package]] -name = "memmap2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" -dependencies = [ - "libc", -] - [[package]] name = "memoffset" version = "0.8.0" @@ -2259,17 +1585,11 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] @@ -2292,16 +1612,6 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "num" version = "0.4.0" @@ -2389,15 +1699,6 @@ dependencies = [ "libc", ] -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - [[package]] name = "object_store" version = "0.5.6" @@ -2466,9 +1767,9 @@ dependencies = [ [[package]] name = "parquet" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cbd51311f8d9ff3d2697b1522b18a588782e097d313a1a278b0faf2ccf2d3f6" +checksum = "b0a1e6fa27f09ebddba280f5966ef435f3ac4d74cfc3ffe370fd3fd59c2e004d" dependencies = [ "ahash", "arrow-array", @@ -2581,9 +1882,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "ppv-lite86" @@ -2593,12 +1894,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ceca8aaf45b5c46ec7ed39fff75f57290368c1846d33d24a122ca81416ab058" +checksum = "617feabb81566b593beb4886fb8c1f38064169dae4dccad0e3220160c3b37203" dependencies = [ "proc-macro2", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -2609,19 +1910,13 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" dependencies = [ "unicode-ident", ] -[[package]] -name = "prodash" -version = "23.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9516b775656bc3e8985e19cd4b8c0c0de045095074e453d2c0a513b5f978392d" - [[package]] name = "prost" version = "0.11.9" @@ -2761,9 +2056,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" dependencies = [ "proc-macro2", ] @@ -2818,38 +2113,26 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" +checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.1", + "regex-syntax", ] -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" - -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" +checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" [[package]] name = "regress" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d995d590bd8ec096d1893f414bf3f5e8b0ee4c9eed9a5642b9766ef2c8e2e8e9" +checksum = "82a9ecfa0cb04d0b04dddb99b8ccf4f66bc8dfd23df694b398570bd8ae3a50fb" dependencies = [ "hashbrown 0.13.2", "memchr", @@ -2857,9 +2140,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.16" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b71749df584b7f4cac2c426c127a7c785a5106cc98f7a8feb044115f0fa254" +checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" dependencies = [ "base64", "bytes", @@ -2928,9 +2211,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.14" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b864d3c18a5785a05953adeed93e2dca37ed30f18e69bba9f30079d51f363f" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags 1.3.2", "errno", @@ -2942,14 +2225,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.8" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" +checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" dependencies = [ "log", "ring", + "rustls-webpki", "sct", - "webpki", ] [[package]] @@ -2961,6 +2244,16 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -3012,12 +2305,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "scratch" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" - [[package]] name = "sct" version = "0.7.0" @@ -3042,22 +2329,22 @@ checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" [[package]] name = "serde" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" +checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.160" +version = "1.0.163" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" +checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -3084,13 +2371,14 @@ dependencies = [ [[package]] name = "serde_tokenstream" -version = "0.1.7" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "797ba1d80299b264f3aac68ab5d12e5825a561749db4df7cd7c8083900c5d4e9" +checksum = "8a00ffd23fd882d096f09fcaae2a9de8329a328628e86027e049ee051dc1621f" dependencies = [ "proc-macro2", + "quote", "serde", - "syn 1.0.109", + "syn 2.0.16", ] [[package]] @@ -3118,12 +2406,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sha2" version = "0.10.6" @@ -3135,25 +2417,6 @@ dependencies = [ "digest", ] -[[package]] -name = "signal-hook" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "732768f1176d21d09e076c23a93123d40bba92d50c4058da34d45c8de8e682b9" -dependencies = [ - "libc", - "signal-hook-registry", -] - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - [[package]] name = "siphasher" version = "0.3.10" @@ -3267,11 +2530,11 @@ dependencies = [ [[package]] name = "substrait" -version = "0.8.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cd43d44620f716d55d46b998b3cf1baab2935aaa8adc14e3d3d9a465ddae15" +checksum = "9df5d9e071804204172dc77e707c363f187e7f6566f9c78e5100c9a8f5ea434e" dependencies = [ - "gix", + "git2", "heck", "prettyplease", "prost", @@ -3283,16 +2546,16 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.15", + "syn 2.0.16", "typify", "walkdir", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" @@ -3307,9 +2570,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.15" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" dependencies = [ "proc-macro2", "quote", @@ -3318,9 +2581,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.6" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" [[package]] name = "tempfile" @@ -3335,15 +2598,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.40" @@ -3361,7 +2615,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] @@ -3375,35 +2629,6 @@ dependencies = [ "ordered-float", ] -[[package]] -name = "time" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" -dependencies = [ - "itoa", - "libc", - "num_threads", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" - -[[package]] -name = "time-macros" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" -dependencies = [ - "time-core", -] - [[package]] name = "tiny-keccak" version = "2.0.2" @@ -3430,9 +2655,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.27.0" +version = "1.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0de47a4eecbe11f498978a9b29d792f0d2692d1dd003650c24c76510e3bc001" +checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" dependencies = [ "autocfg", "bytes", @@ -3443,36 +2668,35 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a573bdc87985e9d6ddeed1b3d864e8a302c847e40d647746df2f1de209d1ce" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] name = "tokio-stream" -version = "0.1.12" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", "pin-project-lite", @@ -3481,9 +2705,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ "bytes", "futures-core", @@ -3519,14 +2743,14 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.15", + "syn 2.0.16", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -3566,9 +2790,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "typify" -version = "0.0.11" +version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bfde96849e25d7feef1bbf652e9cfc51deb63203fdc07b115b8bc3bcfe20b9" +checksum = "a6658d09e71bfe59e7987dc95ee7f71809fdb5793ab0cdc1503cc0073990484d" dependencies = [ "typify-impl", "typify-macro", @@ -3576,9 +2800,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.11" +version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95d27d749378ceab6ec22188ed7ad102205c89ddb92ab662371c850ffc71aa1a" +checksum = "34d3bb47587b13edf526d6ed02bf360ecefe083ab47a4ef29fc43112828b2bef" dependencies = [ "heck", "log", @@ -3587,16 +2811,16 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 1.0.109", + "syn 2.0.16", "thiserror", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.0.11" +version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35db6fc2bd9220ecdac6eeb88158824b83610de3dda0c6d0f2142b49efd858b0" +checksum = "d3f7e627c18be12d53bc1f261830b9c2763437b6a86ac57293b9085af2d32ffe" dependencies = [ "proc-macro2", "quote", @@ -3604,7 +2828,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 1.0.109", + "syn 2.0.16", "typify-impl", ] @@ -3614,12 +2838,6 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" -[[package]] -name = "unicode-bom" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98e90c70c9f0d4d1ee6d0a7d04aa06cb9bbd53d8cfbdd62a0269a7c2eb640552" - [[package]] name = "unicode-ident" version = "1.0.8" @@ -3678,14 +2896,20 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dad5567ad0cf5b760e5665964bec1b47dfd077ba8a2544b513f3556d3d239a2" +checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" dependencies = [ "getrandom", "serde", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.4" @@ -3720,9 +2944,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3730,24 +2954,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.34" +version = "0.4.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" +checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" dependencies = [ "cfg-if", "js-sys", @@ -3757,9 +2981,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3767,22 +2991,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.16", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" [[package]] name = "wasm-streams" @@ -3799,9 +3023,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.61" +version = "0.3.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" +checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index cad16cc86..8745d9263 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "24.0.0" +version = "25.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "24.0.0" , features = ["pyarrow", "avro"] } -datafusion-common = { version = "24.0.0", features = ["pyarrow"] } -datafusion-expr = "24.0.0" -datafusion-optimizer = "24.0.0" -datafusion-sql = "24.0.0" -datafusion-substrait = "24.0.0" +datafusion = { version = "25.0.0" , features = ["pyarrow", "avro"] } +datafusion-common = { version = "25.0.0", features = ["pyarrow"] } +datafusion-expr = "25.0.0" +datafusion-optimizer = "25.0.0" +datafusion-sql = "25.0.0" +datafusion-substrait = "25.0.0" prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } diff --git a/dev/release/generate-changelog.md b/dev/release/generate-changelog.py old mode 100644 new mode 100755 similarity index 69% rename from dev/release/generate-changelog.md rename to dev/release/generate-changelog.py index caa6ae647..58e6781a3 --- a/dev/release/generate-changelog.md +++ b/dev/release/generate-changelog.py @@ -23,19 +23,27 @@ def print_pulls(repo_name, title, pulls): -if len(pulls) > 0: -print("**{}:**".format(title)) -print() -for (pull, commit) in pulls: -url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) -print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) -print() + if len(pulls) > 0: + print("**{}:**".format(title)) + print() + for (pull, commit) in pulls: + url = "https://github.com/{}/pull/{}".format( + repo_name, pull.number + ) + print( + "- {} [#{}]({}) ({})".format( + pull.title, pull.number, url, commit.author.login + ) + ) + print() def generate_changelog(repo, repo_name, tag1, tag2): # get a list of commits between two tags - print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) + print( + f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr + ) comparison = repo.compare(tag1, tag2) # get the pull requests for these commits @@ -52,7 +60,7 @@ def generate_changelog(repo, repo_name, tag1, tag2): all_pulls.append((pull, commit)) # we split the pulls into categories - #TODO: make categories configurable + # TODO: make categories configurable breaking = [] bugs = [] docs = [] @@ -63,25 +71,25 @@ def generate_changelog(repo, repo_name, tag1, tag2): for (pull, commit) in all_pulls: # see if PR title uses Conventional Commits - cc_type = '' - cc_scope = '' - cc_breaking = '' - parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + cc_type = "" + # cc_scope = '' + cc_breaking = "" + parts = re.findall(r"^([a-z]+)(\([a-z]+\))?(!)?:", pull.title) if len(parts) == 1: parts_tuple = parts[0] - cc_type = parts_tuple[0] # fix, feat, docs, chore - cc_scope = parts_tuple[1] # component within project - cc_breaking = parts_tuple[2] == '!' + cc_type = parts_tuple[0] # fix, feat, docs, chore + # cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == "!" labels = [label.name for label in pull.labels] - #print(pull.number, labels, parts, file=sys.stderr) - if 'api change' in labels or cc_breaking: + # print(pull.number, labels, parts, file=sys.stderr) + if "api change" in labels or cc_breaking: breaking.append((pull, commit)) - elif 'bug' in labels or cc_type == 'fix': + elif "bug" in labels or cc_type == "fix": bugs.append((pull, commit)) - elif 'enhancement' in labels or cc_type == 'feat': + elif "enhancement" in labels or cc_type == "feat": enhancements.append((pull, commit)) - elif 'documentation' in labels or cc_type == 'docs': + elif "documentation" in labels or cc_type == "docs": docs.append((pull, commit)) # produce the changelog content @@ -94,12 +102,14 @@ def generate_changelog(repo, repo_name, tag1, tag2): def cli(args=None): -"""Process command line arguments.""" -if not args: -args = sys.argv[1:] + """Process command line arguments.""" + if not args: + args = sys.argv[1:] parser = argparse.ArgumentParser() - parser.add_argument("project", help="The project name e.g. apache/arrow-datafusion-python") + parser.add_argument( + "project", help="The project name e.g. apache/arrow-datafusion-python" + ) parser.add_argument("tag1", help="The previous release tag") parser.add_argument("tag2", help="The current release tag") args = parser.parse_args() @@ -110,5 +120,6 @@ def cli(args=None): repo = g.get_repo(args.project) generate_changelog(repo, args.project, args.tag1, args.tag2) + if __name__ == "__main__": -cli() \ No newline at end of file + cli() diff --git a/src/expr.rs b/src/expr.rs index c002b3291..1a9adf892 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,17 +15,22 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::DFField; -use datafusion_expr::expr::{AggregateFunction, Sort, WindowFunction}; -use datafusion_expr::utils::exprlist_to_fields; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::PyArrowType; +use datafusion::scalar::ScalarValue; +use datafusion_common::DFField; use datafusion_expr::{ - col, lit, Between, BinaryExpr, Case, Cast, Expr, GetIndexedField, Like, LogicalPlan, Operator, - TryCast, + col, + expr::{ + AggregateFunction, AggregateUDF, InList, InSubquery, ScalarFunction, ScalarUDF, Sort, + WindowFunction, + }, + lit, + utils::exprlist_to_fields, + Between, BinaryExpr, Case, Cast, Expr, GetIndexedField, Like, LogicalPlan, Operator, TryCast, }; use crate::common::data_type::{DataTypeMap, RexType}; @@ -35,7 +40,6 @@ use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; use crate::sql::logical::PyLogicalPlan; -use datafusion::scalar::ScalarValue; use self::alias::PyAlias; use self::bool_expr::{ @@ -360,13 +364,13 @@ impl PyExpr { | Expr::Cast(Cast { expr, .. }) | Expr::TryCast(TryCast { expr, .. }) | Expr::Sort(Sort { expr, .. }) - | Expr::InSubquery { expr, .. } => Ok(vec![PyExpr::from(*expr.clone())]), + | Expr::InSubquery(InSubquery { expr, .. }) => Ok(vec![PyExpr::from(*expr.clone())]), // Expr variants containing a collection of Expr(s) for operands Expr::AggregateFunction(AggregateFunction { args, .. }) - | Expr::AggregateUDF { args, .. } - | Expr::ScalarFunction { args, .. } - | Expr::ScalarUDF { args, .. } + | Expr::AggregateUDF(AggregateUDF { args, .. }) + | Expr::ScalarFunction(ScalarFunction { args, .. }) + | Expr::ScalarUDF(ScalarUDF { args, .. }) | Expr::WindowFunction(WindowFunction { args, .. }) => { Ok(args.iter().map(|arg| PyExpr::from(arg.clone())).collect()) } @@ -394,7 +398,7 @@ impl PyExpr { Ok(operands) } - Expr::InList { expr, list, .. } => { + Expr::InList(InList { expr, list, .. }) => { let mut operands: Vec = vec![PyExpr::from(*expr.clone())]; for list_elem in list { operands.push(PyExpr::from(list_elem.clone())); @@ -451,8 +455,8 @@ impl PyExpr { op, right: _, }) => format!("{op}"), - Expr::ScalarFunction { fun, args: _ } => format!("{fun}"), - Expr::ScalarUDF { fun, .. } => fun.name.clone(), + Expr::ScalarFunction(ScalarFunction { fun, args: _ }) => format!("{fun}"), + Expr::ScalarUDF(ScalarUDF { fun, .. }) => fun.name.clone(), Expr::Cast { .. } => "cast".to_string(), Expr::Between { .. } => "between".to_string(), Expr::Case { .. } => "case".to_string(), diff --git a/src/functions.rs b/src/functions.rs index 8acffeb55..c82361a3b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -18,10 +18,13 @@ use pyo3::{prelude::*, wrap_pyfunction}; use datafusion_common::Column; -use datafusion_expr::expr::AggregateFunction; -use datafusion_expr::expr::{Sort, WindowFunction}; -use datafusion_expr::window_function::find_df_window_func; -use datafusion_expr::{aggregate_function, lit, BuiltinScalarFunction, Expr, WindowFrame}; +use datafusion_expr::{ + aggregate_function, + expr::{AggregateFunction, ScalarFunction, Sort, WindowFunction}, + lit, + window_function::find_df_window_func, + BuiltinScalarFunction, Expr, WindowFrame, +}; use crate::errors::DataFusionError; use crate::expr::PyExpr; @@ -106,6 +109,7 @@ fn count_star() -> PyResult { args: vec![lit(1)], distinct: false, filter: None, + order_by: None, }), }) } @@ -153,10 +157,10 @@ macro_rules! scalar_function { #[pyfunction] #[pyo3(signature = (*args))] fn $NAME(args: Vec) -> PyExpr { - let expr = datafusion_expr::Expr::ScalarFunction { + let expr = datafusion_expr::Expr::ScalarFunction(ScalarFunction { fun: BuiltinScalarFunction::$FUNC, args: args.into_iter().map(|e| e.into()).collect(), - }; + }); expr.into() } }; @@ -176,6 +180,7 @@ macro_rules! aggregate_function { args: args.into_iter().map(|e| e.into()).collect(), distinct, filter: None, + order_by: None, }); expr.into() } diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index d35101237..64124fb12 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -22,7 +22,7 @@ use std::convert::TryFrom; use std::result::Result; use datafusion_common::{Column, ScalarValue}; -use datafusion_expr::{Between, BinaryExpr, Expr, Operator}; +use datafusion_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; use crate::errors::DataFusionError; @@ -161,11 +161,11 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { Ok(if *negated { invert.call1((ret,))? } else { ret }) } - Expr::InList { + Expr::InList(InList { expr, list, negated, - } => { + }) => { let expr = PyArrowFilterExpression::try_from(expr.as_ref())? .0 .into_ref(py); From d912db5ea6ba5fc373a8412e505399bfe21c9a90 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Fri, 26 May 2023 18:44:34 -0400 Subject: [PATCH 048/413] Add Expr::Case when_then_else support to rex_call_operands function (#388) * Add Expr::Case when_then_else support to rex_call_operands function * Update gitignore and formatting * Update gitignore and formatting --- .gitignore | 5 ++++- src/expr.rs | 19 +++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 1d0a84a43..365b89d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ __pycache__/ *.py[cod] *$py.class +# Python dist ignore +dist + # C extensions *.so @@ -24,4 +27,4 @@ apache-rat-*.jar .env CHANGELOG.md.bak -docs/mdbook/book \ No newline at end of file +docs/mdbook/book diff --git a/src/expr.rs b/src/expr.rs index 1a9adf892..819edff2a 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -384,14 +384,21 @@ impl PyExpr { let mut operands: Vec = Vec::new(); if let Some(e) = expr { - operands.push(PyExpr::from(*e.clone())); + for (when, then) in when_then_expr { + operands.push(PyExpr::from(Expr::BinaryExpr(BinaryExpr::new( + Box::new(*e.clone()), + Operator::Eq, + Box::new(*when.clone()), + )))); + operands.push(PyExpr::from(*then.clone())); + } + } else { + for (when, then) in when_then_expr { + operands.push(PyExpr::from(*when.clone())); + operands.push(PyExpr::from(*then.clone())); + } }; - for (when, then) in when_then_expr { - operands.push(PyExpr::from(*when.clone())); - operands.push(PyExpr::from(*then.clone())); - } - if let Some(e) = else_expr { operands.push(PyExpr::from(*e.clone())); }; From 5664a1e38f8b45af15afd60bcef841df01da655e Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Fri, 26 May 2023 18:45:05 -0400 Subject: [PATCH 049/413] Introduce BaseSessionContext abstract class (#390) --- datafusion/context.py | 45 +++++++++++++++++++++++++++ datafusion/cudf.py | 14 ++++++--- datafusion/pandas.py | 14 ++++++--- datafusion/polars.py | 14 ++++++--- docs/mdbook/src/usage/create-table.md | 2 +- examples/sql-on-cudf.py | 2 +- examples/sql-on-pandas.py | 2 +- examples/sql-on-polars.py | 2 +- 8 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 datafusion/context.py diff --git a/datafusion/context.py b/datafusion/context.py new file mode 100644 index 000000000..aa9c9a8af --- /dev/null +++ b/datafusion/context.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod + + +class BaseSessionContext(ABC): + """ + Abstraction defining all methods, properties, and common functionality + shared amongst implementations using DataFusion as their SQL Parser/Engine + """ + + @abstractmethod + def register_table( + self, + table_name: str, + path: str, + **kwargs, + ): + pass + + # TODO: Remove abstraction, this functionality can be shared + # between all implementing classes since it just prints the + # logical plan from DataFusion + @abstractmethod + def explain(self, sql): + pass + + @abstractmethod + def sql(self, sql): + pass diff --git a/datafusion/cudf.py b/datafusion/cudf.py index d5f02156f..594e5efea 100644 --- a/datafusion/cudf.py +++ b/datafusion/cudf.py @@ -17,18 +17,15 @@ import cudf import datafusion +from datafusion.context import BaseSessionContext from datafusion.expr import Projection, TableScan, Column -class SessionContext: +class SessionContext(BaseSessionContext): def __init__(self): self.datafusion_ctx = datafusion.SessionContext() self.parquet_tables = {} - def register_parquet(self, name, path): - self.parquet_tables[name] = path - self.datafusion_ctx.register_parquet(name, path) - def to_cudf_expr(self, expr): # get Python wrapper for logical expression expr = expr.to_variant() @@ -55,6 +52,13 @@ def to_cudf_df(self, plan): "unsupported logical operator: {}".format(type(node)) ) + def register_table(self, name, path, **kwargs): + self.parquet_tables[name] = path + self.datafusion_ctx.register_parquet(name, path) + + def explain(self, sql): + super.explain() + def sql(self, sql): datafusion_df = self.datafusion_ctx.sql(sql) plan = datafusion_df.logical_plan() diff --git a/datafusion/pandas.py b/datafusion/pandas.py index f8e56512b..935d9619b 100644 --- a/datafusion/pandas.py +++ b/datafusion/pandas.py @@ -17,18 +17,15 @@ import pandas as pd import datafusion +from datafusion.context import BaseSessionContext from datafusion.expr import Projection, TableScan, Column -class SessionContext: +class SessionContext(BaseSessionContext): def __init__(self): self.datafusion_ctx = datafusion.SessionContext() self.parquet_tables = {} - def register_parquet(self, name, path): - self.parquet_tables[name] = path - self.datafusion_ctx.register_parquet(name, path) - def to_pandas_expr(self, expr): # get Python wrapper for logical expression expr = expr.to_variant() @@ -55,6 +52,13 @@ def to_pandas_df(self, plan): "unsupported logical operator: {}".format(type(node)) ) + def register_table(self, name, path, **kwargs): + self.parquet_tables[name] = path + self.datafusion_ctx.register_parquet(name, path) + + def explain(self, sql): + super.explain() + def sql(self, sql): datafusion_df = self.datafusion_ctx.sql(sql) plan = datafusion_df.logical_plan() diff --git a/datafusion/polars.py b/datafusion/polars.py index a1bafbef8..bbc1fd7c2 100644 --- a/datafusion/polars.py +++ b/datafusion/polars.py @@ -17,19 +17,16 @@ import polars import datafusion +from datafusion.context import BaseSessionContext from datafusion.expr import Projection, TableScan, Aggregate from datafusion.expr import Column, AggregateFunction -class SessionContext: +class SessionContext(BaseSessionContext): def __init__(self): self.datafusion_ctx = datafusion.SessionContext() self.parquet_tables = {} - def register_parquet(self, name, path): - self.parquet_tables[name] = path - self.datafusion_ctx.register_parquet(name, path) - def to_polars_expr(self, expr): # get Python wrapper for logical expression expr = expr.to_variant() @@ -78,6 +75,13 @@ def to_polars_df(self, plan): "unsupported logical operator: {}".format(type(node)) ) + def register_table(self, name, path, **kwargs): + self.parquet_tables[name] = path + self.datafusion_ctx.register_parquet(name, path) + + def explain(self, sql): + super.explain() + def sql(self, sql): datafusion_df = self.datafusion_ctx.sql(sql) plan = datafusion_df.logical_plan() diff --git a/docs/mdbook/src/usage/create-table.md b/docs/mdbook/src/usage/create-table.md index 332863a16..98870fac0 100644 --- a/docs/mdbook/src/usage/create-table.md +++ b/docs/mdbook/src/usage/create-table.md @@ -55,5 +55,5 @@ ctx.register_csv("csv_1e8", "G1_1e8_1e2_0_0.csv") You can read a Parquet file into a DataFusion DataFrame. Here's how to read the `yellow_tripdata_2021-01.parquet` file into a table named `taxi`. ```python -ctx.register_parquet("taxi", "yellow_tripdata_2021-01.parquet") +ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") ``` diff --git a/examples/sql-on-cudf.py b/examples/sql-on-cudf.py index 999756fc8..b64d8f046 100644 --- a/examples/sql-on-cudf.py +++ b/examples/sql-on-cudf.py @@ -19,6 +19,6 @@ ctx = SessionContext() -ctx.register_parquet("taxi", "yellow_tripdata_2021-01.parquet") +ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") df = ctx.sql("select passenger_count from taxi") print(df) diff --git a/examples/sql-on-pandas.py b/examples/sql-on-pandas.py index 0efd77631..e3312a201 100644 --- a/examples/sql-on-pandas.py +++ b/examples/sql-on-pandas.py @@ -19,6 +19,6 @@ ctx = SessionContext() -ctx.register_parquet("taxi", "yellow_tripdata_2021-01.parquet") +ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") df = ctx.sql("select passenger_count from taxi") print(df) diff --git a/examples/sql-on-polars.py b/examples/sql-on-polars.py index c208114c1..dd7a9e021 100644 --- a/examples/sql-on-polars.py +++ b/examples/sql-on-polars.py @@ -19,7 +19,7 @@ ctx = SessionContext() -ctx.register_parquet("taxi", "yellow_tripdata_2021-01.parquet") +ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") df = ctx.sql( "select passenger_count, count(*) from taxi group by passenger_count" ) From 931cabc629a28bb47c030e7e17de59a19feb38dd Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 30 May 2023 14:51:27 -0400 Subject: [PATCH 050/413] CRUD Schema support for `BaseSessionContext` (#392) * checkpoint commit * Introduce BaseSessionContext abstract class * Introduce abstract methods for CRUD schema operations * Clean up schema.rs file --- .gitignore | 3 + datafusion/context.py | 52 ++++++++++ datafusion/cudf.py | 42 +++++++- datafusion/pandas.py | 32 +++++- datafusion/polars.py | 22 +++- src/common.rs | 7 ++ src/common/function.rs | 55 ++++++++++ src/common/schema.rs | 222 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 src/common/function.rs create mode 100644 src/common/schema.rs diff --git a/.gitignore b/.gitignore index 365b89d5c..0030b907b 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,9 @@ dist # C extensions *.so +# Python dist +dist + # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: diff --git a/datafusion/context.py b/datafusion/context.py index aa9c9a8af..a36462123 100644 --- a/datafusion/context.py +++ b/datafusion/context.py @@ -16,6 +16,9 @@ # under the License. from abc import ABC, abstractmethod +from typing import Dict + +from datafusion.common import SqlSchema class BaseSessionContext(ABC): @@ -24,6 +27,55 @@ class BaseSessionContext(ABC): shared amongst implementations using DataFusion as their SQL Parser/Engine """ + DEFAULT_CATALOG_NAME = "root" + DEFAULT_SCHEMA_NAME = "datafusion" + + @abstractmethod + def create_schema( + self, + schema_name: str, + **kwargs, + ): + """ + Creates/Registers a logical container that holds database + objects such as tables, views, indexes, and other + related objects. It provides a way to group related database + objects together. A schema can be owned by a database + user and can be used to separate objects in different + logical groups for easy management. + """ + pass + + @abstractmethod + def update_schema( + self, + schema_name: str, + new_schema: SqlSchema, + **kwargs, + ): + """ + Updates an existing schema in the SessionContext + """ + pass + + @abstractmethod + def drop_schema( + self, + schema_name: str, + **kwargs, + ): + """ + Drops the specified Schema, based on name, from the current context + """ + pass + + @abstractmethod + def show_schemas(self, **kwargs) -> Dict[str, SqlSchema]: + """ + Return all schemas in the current SessionContext impl. + """ + pass + @abstractmethod def register_table( self, diff --git a/datafusion/cudf.py b/datafusion/cudf.py index 594e5efea..e39daea31 100644 --- a/datafusion/cudf.py +++ b/datafusion/cudf.py @@ -15,16 +15,36 @@ # specific language governing permissions and limitations # under the License. +import logging import cudf -import datafusion from datafusion.context import BaseSessionContext from datafusion.expr import Projection, TableScan, Column +from datafusion.common import SqlSchema + +logger = logging.getLogger(__name__) + class SessionContext(BaseSessionContext): - def __init__(self): - self.datafusion_ctx = datafusion.SessionContext() - self.parquet_tables = {} + def __init__(self, context, logging_level=logging.INFO): + """ + Create a new Session. + """ + # Cudf requires a provided context + self.context = context + + # Set the logging level for this SQL context + logging.basicConfig(level=logging_level) + + # Name of the root catalog + self.catalog_name = self.DEFAULT_CATALOG_NAME + # Name of the root schema + self.schema_name = self.DEFAULT_SCHEMA_NAME + # Add the schema to the context + sch = SqlSchema(self.schema_name) + self.schemas = {} + self.schemas[self.schema_name] = sch + self.context.register_schema(self.schema_name, sch) def to_cudf_expr(self, expr): # get Python wrapper for logical expression @@ -52,6 +72,20 @@ def to_cudf_df(self, plan): "unsupported logical operator: {}".format(type(node)) ) + def create_schema(self, schema_name: str, **kwargs): + logger.debug(f"Creating schema: {schema_name}") + self.schemas[schema_name] = SqlSchema(schema_name) + self.context.register_schema(schema_name, SqlSchema(schema_name)) + + def update_schema(self, schema_name: str, new_schema: SqlSchema, **kwargs): + self.schemas[schema_name] = new_schema + + def drop_schema(self, schema_name, **kwargs): + del self.schemas[schema_name] + + def show_schemas(self, **kwargs): + return self.schemas + def register_table(self, name, path, **kwargs): self.parquet_tables[name] = path self.datafusion_ctx.register_parquet(name, path) diff --git a/datafusion/pandas.py b/datafusion/pandas.py index 935d9619b..c2da83ff6 100644 --- a/datafusion/pandas.py +++ b/datafusion/pandas.py @@ -15,17 +15,33 @@ # specific language governing permissions and limitations # under the License. +import logging import pandas as pd import datafusion +from datafusion.common import SqlSchema from datafusion.context import BaseSessionContext from datafusion.expr import Projection, TableScan, Column +logger = logging.getLogger(__name__) + class SessionContext(BaseSessionContext): - def __init__(self): + def __init__(self, logging_level=logging.INFO): self.datafusion_ctx = datafusion.SessionContext() self.parquet_tables = {} + # Set the logging level for this SQL context + logging.basicConfig(level=logging_level) + + # Name of the root catalog + self.catalog_name = self.DEFAULT_CATALOG_NAME + # Name of the root schema + self.schema_name = self.DEFAULT_SCHEMA_NAME + # Add the schema to the context + sch = SqlSchema(self.schema_name) + self.schemas[self.schema_name] = sch + self.context.register_schema(self.schema_name, sch) + def to_pandas_expr(self, expr): # get Python wrapper for logical expression expr = expr.to_variant() @@ -52,6 +68,20 @@ def to_pandas_df(self, plan): "unsupported logical operator: {}".format(type(node)) ) + def create_schema(self, schema_name: str, **kwargs): + logger.debug(f"Creating schema: {schema_name}") + self.schemas[schema_name] = SqlSchema(schema_name) + self.context.register_schema(schema_name, SqlSchema(schema_name)) + + def update_schema(self, schema_name: str, new_schema: SqlSchema, **kwargs): + self.schemas[schema_name] = new_schema + + def drop_schema(self, schema_name, **kwargs): + del self.schemas[schema_name] + + def show_schemas(self, **kwargs): + return self.schemas + def register_table(self, name, path, **kwargs): self.parquet_tables[name] = path self.datafusion_ctx.register_parquet(name, path) diff --git a/datafusion/polars.py b/datafusion/polars.py index bbc1fd7c2..e4eb966fc 100644 --- a/datafusion/polars.py +++ b/datafusion/polars.py @@ -14,16 +14,20 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import logging import polars import datafusion from datafusion.context import BaseSessionContext from datafusion.expr import Projection, TableScan, Aggregate from datafusion.expr import Column, AggregateFunction +from datafusion.common import SqlSchema + +logger = logging.getLogger(__name__) + class SessionContext(BaseSessionContext): - def __init__(self): + def __init__(self, logging_level=logging.INFO): self.datafusion_ctx = datafusion.SessionContext() self.parquet_tables = {} @@ -75,6 +79,20 @@ def to_polars_df(self, plan): "unsupported logical operator: {}".format(type(node)) ) + def create_schema(self, schema_name: str, **kwargs): + logger.debug(f"Creating schema: {schema_name}") + self.schemas[schema_name] = SqlSchema(schema_name) + self.context.register_schema(schema_name, SqlSchema(schema_name)) + + def update_schema(self, schema_name: str, new_schema: SqlSchema, **kwargs): + self.schemas[schema_name] = new_schema + + def drop_schema(self, schema_name, **kwargs): + del self.schemas[schema_name] + + def show_schemas(self, **kwargs): + return self.schemas + def register_table(self, name, path, **kwargs): self.parquet_tables[name] = path self.datafusion_ctx.register_parquet(name, path) diff --git a/src/common.rs b/src/common.rs index 8a8e2adf5..45523173c 100644 --- a/src/common.rs +++ b/src/common.rs @@ -20,6 +20,8 @@ use pyo3::prelude::*; pub mod data_type; pub mod df_field; pub mod df_schema; +pub mod function; +pub mod schema; /// Initializes the `common` module to match the pattern of `datafusion-common` https://docs.rs/datafusion-common/18.0.0/datafusion_common/index.html pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { @@ -29,5 +31,10 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/src/common/function.rs b/src/common/function.rs new file mode 100644 index 000000000..a8d752f16 --- /dev/null +++ b/src/common/function.rs @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; + +use datafusion::arrow::datatypes::DataType; +use pyo3::prelude::*; + +use super::data_type::PyDataType; + +#[pyclass(name = "SqlFunction", module = "datafusion.common", subclass)] +#[derive(Debug, Clone)] +pub struct SqlFunction { + pub name: String, + pub return_types: HashMap, DataType>, + pub aggregation: bool, +} + +impl SqlFunction { + pub fn new( + function_name: String, + input_types: Vec, + return_type: PyDataType, + aggregation_bool: bool, + ) -> Self { + let mut func = Self { + name: function_name, + return_types: HashMap::new(), + aggregation: aggregation_bool, + }; + func.add_type_mapping(input_types, return_type); + func + } + + pub fn add_type_mapping(&mut self, input_types: Vec, return_type: PyDataType) { + self.return_types.insert( + input_types.iter().map(|t| t.clone().into()).collect(), + return_type.into(), + ); + } +} diff --git a/src/common/schema.rs b/src/common/schema.rs new file mode 100644 index 000000000..304319369 --- /dev/null +++ b/src/common/schema.rs @@ -0,0 +1,222 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; + +use datafusion::arrow::datatypes::SchemaRef; +use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource}; +use pyo3::prelude::*; + +use datafusion_optimizer::utils::split_conjunction; + +use super::{data_type::DataTypeMap, function::SqlFunction}; + +#[pyclass(name = "SqlSchema", module = "datafusion.common", subclass)] +#[derive(Debug, Clone)] +pub struct SqlSchema { + #[pyo3(get, set)] + pub name: String, + #[pyo3(get, set)] + pub tables: Vec, + #[pyo3(get, set)] + pub views: Vec, + #[pyo3(get, set)] + pub functions: Vec, +} + +#[pyclass(name = "SqlTable", module = "datafusion.common", subclass)] +#[derive(Debug, Clone)] +pub struct SqlTable { + #[pyo3(get, set)] + pub name: String, + #[pyo3(get, set)] + pub columns: Vec<(String, DataTypeMap)>, + #[pyo3(get, set)] + pub primary_key: Option, + #[pyo3(get, set)] + pub foreign_keys: Vec, + #[pyo3(get, set)] + pub indexes: Vec, + #[pyo3(get, set)] + pub constraints: Vec, + #[pyo3(get, set)] + pub statistics: SqlStatistics, + #[pyo3(get, set)] + pub filepath: Option, +} + +#[pymethods] +impl SqlTable { + #[new] + pub fn new( + _schema_name: String, + table_name: String, + columns: Vec<(String, DataTypeMap)>, + row_count: f64, + filepath: Option, + ) -> Self { + Self { + name: table_name, + columns, + primary_key: None, + foreign_keys: Vec::new(), + indexes: Vec::new(), + constraints: Vec::new(), + statistics: SqlStatistics::new(row_count), + filepath, + } + } +} + +#[pyclass(name = "SqlView", module = "datafusion.common", subclass)] +#[derive(Debug, Clone)] +pub struct SqlView { + #[pyo3(get, set)] + pub name: String, + #[pyo3(get, set)] + pub definition: String, // SQL code that defines the view +} + +#[pymethods] +impl SqlSchema { + #[new] + pub fn new(schema_name: &str) -> Self { + Self { + name: schema_name.to_owned(), + tables: Vec::new(), + views: Vec::new(), + functions: Vec::new(), + } + } + + pub fn table_by_name(&self, table_name: &str) -> Option { + for tbl in &self.tables { + if tbl.name.eq(table_name) { + return Some(tbl.clone()); + } + } + None + } + + pub fn add_table(&mut self, table: SqlTable) { + self.tables.push(table); + } +} + +/// SqlTable wrapper that is compatible with DataFusion logical query plans +pub struct SqlTableSource { + schema: SchemaRef, + statistics: Option, + filepath: Option, +} + +impl SqlTableSource { + /// Initialize a new `EmptyTable` from a schema + pub fn new( + schema: SchemaRef, + statistics: Option, + filepath: Option, + ) -> Self { + Self { + schema, + statistics, + filepath, + } + } + + /// Access optional statistics associated with this table source + pub fn statistics(&self) -> Option<&SqlStatistics> { + self.statistics.as_ref() + } + + /// Access optional filepath associated with this table source + #[allow(dead_code)] + pub fn filepath(&self) -> Option<&String> { + self.filepath.as_ref() + } +} + +/// Implement TableSource, used in the logical query plan and in logical query optimizations +impl TableSource for SqlTableSource { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + self.schema.clone() + } + + fn supports_filter_pushdown( + &self, + filter: &Expr, + ) -> datafusion_common::Result { + let filters = split_conjunction(filter); + if filters.iter().all(|f| is_supported_push_down_expr(f)) { + // Push down filters to the tablescan operation if all are supported + Ok(TableProviderFilterPushDown::Exact) + } else if filters.iter().any(|f| is_supported_push_down_expr(f)) { + // Partially apply the filter in the TableScan but retain + // the Filter operator in the plan as well + Ok(TableProviderFilterPushDown::Inexact) + } else { + Ok(TableProviderFilterPushDown::Unsupported) + } + } + + fn table_type(&self) -> datafusion_expr::TableType { + datafusion_expr::TableType::Base + } + + #[allow(deprecated)] + fn supports_filters_pushdown( + &self, + filters: &[&Expr], + ) -> datafusion_common::Result> { + filters + .iter() + .map(|f| self.supports_filter_pushdown(f)) + .collect() + } + + fn get_logical_plan(&self) -> Option<&datafusion_expr::LogicalPlan> { + None + } +} + +fn is_supported_push_down_expr(_expr: &Expr) -> bool { + // For now we support all kinds of expr's at this level + true +} + +#[pyclass(name = "SqlStatistics", module = "datafusion.common", subclass)] +#[derive(Debug, Clone)] +pub struct SqlStatistics { + row_count: f64, +} + +#[pymethods] +impl SqlStatistics { + #[new] + pub fn new(row_count: f64) -> Self { + Self { row_count } + } + + #[pyo3(name = "getRowCount")] + pub fn get_row_count(&self) -> f64 { + self.row_count + } +} From 51158bd495d215d2260c091af7c0109dbe3b9432 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 30 May 2023 18:21:22 -0400 Subject: [PATCH 051/413] CRUD Table support for `BaseSessionContext` (#394) * checkpoint commit * Introduce BaseSessionContext abstract class * Introduce abstract methods for CRUD schema operations * Clean up schema.rs file * Introduce CRUD methods for table instances * Add function to drop_table * Add schema_name to drop_table function * remove unused parameter in SqlTable new * Update function to allow for modifying existing tables --- datafusion/context.py | 48 +++++++++++++++++++++++++++++++++++++++++-- src/common/schema.rs | 5 ++++- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/datafusion/context.py b/datafusion/context.py index a36462123..6292177f7 100644 --- a/datafusion/context.py +++ b/datafusion/context.py @@ -16,9 +16,9 @@ # under the License. from abc import ABC, abstractmethod -from typing import Dict +from typing import Dict, List -from datafusion.common import SqlSchema +from datafusion.common import SqlSchema, SqlTable class BaseSessionContext(ABC): @@ -76,6 +76,50 @@ def show_schemas(self, **kwargs) -> Dict[str, SqlSchema]: """ pass + @abstractmethod + def create_table( + self, + table_name: str, + schema_name: str = None, + **kwargs, + ): + """ + Creates/Registers a table in the specied schema instance + """ + pass + + @abstractmethod + def update_table( + self, + schema_name: str, + table_name: str, + new_table: SqlTable, + **kwargs, + ): + """ + Updates an existing table in the SessionContext + """ + pass + + @abstractmethod + def drop_table( + self, + schema_name: str, + table_name: str, + **kwargs, + ): + """ + Drops the specified table, based on name, from the current context + """ + pass + + @abstractmethod + def show_tables(self, **kwargs) -> List[SqlTable]: + """ + Return all tables in the current SessionContext impl. + """ + pass + @abstractmethod def register_table( self, diff --git a/src/common/schema.rs b/src/common/schema.rs index 304319369..a003d0ca1 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -63,7 +63,6 @@ pub struct SqlTable { impl SqlTable { #[new] pub fn new( - _schema_name: String, table_name: String, columns: Vec<(String, DataTypeMap)>, row_count: f64, @@ -115,6 +114,10 @@ impl SqlSchema { pub fn add_table(&mut self, table: SqlTable) { self.tables.push(table); } + + pub fn drop_table(&mut self, table_name: String) { + self.tables.retain(|x| !x.name.eq(&table_name)); + } } /// SqlTable wrapper that is compatible with DataFusion logical query plans From 1174969f70ae938f12a1a0372770b994f85b0014 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 11 Jun 2023 10:31:34 -0600 Subject: [PATCH 052/413] Prepare for 26.0.0 release (#410) * Upgrade to DF 26 * cargo update * changelog * python lint --- CHANGELOG.md | 11 + Cargo.lock | 436 ++++++++++++----------------- Cargo.toml | 16 +- datafusion/tests/test_dataframe.py | 1 - datafusion/tests/test_substrait.py | 4 +- dev/release/generate-changelog.py | 6 +- src/common/df_field.rs | 2 +- src/expr.rs | 6 +- src/expr/scalar_function.rs | 2 +- src/substrait.rs | 6 +- 10 files changed, 209 insertions(+), 281 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6d58e4a8..d2f2de219 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,17 @@ # DataFusion Python Changelog +## [26.0.0](https://github.com/apache/arrow-datafusion-python/tree/26.0.0) (2023-06-11) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/25.0.0...26.0.0) + +**Merged pull requests:** + +- Add Expr::Case when_then_else support to rex_call_operands function [#388](https://github.com/apache/arrow-datafusion-python/pull/388) (jdye64) +- Introduce BaseSessionContext abstract class [#390](https://github.com/apache/arrow-datafusion-python/pull/390) (jdye64) +- CRUD Schema support for `BaseSessionContext` [#392](https://github.com/apache/arrow-datafusion-python/pull/392) (jdye64) +- CRUD Table support for `BaseSessionContext` [#394](https://github.com/apache/arrow-datafusion-python/pull/394) (jdye64) + ## [25.0.0](https://github.com/apache/arrow-datafusion-python/tree/25.0.0) (2023-05-23) [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/24.0.0...25.0.0) diff --git a/Cargo.lock b/Cargo.lock index d8fa65e64..2392f5796 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr", ] @@ -51,6 +51,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -107,9 +113,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "218ca81dd088b102c0fd6687c72e73fad1ba93d2ef7b3cf9a1043b04b2c39dbf" +checksum = "6619cab21a0cdd8c9b9f1d9e09bfaa9b1974e5ef809a6566aef0b998caf38ace" dependencies = [ "ahash", "arrow-arith", @@ -130,9 +136,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d49309fa2299ec34a709cfc9f487c41ecaead96d1ab70e21857466346bbbd690" +checksum = "e0dc95485623a76e00929bda8caa40c1f838190952365c4f43a7b9ae86d03e94" dependencies = [ "arrow-array", "arrow-buffer", @@ -145,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a27466d897d99654357a6d95dc0a26931d9e4306e60c14fc31a894edb86579" +checksum = "3267847f53d3042473cfd2c769afd8d74a6d7d201fc3a34f5cb84c0282ef47a7" dependencies = [ "ahash", "arrow-buffer", @@ -162,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9405b78106a9d767c7b97c78a70ee1b23ee51a74f5188a821a716d9a85d1af2b" +checksum = "c5f66553e66e120ac4b21570368ee9ebf35ff3f5399f872b0667699e145678f5" dependencies = [ "half", "num", @@ -172,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0ec5a79a87783dc828b7ff8f89f62880b3f553bc5f5b932a82f4a1035024b4" +checksum = "65e6f3579dbf0d97c683d451b2550062b0f0e62a3169bf74238b5f59f44ad6d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -189,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350d8e55c3b2d602a0a04389bcc1da40167657143a9922a7103190603e7b7692" +checksum = "373579c4c1a8f5307d3125b7a89c700fcf8caf85821c77eb4baab3855ae0aba5" dependencies = [ "arrow-array", "arrow-buffer", @@ -208,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f710d98964d2c069b8baf566130045e79e11baa105623f038a6c942f805681" +checksum = "61bc8df9912cca6642665fdf989d6fa0de2570f18a7f709bcf59d29de96d2097" dependencies = [ "arrow-buffer", "arrow-schema", @@ -220,9 +226,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c99787cb8fabc187285da9e7182d22f2b80ecfac61ca0a42c4299e9eecdf903" +checksum = "0105dcf5f91daa7182d87b713ee0b32b3bfc88e0c48e7dc3e9d6f1277a07d1ae" dependencies = [ "arrow-array", "arrow-buffer", @@ -234,9 +240,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c95a58ce63f60d80d7a3a1222d65df0bc060b71d31353c34a8118c2a6eae7b" +checksum = "e73134fb5b5ec8770f8cbb214c2c487b2d350081e403ca4eeeb6f8f5e19846ac" dependencies = [ "arrow-array", "arrow-buffer", @@ -254,9 +260,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4141e6488610cc144e841da3de5f5371488f3cf5bc6bc7b3e752c64e7639c31b" +checksum = "89f25bc66e18d4c2aa1fe2f9bb03e2269da60e636213210385ae41a107f9965a" dependencies = [ "arrow-array", "arrow-buffer", @@ -269,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940191a3c636c111c41e816325b0941484bf904c46de72cd9553acd1afd24d33" +checksum = "1095ff85ea4f5ff02d17b30b089de31b51a50be01c6b674f0a0509ab771232f1" dependencies = [ "ahash", "arrow-array", @@ -284,18 +290,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18c41d058b2895a12f46dfafc306ee3529ad9660406be0ab8a7967d5e27c417e" +checksum = "25187bbef474151a2e4ddec67b9e34bda5cbfba292dc571392fa3a1f71ff5a82" dependencies = [ "bitflags 2.3.1", ] [[package]] name = "arrow-select" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fcbdda2772b7e712e77444f3a71f4ee517095aceb993b35de71de41c70d9b4f" +checksum = "fd0d4ee884aec3aa05e41478e3cd312bf609de9babb5d187a43fb45931da4da4" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7081c34f4b534ad320a03db79d58e38972041bb7c65686b98bbcc2f9a67a9cee" +checksum = "d6d71c3ffe4c07e66ce8fdc6aed5b00e0e60c5144911879b10546f5b72d8fa1c" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,9 +327,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.3.15" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" dependencies = [ "bzip2", "flate2", @@ -333,8 +339,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.11.2+zstd.1.5.2", - "zstd-safe 5.0.2+zstd.1.5.2", + "zstd", + "zstd-safe", ] [[package]] @@ -345,7 +351,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -356,7 +362,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -376,9 +382,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105" +checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" [[package]] name = "bitflags" @@ -403,9 +409,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" +checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" dependencies = [ "arrayref", "arrayvec", @@ -501,12 +507,12 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.24" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", - "num-integer", "num-traits", "serde", "winapi", @@ -536,9 +542,9 @@ dependencies = [ [[package]] name = "comfy-table" -version = "6.1.4" +version = "6.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7b787b0dc42e8111badfdbe4c3059158ccb2db8780352fa1b01e8ccf45cc4d" +checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" dependencies = [ "strum", "strum_macros", @@ -569,9 +575,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" +checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" [[package]] name = "core-foundation-sys" @@ -615,9 +621,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", "itoa", @@ -649,9 +655,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a99e1dc003c0b310b203e4dc2aadc6531dccb37fa681630b588c5154d1b7637" +checksum = "9992c267436551d40b52d65289b144712e7b0ebdc62c8c859fd1574e5f73efbb" dependencies = [ "ahash", "apache-avro", @@ -696,14 +702,14 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.12.3+zstd.1.5.2", + "zstd", ] [[package]] name = "datafusion-common" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "362811fd9cb830877f5a538a19e5f9a1e5519486dcd6dc4f39a3d6192d2f4ba6" +checksum = "c3be97f7a7c720cdbb71e9eeabf814fa6ad8102b9022390f6cac74d3b4af6392" dependencies = [ "apache-avro", "arrow", @@ -718,9 +724,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b052014fe38299019dd2e7dec6c2dd44c7ad9a5375ab4a36e5ed800700d7eb7a" +checksum = "c77c4b14b809b0e4c5bb101b6834504f06cdbb0d3c643400c61d0d844b33264e" dependencies = [ "dashmap", "datafusion-common", @@ -736,21 +742,24 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55b590b184c6f59a65ee3ada666475ff8173bfd409c618c3c29b5abf7ef4a92c" +checksum = "e6ec7409bd45cf4fae6395d7d1024c8a97e543cadc88363e405d2aad5330e5e7" dependencies = [ "ahash", "arrow", "datafusion-common", + "lazy_static", "sqlparser", + "strum", + "strum_macros", ] [[package]] name = "datafusion-optimizer" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "543311151a4c068b2b663f07905dd4d4a605279dacb24ab9434f8f29a1917cd3" +checksum = "64b537c93f87989c212db92a448a0f5eb4f0995e27199bb7687ae94f8b64a7a8" dependencies = [ "arrow", "async-trait", @@ -766,9 +775,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49cfd92e225d948a2199650e8e292ef1f0a3d62ab9b540795c3d3ba9edc2b2e" +checksum = "f60ee3f53340fdef36ee54d9e12d446ae2718b1d0196ac581f791d34808ec876" dependencies = [ "ahash", "arrow", @@ -799,7 +808,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "25.0.0" +version = "26.0.0" dependencies = [ "async-trait", "datafusion", @@ -818,7 +827,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.16", + "syn 2.0.18", "tokio", "url", "uuid", @@ -826,9 +835,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e98cdd6f0d005bce0035bebe03a35ccb7eca3cf40a9bce27e24b1e2a56941ffc" +checksum = "d58fc64058aa3bcb00077a0d19474a0d584d31dec8c7ac3406868f485f659af9" dependencies = [ "arrow", "datafusion-common", @@ -838,9 +847,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9585219bbbad5287c762ad9ede044ab6c0b778fde980c9858b15d6dd5bd80f35" +checksum = "1531f0314151a34bf6c0a83c7261525688b7c729876f53e7896b8f4ca8f57d07" dependencies = [ "arrow", "arrow-schema", @@ -852,9 +861,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "25.0.0" +version = "26.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0f1a60b63cca4fb2efbf572c3f2875bccbb2dfe448dd562f103ca5d9830216" +checksum = "079d5be5ec59580777bfa16d79187fea99b6498e3e8e07eb36d504a5fe708f13" dependencies = [ "async-recursion", "chrono", @@ -862,6 +871,7 @@ dependencies = [ "itertools", "object_store", "prost", + "prost-types", "substrait", "tokio", ] @@ -912,7 +922,7 @@ checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -942,9 +952,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.1.21" +version = "23.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f5399c2c9c50ae9418e522842ad362f61ee48b346ac106807bd355a8a7c619" +checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -968,9 +978,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" dependencies = [ "percent-encoding", ] @@ -1031,7 +1041,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -1076,9 +1086,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -1087,9 +1097,9 @@ dependencies = [ [[package]] name = "git2" -version = "0.17.1" +version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7905cdfe33d31a88bb2e8419ddd054451f5432d1da9eaf2ac7804ee1ea12d5" +checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" dependencies = [ "bitflags 1.3.2", "libc", @@ -1242,9 +1252,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0722cd7114b7de04316e7ea5456a0bbb20e4adb46fd27a3697adb812cff0f37c" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1265,9 +1275,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1306,13 +1316,13 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "io-lifetimes" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ "hermit-abi 0.3.1", "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -1426,9 +1436,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.144" +version = "0.2.146" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b00cc1c228a6782d0f076e7b232802e0c5689d41bb5df366f2a6b6621cfdfe1" +checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libflate" @@ -1452,9 +1462,9 @@ dependencies = [ [[package]] name = "libgit2-sys" -version = "0.15.1+1.6.4" +version = "0.15.2+1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4577bde8cdfc7d6a2a4bcb7b049598597de33ffd337276e9c7db6cd4a2cee7" +checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" dependencies = [ "cc", "libc", @@ -1498,9 +1508,9 @@ checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1508,12 +1518,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lz4" @@ -1596,14 +1603,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -1729,9 +1735,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-float" @@ -1754,22 +1760,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "parquet" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0a1e6fa27f09ebddba280f5966ef435f3ac4d74cfc3ffe370fd3fd59c2e004d" +checksum = "d6a656fcc17e641657c955742c689732684e096f790ff30865d9f8dcc39f7c4a" dependencies = [ "ahash", "arrow-array", @@ -1796,7 +1802,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.12.3+zstd.1.5.2", + "zstd", ] [[package]] @@ -1816,9 +1822,9 @@ checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" @@ -1894,12 +1900,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "617feabb81566b593beb4886fb8c1f38064169dae4dccad0e3220160c3b37203" +checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" dependencies = [ "proc-macro2", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -1910,9 +1916,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa1fb82fc0c281dd9671101b66b771ebbe1eaf967b96ac8740dcba4b70005ca8" +checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" dependencies = [ "unicode-ident", ] @@ -2056,9 +2062,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f4f29d145265ec1c483c7c654450edde0bfe043d3938d6972630663356d9500" +checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" dependencies = [ "proc-macro2", ] @@ -2093,15 +2099,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.3.5" @@ -2113,9 +2110,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.2" +version = "1.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1a59b5d8e97dee33696bf13c5ba8ab85341c002922fba050069326b9c498974" +checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" dependencies = [ "aho-corasick", "memchr", @@ -2211,16 +2208,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -2329,22 +2326,22 @@ checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" [[package]] name = "serde" -version = "1.0.163" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2" +checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.163" +version = "1.0.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e" +checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -2378,7 +2375,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -2484,9 +2481,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355dc4d4b6207ca8a3434fc587db0a8016130a574dbcdbfb93d7f7b5bc5b211a" +checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" dependencies = [ "log", "sqlparser_derive", @@ -2514,6 +2511,9 @@ name = "strum" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +dependencies = [ + "strum_macros", +] [[package]] name = "strum_macros" @@ -2546,7 +2546,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.16", + "syn 2.0.18", "typify", "walkdir", ] @@ -2570,9 +2570,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.16" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f671d4b5ffdb8eadec19c0ae67fe2639df8684bd7bc4b83d986b8db549cf01" +checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" dependencies = [ "proc-macro2", "quote", @@ -2587,15 +2587,16 @@ checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" [[package]] name = "tempfile" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" dependencies = [ + "autocfg", "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -2615,7 +2616,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -2655,9 +2656,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.1" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa32867d44e6f2ce3385e89dceb990188b8bb0fb25b0cf576647a6f98ac5105" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", @@ -2668,7 +2669,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -2679,14 +2680,14 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] name = "tokio-rustls" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d409377ff5b1e3ca6437aa86c1eb7d40c134bfec254e44c830defa92669db5" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", "tokio", @@ -2743,7 +2744,7 @@ checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", ] [[package]] @@ -2811,7 +2812,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.16", + "syn 2.0.18", "thiserror", "unicode-ident", ] @@ -2828,7 +2829,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.16", + "syn 2.0.18", "typify-impl", ] @@ -2840,9 +2841,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" [[package]] name = "unicode-normalization" @@ -2885,9 +2886,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" dependencies = [ "form_urlencoded", "idna", @@ -2963,7 +2964,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", "wasm-bindgen-shared", ] @@ -2997,7 +2998,7 @@ checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.16", + "syn 2.0.18", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3098,16 +3099,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets 0.42.2", + "windows-targets", ] [[package]] @@ -3116,22 +3108,7 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "windows-targets", ] [[package]] @@ -3140,93 +3117,51 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - [[package]] name = "windows_aarch64_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - [[package]] name = "windows_i686_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - [[package]] name = "windows_i686_msvc" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - [[package]] name = "windows_x86_64_gnu" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - [[package]] name = "windows_x86_64_gnullvm" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - [[package]] name = "windows_x86_64_msvc" version = "0.48.0" @@ -3272,32 +3207,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "zstd" -version = "0.11.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" -dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", -] - [[package]] name = "zstd" version = "0.12.3+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" dependencies = [ - "zstd-safe 6.0.5+zstd.1.5.4", -] - -[[package]] -name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8745d9263..b7f0db42e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "25.0.0" +version = "26.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,19 +36,19 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "25.0.0" , features = ["pyarrow", "avro"] } -datafusion-common = { version = "25.0.0", features = ["pyarrow"] } -datafusion-expr = "25.0.0" -datafusion-optimizer = "25.0.0" -datafusion-sql = "25.0.0" -datafusion-substrait = "25.0.0" +datafusion = { version = "26.0.0" , features = ["pyarrow", "avro"] } +datafusion-common = { version = "26.0.0", features = ["pyarrow"] } +datafusion-expr = "26.0.0" +datafusion-optimizer = "26.0.0" +datafusion-sql = "26.0.0" +datafusion-substrait = "26.0.0" prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.5.3", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.5.4", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.7.1" syn = "2.0.11" diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index 221b0cc09..4df2061e1 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -614,7 +614,6 @@ def test_to_pydict(df): def test_describe(df): - # Calculate statistics df = df.describe() diff --git a/datafusion/tests/test_substrait.py b/datafusion/tests/test_substrait.py index 01df2d746..7c7a2c1f2 100644 --- a/datafusion/tests/test_substrait.py +++ b/datafusion/tests/test_substrait.py @@ -54,4 +54,6 @@ def test_substrait_serialization(ctx): # demonstrate how to create a DataFrame from a deserialized logical plan df = ctx.create_dataframe_from_logical_plan(logical_plan) - substrait_plan = ss.substrait.producer.to_substrait_plan(df.logical_plan()) + substrait_plan = ss.substrait.producer.to_substrait_plan( + df.logical_plan(), ctx + ) diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 58e6781a3..e97f00304 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -26,7 +26,7 @@ def print_pulls(repo_name, title, pulls): if len(pulls) > 0: print("**{}:**".format(title)) print() - for (pull, commit) in pulls: + for pull, commit in pulls: url = "https://github.com/{}/pull/{}".format( repo_name, pull.number ) @@ -39,7 +39,6 @@ def print_pulls(repo_name, title, pulls): def generate_changelog(repo, repo_name, tag1, tag2): - # get a list of commits between two tags print( f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr @@ -68,8 +67,7 @@ def generate_changelog(repo, repo_name, tag1, tag2): # categorize the pull requests based on GitHub labels print("Categorizing pull requests", file=sys.stderr) - for (pull, commit) in all_pulls: - + for pull, commit in all_pulls: # see if PR title uses Conventional Commits cc_type = "" # cc_scope = '' diff --git a/src/common/df_field.rs b/src/common/df_field.rs index d7745bfb5..703af0aa2 100644 --- a/src/common/df_field.rs +++ b/src/common/df_field.rs @@ -49,7 +49,7 @@ impl PyDFField { fn new(qualifier: Option, name: &str, data_type: PyDataType, nullable: bool) -> Self { PyDFField { field: DFField::new( - qualifier.map(|q| OwnedTableReference::from(q)), + qualifier.map(OwnedTableReference::from), name, data_type.into(), nullable, diff --git a/src/expr.rs b/src/expr.rs index 819edff2a..d519d0c1c 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -186,7 +186,8 @@ impl PyExpr { } fn __mod__(&self, rhs: PyExpr) -> PyResult { - Ok(self.expr.clone().modulus(rhs.expr).into()) + let expr = self.expr.clone() % rhs.expr; + Ok(expr.into()) } fn __and__(&self, rhs: PyExpr) -> PyResult { @@ -198,7 +199,8 @@ impl PyExpr { } fn __invert__(&self) -> PyResult { - Ok(self.expr.clone().not().into()) + let expr = !self.expr.clone(); + Ok(expr.into()) } fn __getitem__(&self, key: &str) -> PyResult { diff --git a/src/expr/scalar_function.rs b/src/expr/scalar_function.rs index 1a71d5692..776ca3297 100644 --- a/src/expr/scalar_function.rs +++ b/src/expr/scalar_function.rs @@ -56,7 +56,7 @@ impl From for BuiltinScalarFunction { #[pymethods] impl PyScalarFunction { fn fun(&self) -> PyResult { - Ok(self.scalar_function.clone().into()) + Ok(self.scalar_function.into()) } fn args(&self) -> PyResult> { diff --git a/src/substrait.rs b/src/substrait.rs index 5d2e7a485..d027ad21f 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -39,7 +39,7 @@ impl PyPlan { let mut proto_bytes = Vec::::new(); self.plan .encode(&mut proto_bytes) - .map_err(|e| DataFusionError::EncodeError(e))?; + .map_err(DataFusionError::EncodeError)?; Ok(PyBytes::new(py, &proto_bytes).into()) } } @@ -113,8 +113,8 @@ pub(crate) struct PySubstraitProducer; impl PySubstraitProducer { /// Convert DataFusion LogicalPlan to Substrait Plan #[staticmethod] - pub fn to_substrait_plan(plan: PyLogicalPlan) -> PyResult { - match producer::to_substrait_plan(&plan.plan) { + pub fn to_substrait_plan(plan: PyLogicalPlan, ctx: &PySessionContext) -> PyResult { + match producer::to_substrait_plan(&plan.plan, &ctx.ctx) { Ok(plan) => Ok(PyPlan { plan: *plan }), Err(e) => Err(py_datafusion_err(e)), } From c0be61bd575ae717ec8c949693abd6cdec39a50c Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Mon, 3 Jul 2023 11:59:59 -0400 Subject: [PATCH 053/413] LogicalPlan.to_variant() make public (#412) * Make to_variant public * Add to_variant() coverage for Subquery and SubqueryAlias * Add variant_name() function to Expr * Add function for friendly display of arrow DataType to Python layer since DataType enum values cannot be directly accessed * Change signature * Updated to include Decimal128 variant coverage * Add try_from to PyCreateView * Cargo lint fixes --- src/common/data_type.rs | 44 +++++++++++++++++++++++++++++++++++++++++ src/expr.rs | 16 ++++++++++----- src/expr/create_view.rs | 15 ++++++++++++-- src/expr/subquery.rs | 44 +++++++++++++++++++++++++++++++++++++++++ src/sql/logical.rs | 6 +++++- 5 files changed, 117 insertions(+), 8 deletions(-) diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 622e1aa46..85d2febb5 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -503,6 +503,50 @@ impl DataTypeMap { )), } } + + /// Unfortunately PyO3 does not allow for us to expose the DataType as an enum since + /// we cannot directly annotae the Enum instance of dependency code. Therefore, here + /// we provide an enum to mimic it. + #[pyo3(name = "friendly_arrow_type_name")] + pub fn friendly_arrow_type_name(&self) -> PyResult<&str> { + Ok(match &self.arrow_type.data_type { + DataType::Null => "Null", + DataType::Boolean => "Boolean", + DataType::Int8 => "Int8", + DataType::Int16 => "Int16", + DataType::Int32 => "Int32", + DataType::Int64 => "Int64", + DataType::UInt8 => "UInt8", + DataType::UInt16 => "UInt16", + DataType::UInt32 => "UInt32", + DataType::UInt64 => "UInt64", + DataType::Float16 => "Float16", + DataType::Float32 => "Float32", + DataType::Float64 => "Float64", + DataType::Timestamp(_, _) => "Timestamp", + DataType::Date32 => "Date32", + DataType::Date64 => "Date64", + DataType::Time32(_) => "Time32", + DataType::Time64(_) => "Time64", + DataType::Duration(_) => "Duration", + DataType::Interval(_) => "Interval", + DataType::Binary => "Binary", + DataType::FixedSizeBinary(_) => "FixedSizeBinary", + DataType::LargeBinary => "LargeBinary", + DataType::Utf8 => "Utf8", + DataType::LargeUtf8 => "LargeUtf8", + DataType::List(_) => "List", + DataType::FixedSizeList(_, _) => "FixedSizeList", + DataType::LargeList(_) => "LargeList", + DataType::Struct(_) => "Struct", + DataType::Union(_, _) => "Union", + DataType::Dictionary(_, _) => "Dictionary", + DataType::Decimal128(_, _) => "Decimal128", + DataType::Decimal256(_, _) => "Decimal256", + DataType::Map(_, _) => "Map", + DataType::RunEndEncoded(_, _) => "RunEndEncoded", + }) + } } /// PyO3 requires that objects passed between Rust and Python implement the trait `PyClass` diff --git a/src/expr.rs b/src/expr.rs index d519d0c1c..17b6c34f5 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -153,6 +153,12 @@ impl PyExpr { Ok(self.expr.canonical_name()) } + /// Returns the name of the Expr variant. + /// Ex: 'IsNotNull', 'Literal', 'BinaryExpr', etc + fn variant_name(&self) -> PyResult<&str> { + Ok(self.expr.variant_name()) + } + fn __richcmp__(&self, other: PyExpr, op: CompareOp) -> PyExpr { let expr = match op { CompareOp::Lt => self.expr.clone().lt(other.expr), @@ -302,7 +308,7 @@ impl PyExpr { ScalarValue::Boolean(v) => v.into_py(py), ScalarValue::Float32(v) => v.into_py(py), ScalarValue::Float64(v) => v.into_py(py), - ScalarValue::Decimal128(_, _, _) => todo!(), + ScalarValue::Decimal128(v, _, _) => v.into_py(py), ScalarValue::Int8(v) => v.into_py(py), ScalarValue::Int16(v) => v.into_py(py), ScalarValue::Int32(v) => v.into_py(py), @@ -323,10 +329,10 @@ impl PyExpr { ScalarValue::Time32Millisecond(v) => v.into_py(py), ScalarValue::Time64Microsecond(v) => v.into_py(py), ScalarValue::Time64Nanosecond(v) => v.into_py(py), - ScalarValue::TimestampSecond(_, _) => todo!(), - ScalarValue::TimestampMillisecond(_, _) => todo!(), - ScalarValue::TimestampMicrosecond(_, _) => todo!(), - ScalarValue::TimestampNanosecond(_, _) => todo!(), + ScalarValue::TimestampSecond(v, _) => v.into_py(py), + ScalarValue::TimestampMillisecond(v, _) => v.into_py(py), + ScalarValue::TimestampMicrosecond(v, _) => v.into_py(py), + ScalarValue::TimestampNanosecond(v, _) => v.into_py(py), ScalarValue::IntervalYearMonth(v) => v.into_py(py), ScalarValue::IntervalDayTime(v) => v.into_py(py), ScalarValue::IntervalMonthDayNano(v) => v.into_py(py), diff --git a/src/expr/create_view.rs b/src/expr/create_view.rs index 9d06239ea..febd723c5 100644 --- a/src/expr/create_view.rs +++ b/src/expr/create_view.rs @@ -17,10 +17,10 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::CreateView; +use datafusion_expr::{CreateView, DdlStatement, LogicalPlan}; use pyo3::prelude::*; -use crate::sql::logical::PyLogicalPlan; +use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; use super::logical_node::LogicalNode; @@ -92,3 +92,14 @@ impl LogicalNode for PyCreateView { Ok(self.clone().into_py(py)) } } + +impl TryFrom for PyCreateView { + type Error = PyErr; + + fn try_from(logical_plan: LogicalPlan) -> Result { + match logical_plan { + LogicalPlan::Ddl(DdlStatement::CreateView(create)) => Ok(PyCreateView { create }), + _ => Err(py_type_err("unexpected plan")), + } + } +} diff --git a/src/expr/subquery.rs b/src/expr/subquery.rs index 93ff244f6..f6f7b7fe5 100644 --- a/src/expr/subquery.rs +++ b/src/expr/subquery.rs @@ -15,9 +15,15 @@ // specific language governing permissions and limitations // under the License. +use std::fmt::{self, Display, Formatter}; + use datafusion_expr::Subquery; use pyo3::prelude::*; +use crate::sql::logical::PyLogicalPlan; + +use super::logical_node::LogicalNode; + #[pyclass(name = "Subquery", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PySubquery { @@ -35,3 +41,41 @@ impl From for PySubquery { PySubquery { subquery } } } + +impl Display for PySubquery { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Subquery + Subquery: {:?} + outer_ref_columns: {:?}", + self.subquery.subquery, self.subquery.outer_ref_columns, + ) + } +} + +#[pymethods] +impl PySubquery { + /// Retrieves the input `LogicalPlan` to this `Projection` node + fn input(&self) -> PyResult> { + Ok(Self::inputs(self)) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("Subquery({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("Subquery".to_string()) + } +} + +impl LogicalNode for PySubquery { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant(&self, py: Python) -> PyResult { + Ok(self.clone().into_py(py)) + } +} diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 07a3f65b1..2183155bf 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -28,6 +28,8 @@ use crate::expr::filter::PyFilter; use crate::expr::limit::PyLimit; use crate::expr::projection::PyProjection; use crate::expr::sort::PySort; +use crate::expr::subquery::PySubquery; +use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; use datafusion_expr::LogicalPlan; use pyo3::prelude::*; @@ -56,7 +58,7 @@ impl PyLogicalPlan { #[pymethods] impl PyLogicalPlan { /// Return the specific logical operator - fn to_variant(&self, py: Python) -> PyResult { + pub fn to_variant(&self, py: Python) -> PyResult { Python::with_gil(|_| match self.plan.as_ref() { LogicalPlan::Aggregate(plan) => PyAggregate::from(plan.clone()).to_variant(py), LogicalPlan::Analyze(plan) => PyAnalyze::from(plan.clone()).to_variant(py), @@ -69,6 +71,8 @@ impl PyLogicalPlan { LogicalPlan::Projection(plan) => PyProjection::from(plan.clone()).to_variant(py), LogicalPlan::Sort(plan) => PySort::from(plan.clone()).to_variant(py), LogicalPlan::TableScan(plan) => PyTableScan::from(plan.clone()).to_variant(py), + LogicalPlan::Subquery(plan) => PySubquery::from(plan.clone()).to_variant(py), + LogicalPlan::SubqueryAlias(plan) => PySubqueryAlias::from(plan.clone()).to_variant(py), other => Err(py_unsupported_variant_err(format!( "Cannot convert this plan to a LogicalNode: {:?}", other From 3f81513d6c5fd109bdf8c509f81c0a587924d354 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 3 Jul 2023 11:26:58 -0600 Subject: [PATCH 054/413] Prepare 27.0.0 release (#423) --- Cargo.lock | 534 +++++++++++++++++++-------------- Cargo.toml | 20 +- datafusion/tests/test_store.py | 2 +- src/catalog.rs | 4 +- src/context.rs | 2 +- src/dataframe.rs | 2 +- src/dataset.rs | 2 +- src/dataset_exec.rs | 2 +- src/physical_plan.rs | 2 +- src/record_batch.rs | 2 +- src/udaf.rs | 6 +- src/udf.rs | 2 +- 12 files changed, 337 insertions(+), 243 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2392f5796..21124d4fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -51,6 +60,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -91,8 +106,8 @@ dependencies = [ "serde", "serde_json", "snap", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "thiserror", "typed-builder", "uuid", @@ -107,15 +122,15 @@ checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" [[package]] name = "arrayvec" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6619cab21a0cdd8c9b9f1d9e09bfaa9b1974e5ef809a6566aef0b998caf38ace" +checksum = "773d18d72cd290f3f9e2149a714c8ac404b6c3fd614c684f0015449940fca899" dependencies = [ "ahash", "arrow-arith", @@ -136,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0dc95485623a76e00929bda8caa40c1f838190952365c4f43a7b9ae86d03e94" +checksum = "93bc0da4b22ba63807fa2a74998e21209179c93c67856ae65d9218b81f3ef918" dependencies = [ "arrow-array", "arrow-buffer", @@ -151,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3267847f53d3042473cfd2c769afd8d74a6d7d201fc3a34f5cb84c0282ef47a7" +checksum = "ea9a0fd21121304cad96f307c938d861cb1e7f0c151b93047462cd9817d760fb" dependencies = [ "ahash", "arrow-buffer", @@ -162,15 +177,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "num", ] [[package]] name = "arrow-buffer" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f66553e66e120ac4b21570368ee9ebf35ff3f5399f872b0667699e145678f5" +checksum = "30ce342ecf5971004e23cef8b5fb3bacd2bbc48a381464144925074e1472e9eb" dependencies = [ "half", "num", @@ -178,9 +193,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e6f3579dbf0d97c683d451b2550062b0f0e62a3169bf74238b5f59f44ad6d8" +checksum = "4b94a0ce7d27abbb02e2ee4db770f593127610f57b32625b0bc6a1a90d65f085" dependencies = [ "arrow-array", "arrow-buffer", @@ -189,15 +204,16 @@ dependencies = [ "arrow-select", "chrono", "comfy-table", + "half", "lexical-core", "num", ] [[package]] name = "arrow-csv" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373579c4c1a8f5307d3125b7a89c700fcf8caf85821c77eb4baab3855ae0aba5" +checksum = "0f3be10a00a43c4bf0d243c070754ebdde17c5d576b4928d9c3efbe3005a3853" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61bc8df9912cca6642665fdf989d6fa0de2570f18a7f709bcf59d29de96d2097" +checksum = "1d9a83dad6a53d6907765106d3bc61d6d9d313cfe1751701b3ef0948e7283dc2" dependencies = [ "arrow-buffer", "arrow-schema", @@ -226,9 +242,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0105dcf5f91daa7182d87b713ee0b32b3bfc88e0c48e7dc3e9d6f1277a07d1ae" +checksum = "a46da5e438a854e0386b38774da88a98782c0973c6dbc5c949ca4e02faf9b016" dependencies = [ "arrow-array", "arrow-buffer", @@ -240,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e73134fb5b5ec8770f8cbb214c2c487b2d350081e403ca4eeeb6f8f5e19846ac" +checksum = "d5f27a1fbc76553ad92dc1a9583e56b7058d8c418c4089b0b689f5b87e2da5e1" dependencies = [ "arrow-array", "arrow-buffer", @@ -251,7 +267,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap", + "indexmap 1.9.3", "lexical-core", "num", "serde", @@ -260,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89f25bc66e18d4c2aa1fe2f9bb03e2269da60e636213210385ae41a107f9965a" +checksum = "f2373661f6c2233e18f6fa69c40999a9440231d1e8899be8bbbe73c7e24aa3b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -275,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1095ff85ea4f5ff02d17b30b089de31b51a50be01c6b674f0a0509ab771232f1" +checksum = "377cd5158b7de4034a175e296726c40c3236e65d71d90a5dab2fb4fab526a8f4" dependencies = [ "ahash", "arrow-array", @@ -285,23 +301,23 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.13.2", + "hashbrown 0.14.0", ] [[package]] name = "arrow-schema" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25187bbef474151a2e4ddec67b9e34bda5cbfba292dc571392fa3a1f71ff5a82" +checksum = "ba9ed245bd2d7d97ad1457cb281d4296e8b593588758b8fec6d67b2b2b0f2265" dependencies = [ - "bitflags 2.3.1", + "bitflags 2.3.3", ] [[package]] name = "arrow-select" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd0d4ee884aec3aa05e41478e3cd312bf609de9babb5d187a43fb45931da4da4" +checksum = "0dc9bd6aebc565b1d04bae64a0f4dda3abc677190eb7d960471b1b20e1cebed0" dependencies = [ "arrow-array", "arrow-buffer", @@ -312,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6d71c3ffe4c07e66ce8fdc6aed5b00e0e60c5144911879b10546f5b72d8fa1c" +checksum = "23cf2baea2ef53787332050decf7d71aca836a352e188c8ad062892405955d2b" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,7 +367,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -362,7 +378,7 @@ checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -380,6 +396,21 @@ dependencies = [ "cc", ] +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.21.2" @@ -394,9 +425,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.1" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6776fc96284a0bb647b615056fc496d1fe1644a7ab01829818a6d91cae888b84" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "blake2" @@ -520,9 +551,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9cc2b23599e6d7479755f3594285efb3f74a1bdca7a7374948bc831e23a552" +checksum = "f1369bc6b9e9a7dfdae2055f6ec151fe9c554a9d23d357c0237cee2e25eaabb7" dependencies = [ "chrono", "chrono-tz-build", @@ -531,9 +562,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" +checksum = "e2f5ebdc942f57ed96d560a6d1a459bae5851102a25d5bf89dc04ae453e31ecf" dependencies = [ "parse-zoneinfo", "phf", @@ -542,12 +573,12 @@ dependencies = [ [[package]] name = "comfy-table" -version = "6.2.0" +version = "7.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e959d788268e3bf9d35ace83e81b124190378e4c91c9067524675e33394b8ba" +checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" dependencies = [ - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "unicode-width", ] @@ -587,9 +618,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" dependencies = [ "libc", ] @@ -655,9 +686,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9992c267436551d40b52d65289b144712e7b0ebdc62c8c859fd1574e5f73efbb" +checksum = "e96f6e4eb10bd3e6b709686858246466983e8c5354a928ff77ee34919aa60d00" dependencies = [ "ahash", "apache-avro", @@ -680,9 +711,9 @@ dependencies = [ "flate2", "futures", "glob", - "hashbrown 0.13.2", - "indexmap", - "itertools", + "hashbrown 0.14.0", + "indexmap 1.9.3", + "itertools 0.11.0", "lazy_static", "log", "num-traits", @@ -697,7 +728,6 @@ dependencies = [ "sqlparser", "tempfile", "tokio", - "tokio-stream", "tokio-util", "url", "uuid", @@ -707,9 +737,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3be97f7a7c720cdbb71e9eeabf814fa6ad8102b9022390f6cac74d3b4af6392" +checksum = "00e5fddcc0dd49bbe199e43aa406f39c46c790bb2a43c7b36a478e5f3f971235" dependencies = [ "apache-avro", "arrow", @@ -724,14 +754,14 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77c4b14b809b0e4c5bb101b6834504f06cdbb0d3c643400c61d0d844b33264e" +checksum = "cfd50b6cb17acc78d2473c0d28014b8fd4e2e0a2c067c07645d6547b33b0aeeb" dependencies = [ "dashmap", "datafusion-common", "datafusion-expr", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "log", "object_store", "parking_lot", @@ -742,24 +772,24 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ec7409bd45cf4fae6395d7d1024c8a97e543cadc88363e405d2aad5330e5e7" +checksum = "e1a35dc2cd9eac18063d636f7ddf4f090fe1f34284d80192ac7ade38cc3c6991" dependencies = [ "ahash", "arrow", "datafusion-common", "lazy_static", "sqlparser", - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.0", ] [[package]] name = "datafusion-optimizer" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64b537c93f87989c212db92a448a0f5eb4f0995e27199bb7687ae94f8b64a7a8" +checksum = "5f5043afeb45ec1c0f45519e1eed6a477f2d30732e8f975d9cf9a75fba0ca716" dependencies = [ "arrow", "async-trait", @@ -767,17 +797,17 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.13.2", - "itertools", + "hashbrown 0.14.0", + "itertools 0.11.0", "log", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60ee3f53340fdef36ee54d9e12d446ae2718b1d0196ac581f791d34808ec876" +checksum = "6cc892a24f4b829ee7718ad3950884c0346dbdf1517f3df153af4bcf54d8ca4d" dependencies = [ "ahash", "arrow", @@ -791,9 +821,9 @@ dependencies = [ "datafusion-expr", "datafusion-row", "half", - "hashbrown 0.13.2", - "indexmap", - "itertools", + "hashbrown 0.14.0", + "indexmap 1.9.3", + "itertools 0.11.0", "lazy_static", "libc", "md-5", @@ -808,7 +838,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "26.0.0" +version = "27.0.0" dependencies = [ "async-trait", "datafusion", @@ -827,7 +857,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.18", + "syn 2.0.22", "tokio", "url", "uuid", @@ -835,9 +865,9 @@ dependencies = [ [[package]] name = "datafusion-row" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d58fc64058aa3bcb00077a0d19474a0d584d31dec8c7ac3406868f485f659af9" +checksum = "ce75c660bbddfdd254109e668e5b5bd69df31ea26e3768e15cef0c68015e650e" dependencies = [ "arrow", "datafusion-common", @@ -847,9 +877,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1531f0314151a34bf6c0a83c7261525688b7c729876f53e7896b8f4ca8f57d07" +checksum = "49cab87e4933a452e0b7b3f0cbd0e760daf7d33fb54d09d70d3ffba229eaa652" dependencies = [ "arrow", "arrow-schema", @@ -861,14 +891,14 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "26.0.0" +version = "27.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "079d5be5ec59580777bfa16d79187fea99b6498e3e8e07eb36d504a5fe708f13" +checksum = "ba77d22232053f6cdd98bd6f5328940850844450253f25b8c50bfc5199c505d4" dependencies = [ "async-recursion", "chrono", "datafusion", - "itertools", + "itertools 0.11.0", "object_store", "prost", "prost-types", @@ -914,6 +944,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" + [[package]] name = "errno" version = "0.3.1" @@ -1041,7 +1077,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1095,6 +1131,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "git2" version = "0.17.2" @@ -1116,9 +1158,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" +checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" dependencies = [ "bytes", "fnv", @@ -1126,7 +1168,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -1135,10 +1177,11 @@ dependencies = [ [[package]] name = "half" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4af3693f1b705df946e9fe5631932443781d0aabb423b62fcd4d73f6d2fd0" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" dependencies = [ + "cfg-if", "crunchy", "num-traits", ] @@ -1159,19 +1202,20 @@ dependencies = [ ] [[package]] -name = "heck" -version = "0.4.1" +name = "hashbrown" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] -name = "hermit-abi" -version = "0.2.6" +name = "heck" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -1213,11 +1257,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" -version = "0.14.26" +version = "0.14.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" dependencies = [ "bytes", "futures-channel", @@ -1293,6 +1343,16 @@ dependencies = [ "hashbrown 0.12.3", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "indoc" version = "1.0.9" @@ -1320,16 +1380,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.1", + "hermit-abi", "libc", "windows-sys", ] [[package]] name = "ipnet" -version = "2.7.2" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" +checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" [[package]] name = "itertools" @@ -1340,6 +1400,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.6" @@ -1357,9 +1426,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f37a4a5928311ac501dee68b3c7613a1037d0edb30c8e5427bd832d55d1b790" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -1436,9 +1505,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.146" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libflate" @@ -1570,9 +1639,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -1697,26 +1766,37 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi", "libc", ] +[[package]] +name = "object" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" -version = "0.5.6" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec9cd6ca25e796a49fa242876d1c4de36a24a6da5258e9f0bc062dbf5e81c53b" +checksum = "27c776db4f332b571958444982ff641d2531417a326ca368995073b639205d58" dependencies = [ "async-trait", "base64", "bytes", "chrono", "futures", - "itertools", + "humantime", + "hyper", + "itertools 0.10.5", "parking_lot", "percent-encoding", "quick-xml", @@ -1773,9 +1853,9 @@ dependencies = [ [[package]] name = "parquet" -version = "40.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a656fcc17e641657c955742c689732684e096f790ff30865d9f8dcc39f7c4a" +checksum = "baab9c36b1c8300b81b4d577d306a0a733f9d34021363098d3548e37757ed6c8" dependencies = [ "ahash", "arrow-array", @@ -1791,7 +1871,7 @@ dependencies = [ "chrono", "flate2", "futures", - "hashbrown 0.13.2", + "hashbrown 0.14.0", "lz4", "num", "num-bigint", @@ -1833,23 +1913,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" dependencies = [ "phf_generator", "phf_shared", @@ -1857,9 +1937,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", "rand", @@ -1867,9 +1947,9 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] @@ -1900,12 +1980,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.6" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b69d39aab54d069e7f2fe8cb970493e7834601ca2d8c65fd7bbd183578080d1" +checksum = "9825a04601d60621feed79c4e6b56d65db77cdca55cef43b46b0de1096d1c282" dependencies = [ "proc-macro2", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -1916,9 +1996,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" dependencies = [ "unicode-ident", ] @@ -1941,7 +2021,7 @@ checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -1960,7 +2040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", - "itertools", + "itertools 0.10.5", "proc-macro2", "quote", "syn 1.0.109", @@ -1986,9 +2066,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b1ac5b3731ba34fdaa9785f8d74d17448cd18f30cf19e0c7e7b1fdb5272109" +checksum = "cffef52f74ec3b1a1baf295d9b8fcc3070327aefc39a6d00656b13c1d0b8885c" dependencies = [ "cfg-if", "indoc", @@ -2003,9 +2083,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cb946f5ac61bb61a5014924910d936ebd2b23b705f7a4a3c40b05c720b079a3" +checksum = "713eccf888fb05f1a96eb78c0dbc51907fee42b3377272dc902eb38985f418d5" dependencies = [ "once_cell", "target-lexicon", @@ -2013,9 +2093,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd4d7c5337821916ea2a1d21d1092e8443cf34879e53a0ac653fbb98f44ff65c" +checksum = "5b2ecbdcfb01cbbf56e179ce969a048fd7305a66d4cdf3303e0da09d69afe4c3" dependencies = [ "libc", "pyo3-build-config", @@ -2023,9 +2103,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d39c55dab3fc5a4b25bbd1ac10a2da452c4aca13bb450f22818a002e29648d" +checksum = "b78fdc0899f2ea781c463679b20cb08af9247febc8d052de941951024cd8aea0" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2035,9 +2115,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.18.3" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97daff08a4c48320587b5224cc98d609e3c27b6d437315bd40b605c98eeb5918" +checksum = "60da7b84f1227c3e2fe7593505de274dcf4c8928b4e0a1c23d551a14e4e80a0f" dependencies = [ "proc-macro2", "quote", @@ -2062,9 +2142,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.28" +version = "1.0.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9ab9c7eadfd8df19006f1cf1a4aed13540ed5cbc047010ece5826e10825488" +checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" dependencies = [ "proc-macro2", ] @@ -2197,6 +2277,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2208,9 +2294,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.20" +version = "0.37.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e891d04aa506a6d1f318d2771bcb1c7dfda84e126660ace067c9b474bb2c0" +checksum = "62f25693a73057a1b4cb56179dd3c7ea21a7c6c5ee7d85781f5749b46f34b79c" dependencies = [ "bitflags 1.3.2", "errno", @@ -2222,9 +2308,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.1" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" dependencies = [ "log", "ring", @@ -2234,9 +2320,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ "base64", ] @@ -2341,7 +2427,7 @@ checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2357,9 +2443,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.96" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" +checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" dependencies = [ "itoa", "ryu", @@ -2375,7 +2461,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2392,11 +2478,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.21" +version = "0.9.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" +checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" dependencies = [ - "indexmap", + "indexmap 2.0.0", "itoa", "ryu", "serde", @@ -2405,9 +2491,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" dependencies = [ "cfg-if", "cpufeatures", @@ -2481,9 +2567,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3706eefb17039056234df6b566b0014f303f867f2656108334a55b8096f59" +checksum = "ca597d77c98894be1f965f2e4e2d2a61575d4998088e655476c73715c54b2b43" dependencies = [ "log", "sqlparser_derive", @@ -2511,8 +2597,14 @@ name = "strum" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.0", ] [[package]] @@ -2528,11 +2620,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe9f3bd7d2e45dcc5e265fbb88d6513e4747d8ef9444cf01a533119bce28a157" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.22", +] + [[package]] name = "substrait" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9df5d9e071804204172dc77e707c363f187e7f6566f9c78e5100c9a8f5ea434e" +checksum = "7d3b77ddddd080d1bb5ebfe6b62d1c4e2f33c9f6a4586d5eac5306a08f3d4585" dependencies = [ "git2", "heck", @@ -2546,7 +2651,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.18", + "syn 2.0.22", "typify", "walkdir", ] @@ -2570,9 +2675,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" dependencies = [ "proc-macro2", "quote", @@ -2581,9 +2686,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd1ba337640d60c3e96bc6f0638a939b9c9a7f2c316a1598c279828b3d1dc8c5" +checksum = "1b1c7f239eb94671427157bd93b3694320f3668d4e1eff08c7285366fd777fac" [[package]] name = "tempfile" @@ -2616,7 +2721,7 @@ checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2656,11 +2761,12 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.2" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", + "backtrace", "bytes", "libc", "mio", @@ -2680,7 +2786,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2693,17 +2799,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.8" @@ -2738,13 +2833,13 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74" +checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", ] [[package]] @@ -2791,9 +2886,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "typify" -version = "0.0.12" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6658d09e71bfe59e7987dc95ee7f71809fdb5793ab0cdc1503cc0073990484d" +checksum = "be9bb640c0eece20cac2028ebbc2ca1a3d17e3b1ddd98540309c309ed178d158" dependencies = [ "typify-impl", "typify-macro", @@ -2801,9 +2896,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.12" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34d3bb47587b13edf526d6ed02bf360ecefe083ab47a4ef29fc43112828b2bef" +checksum = "5c8d9ecedde2fd77e975c38eeb9ca40b34ad0247b2259c6e6bbd2a8d6cc2444f" dependencies = [ "heck", "log", @@ -2812,16 +2907,16 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.18", + "syn 2.0.22", "thiserror", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.0.12" +version = "0.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3f7e627c18be12d53bc1f261830b9c2763437b6a86ac57293b9085af2d32ffe" +checksum = "c08942cd65d458d2da15777a649cb6400cb545f17964f1ca965583f22e9cc3a9" dependencies = [ "proc-macro2", "quote", @@ -2829,7 +2924,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.18", + "syn 2.0.22", "typify-impl", ] @@ -2897,9 +2992,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.3.3" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345444e32442451b267fc254ae85a209c64be56d2890e601a0c37ff0c3c5ecd2" +checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" dependencies = [ "getrandom", "serde", @@ -2929,11 +3024,10 @@ dependencies = [ [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] @@ -2945,9 +3039,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bba0e8cb82ba49ff4e229459ff22a191bbe9a1cb3a341610c9c33efc27ddf73" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2955,24 +3049,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b04bc93f9d6bdee709f6bd2118f57dd6679cf1176a1af464fca3ab0d66d8fb" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.36" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1985d03709c53167ce907ff394f5316aa22cb4e12761295c5dc57dacb6297e" +checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" dependencies = [ "cfg-if", "js-sys", @@ -2982,9 +3076,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d6b024f1a526bb0234f52840389927257beb670610081360e5a03c5df9c258" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2992,22 +3086,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e128beba882dd1eb6200e1dc92ae6c5dbaa4311aa7bb211ca035779e5efc39f8" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.22", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.86" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9d5b4305409d1fc9482fee2d7f9bcbf24b3972bf59817ef757e23982242a93" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" @@ -3024,9 +3118,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.63" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bdd9ef4e984da1187bf8110c5cf5b845fbc87a23602cdf912386a76fcd3a7c2" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" dependencies = [ "js-sys", "wasm-bindgen", @@ -3113,9 +3207,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", diff --git a/Cargo.toml b/Cargo.toml index b7f0db42e..5ca57c68d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "26.0.0" +version = "27.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -35,27 +35,27 @@ protoc = [ "datafusion-substrait/protoc" ] [dependencies] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" -pyo3 = { version = "0.18.1", features = ["extension-module", "abi3", "abi3-py37"] } -datafusion = { version = "26.0.0" , features = ["pyarrow", "avro"] } -datafusion-common = { version = "26.0.0", features = ["pyarrow"] } -datafusion-expr = "26.0.0" -datafusion-optimizer = "26.0.0" -datafusion-sql = "26.0.0" -datafusion-substrait = "26.0.0" +pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } +datafusion = { version = "27.0.0" , features = ["pyarrow", "avro"] } +datafusion-common = { version = "27.0.0", features = ["pyarrow"] } +datafusion-expr = "27.0.0" +datafusion-optimizer = "27.0.0" +datafusion-sql = "27.0.0" +datafusion-substrait = "27.0.0" prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.5.4", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.6.1", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.7.1" syn = "2.0.11" url = "2.2" [build-dependencies] -pyo3-build-config = "0.18.3" +pyo3-build-config = "0.19.0" [lib] name = "datafusion_python" diff --git a/datafusion/tests/test_store.py b/datafusion/tests/test_store.py index 9174c246f..3ffd9ee49 100644 --- a/datafusion/tests/test_store.py +++ b/datafusion/tests/test_store.py @@ -37,7 +37,7 @@ def ctx(local): def test_read_parquet(ctx): ctx.register_parquet( "test", - f"file://{os.getcwd()}/testing/data/parquet", + f"file://{os.getcwd()}/parquet/data/alltypes_plain.parquet", [], True, ".parquet", diff --git a/src/catalog.rs b/src/catalog.rs index 76521e9af..94faea067 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -24,8 +24,8 @@ use pyo3::prelude::*; use crate::errors::DataFusionError; use crate::utils::wait_for_future; use datafusion::{ - arrow::pyarrow::PyArrowConvert, - catalog::{catalog::CatalogProvider, schema::SchemaProvider}, + arrow::pyarrow::ToPyArrow, + catalog::{schema::SchemaProvider, CatalogProvider}, datasource::{TableProvider, TableType}, }; diff --git a/src/context.rs b/src/context.rs index b603c010e..cf133d79a 100644 --- a/src/context.rs +++ b/src/context.rs @@ -40,8 +40,8 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::datasource::datasource::TableProvider; use datafusion::datasource::MemTable; +use datafusion::datasource::TableProvider; use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext}; use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; diff --git a/src/dataframe.rs b/src/dataframe.rs index 7e1ce03b5..00c12e854 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -20,7 +20,7 @@ use crate::sql::logical::PyLogicalPlan; use crate::utils::wait_for_future; use crate::{errors::DataFusionError, expr::PyExpr}; use datafusion::arrow::datatypes::Schema; -use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType}; +use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; use datafusion::dataframe::DataFrame; use datafusion::prelude::*; diff --git a/src/dataset.rs b/src/dataset.rs index 0a2c7f50f..713610c51 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -28,10 +28,10 @@ use async_trait::async_trait; use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::pyarrow::PyArrowType; -use datafusion::datasource::datasource::TableProviderFilterPushDown; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::{DataFusionError, Result as DFResult}; use datafusion::execution::context::SessionState; +use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion::physical_plan::ExecutionPlan; use datafusion_expr::Expr; diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 859678856..b9d348f79 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -239,7 +239,7 @@ impl ExecutionPlan for DatasetExec { Python::with_gil(|py| { let number_of_fragments = self.fragments.as_ref(py).len(); match t { - DisplayFormatType::Default => { + DisplayFormatType::Default | DisplayFormatType::Verbose => { let projected_columns: Vec = self .schema .fields() diff --git a/src/physical_plan.rs b/src/physical_plan.rs index 6f02cefaa..ab783221b 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -51,7 +51,7 @@ impl PyExecutionPlan { pub fn display_indent(&self) -> String { let d = displayable(self.plan.as_ref()); - format!("{}", d.indent()) + format!("{}", d.indent(false)) } fn __repr__(&self) -> String { diff --git a/src/record_batch.rs b/src/record_batch.rs index 15b70e8ce..aa3a392da 100644 --- a/src/record_batch.rs +++ b/src/record_batch.rs @@ -16,7 +16,7 @@ // under the License. use crate::utils::wait_for_future; -use datafusion::arrow::pyarrow::PyArrowConvert; +use datafusion::arrow::pyarrow::ToPyArrow; use datafusion::arrow::record_batch::RecordBatch; use datafusion::physical_plan::SendableRecordBatchStream; use futures::StreamExt; diff --git a/src/udaf.rs b/src/udaf.rs index 756fa5659..ae2e81a08 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -21,10 +21,10 @@ use pyo3::{prelude::*, types::PyTuple}; use datafusion::arrow::array::{Array, ArrayRef}; use datafusion::arrow::datatypes::DataType; -use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType}; +use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::common::ScalarValue; use datafusion::error::{DataFusionError, Result}; -use datafusion_expr::{create_udaf, Accumulator, AccumulatorFunctionImplementation, AggregateUDF}; +use datafusion_expr::{create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF}; use crate::expr::PyExpr; use crate::utils::parse_volatility; @@ -95,7 +95,7 @@ impl Accumulator for RustAccumulator { } } -pub fn to_rust_accumulator(accum: PyObject) -> AccumulatorFunctionImplementation { +pub fn to_rust_accumulator(accum: PyObject) -> AccumulatorFactoryFunction { Arc::new(move |_| -> Result> { let accum = Python::with_gil(|py| { accum diff --git a/src/udf.rs b/src/udf.rs index dcb9cd572..417e5f327 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -21,7 +21,7 @@ use pyo3::{prelude::*, types::PyTuple}; use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::arrow::datatypes::DataType; -use datafusion::arrow::pyarrow::{PyArrowConvert, PyArrowType}; +use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; use datafusion::physical_plan::functions::make_scalar_function; use datafusion::physical_plan::udf::ScalarUDF; From 58bdbd88d52a57b98a50eb5f5155b7ee3ecce47e Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sat, 22 Jul 2023 10:05:06 -0400 Subject: [PATCH 055/413] File based input utils (#433) * checkpoint commit * Introduce BaseSessionContext abstract class * Introduce abstract methods for CRUD schema operations * Clean up schema.rs file * Introduce CRUD methods for table instances * Add function to drop_table * Add schema_name to drop_table function * remove unused parameter in SqlTable new * Update function to allow for modifying existing tables * Add functionality for generating SqlTable information from input sources * Add functionality for generating SqlTable information from input sources * Adding a utility method to convert arrow type strings to DataType instances * Add method to DataTypeMap for getting the DataType from an Arrow type string instance * Add function fqn which retrieves the table catalog, schema, and name from the TableReference * Add datatype mapping for parquet int96 -> nanosecond timestamp * Cargo check fixes --- Cargo.lock | 255 +++++++++++++++++---------------- datafusion/context.py | 5 +- datafusion/input/__init__.py | 22 +++ datafusion/input/base.py | 43 ++++++ datafusion/input/location.py | 88 ++++++++++++ datafusion/tests/test_input.py | 33 +++++ src/common/data_type.rs | 56 +++++++- src/expr/table_scan.rs | 26 +++- 8 files changed, 398 insertions(+), 130 deletions(-) create mode 100644 datafusion/input/__init__.py create mode 100644 datafusion/input/base.py create mode 100644 datafusion/input/location.py create mode 100644 datafusion/tests/test_input.py diff --git a/Cargo.lock b/Cargo.lock index 21124d4fd..a6c80f61e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -62,9 +62,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56fc6cf8dc8c4158eed8649f9b8b0ea1518eb62b544fe9490d66fa0b349eafe9" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] name = "android-tzdata" @@ -83,9 +83,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "apache-avro" @@ -343,9 +343,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" +checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" dependencies = [ "bzip2", "flate2", @@ -367,18 +367,18 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -440,9 +440,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "729b71f35bd3fa1a4c86b85d32c8b9069ea7fe14f7a53cfabb65f62d4265b888" +checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" dependencies = [ "arrayref", "arrayvec", @@ -606,9 +606,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.2.6" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21a53c0a4d288377e7415b53dcfc3c04da5cdc2cc95c8d5ac178b58f0b861ad6" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation-sys" @@ -618,9 +618,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03e69e28e9f7f77debdedbaafa2866e1de9ba56df55a8bd7cfc724c25a09987c" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -673,12 +673,12 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.0", "lock_api", "once_cell", "parking_lot_core", @@ -782,7 +782,7 @@ dependencies = [ "lazy_static", "sqlparser", "strum 0.25.0", - "strum_macros 0.25.0", + "strum_macros 0.25.1", ] [[package]] @@ -857,7 +857,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.22", + "syn 2.0.26", "tokio", "url", "uuid", @@ -925,9 +925,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" [[package]] name = "either" @@ -946,9 +946,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -1077,7 +1077,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -1219,9 +1219,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "http" @@ -1289,10 +1289,11 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0646026eb1b3eea4cd9ba47912ea5ce9cc07713d105b1a14698f4e6433d348b7" +checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" dependencies = [ + "futures-util", "http", "hyper", "rustls", @@ -1411,9 +1412,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" @@ -1689,9 +1690,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "num" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" dependencies = [ "num-bigint", "num-complex", @@ -1896,9 +1897,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" @@ -1956,9 +1957,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -1980,12 +1981,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9825a04601d60621feed79c4e6b56d65db77cdca55cef43b46b0de1096d1c282" +checksum = "92139198957b410250d43fad93e630d956499a625c527eda65175c8680f83387" dependencies = [ "proc-macro2", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -1996,9 +1997,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -2066,9 +2067,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cffef52f74ec3b1a1baf295d9b8fcc3070327aefc39a6d00656b13c1d0b8885c" +checksum = "ffb88ae05f306b4bfcde40ac4a51dc0b05936a9207a4b75b798c7729c4258a59" dependencies = [ "cfg-if", "indoc", @@ -2083,9 +2084,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713eccf888fb05f1a96eb78c0dbc51907fee42b3377272dc902eb38985f418d5" +checksum = "554db24f0b3c180a9c0b1268f91287ab3f17c162e15b54caaae5a6b3773396b0" dependencies = [ "once_cell", "target-lexicon", @@ -2093,9 +2094,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b2ecbdcfb01cbbf56e179ce969a048fd7305a66d4cdf3303e0da09d69afe4c3" +checksum = "922ede8759e8600ad4da3195ae41259654b9c55da4f7eec84a0ccc7d067a70a4" dependencies = [ "libc", "pyo3-build-config", @@ -2103,9 +2104,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b78fdc0899f2ea781c463679b20cb08af9247febc8d052de941951024cd8aea0" +checksum = "8a5caec6a1dd355964a841fcbeeb1b89fe4146c87295573f94228911af3cc5a2" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2115,9 +2116,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.19.0" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60da7b84f1227c3e2fe7593505de274dcf4c8928b4e0a1c23d551a14e4e80a0f" +checksum = "e0b78ccbb160db1556cdb6fd96c50334c5d4ec44dc5e0a968d0a1208fa0efa8b" dependencies = [ "proc-macro2", "quote", @@ -2142,9 +2143,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.29" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" dependencies = [ "proc-macro2", ] @@ -2190,9 +2191,21 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.4" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" dependencies = [ "aho-corasick", "memchr", @@ -2201,9 +2214,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "regress" @@ -2294,9 +2307,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.21" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f25693a73057a1b4cb56179dd3c7ea21a7c6c5ee7d85781f5749b46f34b79c" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ "bitflags 1.3.2", "errno", @@ -2308,9 +2321,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.2" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" dependencies = [ "log", "ring", @@ -2329,9 +2342,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.101.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" dependencies = [ "ring", "untrusted", @@ -2339,15 +2352,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -2384,9 +2397,9 @@ dependencies = [ [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" @@ -2400,34 +2413,34 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "seq-macro" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b44e8fc93a14e66336d230954dda83d18b4605ccace8fe09bc7514a71ad0bc" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.173" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "e91f70896d6720bc714a4a57d22fc91f1db634680e65c8efe13323f1fa38d53f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.173" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "a6250dde8342e0232232be9ca3db7aa40aceb5a3e5dd9bddbc00d99a007cde49" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -2443,9 +2456,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.99" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ "itoa", "ryu", @@ -2461,7 +2474,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -2478,9 +2491,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.22" +version = "0.9.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" +checksum = "bd5f51e3fdb5b9cdd1577e1cb7a733474191b1aca6a72c2e50913241632c1180" dependencies = [ "indexmap 2.0.0", "itoa", @@ -2517,15 +2530,15 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "snafu" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", "snafu-derive", @@ -2533,9 +2546,9 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ "heck", "proc-macro2", @@ -2604,7 +2617,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros 0.25.0", + "strum_macros 0.25.1", ] [[package]] @@ -2622,15 +2635,15 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9f3bd7d2e45dcc5e265fbb88d6513e4747d8ef9444cf01a533119bce28a157" +checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -2651,7 +2664,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.22", + "syn 2.0.26", "typify", "walkdir", ] @@ -2675,9 +2688,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.22" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efbeae7acf4eabd6bcdcbd11c92f45231ddda7539edc7806bd1a04a03b24616" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ "proc-macro2", "quote", @@ -2686,9 +2699,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.8" +version = "0.12.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1c7f239eb94671427157bd93b3694320f3668d4e1eff08c7285366fd777fac" +checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e" [[package]] name = "tempfile" @@ -2706,22 +2719,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -2786,7 +2799,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -2839,7 +2852,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", ] [[package]] @@ -2907,7 +2920,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.22", + "syn 2.0.26", "thiserror", "unicode-ident", ] @@ -2924,7 +2937,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.22", + "syn 2.0.26", "typify-impl", ] @@ -2936,9 +2949,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.9" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15811caf2415fb889178633e7724bad2509101cde276048e013b9def5e51fa0" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-normalization" @@ -2969,9 +2982,9 @@ checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" [[package]] name = "unsafe-libyaml" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" [[package]] name = "untrusted" @@ -2992,9 +3005,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom", "serde", @@ -3058,7 +3071,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", "wasm-bindgen-shared", ] @@ -3092,7 +3105,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.22", + "syn 2.0.26", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3303,18 +3316,18 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "6.0.5+zstd.1.5.4" +version = "6.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" dependencies = [ "libc", "zstd-sys", diff --git a/datafusion/context.py b/datafusion/context.py index 6292177f7..30602402c 100644 --- a/datafusion/context.py +++ b/datafusion/context.py @@ -16,7 +16,7 @@ # under the License. from abc import ABC, abstractmethod -from typing import Dict, List +from typing import Any, Dict, List from datafusion.common import SqlSchema, SqlTable @@ -79,8 +79,9 @@ def show_schemas(self, **kwargs) -> Dict[str, SqlSchema]: @abstractmethod def create_table( self, + schema_name: str, table_name: str, - schema_name: str = None, + input_source: Any, **kwargs, ): """ diff --git a/datafusion/input/__init__.py b/datafusion/input/__init__.py new file mode 100644 index 000000000..27e39b8ca --- /dev/null +++ b/datafusion/input/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from .location import LocationInputPlugin + +__all__ = [ + LocationInputPlugin, +] diff --git a/datafusion/input/base.py b/datafusion/input/base.py new file mode 100644 index 000000000..fb1207896 --- /dev/null +++ b/datafusion/input/base.py @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from abc import ABC, abstractmethod +from typing import Any + +from datafusion.common import SqlTable + + +class BaseInputSource(ABC): + """ + If a consuming library would like to provider their own InputSource + this is the class they should extend to write their own. Once + completed the Plugin InputSource can be registered with the + SessionContext to ensure that it will be used in order + to obtain the SqlTable information from the custom datasource. + """ + + @abstractmethod + def is_correct_input( + self, input_item: Any, table_name: str, **kwargs + ) -> bool: + pass + + @abstractmethod + def build_table( + self, input_item: Any, table_name: str, **kwarg + ) -> SqlTable: + pass diff --git a/datafusion/input/location.py b/datafusion/input/location.py new file mode 100644 index 000000000..121407581 --- /dev/null +++ b/datafusion/input/location.py @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from typing import Any + +from datafusion.common import DataTypeMap, SqlTable +from datafusion.input.base import BaseInputSource + + +class LocationInputPlugin(BaseInputSource): + """ + Input Plugin for everything, which can be read + in from a file (on disk, remote etc.) + """ + + def is_correct_input(self, input_item: Any, table_name: str, **kwargs): + return isinstance(input_item, str) + + def build_table( + self, + input_file: str, + table_name: str, + **kwargs, + ) -> SqlTable: + _, extension = os.path.splitext(input_file) + format = extension.lstrip(".").lower() + num_rows = 0 # Total number of rows in the file. Used for statistics + columns = [] + + if format == "parquet": + import pyarrow.parquet as pq + + # Read the Parquet metadata + metadata = pq.read_metadata(input_file) + num_rows = metadata.num_rows + + # Iterate through the schema and build the SqlTable + print(f"Metadata Schema: {metadata.schema}") + for col in metadata.schema: + columns.append( + ( + col.name, + DataTypeMap.from_parquet_type_str(col.physical_type), + ) + ) + + elif format == "csv": + import csv + + # Consume header row and count number of rows for statistics. + # TODO: Possibly makes sense to have the eager number of rows + # calculated as a configuration since you must read the entire file + # to get that information. However, this should only be occuring + # at table creation time and therefore shouldn't + # slow down query performance. + with open(input_file, "r") as file: + reader = csv.reader(file) + header_row = next(reader) + print(header_row) + for _ in reader: + num_rows += 1 + + # TODO: Need to actually consume this row into resonable columns + raise RuntimeError( + "TODO: Currently unable to support CSV input files." + ) + else: + raise RuntimeError( + f"Input of format: `{format}` is currently not supported.\ + Only Parquet and CSV." + ) + + return SqlTable(table_name, columns, num_rows, input_file) diff --git a/datafusion/tests/test_input.py b/datafusion/tests/test_input.py new file mode 100644 index 000000000..1e2ef4166 --- /dev/null +++ b/datafusion/tests/test_input.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +from datafusion.input.location import LocationInputPlugin + + +def test_location_input(): + location_input = LocationInputPlugin() + + cwd = os.getcwd() + input_file = ( + cwd + "/testing/data/parquet/generated_simple_numerics/blogs.parquet" + ) + table_name = "blog" + tbl = location_input.build_table(input_file, table_name) + assert "blog" == tbl.name + assert 3 == len(tbl.columns) + assert "blogs.parquet" in tbl.filepath diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 85d2febb5..d7db138a4 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -17,7 +17,7 @@ use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; use datafusion_common::{DataFusionError, ScalarValue}; -use pyo3::prelude::*; +use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; @@ -303,12 +303,43 @@ impl DataTypeMap { } } + #[staticmethod] + #[pyo3(name = "from_parquet_type_str")] + /// When using pyarrow.parquet.read_metadata().schema.column(x).physical_type you are presented + /// with a String type for schema rather than an object type. Here we make a best effort + /// to convert that to a physical type. + pub fn py_map_from_parquet_type_str(parquet_str_type: String) -> PyResult { + let arrow_dtype = match parquet_str_type.to_lowercase().as_str() { + "boolean" => Ok(DataType::Boolean), + "int32" => Ok(DataType::Int32), + "int64" => Ok(DataType::Int64), + "int96" => { + // Int96 is an old datatype that is now deprecated. We convert to nanosecond timestamp + Ok(DataType::Timestamp(TimeUnit::Nanosecond, None)) + } + "float" => Ok(DataType::Float32), + "double" => Ok(DataType::Float64), + _ => Err(PyValueError::new_err(format!( + "Unable to determine Arrow Data Type from Parquet String type: {:?}", + parquet_str_type + ))), + }; + DataTypeMap::map_from_arrow_type(&arrow_dtype?) + } + #[staticmethod] #[pyo3(name = "arrow")] pub fn py_map_from_arrow_type(arrow_type: &PyDataType) -> PyResult { DataTypeMap::map_from_arrow_type(&arrow_type.data_type) } + #[staticmethod] + #[pyo3(name = "arrow_str")] + pub fn py_map_from_arrow_type_str(arrow_type_str: String) -> PyResult { + let data_type = PyDataType::py_map_from_arrow_type_str(arrow_type_str); + DataTypeMap::map_from_arrow_type(&data_type?.data_type) + } + #[staticmethod] #[pyo3(name = "sql")] pub fn py_map_from_sql_type(sql_type: &SqlType) -> PyResult { @@ -558,6 +589,29 @@ pub struct PyDataType { pub data_type: DataType, } +impl PyDataType { + /// There are situations when obtaining dtypes on the Python side where the Arrow type + /// is presented as a String rather than an actual DataType. This function is used to + /// convert that String to a DataType for the Python side to use. + pub fn py_map_from_arrow_type_str(arrow_str_type: String) -> PyResult { + let arrow_dtype = match arrow_str_type.to_lowercase().as_str() { + "boolean" => Ok(DataType::Boolean), + "int32" => Ok(DataType::Int32), + "int64" => Ok(DataType::Int64), + "float" => Ok(DataType::Float32), + "double" => Ok(DataType::Float64), + "float64" => Ok(DataType::Float64), + _ => Err(PyValueError::new_err(format!( + "Unable to determine Arrow Data Type from Arrow String type: {:?}", + arrow_str_type + ))), + }; + Ok(PyDataType { + data_type: arrow_dtype?, + }) + } +} + impl From for DataType { fn from(data_type: PyDataType) -> DataType { data_type.data_type diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index ac848d9eb..8fafd12ee 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion_common::TableReference; use datafusion_expr::logical_plan::TableScan; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; @@ -71,12 +72,25 @@ impl PyTableScan { Ok(format!("{}", self.table_scan.table_name)) } - /// TODO: Bindings for `TableSource` need to exist first. Left as a - /// placeholder to display intention to add when able to. - // #[pyo3(name = "source")] - // fn py_source(&self) -> PyResult> { - // Ok(self.table_scan.source) - // } + #[pyo3(name = "fqn")] + fn fqn(&self) -> PyResult<(Option, Option, String)> { + let table_ref: TableReference = self.table_scan.table_name.clone(); + Ok(match table_ref { + TableReference::Bare { table } => (None, None, table.to_string()), + TableReference::Partial { schema, table } => { + (None, Some(schema.to_string()), table.to_string()) + } + TableReference::Full { + catalog, + schema, + table, + } => ( + Some(catalog.to_string()), + Some(schema.to_string()), + table.to_string(), + ), + }) + } /// The column indexes that should be. Note if this is empty then /// all columns should be read by the `TableProvider`. This function From 5793db39bd785479209bcbbce6e1740fc7e08215 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 22 Jul 2023 19:43:04 +0100 Subject: [PATCH 056/413] Upgrade to 28.0.0-rc1 (#434) --- Cargo.lock | 252 +++++++++++++---------------- Cargo.toml | 14 +- datafusion/tests/test_functions.py | 4 +- src/common/data_type.rs | 7 + src/dataset_exec.rs | 13 +- src/expr.rs | 35 ++-- src/expr/projection.rs | 4 +- src/functions.rs | 3 +- 8 files changed, 153 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6c80f61e..a491051f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -128,9 +128,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773d18d72cd290f3f9e2149a714c8ac404b6c3fd614c684f0015449940fca899" +checksum = "2feeebd77b34b0bc88f224e06d01c27da4733997cc4789a4e056196656cdc59a" dependencies = [ "ahash", "arrow-arith", @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93bc0da4b22ba63807fa2a74998e21209179c93c67856ae65d9218b81f3ef918" +checksum = "7173f5dc49c0ecb5135f52565af33afd3fdc9a12d13bd6f9973e8b96305e4b2e" dependencies = [ "arrow-array", "arrow-buffer", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9a0fd21121304cad96f307c938d861cb1e7f0c151b93047462cd9817d760fb" +checksum = "63d7ea725f7d1f8bb2cffc53ef538557e95fc802e217d5be25122d402e22f3d0" dependencies = [ "ahash", "arrow-buffer", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ce342ecf5971004e23cef8b5fb3bacd2bbc48a381464144925074e1472e9eb" +checksum = "bdbe439e077f484e5000b9e1d47b5e4c0d15f2b311a8f5bcc682553d5d67a722" dependencies = [ "half", "num", @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b94a0ce7d27abbb02e2ee4db770f593127610f57b32625b0bc6a1a90d65f085" +checksum = "93913cc14875770aa1eef5e310765e855effa352c094cb1c7c00607d0f37b4e1" dependencies = [ "arrow-array", "arrow-buffer", @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3be10a00a43c4bf0d243c070754ebdde17c5d576b4928d9c3efbe3005a3853" +checksum = "ef55b67c55ed877e6fe7b923121c19dae5e31ca70249ea2779a17b58fb0fbd9a" dependencies = [ "arrow-array", "arrow-buffer", @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d9a83dad6a53d6907765106d3bc61d6d9d313cfe1751701b3ef0948e7283dc2" +checksum = "d4f4f4a3c54614126a71ab91f6631c9743eb4643d6e9318b74191da9dc6e028b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a46da5e438a854e0386b38774da88a98782c0973c6dbc5c949ca4e02faf9b016" +checksum = "d41a3659f984a524ef1c2981d43747b24d8eec78e2425267fcd0ef34ce71cd18" dependencies = [ "arrow-array", "arrow-buffer", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f27a1fbc76553ad92dc1a9583e56b7058d8c418c4089b0b689f5b87e2da5e1" +checksum = "10b95faa95a378f56ef32d84cc0104ea998c39ef7cd1faaa6b4cebf8ea92846d" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,7 +267,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 1.9.3", + "indexmap 2.0.0", "lexical-core", "num", "serde", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2373661f6c2233e18f6fa69c40999a9440231d1e8899be8bbbe73c7e24aa3b4" +checksum = "c68549a4284d9f8b39586afb8d5ff8158b8f0286353a4844deb1d11cf1ba1f26" dependencies = [ "arrow-array", "arrow-buffer", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377cd5158b7de4034a175e296726c40c3236e65d71d90a5dab2fb4fab526a8f4" +checksum = "0a75a4a757afc301ce010adadff54d79d66140c4282ed3de565f6ccb716a5cf3" dependencies = [ "ahash", "arrow-array", @@ -306,18 +306,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba9ed245bd2d7d97ad1457cb281d4296e8b593588758b8fec6d67b2b2b0f2265" +checksum = "2bebcb57eef570b15afbcf2d07d813eb476fde9f6dd69c81004d6476c197e87e" dependencies = [ "bitflags 2.3.3", ] [[package]] name = "arrow-select" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc9bd6aebc565b1d04bae64a0f4dda3abc677190eb7d960471b1b20e1cebed0" +checksum = "f6e2943fa433a48921e914417173816af64eef61c0a3d448280e6c40a62df221" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,15 +328,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cf2baea2ef53787332050decf7d71aca836a352e188c8ad062892405955d2b" +checksum = "bbc92ed638851774f6d7af1ad900b92bc1486746497511868b4298fcbcfa35af" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "num", "regex", "regex-syntax", ] @@ -367,18 +368,18 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] name = "async-trait" -version = "0.1.71" +version = "0.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -686,9 +687,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96f6e4eb10bd3e6b709686858246466983e8c5354a928ff77ee34919aa60d00" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "ahash", "apache-avro", @@ -706,13 +706,13 @@ dependencies = [ "datafusion-expr", "datafusion-optimizer", "datafusion-physical-expr", - "datafusion-row", "datafusion-sql", "flate2", "futures", "glob", + "half", "hashbrown 0.14.0", - "indexmap 1.9.3", + "indexmap 2.0.0", "itertools 0.11.0", "lazy_static", "log", @@ -737,9 +737,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e5fddcc0dd49bbe199e43aa406f39c46c790bb2a43c7b36a478e5f3f971235" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "apache-avro", "arrow", @@ -754,9 +753,8 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd50b6cb17acc78d2473c0d28014b8fd4e2e0a2c067c07645d6547b33b0aeeb" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "dashmap", "datafusion-common", @@ -772,9 +770,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1a35dc2cd9eac18063d636f7ddf4f090fe1f34284d80192ac7ade38cc3c6991" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "ahash", "arrow", @@ -787,9 +784,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5043afeb45ec1c0f45519e1eed6a477f2d30732e8f975d9cf9a75fba0ca716" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "arrow", "async-trait", @@ -805,27 +801,28 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cc892a24f4b829ee7718ad3950884c0346dbdf1517f3df153af4bcf54d8ca4d" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", "arrow-schema", + "base64", "blake2", "blake3", "chrono", "datafusion-common", "datafusion-expr", - "datafusion-row", "half", "hashbrown 0.14.0", - "indexmap 1.9.3", + "hex", + "indexmap 2.0.0", "itertools 0.11.0", "lazy_static", "libc", + "log", "md-5", "paste", "petgraph", @@ -838,7 +835,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "27.0.0" +version = "28.0.0" dependencies = [ "async-trait", "datafusion", @@ -857,29 +854,16 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.26", + "syn 2.0.27", "tokio", "url", "uuid", ] -[[package]] -name = "datafusion-row" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce75c660bbddfdd254109e668e5b5bd69df31ea26e3768e15cef0c68015e650e" -dependencies = [ - "arrow", - "datafusion-common", - "paste", - "rand", -] - [[package]] name = "datafusion-sql" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cab87e4933a452e0b7b3f0cbd0e760daf7d33fb54d09d70d3ffba229eaa652" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "arrow", "arrow-schema", @@ -891,9 +875,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "27.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba77d22232053f6cdd98bd6f5328940850844450253f25b8c50bfc5199c505d4" +version = "28.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" dependencies = [ "async-recursion", "chrono", @@ -931,9 +914,9 @@ checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" [[package]] name = "either" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" @@ -973,12 +956,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "fixedbitset" @@ -1077,7 +1057,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -1223,6 +1203,12 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.9" @@ -1360,32 +1346,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "integer-encoding" version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" -[[package]] -name = "io-lifetimes" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys", -] - [[package]] name = "ipnet" version = "2.8.0" @@ -1572,9 +1538,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" [[package]] name = "lock_api" @@ -1757,9 +1723,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", "libm", @@ -1854,9 +1820,9 @@ dependencies = [ [[package]] name = "parquet" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baab9c36b1c8300b81b4d577d306a0a733f9d34021363098d3548e37757ed6c8" +checksum = "ec7267a9607c3f955d4d0ac41b88a67cecc0d8d009173ad3da390699a6cb3750" dependencies = [ "ahash", "arrow-array", @@ -1981,12 +1947,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92139198957b410250d43fad93e630d956499a625c527eda65175c8680f83387" +checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -2307,13 +2273,12 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.23" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.3.3", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -2425,22 +2390,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.173" +version = "1.0.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91f70896d6720bc714a4a57d22fc91f1db634680e65c8efe13323f1fa38d53f" +checksum = "3b88756493a5bd5e5395d53baa70b194b05764ab85b59e43e4b8f4e1192fa9b1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.173" +version = "1.0.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6250dde8342e0232232be9ca3db7aa40aceb5a3e5dd9bddbc00d99a007cde49" +checksum = "6e5c3a298c7f978e53536f95a63bdc4c4a64550582f31a0359a9afda6aede62e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -2474,7 +2439,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -2491,9 +2456,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.24" +version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd5f51e3fdb5b9cdd1577e1cb7a733474191b1aca6a72c2e50913241632c1180" +checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ "indexmap 2.0.0", "itoa", @@ -2643,14 +2608,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] name = "substrait" -version = "0.11.0" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3b77ddddd080d1bb5ebfe6b62d1c4e2f33c9f6a4586d5eac5306a08f3d4585" +checksum = "2ac1ce8315086b127ca0abf162c62279550942bb26ebf7946fe17fe114446472" dependencies = [ "git2", "heck", @@ -2664,7 +2629,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.26", + "syn 2.0.27", "typify", "walkdir", ] @@ -2688,9 +2653,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.26" +version = "2.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" +checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" dependencies = [ "proc-macro2", "quote", @@ -2705,11 +2670,10 @@ checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e" [[package]] name = "tempfile" -version = "3.6.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ - "autocfg", "cfg-if", "fastrand", "redox_syscall", @@ -2719,22 +2683,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -2799,7 +2763,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -2852,7 +2816,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", ] [[package]] @@ -2920,7 +2884,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.26", + "syn 2.0.27", "thiserror", "unicode-ident", ] @@ -2937,7 +2901,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.26", + "syn 2.0.27", "typify-impl", ] @@ -3071,7 +3035,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", "wasm-bindgen-shared", ] @@ -3105,7 +3069,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.26", + "syn 2.0.27", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index 5ca57c68d..a460d2ccc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "27.0.0" +version = "28.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "27.0.0" , features = ["pyarrow", "avro"] } -datafusion-common = { version = "27.0.0", features = ["pyarrow"] } -datafusion-expr = "27.0.0" -datafusion-optimizer = "27.0.0" -datafusion-sql = "27.0.0" -datafusion-substrait = "27.0.0" +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1", features = ["pyarrow", "avro"] } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1", features = ["pyarrow"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index bea580859..ec334747c 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -382,11 +382,11 @@ def test_temporal_functions(df): assert result.column(1) == pa.array([2022, 2027, 2020], type=pa.float64()) assert result.column(2) == pa.array( [datetime(2022, 12, 1), datetime(2027, 6, 1), datetime(2020, 7, 1)], - type=pa.timestamp("ns"), + type=pa.timestamp("us"), ) assert result.column(3) == pa.array( [datetime(2022, 12, 31), datetime(2027, 6, 26), datetime(2020, 7, 2)], - type=pa.timestamp("ns"), + type=pa.timestamp("us"), ) assert result.column(4) == pa.array( [ diff --git a/src/common/data_type.rs b/src/common/data_type.rs index d7db138a4..fe5f6f060 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -288,6 +288,13 @@ impl DataTypeMap { ScalarValue::List(_val, field_ref) => Ok(DataType::List(field_ref.to_owned())), ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())), ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)), + ScalarValue::Fixedsizelist(_, field_ref, size) => { + Ok(DataType::FixedSizeList(field_ref.to_owned(), *size)) + } + ScalarValue::DurationSecond(_) => Ok(DataType::Duration(TimeUnit::Second)), + ScalarValue::DurationMillisecond(_) => Ok(DataType::Duration(TimeUnit::Millisecond)), + ScalarValue::DurationMicrosecond(_) => Ok(DataType::Duration(TimeUnit::Microsecond)), + ScalarValue::DurationNanosecond(_) => Ok(DataType::Duration(TimeUnit::Nanosecond)), } } } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index b9d348f79..686bbf74c 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -35,7 +35,8 @@ use datafusion::execution::context::TaskContext; use datafusion::physical_expr::PhysicalSortExpr; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, Statistics, + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, + Statistics, }; use datafusion_expr::Expr; use datafusion_optimizer::utils::conjunction; @@ -235,6 +236,12 @@ impl ExecutionPlan for DatasetExec { }) } + fn statistics(&self) -> Statistics { + self.projected_statistics.clone() + } +} + +impl DisplayAs for DatasetExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { Python::with_gil(|py| { let number_of_fragments = self.fragments.as_ref(py).len(); @@ -267,8 +274,4 @@ impl ExecutionPlan for DatasetExec { } }) } - - fn statistics(&self) -> Statistics { - self.projected_statistics.clone() - } } diff --git a/src/expr.rs b/src/expr.rs index 17b6c34f5..2fd638a13 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -115,7 +115,7 @@ impl PyExpr { /// Return the specific expression fn to_variant(&self, py: Python) -> PyResult { Python::with_gil(|_| match &self.expr { - Expr::Alias(alias, name) => Ok(PyAlias::new(alias, name).into_py(py)), + Expr::Alias(alias) => Ok(PyAlias::new(&alias.expr, &alias.name).into_py(py)), Expr::Column(col) => Ok(PyColumn::from(col.clone()).into_py(py)), Expr::ScalarVariable(data_type, variables) => { Ok(PyScalarVariable::new(data_type, variables).into_py(py)) @@ -265,7 +265,6 @@ impl PyExpr { | Expr::Negative(..) | Expr::IsNull(..) | Expr::Like { .. } - | Expr::ILike { .. } | Expr::SimilarTo { .. } | Expr::Between { .. } | Expr::Case { .. } @@ -336,8 +335,13 @@ impl PyExpr { ScalarValue::IntervalYearMonth(v) => v.into_py(py), ScalarValue::IntervalDayTime(v) => v.into_py(py), ScalarValue::IntervalMonthDayNano(v) => v.into_py(py), + ScalarValue::DurationSecond(v) => v.into_py(py), + ScalarValue::DurationMicrosecond(v) => v.into_py(py), + ScalarValue::DurationNanosecond(v) => v.into_py(py), + ScalarValue::DurationMillisecond(v) => v.into_py(py), ScalarValue::Struct(_, _) => todo!(), ScalarValue::Dictionary(_, _) => todo!(), + ScalarValue::Fixedsizelist(_, _, _) => todo!(), }), _ => Err(py_type_err(format!( "Non Expr::Literal encountered in types: {:?}", @@ -356,9 +360,10 @@ impl PyExpr { Ok(vec![PyExpr::from(self.expr.clone())]) } + Expr::Alias(alias) => Ok(vec![PyExpr::from(*alias.expr.clone())]), + // Expr(s) that house the Expr instance to return in their bounded params - Expr::Alias(expr, ..) - | Expr::Not(expr) + Expr::Not(expr) | Expr::IsNull(expr) | Expr::IsNotNull(expr) | Expr::IsTrue(expr) @@ -429,10 +434,6 @@ impl PyExpr { PyExpr::from(*expr.clone()), PyExpr::from(*pattern.clone()), ]), - Expr::ILike(Like { expr, pattern, .. }) => Ok(vec![ - PyExpr::from(*expr.clone()), - PyExpr::from(*pattern.clone()), - ]), Expr::SimilarTo(Like { expr, pattern, .. }) => Ok(vec![ PyExpr::from(*expr.clone()), PyExpr::from(*pattern.clone()), @@ -486,18 +487,16 @@ impl PyExpr { Expr::InList { .. } => "in list".to_string(), Expr::Negative(..) => "negative".to_string(), Expr::Not(..) => "not".to_string(), - Expr::Like(Like { negated, .. }) => { - if *negated { - "not like".to_string() - } else { - "like".to_string() - } - } - Expr::ILike(Like { negated, .. }) => { + Expr::Like(Like { + negated, + case_insensitive, + .. + }) => { + let name = if *case_insensitive { "ilike" } else { "like" }; if *negated { - "not ilike".to_string() + format!("not {name}") } else { - "ilike".to_string() + name.to_string() } } Expr::SimilarTo(Like { negated, .. }) => { diff --git a/src/expr/projection.rs b/src/expr/projection.rs index b3296618f..8c1423df4 100644 --- a/src/expr/projection.rs +++ b/src/expr/projection.rs @@ -98,8 +98,8 @@ impl PyProjection { pub fn projected_expressions(local_expr: &PyExpr) -> Vec { let mut projs: Vec = Vec::new(); match &local_expr.expr { - Expr::Alias(expr, _name) => { - let py_expr: PyExpr = PyExpr::from(*expr.clone()); + Expr::Alias(alias) => { + let py_expr: PyExpr = PyExpr::from(*alias.expr.clone()); projs.extend_from_slice(Self::projected_expressions(&py_expr).as_slice()); } _ => projs.push(local_expr.clone()), diff --git a/src/functions.rs b/src/functions.rs index c82361a3b..8b60e6433 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -18,6 +18,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; use datafusion_common::Column; +use datafusion_expr::expr::Alias; use datafusion_expr::{ aggregate_function, expr::{AggregateFunction, ScalarFunction, Sort, WindowFunction}, @@ -85,7 +86,7 @@ fn order_by(expr: PyExpr, asc: Option, nulls_first: Option) -> PyRes #[pyfunction] fn alias(expr: PyExpr, name: &str) -> PyResult { Ok(PyExpr { - expr: datafusion_expr::Expr::Alias(Box::new(expr.expr), String::from(name)), + expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, name)), }) } From 93f8063f5a0d6058b3a3406ab87c84426b37a3b7 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Mon, 24 Jul 2023 08:46:26 -0400 Subject: [PATCH 057/413] Introduces utility for obtaining SqlTable information from a file like location (#398) * checkpoint commit * Introduce BaseSessionContext abstract class * Introduce abstract methods for CRUD schema operations * Clean up schema.rs file * Introduce CRUD methods for table instances * Add function to drop_table * Add schema_name to drop_table function * remove unused parameter in SqlTable new * Update function to allow for modifying existing tables * Add functionality for generating SqlTable information from input sources * Add functionality for generating SqlTable information from input sources * Adding a utility method to convert arrow type strings to DataType instances * Add method to DataTypeMap for getting the DataType from an Arrow type string instance * Adjust pytests * Add back deprecated int96 parquet datatype --- datafusion/input/location.py | 1 - src/common/data_type.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion/input/location.py b/datafusion/input/location.py index 121407581..efbc82f23 100644 --- a/datafusion/input/location.py +++ b/datafusion/input/location.py @@ -50,7 +50,6 @@ def build_table( num_rows = metadata.num_rows # Iterate through the schema and build the SqlTable - print(f"Metadata Schema: {metadata.schema}") for col in metadata.schema: columns.append( ( diff --git a/src/common/data_type.rs b/src/common/data_type.rs index fe5f6f060..199fb616e 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -321,7 +321,7 @@ impl DataTypeMap { "int32" => Ok(DataType::Int32), "int64" => Ok(DataType::Int64), "int96" => { - // Int96 is an old datatype that is now deprecated. We convert to nanosecond timestamp + // Int96 is an old parquet datatype that is now deprecated. We convert to nanosecond timestamp Ok(DataType::Timestamp(TimeUnit::Nanosecond, None)) } "float" => Ok(DataType::Float32), From 309fc486c47d86776aeec07d86cd04b5d70d97a1 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Mon, 24 Jul 2023 05:47:17 -0700 Subject: [PATCH 058/413] feat: expose offset in python API (#437) --- datafusion/tests/test_dataframe.py | 11 +++++++++++ datafusion/tests/test_expr.py | 8 ++++++++ src/dataframe.rs | 5 +++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index 4df2061e1..78cb50f12 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -115,6 +115,17 @@ def test_limit(df): assert len(result.column(1)) == 1 +def test_limit_with_offset(df): + # only 3 rows, but limit past the end to ensure that offset is working + df = df.limit(5, offset=2) + + # execute and collect the first (and only) batch + result = df.collect()[0] + + assert len(result.column(0)) == 1 + assert len(result.column(1)) == 1 + + def test_with_column(df): df = df.with_column("c", column("a") + column("b")) diff --git a/datafusion/tests/test_expr.py b/datafusion/tests/test_expr.py index 0c4869f27..73f7d087a 100644 --- a/datafusion/tests/test_expr.py +++ b/datafusion/tests/test_expr.py @@ -81,6 +81,14 @@ def test_limit(test_ctx): plan = plan.to_variant() assert isinstance(plan, Limit) + assert plan.skip() == 0 + + df = test_ctx.sql("select c1 from test LIMIT 10 OFFSET 5") + plan = df.logical_plan() + + plan = plan.to_variant() + assert isinstance(plan, Limit) + assert plan.skip() == 5 def test_aggregate_query(test_ctx): diff --git a/src/dataframe.rs b/src/dataframe.rs index 00c12e854..b8d8ddc3c 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -136,8 +136,9 @@ impl PyDataFrame { Ok(Self::new(df)) } - fn limit(&self, count: usize) -> PyResult { - let df = self.df.as_ref().clone().limit(0, Some(count))?; + #[pyo3(signature = (count, offset=0))] + fn limit(&self, count: usize, offset: usize) -> PyResult { + let df = self.df.as_ref().clone().limit(offset, Some(count))?; Ok(Self::new(df)) } From ffd15410c01868f5ed62c5fb2db2a460b42e06b3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 25 Jul 2023 17:01:34 +0100 Subject: [PATCH 059/413] Use DataFusion 28 (#439) --- Cargo.lock | 40 ++++++++++++++++++++++++---------------- Cargo.toml | 12 ++++++------ 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a491051f4..85634fd11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -688,7 +688,8 @@ dependencies = [ [[package]] name = "datafusion" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ddbcb2dda5b5033537457992ebde78938014390b2b19f9f4282e3be0e18b0c3" dependencies = [ "ahash", "apache-avro", @@ -738,7 +739,8 @@ dependencies = [ [[package]] name = "datafusion-common" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fbb7b4da925031311743ab96662d55f0f7342d3692744f184f99b2257ef435" dependencies = [ "apache-avro", "arrow", @@ -754,7 +756,8 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb3617466d894eb0ad11d06bab1e6e89c571c0a27d660685d327d0c6e1e1ccd" dependencies = [ "dashmap", "datafusion-common", @@ -771,7 +774,8 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bd8220a0dfcdfddcc785cd7e71770ef1ce54fbe1e08984e5adf537027ecb6de" dependencies = [ "ahash", "arrow", @@ -785,7 +789,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d685a100c66952aaadd0cbe766df46d1887d58fc8bcf3589e6387787f18492b" dependencies = [ "arrow", "async-trait", @@ -802,7 +807,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2c635da9b05b4b4c6c8d935f46fd99f9b6225f834091cf4e3c8a045b68beab" dependencies = [ "ahash", "arrow", @@ -863,7 +869,8 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3ef8abf4dd84d3f20c910822b52779c035ab7f4f2d5e7125ede3bae618e9de8" dependencies = [ "arrow", "arrow-schema", @@ -876,7 +883,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "28.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=28.0.0-rc1#51b4392577554becf637a8adcefa0e7fdc79e41f" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c97d351bbd6bd6497e7c9606ddd3c00cd63e9d185d7ab96fc8a66cf3c449177" dependencies = [ "async-recursion", "chrono", @@ -1526,9 +1534,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56ee889ecc9568871456d42f603d6a0ce59ff328d291063a45cbdf0036baf6db" +checksum = "24e6ab01971eb092ffe6a7d42f49f9ff42662f17604681e2843ad65077ba47dc" dependencies = [ "cc", "libc", @@ -2109,9 +2117,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] @@ -2390,18 +2398,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.174" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b88756493a5bd5e5395d53baa70b194b05764ab85b59e43e4b8f4e1192fa9b1" +checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.174" +version = "1.0.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e5c3a298c7f978e53536f95a63bdc4c4a64550582f31a0359a9afda6aede62e" +checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index a460d2ccc..8b09bc50c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1", features = ["pyarrow", "avro"] } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1", features = ["pyarrow"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "28.0.0-rc1" } +datafusion = { version = "28.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "28.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "28.0.0" } +datafusion-optimizer = { version = "28.0.0" } +datafusion-sql = { version = "28.0.0" } +datafusion-substrait = { version = "28.0.0" } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } From 92ca34bd18f72b533066712dffd51ac23a95fb81 Mon Sep 17 00:00:00 2001 From: Goksel Kabadayi Date: Thu, 3 Aug 2023 06:39:56 +0300 Subject: [PATCH 060/413] Build Linux aarch64 wheel (#443) * Build Linux aarch64 wheel * Remove unnecessary platform tag --- .github/workflows/build.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fe06b9c86..e72c4805a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -180,6 +180,36 @@ jobs: name: dist path: target/wheels/* + build-manylinux-aarch64: + needs: [generate-license] + name: Manylinux arm64 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - run: rm LICENSE.txt + - name: Download LICENSE.txt + uses: actions/download-artifact@v3 + with: + name: python-wheel-license + path: . + - run: cat LICENSE.txt + - name: Build wheels + uses: PyO3/maturin-action@v1 + env: + RUST_BACKTRACE: 1 + with: + rust-toolchain: nightly + target: aarch64 + # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail + manylinux: 2_28 + rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 + args: --release --features protoc + - name: Archive wheels + uses: actions/upload-artifact@v3 + with: + name: dist + path: target/wheels/* + build-sdist: needs: [generate-license] name: Source distribution From 1fde8e47ae6714f02c2d2a088577e92e814dfa34 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Sat, 5 Aug 2023 20:12:30 +0200 Subject: [PATCH 061/413] feat: add case function (#447) (#448) --- datafusion/tests/test_functions.py | 22 ++++++++++++ src/expr.rs | 1 + src/expr/conditional_expr.rs | 54 ++++++++++++++++++++++++++++++ src/functions.rs | 15 +++++++-- 4 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 src/expr/conditional_expr.rs diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index ec334747c..c5ddea352 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -411,3 +411,25 @@ def test_temporal_functions(df): assert result.column(9) == pa.array( [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") ) + + +def test_case(df): + df = df.select( + f.case(column("b")) + .when(literal(4), literal(10)) + .otherwise(literal(8)), + f.case(column("a")) + .when(literal("Hello"), literal("Hola")) + .when(literal("World"), literal("Mundo")) + .otherwise(literal("!!")), + f.case(column("a")) + .when(literal("Hello"), literal("Hola")) + .when(literal("World"), literal("Mundo")) + .end(), + ) + + result = df.collect() + result = result[0] + assert result.column(0) == pa.array([10, 8, 8]) + assert result.column(1) == pa.array(["Hola", "Mundo", "!!"]) + assert result.column(2) == pa.array(["Hola", "Mundo", None]) diff --git a/src/expr.rs b/src/expr.rs index 2fd638a13..d1022e905 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -59,6 +59,7 @@ pub mod bool_expr; pub mod case; pub mod cast; pub mod column; +pub mod conditional_expr; pub mod create_memory_table; pub mod create_view; pub mod cross_join; diff --git a/src/expr/conditional_expr.rs b/src/expr/conditional_expr.rs new file mode 100644 index 000000000..96ef58f56 --- /dev/null +++ b/src/expr/conditional_expr.rs @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::expr::PyExpr; +use datafusion_expr::conditional_expressions::CaseBuilder; +use pyo3::prelude::*; + +#[pyclass(name = "CaseBuilder", module = "datafusion.expr", subclass)] +pub struct PyCaseBuilder { + pub case_builder: CaseBuilder, +} + +impl From for CaseBuilder { + fn from(case_builder: PyCaseBuilder) -> Self { + case_builder.case_builder + } +} + +impl From for PyCaseBuilder { + fn from(case_builder: CaseBuilder) -> PyCaseBuilder { + PyCaseBuilder { case_builder } + } +} + +#[pymethods] +impl PyCaseBuilder { + fn when(&mut self, when: PyExpr, then: PyExpr) -> PyCaseBuilder { + PyCaseBuilder { + case_builder: self.case_builder.when(when.expr, then.expr), + } + } + + fn otherwise(&mut self, else_expr: PyExpr) -> PyResult { + Ok(self.case_builder.otherwise(else_expr.expr)?.clone().into()) + } + + fn end(&mut self) -> PyResult { + Ok(self.case_builder.end()?.clone().into()) + } +} diff --git a/src/functions.rs b/src/functions.rs index 8b60e6433..06d35ef3a 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -17,6 +17,9 @@ use pyo3::{prelude::*, wrap_pyfunction}; +use crate::errors::DataFusionError; +use crate::expr::conditional_expr::PyCaseBuilder; +use crate::expr::PyExpr; use datafusion_common::Column; use datafusion_expr::expr::Alias; use datafusion_expr::{ @@ -27,9 +30,6 @@ use datafusion_expr::{ BuiltinScalarFunction, Expr, WindowFrame, }; -use crate::errors::DataFusionError; -use crate::expr::PyExpr; - #[pyfunction] fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { datafusion_expr::in_list( @@ -115,6 +115,14 @@ fn count_star() -> PyResult { }) } +/// Create a CASE WHEN statement with literal WHEN expressions for comparison to the base expression. +#[pyfunction] +fn case(expr: PyExpr) -> PyResult { + Ok(PyCaseBuilder { + case_builder: datafusion_expr::case(expr.expr), + }) +} + /// Creates a new Window function expression #[pyfunction] fn window( @@ -355,6 +363,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(chr))?; m.add_wrapped(wrap_pyfunction!(char_length))?; m.add_wrapped(wrap_pyfunction!(coalesce))?; + m.add_wrapped(wrap_pyfunction!(case))?; m.add_wrapped(wrap_pyfunction!(col))?; m.add_wrapped(wrap_pyfunction!(concat_ws))?; m.add_wrapped(wrap_pyfunction!(concat))?; From e34d203fd63760f8660adee251817df2afb542b0 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Thu, 10 Aug 2023 02:03:34 +0200 Subject: [PATCH 062/413] enhancement(docs): Add user guide (#432) (#445) --- docs/requirements.txt | 4 +- docs/source/api.rst | 1 - docs/source/conf.py | 1 + .../source/contributor-guide/introduction.rst | 85 ++++++ docs/source/index.rst | 263 +++--------------- docs/source/user-guide/basics.rst | 89 ++++++ .../common-operations/aggregations.rst | 60 ++++ .../common-operations/basic-info.rst | 61 ++++ .../common-operations/functions.rst | 117 ++++++++ .../user-guide/common-operations/index.rst | 30 ++ .../user-guide/common-operations/joins.rst | 104 +++++++ .../common-operations/select-and-filter.rst | 67 +++++ .../common-operations/udf-and-udfa.rst | 85 ++++++ .../user-guide/common-operations/windows.rst | 93 +++++++ docs/source/user-guide/introduction.rst | 43 +++ docs/source/user-guide/io/avro.rst | 30 ++ docs/source/user-guide/io/csv.rst | 36 +++ docs/source/user-guide/io/index.rst | 28 ++ docs/source/user-guide/io/json.rst | 29 ++ docs/source/user-guide/io/parquet.rst | 36 +++ docs/source/user-guide/sql.rst | 39 +++ 21 files changed, 1075 insertions(+), 226 deletions(-) create mode 100644 docs/source/contributor-guide/introduction.rst create mode 100644 docs/source/user-guide/basics.rst create mode 100644 docs/source/user-guide/common-operations/aggregations.rst create mode 100644 docs/source/user-guide/common-operations/basic-info.rst create mode 100644 docs/source/user-guide/common-operations/functions.rst create mode 100644 docs/source/user-guide/common-operations/index.rst create mode 100644 docs/source/user-guide/common-operations/joins.rst create mode 100644 docs/source/user-guide/common-operations/select-and-filter.rst create mode 100644 docs/source/user-guide/common-operations/udf-and-udfa.rst create mode 100644 docs/source/user-guide/common-operations/windows.rst create mode 100644 docs/source/user-guide/introduction.rst create mode 100644 docs/source/user-guide/io/avro.rst create mode 100644 docs/source/user-guide/io/csv.rst create mode 100644 docs/source/user-guide/io/index.rst create mode 100644 docs/source/user-guide/io/json.rst create mode 100644 docs/source/user-guide/io/parquet.rst create mode 100644 docs/source/user-guide/sql.rst diff --git a/docs/requirements.txt b/docs/requirements.txt index 8eb744968..eb7f5b7bb 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -19,4 +19,6 @@ sphinx==5.3.0 pydata-sphinx-theme==0.8.0 myst-parser maturin -jinja2 \ No newline at end of file +jinja2 +ipython +pandas \ No newline at end of file diff --git a/docs/source/api.rst b/docs/source/api.rst index a5d65433d..d9f4a09dd 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -24,7 +24,6 @@ API Reference .. toctree:: :maxdepth: 2 - api/config api/dataframe api/execution_context api/expression diff --git a/docs/source/conf.py b/docs/source/conf.py index 929c24930..0822e0ab6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -52,6 +52,7 @@ "sphinx.ext.viewcode", "sphinx.ext.napoleon", "myst_parser", + "IPython.sphinxext.ipython_directive", ] source_suffix = { diff --git a/docs/source/contributor-guide/introduction.rst b/docs/source/contributor-guide/introduction.rst new file mode 100644 index 000000000..dd61ad8fc --- /dev/null +++ b/docs/source/contributor-guide/introduction.rst @@ -0,0 +1,85 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Introduction +============ +We welcome and encourage contributions of all kinds, such as: + +1. Tickets with issue reports of feature requests +2. Documentation improvements +3. Code, both PR and (especially) PR Review. + +In addition to submitting new PRs, we have a healthy tradition of community members reviewing each other’s PRs. +Doing so is a great way to help the community as well as get more familiar with Rust and the relevant codebases. + +How to develop +-------------- + +This assumes that you have rust and cargo installed. We use the workflow recommended by `pyo3 `_ and `maturin `_. + +Bootstrap: + +.. code-block:: shell + + # fetch this repo + git clone git@github.com:apache/arrow-datafusion-python.git + # prepare development environment (used to build wheel / install in development) + python3 -m venv venv + # activate the venv + source venv/bin/activate + # update pip itself if necessary + python -m pip install -U pip + # install dependencies (for Python 3.8+) + python -m pip install -r requirements-310.txt + +The tests rely on test data in git submodules. + +.. code-block:: shell + + git submodule init + git submodule update + + +Whenever rust code changes (your changes or via `git pull`): + +.. code-block:: shell + + # make sure you activate the venv using "source venv/bin/activate" first + maturin develop + python -m pytest + + +Update Dependencies +------------------- + +To change test dependencies, change the `requirements.in` and run + +.. code-block:: shell + + # install pip-tools (this can be done only once), also consider running in venv + python -m pip install pip-tools + python -m piptools compile --generate-hashes -o requirements-310.txt + + +To update dependencies, run with `-U` + +.. code-block:: shell + + python -m piptools compile -U --generate-hashes -o requirements-310.txt + + +More details about pip-tools `here `_ diff --git a/docs/source/index.rst b/docs/source/index.rst index 78f44ea17..155b0cf9f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -31,12 +31,17 @@ Its query engine, DataFusion, is written in `Rust `_, Technically, zero-copy is achieved via the `c data interface `_. -How to use it -============= +Install +------- -Simple usage: +.. code-block:: shell + + pip install datafusion -.. code-block:: python +Example +------- + +.. ipython:: python import datafusion from datafusion import col @@ -50,7 +55,7 @@ Simple usage: [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], names=["a", "b"], ) - df = ctx.create_dataframe([[batch]]) + df = ctx.create_dataframe([[batch]], name="batch_array") # create a new statement df = df.select( @@ -58,234 +63,44 @@ Simple usage: col("a") - col("b"), ) - # execute and collect the first (and only) batch - result = df.collect()[0] - - assert result.column(0) == pyarrow.array([5, 7, 9]) - assert result.column(1) == pyarrow.array([-3, -3, -3]) - - -We can also execute a query against data stored in CSV - -.. code-block:: bash - - echo "a,b\n1,4\n2,5\n3,6" > example.csv - - -.. code-block:: python - - import datafusion - from datafusion import col - import pyarrow - - # create a context - ctx = datafusion.SessionContext() - - # register a CSV - ctx.register_csv('example', 'example.csv') - - # create a new statement - df = ctx.table('example').select( - col("a") + col("b"), - col("a") - col("b"), - ) - - # execute and collect the first (and only) batch - result = df.collect()[0] - - assert result.column(0) == pyarrow.array([5, 7, 9]) - assert result.column(1) == pyarrow.array([-3, -3, -3]) - - -And how to execute a query against a CSV using SQL: - - -.. code-block:: python - - import datafusion - from datafusion import col - import pyarrow - - # create a context - ctx = datafusion.SessionContext() - - # register a CSV - ctx.register_csv('example', 'example.csv') - - # create a new statement via SQL - df = ctx.sql("SELECT a+b, a-b FROM example") - - # execute and collect the first (and only) batch - result = df.collect()[0] - - assert result.column(0) == pyarrow.array([5, 7, 9]) - assert result.column(1) == pyarrow.array([-3, -3, -3]) - - - -UDFs ----- - -.. code-block:: python - - import pyarrow - from datafusion import udf - - def is_null(array: pyarrow.Array) -> pyarrow.Array: - return array.is_null() - - is_null_arr = udf(is_null, [pyarrow.int64()], pyarrow.bool_(), 'stable') - - # create a context - ctx = datafusion.SessionContext() - - # create a RecordBatch and a new DataFrame from it - batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], - names=["a", "b"], - ) - df = ctx.create_dataframe([[batch]]) - - df = df.select(is_null_arr(col("a"))) - - result = df.collect()[0] - - assert result.column(0) == pyarrow.array([False] * 3) - + df -UDAF ----- - -.. code-block:: python - - import pyarrow - import pyarrow.compute - import datafusion - from datafusion import udaf, Accumulator - from datafusion import col - - - class MyAccumulator(Accumulator): - """ - Interface of a user-defined accumulation. - """ - def __init__(self): - self._sum = pyarrow.scalar(0.0) - - def update(self, values: pyarrow.Array) -> None: - # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(values).as_py()) - - def merge(self, states: pyarrow.Array) -> None: - # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(states).as_py()) - - def state(self) -> pyarrow.Array: - return pyarrow.array([self._sum.as_py()]) - - def evaluate(self) -> pyarrow.Scalar: - return self._sum - - # create a context - ctx = datafusion.SessionContext() - - # create a RecordBatch and a new DataFrame from it - batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], - names=["a", "b"], - ) - df = ctx.create_dataframe([[batch]]) - - my_udaf = udaf(MyAccumulator, pyarrow.float64(), pyarrow.float64(), [pyarrow.float64()], 'stable') - - df = df.aggregate( - [], - [my_udaf(col("a"))] - ) - - result = df.collect()[0] - - assert result.column(0) == pyarrow.array([6.0]) - -How to install (from pip) -========================= - -.. code-block:: shell - - pip install datafusion - -You can verify the installation by running: - -.. code-block:: python - - >>> import datafusion - >>> datafusion.__version__ - '0.6.0' - - -How to develop -============== - -This assumes that you have rust and cargo installed. We use the workflow recommended by `pyo3 `_ and `maturin `_. - -Bootstrap: - -.. code-block:: shell - - # fetch this repo - git clone git@github.com:apache/arrow-datafusion-python.git - # prepare development environment (used to build wheel / install in development) - python3 -m venv venv - # activate the venv - source venv/bin/activate - # update pip itself if necessary - python -m pip install -U pip - # install dependencies (for Python 3.8+) - python -m pip install -r requirements-310.txt - -The tests rely on test data in git submodules. - -.. code-block:: shell - - git submodule init - git submodule update - - -Whenever rust code changes (your changes or via `git pull`): - -.. code-block:: shell - - # make sure you activate the venv using "source venv/bin/activate" first - maturin develop - python -m pytest - - -How to update dependencies -========================== - -To change test dependencies, change the `requirements.in` and run - -.. code-block:: shell - - # install pip-tools (this can be done only once), also consider running in venv - python -m pip install pip-tools - python -m piptools compile --generate-hashes -o requirements-310.txt +.. _toc.links: +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: LINKS -To update dependencies, run with `-U` + Github and Issue Tracker + Rust's API Docs + Code of conduct -.. code-block:: shell - - python -m piptools compile -U --generate-hashes -o requirements-310.txt +.. _toc.guide: +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: USER GUIDE + user-guide/introduction + user-guide/basics + user-guide/common-operations/index + user-guide/io/index + user-guide/sql -More details about pip-tools `here `_ +.. _toc.contributor_guide: +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: CONTRIBUTOR GUIDE -API reference -============= + contributor-guide/introduction +.. _toc.api: .. toctree:: - :maxdepth: 2 + :hidden: + :maxdepth: 1 + :caption: API api diff --git a/docs/source/user-guide/basics.rst b/docs/source/user-guide/basics.rst new file mode 100644 index 000000000..438b23199 --- /dev/null +++ b/docs/source/user-guide/basics.rst @@ -0,0 +1,89 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Concepts +======== + +In this section, we will cover a basic example to introduce a few key concepts. + +.. code-block:: python + + import datafusion + from datafusion import col + import pyarrow + + # create a context + ctx = datafusion.SessionContext() + + # create a RecordBatch and a new DataFrame from it + batch = pyarrow.RecordBatch.from_arrays( + [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]]) + + # create a new statement + df = df.select( + col("a") + col("b"), + col("a") - col("b"), + ) + + # execute and collect the first (and only) batch + result = df.collect()[0] + +The first statement group: + +.. code-block:: python + + # create a context + ctx = datafusion.SessionContext() + +creates a :code:`SessionContext`, that is, the main interface for executing queries with DataFusion. It maintains the state +of the connection between a user and an instance of the DataFusion engine. Additionally it provides the following functionality: + +- Create a DataFrame from a CSV or Parquet data source. +- Register a CSV or Parquet data source as a table that can be referenced from a SQL query. +- Register a custom data source that can be referenced from a SQL query. +- Execute a SQL query + +The second statement group creates a :code:`DataFrame`, + +.. code-block:: python + + # create a RecordBatch and a new DataFrame from it + batch = pyarrow.RecordBatch.from_arrays( + [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]]) + +A DataFrame refers to a (logical) set of rows that share the same column names, similar to a `Pandas DataFrame `_. +DataFrames are typically created by calling a method on :code:`SessionContext`, such as :code:`read_csv`, and can then be modified by +calling the transformation methods, such as :meth:`.DataFrame.filter`, :meth:`.DataFrame.select`, :meth:`.DataFrame.aggregate`, +and :meth:`.DataFrame.limit` to build up a query definition. + +The third statement uses :code:`Expressions` to build up a query definition. + +.. code-block:: python + + df = df.select( + col("a") + col("b"), + col("a") - col("b"), + ) + +Finally the :code:`collect` method converts the logical plan represented by the DataFrame into a physical plan and execute it, +collecting all results into a list of `RecordBatch `_. \ No newline at end of file diff --git a/docs/source/user-guide/common-operations/aggregations.rst b/docs/source/user-guide/common-operations/aggregations.rst new file mode 100644 index 000000000..235d644e6 --- /dev/null +++ b/docs/source/user-guide/common-operations/aggregations.rst @@ -0,0 +1,60 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Aggregation +============ + +An aggregate or aggregation is a function where the values of multiple rows are processed together to form a single summary value. +For performing an aggregation, DataFusion provides the :meth:`.DataFrame.aggregate` + +.. ipython:: python + + from datafusion import SessionContext + from datafusion import column, lit + from datafusion import functions as f + import random + + ctx = SessionContext() + df = ctx.from_pydict( + { + "a": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"], + "b": ["one", "one", "two", "three", "two", "two", "one", "three"], + "c": [random.randint(0, 100) for _ in range(8)], + "d": [random.random() for _ in range(8)], + }, + name="foo_bar" + ) + + col_a = column("a") + col_b = column("b") + col_c = column("c") + col_d = column("d") + + df.aggregate([], [f.approx_distinct(col_c), f.approx_median(col_d), f.approx_percentile_cont(col_d, lit(0.5))]) + +When the :code:`group_by` list is empty the aggregation is done over the whole :class:`.DataFrame`. For grouping +the :code:`group_by` list must contain at least one column + +.. ipython:: python + + df.aggregate([col_a], [f.sum(col_c), f.max(col_d), f.min(col_d)]) + +More than one column can be used for grouping + +.. ipython:: python + + df.aggregate([col_a, col_b], [f.sum(col_c), f.max(col_d), f.min(col_d)]) diff --git a/docs/source/user-guide/common-operations/basic-info.rst b/docs/source/user-guide/common-operations/basic-info.rst new file mode 100644 index 000000000..424e1cc92 --- /dev/null +++ b/docs/source/user-guide/common-operations/basic-info.rst @@ -0,0 +1,61 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Basic Operations +================ + +In this section, you will learn how to display essential details of DataFrames using specific functions. + +.. ipython:: python + + from datafusion import SessionContext + import random + + ctx = SessionContext() + df = ctx.from_pydict({ + "nrs": [1, 2, 3, 4, 5], + "names": ["python", "ruby", "java", "haskell", "go"], + "random": random.sample(range(1000), 5), + "groups": ["A", "A", "B", "C", "B"], + }) + df + +Use :meth:`.DataFrame.limit` to view the top rows of the frame: + +.. ipython:: python + + df.limit(2) + +Display the columns of the DataFrame using :meth:`.DataFrame.schema`: + +.. ipython:: python + + df.schema() + +The method :meth:`.DataFrame.to_pandas` uses pyarrow to convert to pandas DataFrame, by collecting the batches, +passing them to an Arrow table, and then converting them to a pandas DataFrame. + +.. ipython:: python + + df.to_pandas() + +:meth:`.DataFrame.describe` shows a quick statistic summary of your data: + +.. ipython:: python + + df.describe() + diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst new file mode 100644 index 000000000..7e5c592d8 --- /dev/null +++ b/docs/source/user-guide/common-operations/functions.rst @@ -0,0 +1,117 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Functions +========= + +DataFusion provides a large number of built-in functions for performing complex queries without requiring user-defined functions. +In here we will cover some of the more popular use cases. If you want to view all the functions go to the :ref:`Functions` API Reference. + +We'll use the pokemon dataset in the following examples. + +.. ipython:: python + + import urllib.request + from datafusion import SessionContext + + urllib.request.urlretrieve( + "https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv", + "pokemon.csv", + ) + + ctx = SessionContext() + ctx.register_csv("pokemon", "pokemon.csv") + df = ctx.table("pokemon") + +Mathematical +------------ + +DataFusion offers mathematical functions such as :func:`.pow` or :func:`.log` + +.. ipython:: python + + from datafusion import col, literal + from datafusion import functions as f + + df.select( + f.pow(col('"Attack"'), literal(2)) - f.pow(col('"Defense"'), literal(2)) + ).limit(10) + + +Conditional +----------- + +There 3 conditional functions in DataFusion :func:`.coalesce`, :func:`.nullif` and :func:`.case` (not available in Python) + +.. ipython:: python + + df.select( + f.coalesce(col('"Type 1"'), col('"Type 2"')).alias("dominant_type") + ).limit(10) + +Temporal +-------- + +For selecting the current time use :func:`.now` + +.. ipython:: python + + df.select(f.now()) + +Convert to timestamps using :func:`.to_timestamp` + +.. ipython:: python + + df.select(f.to_timestamp(col('"Total"')).alias("timestamp")) + +String +------ + +In the field of data science, working with textual data is a common task. To make string manipulation easier, +DataFusion offers a range of helpful options. + +.. ipython:: python + + df.select( + f.char_length(col('"Name"')).alias("len"), + f.lower(col('"Name"')).alias("lower"), + f.left(col('"Name"'), literal(4)).alias("code") + ) + +This also includes the functions for regular expressions :func:`.regexp_replace` and :func:`.regexp_match` + +.. ipython:: python + + df.select( + f.regexp_match(col('"Name"'), literal("Char")).alias("dragons"), + f.regexp_replace(col('"Name"'), literal("saur"), literal("fleur")).alias("flowers") + ) + + +Other +----- + +The function :func:`.in_list` allows to check a column for the presence of multiple values: + +.. ipython:: python + + types = [literal("Grass"), literal("Fire"), literal("Water")] + ( + df.select(f.in_list(col('"Type 1"'), types, negated=False).alias("basic_types")) + .limit(20) + .to_pandas() + ) diff --git a/docs/source/user-guide/common-operations/index.rst b/docs/source/user-guide/common-operations/index.rst new file mode 100644 index 000000000..950afb93e --- /dev/null +++ b/docs/source/user-guide/common-operations/index.rst @@ -0,0 +1,30 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Common Operations +================= + +.. toctree:: + :maxdepth: 2 + + basic-info + select-and-filter + joins + functions + aggregations + windows + udf-and-udfa diff --git a/docs/source/user-guide/common-operations/joins.rst b/docs/source/user-guide/common-operations/joins.rst new file mode 100644 index 000000000..128203116 --- /dev/null +++ b/docs/source/user-guide/common-operations/joins.rst @@ -0,0 +1,104 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Joins +===== + +DataFusion supports the following join variants via the method :meth:`.DataFrame.join` + +- Inner Join +- Left Join +- Right Join +- Full Join +- Left Semi Join +- Left Anti Join + +For the examples in this section we'll use the following two DataFrames + +.. ipython:: python + + from datafusion import SessionContext + + ctx = SessionContext() + + left = ctx.from_pydict( + { + "customer_id": [1, 2, 3], + "customer": ["Alice", "Bob", "Charlie"], + } + ) + + right = ctx.from_pylist([ + {"id": 1, "name": "CityCabs"}, + {"id": 2, "name": "MetroRide"}, + {"id": 5, "name": "UrbanGo"}, + ]) + +Inner Join +---------- + +When using an inner join, only rows containing the common values between the two join columns present in both DataFrames +will be included in the resulting DataFrame. + +.. ipython:: python + + left.join(right, join_keys=(["customer_id"], ["id"]), how="inner") + +The parameter :code:`join_keys` specifies the columns from the left DataFrame and right DataFrame that contains the values +that should match. + +Left Join +--------- + +A left join combines rows from two DataFrames using the key columns. It returns all rows from the left DataFrame and +matching rows from the right DataFrame. If there's no match in the right DataFrame, it returns null +values for the corresponding columns. + +.. ipython:: python + + left.join(right, join_keys=(["customer_id"], ["id"]), how="left") + +Full Join +--------- + +A full join merges rows from two tables based on a related column, returning all rows from both tables, even if there +is no match. Unmatched rows will have null values. + +.. ipython:: python + + left.join(right, join_keys=(["customer_id"], ["id"]), how="full") + +Left Semi Join +-------------- + +A left semi join retrieves matching rows from the left table while +omitting duplicates with multiple matches in the right table. + +.. ipython:: python + + left.join(right, join_keys=(["customer_id"], ["id"]), how="semi") + +Left Anti Join +-------------- + +A left anti join shows all rows from the left table without any matching rows in the right table, +based on a the specified matching columns. It excludes rows from the left table that have at least one matching row in +the right table. + +.. ipython:: python + + left.join(right, join_keys=(["customer_id"], ["id"]), how="anti") \ No newline at end of file diff --git a/docs/source/user-guide/common-operations/select-and-filter.rst b/docs/source/user-guide/common-operations/select-and-filter.rst new file mode 100644 index 000000000..8ede230e6 --- /dev/null +++ b/docs/source/user-guide/common-operations/select-and-filter.rst @@ -0,0 +1,67 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Column Selections +================= + +Use :meth:`.DataFrame.select_columns` for basic column selection. + +DataFusion can work with several file types, to start simple we can use a subset of the +`TLC Trip Record Data `_ + +.. ipython:: python + + import urllib.request + from datafusion import SessionContext + + urllib.request.urlretrieve("https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet", + "yellow_trip_data.parquet") + + ctx = SessionContext() + df = ctx.read_parquet("yellow_trip_data.parquet") + df.select_columns("trip_distance", "passenger_count") + +For mathematical or logical operations use :func:`.col` to select columns, and give meaningful names to the resulting +operations using :func:`.alias` + + +.. ipython:: python + + from datafusion import col, lit + df.select((col("tip_amount") + col("tolls_amount")).alias("tips_plus_tolls")) + +.. warning:: + + Please be aware that all identifiers are effectively made lower-case in SQL, so if your file has capital letters + (ex: Name) you must put your column name in double quotes or the selection won’t work. As an alternative for simple + column selection use :meth:`.DataFrame.select_columns` without double quotes + +For selecting columns with capital letters use ``'"VendorID"'`` + +.. ipython:: python + + df.select(col('"VendorID"')) + + +To combine it with literal values use the :func:`.lit` + +.. ipython:: python + + large_trip_distance = col("trip_distance") > lit(5.0) + low_passenger_count = col("passenger_count") < lit(4) + df.select((large_trip_distance & low_passenger_count).alias("lonely_trips")) + diff --git a/docs/source/user-guide/common-operations/udf-and-udfa.rst b/docs/source/user-guide/common-operations/udf-and-udfa.rst new file mode 100644 index 000000000..62d249c7e --- /dev/null +++ b/docs/source/user-guide/common-operations/udf-and-udfa.rst @@ -0,0 +1,85 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +User Defined Functions +====================== + +DataFusion provides powerful expressions and functions, reducing the need for custom Python functions. +However you can still incorporate your own functions, i.e. User-Defined Functions (UDFs), with the :func:`.udf` function. + +.. ipython:: python + + import pyarrow + import datafusion + from datafusion import udf, col + + def is_null(array: pyarrow.Array) -> pyarrow.Array: + return array.is_null() + + is_null_arr = udf(is_null, [pyarrow.int64()], pyarrow.bool_(), 'stable') + + ctx = datafusion.SessionContext() + + batch = pyarrow.RecordBatch.from_arrays( + [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]], name="batch_array") + + df.select(is_null_arr(col("a"))).to_pandas() + +Additionally the :func:`.udaf` function allows you to define User-Defined Aggregate Functions (UDAFs) + +.. code-block:: python + + import pyarrow + import pyarrow.compute + import datafusion + from datafusion import col, udaf, Accumulator + + class MyAccumulator(Accumulator): + """ + Interface of a user-defined accumulation. + """ + def __init__(self): + self._sum = pyarrow.scalar(0.0) + + def update(self, values: pyarrow.Array) -> None: + # not nice since pyarrow scalars can't be summed yet. This breaks on `None` + self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(values).as_py()) + + def merge(self, states: pyarrow.Array) -> None: + # not nice since pyarrow scalars can't be summed yet. This breaks on `None` + self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(states).as_py()) + + def state(self) -> pyarrow.Array: + return pyarrow.array([self._sum.as_py()]) + + def evaluate(self) -> pyarrow.Scalar: + return self._sum + + ctx = datafusion.SessionContext() + df = ctx.from_pydict( + { + "a": [1, 2, 3], + "b": [4, 5, 6], + } + ) + + my_udaf = udaf(MyAccumulator, pyarrow.float64(), pyarrow.float64(), [pyarrow.float64()], 'stable') + + df.aggregate([],[my_udaf(col("a"))]) diff --git a/docs/source/user-guide/common-operations/windows.rst b/docs/source/user-guide/common-operations/windows.rst new file mode 100644 index 000000000..f884c7e0d --- /dev/null +++ b/docs/source/user-guide/common-operations/windows.rst @@ -0,0 +1,93 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Window Functions +================ + +In this section you will learn about window functions. A window function utilizes values from one or multiple rows to +produce a result for each individual row, unlike an aggregate function that provides a single value for multiple rows. + +The functionality of window functions in DataFusion is supported by the dedicated :func:`.window` function. + +We'll use the pokemon dataset (from Ritchie Vink) in the following examples. + +.. ipython:: python + + import urllib.request + from datafusion import SessionContext + from datafusion import col + from datafusion import functions as f + + urllib.request.urlretrieve( + "https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv", + "pokemon.csv", + ) + + ctx = SessionContext() + df = ctx.read_csv("pokemon.csv") + +Here is an example that shows how to compare each pokemons’s attack power with the average attack power in its :code:`"Type 1"` + +.. ipython:: python + + df.select( + col('"Name"'), + col('"Attack"'), + f.alias( + f.window("avg", [col('"Attack"')], partition_by=[col('"Type 1"')]), + "Average Attack", + ) + ) + +You can also control the order in which rows are processed by window functions by providing +a list of :func:`.order_by` functions for the :code:`order_by` parameter. + +.. ipython:: python + + df.select( + col('"Name"'), + col('"Attack"'), + f.alias( + f.window( + "rank", + [], + partition_by=[col('"Type 1"')], + order_by=[f.order_by(col('"Attack"'))], + ), + "rank", + ), + ) + +The possible window functions are: + +1. Rank Functions + - rank + - dense_rank + - row_number + - ntile + +2. Analytical Functions + - cume_dist + - percent_rank + - lag + - lead + - first_value + - last_value + - nth_value + +3. Aggregate Functions + - All aggregate functions can be used as window functions. diff --git a/docs/source/user-guide/introduction.rst b/docs/source/user-guide/introduction.rst new file mode 100644 index 000000000..8abb9113e --- /dev/null +++ b/docs/source/user-guide/introduction.rst @@ -0,0 +1,43 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. _guide: + +Introduction +============ + +Welcome to the User Guide for the Python bindings of Arrow DataFusion. This guide aims to provide an introduction to +DataFusion through various examples and highlight the most effective ways of using it. + +Installation +------------ + +DataFusion is a Python library and, as such, can be installed via pip from `PyPI `__. + +.. code-block:: shell + + pip install datafusion + +You can verify the installation by running: + +.. ipython:: python + + import datafusion + datafusion.__version__ + + + diff --git a/docs/source/user-guide/io/avro.rst b/docs/source/user-guide/io/avro.rst new file mode 100644 index 000000000..85d546e2a --- /dev/null +++ b/docs/source/user-guide/io/avro.rst @@ -0,0 +1,30 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Avro +==== + +`Avro `_ is a serialization format for record data. Reading an avro file is very straightforward +with :meth:`.SessionContext.read_avro` + +.. code-block:: python + + + from datafusion import SessionContext + + ctx = SessionContext() + df = ctx.read_avro("file.avro") \ No newline at end of file diff --git a/docs/source/user-guide/io/csv.rst b/docs/source/user-guide/io/csv.rst new file mode 100644 index 000000000..3f95c54a5 --- /dev/null +++ b/docs/source/user-guide/io/csv.rst @@ -0,0 +1,36 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +CSV +=== + +Reading a csv is very straightforward with :meth:`.SessionContext.read_csv` + +.. code-block:: python + + + from datafusion import SessionContext + + ctx = SessionContext() + df = ctx.read_csv("file.csv") + +An alternative is to use :meth:`.SessionContext.register_csv` + +.. code-block:: python + + ctx.register_csv("file", "file.csv") + df = ctx.table("file") \ No newline at end of file diff --git a/docs/source/user-guide/io/index.rst b/docs/source/user-guide/io/index.rst new file mode 100644 index 000000000..af08240ff --- /dev/null +++ b/docs/source/user-guide/io/index.rst @@ -0,0 +1,28 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +IO +== + +.. toctree:: + :maxdepth: 2 + + csv + parquet + json + avro + diff --git a/docs/source/user-guide/io/json.rst b/docs/source/user-guide/io/json.rst new file mode 100644 index 000000000..5949a0380 --- /dev/null +++ b/docs/source/user-guide/io/json.rst @@ -0,0 +1,29 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +JSON +==== +`JSON `_ (JavaScript Object Notation) is a lightweight data-interchange format. +When it comes to reading a JSON file, using :meth:`.SessionContext.read_json` is a simple and easy + +.. code-block:: python + + + from datafusion import SessionContext + + ctx = SessionContext() + df = ctx.read_avro("file.json") \ No newline at end of file diff --git a/docs/source/user-guide/io/parquet.rst b/docs/source/user-guide/io/parquet.rst new file mode 100644 index 000000000..78bba30c5 --- /dev/null +++ b/docs/source/user-guide/io/parquet.rst @@ -0,0 +1,36 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Parquet +======= + +It is quite simple to read a parquet file using the :meth:`.SessionContext.read_parquet` function. + +.. code-block:: python + + + from datafusion import SessionContext + + ctx = SessionContext() + df = ctx.read_parquet("file.parquet") + +An alternative is to use :meth:`.SessionContext.register_parquet` + +.. code-block:: python + + ctx.register_parquet("file", "file.parquet") + df = ctx.table("file") \ No newline at end of file diff --git a/docs/source/user-guide/sql.rst b/docs/source/user-guide/sql.rst new file mode 100644 index 000000000..6fa7f0c6a --- /dev/null +++ b/docs/source/user-guide/sql.rst @@ -0,0 +1,39 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +SQL +=== + +DataFusion also offers a SQL API, read the full reference `here `_ + +.. ipython:: python + + import datafusion + from datafusion import col + import pyarrow + + # create a context + ctx = datafusion.SessionContext() + + # register a CSV + ctx.register_csv('pokemon', 'pokemon.csv') + + # create a new statement via SQL + df = ctx.sql('SELECT "Attack"+"Defense", "Attack"-"Defense" FROM pokemon') + + # collect and convert to pandas DataFrame + df.to_pandas() \ No newline at end of file From 37c91f4458daa11f50c313e036761f7494ab27f9 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Fri, 11 Aug 2023 21:45:21 +0200 Subject: [PATCH 063/413] docs: include pre-commit hooks section in contributor guide (#455) --- README.md | 6 +++--- docs/source/contributor-guide/introduction.rst | 9 +++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 506a38227..a682f73d5 100644 --- a/README.md +++ b/README.md @@ -222,11 +222,11 @@ python -m pytest ### Running & Installing pre-commit hooks -arrow-datafusion-python takes advantage of (pre-commit)[https://pre-commit.com/] to assist developers in with code linting to help reduce the number of commits that ultimately fail in CI due to linter errors. Using the pre-commit hooks is optional for the developer but certainly helpful for keep PRs clean and concise. +arrow-datafusion-python takes advantage of [pre-commit](https://pre-commit.com/) to assist developers with code linting to help reduce the number of commits that ultimately fail in CI due to linter errors. Using the pre-commit hooks is optional for the developer but certainly helpful for keeping PRs clean and concise. -Our pre-commit hooks can be installed by running `pre-commit install` which will install the configurations in your ARROW_DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit failing to perform the commit if an offending lint is found giving you the opportunity to make changes locally before pushing. +Our pre-commit hooks can be installed by running `pre-commit install`, which will install the configurations in your ARROW_DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit, failing to complete the commit if an offending lint is found allowing you to make changes locally before pushing. -The pre-commit hooks can also be ran ad-hoc without installing them by simply running `pre-commit run --all-files` +The pre-commit hooks can also be run adhoc without installing them by simply running `pre-commit run --all-files` ## How to update dependencies diff --git a/docs/source/contributor-guide/introduction.rst b/docs/source/contributor-guide/introduction.rst index dd61ad8fc..6de2b87bc 100644 --- a/docs/source/contributor-guide/introduction.rst +++ b/docs/source/contributor-guide/introduction.rst @@ -62,6 +62,15 @@ Whenever rust code changes (your changes or via `git pull`): maturin develop python -m pytest +Running & Installing pre-commit hooks +------------------------------------- + +arrow-datafusion-python takes advantage of `pre-commit `_ to assist developers with code linting to help reduce the number of commits that ultimately fail in CI due to linter errors. Using the pre-commit hooks is optional for the developer but certainly helpful for keeping PRs clean and concise. + +Our pre-commit hooks can be installed by running :code:`pre-commit install`, which will install the configurations in your ARROW_DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit, failing to complete the commit if an offending lint is found allowing you to make changes locally before pushing. + +The pre-commit hooks can also be run adhoc without installing them by simply running :code:`pre-commit run --all-files` + Update Dependencies ------------------- From e1b37401a2d1af86ab16b899f1dda8237a0d3535 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Fri, 11 Aug 2023 21:46:22 +0200 Subject: [PATCH 064/413] feat: add compression options (#456) --- datafusion/tests/test_context.py | 38 +++++++++++++++++++++++++++++++- datafusion/tests/test_sql.py | 23 +++++++++++++++++-- src/context.rs | 31 +++++++++++++++++++++----- 3 files changed, 83 insertions(+), 9 deletions(-) diff --git a/datafusion/tests/test_context.py b/datafusion/tests/test_context.py index 6b1223a16..55a324aec 100644 --- a/datafusion/tests/test_context.py +++ b/datafusion/tests/test_context.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +import gzip import os import pyarrow as pa @@ -336,11 +336,47 @@ def test_read_json(ctx): assert result[0].column(1) == pa.array([1, 2, 3]) +def test_read_json_compressed(ctx, tmp_path): + path = os.path.dirname(os.path.abspath(__file__)) + test_data_path = os.path.join(path, "data_test_context", "data.json") + + # File compression type + gzip_path = tmp_path / "data.json.gz" + + with open(test_data_path, "rb") as csv_file: + with gzip.open(gzip_path, "wb") as gzipped_file: + gzipped_file.writelines(csv_file) + + df = ctx.read_json( + gzip_path, file_extension=".gz", file_compression_type="gz" + ) + result = df.collect() + + assert result[0].column(0) == pa.array(["a", "b", "c"]) + assert result[0].column(1) == pa.array([1, 2, 3]) + + def test_read_csv(ctx): csv_df = ctx.read_csv(path="testing/data/csv/aggregate_test_100.csv") csv_df.select(column("c1")).show() +def test_read_csv_compressed(ctx, tmp_path): + test_data_path = "testing/data/csv/aggregate_test_100.csv" + + # File compression type + gzip_path = tmp_path / "aggregate_test_100.csv.gz" + + with open(test_data_path, "rb") as csv_file: + with gzip.open(gzip_path, "wb") as gzipped_file: + gzipped_file.writelines(csv_file) + + csv_df = ctx.read_csv( + gzip_path, file_extension=".gz", file_compression_type="gz" + ) + csv_df.select(column("c1")).show() + + def test_read_parquet(ctx): csv_df = ctx.read_parquet(path="parquet/data/alltypes_plain.parquet") csv_df.show() diff --git a/datafusion/tests/test_sql.py b/datafusion/tests/test_sql.py index 638a222dc..608bb1960 100644 --- a/datafusion/tests/test_sql.py +++ b/datafusion/tests/test_sql.py @@ -19,6 +19,7 @@ import pyarrow as pa import pyarrow.dataset as ds import pytest +import gzip from datafusion import udf @@ -32,6 +33,7 @@ def test_no_table(ctx): def test_register_csv(ctx, tmp_path): path = tmp_path / "test.csv" + gzip_path = tmp_path / "test.csv.gz" table = pa.Table.from_arrays( [ @@ -43,6 +45,10 @@ def test_register_csv(ctx, tmp_path): ) pa.csv.write_csv(table, path) + with open(path, "rb") as csv_file: + with gzip.open(gzip_path, "wb") as gzipped_file: + gzipped_file.writelines(csv_file) + ctx.register_csv("csv", path) ctx.register_csv("csv1", str(path)) ctx.register_csv( @@ -52,6 +58,13 @@ def test_register_csv(ctx, tmp_path): delimiter=",", schema_infer_max_records=10, ) + ctx.register_csv( + "csv_gzip", + gzip_path, + file_extension="gz", + file_compression_type="gzip", + ) + alternative_schema = pa.schema( [ ("some_int", pa.int16()), @@ -61,9 +74,9 @@ def test_register_csv(ctx, tmp_path): ) ctx.register_csv("csv3", path, schema=alternative_schema) - assert ctx.tables() == {"csv", "csv1", "csv2", "csv3"} + assert ctx.tables() == {"csv", "csv1", "csv2", "csv3", "csv_gzip"} - for table in ["csv", "csv1", "csv2"]: + for table in ["csv", "csv1", "csv2", "csv_gzip"]: result = ctx.sql(f"SELECT COUNT(int) AS cnt FROM {table}").collect() result = pa.Table.from_batches(result) assert result.to_pydict() == {"cnt": [4]} @@ -77,6 +90,12 @@ def test_register_csv(ctx, tmp_path): ): ctx.register_csv("csv4", path, delimiter="wrong") + with pytest.raises( + ValueError, + match="file_compression_type must one of: gzip, bz2, xz, zstd", + ): + ctx.register_csv("csv4", path, file_compression_type="rar") + def test_register_parquet(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) diff --git a/src/context.rs b/src/context.rs index cf133d79a..1dca8a791 100644 --- a/src/context.rs +++ b/src/context.rs @@ -17,6 +17,7 @@ use std::collections::{HashMap, HashSet}; use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; use object_store::ObjectStore; @@ -40,6 +41,7 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::datasource::file_format::file_type::FileCompressionType; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext}; @@ -469,7 +471,8 @@ impl PySessionContext { has_header=true, delimiter=",", schema_infer_max_records=1000, - file_extension=".csv"))] + file_extension=".csv", + file_compression_type=None))] fn register_csv( &mut self, name: &str, @@ -479,6 +482,7 @@ impl PySessionContext { delimiter: &str, schema_infer_max_records: usize, file_extension: &str, + file_compression_type: Option, py: Python, ) -> PyResult<()> { let path = path @@ -495,7 +499,8 @@ impl PySessionContext { .has_header(has_header) .delimiter(delimiter[0]) .schema_infer_max_records(schema_infer_max_records) - .file_extension(file_extension); + .file_extension(file_extension) + .file_compression_type(parse_file_compression_type(file_compression_type)?); options.schema = schema.as_ref().map(|x| &x.0); let result = self.ctx.register_csv(name, path, options); @@ -559,7 +564,7 @@ impl PySessionContext { } #[allow(clippy::too_many_arguments)] - #[pyo3(signature = (path, schema=None, schema_infer_max_records=1000, file_extension=".json", table_partition_cols=vec![]))] + #[pyo3(signature = (path, schema=None, schema_infer_max_records=1000, file_extension=".json", table_partition_cols=vec![], file_compression_type=None))] fn read_json( &mut self, path: PathBuf, @@ -567,13 +572,15 @@ impl PySessionContext { schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, String)>, + file_compression_type: Option, py: Python, ) -> PyResult { let path = path .to_str() .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; let mut options = NdJsonReadOptions::default() - .table_partition_cols(convert_table_partition_cols(table_partition_cols)?); + .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) + .file_compression_type(parse_file_compression_type(file_compression_type)?); options.schema_infer_max_records = schema_infer_max_records; options.file_extension = file_extension; let df = if let Some(schema) = schema { @@ -595,7 +602,8 @@ impl PySessionContext { delimiter=",", schema_infer_max_records=1000, file_extension=".csv", - table_partition_cols=vec![]))] + table_partition_cols=vec![], + file_compression_type=None))] fn read_csv( &self, path: PathBuf, @@ -605,6 +613,7 @@ impl PySessionContext { schema_infer_max_records: usize, file_extension: &str, table_partition_cols: Vec<(String, String)>, + file_compression_type: Option, py: Python, ) -> PyResult { let path = path @@ -623,7 +632,8 @@ impl PySessionContext { .delimiter(delimiter[0]) .schema_infer_max_records(schema_infer_max_records) .file_extension(file_extension) - .table_partition_cols(convert_table_partition_cols(table_partition_cols)?); + .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) + .file_compression_type(parse_file_compression_type(file_compression_type)?); if let Some(py_schema) = schema { options.schema = Some(&py_schema.0); @@ -743,6 +753,15 @@ fn convert_table_partition_cols( .collect::, _>>() } +fn parse_file_compression_type( + file_compression_type: Option, +) -> Result { + FileCompressionType::from_str(&*file_compression_type.unwrap_or("".to_string()).as_str()) + .map_err(|_| { + PyValueError::new_err("file_compression_type must one of: gzip, bz2, xz, zstd") + }) +} + impl From for SessionContext { fn from(ctx: PySessionContext) -> SessionContext { ctx.ctx From 0b22c977dbdfbfe1e0536e0b1bf79045e7bf0e7e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 11 Aug 2023 13:46:40 -0600 Subject: [PATCH 065/413] Upgrade to DF 28.0.0-rc1 (#457) * Upgrade to DF 28.0.0-rc1 * cargo update --- Cargo.lock | 290 +++++++++++++++++++------------------- Cargo.toml | 14 +- src/common/data_type.rs | 3 + src/expr.rs | 9 +- src/expr/indexed_field.rs | 9 +- src/udaf.rs | 2 +- 6 files changed, 171 insertions(+), 156 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85634fd11..585e64cb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,9 +38,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c" dependencies = [ "memchr", ] @@ -89,9 +89,9 @@ checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "apache-avro" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf4144857f9e4d7dd6cc4ba4c78efd2a46bad682b029bd0d91e76a021af1b2a" +checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7" dependencies = [ "byteorder", "crc32fast", @@ -106,8 +106,8 @@ dependencies = [ "serde", "serde_json", "snap", - "strum 0.24.1", - "strum_macros 0.24.3", + "strum 0.25.0", + "strum_macros 0.25.2", "thiserror", "typed-builder", "uuid", @@ -128,9 +128,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2feeebd77b34b0bc88f224e06d01c27da4733997cc4789a4e056196656cdc59a" +checksum = "b7104b9e9761613ae92fe770c741d6bbf1dbc791a0fe204400aebdd429875741" dependencies = [ "ahash", "arrow-arith", @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7173f5dc49c0ecb5135f52565af33afd3fdc9a12d13bd6f9973e8b96305e4b2e" +checksum = "38e597a8e8efb8ff52c50eaf8f4d85124ce3c1bf20fab82f476d73739d9ab1c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d7ea725f7d1f8bb2cffc53ef538557e95fc802e217d5be25122d402e22f3d0" +checksum = "2a86d9c1473db72896bd2345ebb6b8ad75b8553ba390875c76708e8dc5c5492d" dependencies = [ "ahash", "arrow-buffer", @@ -183,9 +183,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbe439e077f484e5000b9e1d47b5e4c0d15f2b311a8f5bcc682553d5d67a722" +checksum = "234b3b1c8ed00c874bf95972030ac4def6f58e02ea5a7884314388307fb3669b" dependencies = [ "half", "num", @@ -193,9 +193,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93913cc14875770aa1eef5e310765e855effa352c094cb1c7c00607d0f37b4e1" +checksum = "22f61168b853c7faea8cea23a2169fdff9c82fb10ae5e2c07ad1cab8f6884931" dependencies = [ "arrow-array", "arrow-buffer", @@ -211,9 +211,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef55b67c55ed877e6fe7b923121c19dae5e31ca70249ea2779a17b58fb0fbd9a" +checksum = "10b545c114d9bf8569c84d2fbe2020ac4eea8db462c0a37d0b65f41a90d066fe" dependencies = [ "arrow-array", "arrow-buffer", @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4f4f4a3c54614126a71ab91f6631c9743eb4643d6e9318b74191da9dc6e028b" +checksum = "c6b6852635e7c43e5b242841c7470606ff0ee70eef323004cacc3ecedd33dd8f" dependencies = [ "arrow-buffer", "arrow-schema", @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d41a3659f984a524ef1c2981d43747b24d8eec78e2425267fcd0ef34ce71cd18" +checksum = "a66da9e16aecd9250af0ae9717ae8dd7ea0d8ca5a3e788fe3de9f4ee508da751" dependencies = [ "arrow-array", "arrow-buffer", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b95faa95a378f56ef32d84cc0104ea998c39ef7cd1faaa6b4cebf8ea92846d" +checksum = "60ee0f9d8997f4be44a60ee5807443e396e025c23cf14d2b74ce56135cb04474" dependencies = [ "arrow-array", "arrow-buffer", @@ -276,9 +276,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c68549a4284d9f8b39586afb8d5ff8158b8f0286353a4844deb1d11cf1ba1f26" +checksum = "7fcab05410e6b241442abdab6e1035177dc082bdb6f17049a4db49faed986d63" dependencies = [ "arrow-array", "arrow-buffer", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a75a4a757afc301ce010adadff54d79d66140c4282ed3de565f6ccb716a5cf3" +checksum = "91a847dd9eb0bacd7836ac63b3475c68b2210c2c96d0ec1b808237b973bd5d73" dependencies = [ "ahash", "arrow-array", @@ -306,18 +306,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bebcb57eef570b15afbcf2d07d813eb476fde9f6dd69c81004d6476c197e87e" +checksum = "54df8c47918eb634c20e29286e69494fdc20cafa5173eb6dad49c7f6acece733" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", ] [[package]] name = "arrow-select" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e2943fa433a48921e914417173816af64eef61c0a3d448280e6c40a62df221" +checksum = "941dbe481da043c4bd40c805a19ec2fc008846080c4953171b62bcad5ee5f7fb" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbc92ed638851774f6d7af1ad900b92bc1486746497511868b4298fcbcfa35af" +checksum = "359b2cd9e071d5a3bcf44679f9d85830afebc5b9c98a08019a570a65ae933e0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -368,7 +368,7 @@ checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -379,7 +379,7 @@ checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -426,9 +426,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "blake2" @@ -524,11 +524,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.79" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" dependencies = [ "jobserver", + "libc", ] [[package]] @@ -687,9 +688,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ddbcb2dda5b5033537457992ebde78938014390b2b19f9f4282e3be0e18b0c3" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "ahash", "apache-avro", @@ -738,9 +738,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fbb7b4da925031311743ab96662d55f0f7342d3692744f184f99b2257ef435" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "apache-avro", "arrow", @@ -755,13 +754,14 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb3617466d894eb0ad11d06bab1e6e89c571c0a27d660685d327d0c6e1e1ccd" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ + "arrow", "dashmap", "datafusion-common", "datafusion-expr", + "futures", "hashbrown 0.14.0", "log", "object_store", @@ -773,9 +773,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bd8220a0dfcdfddcc785cd7e71770ef1ce54fbe1e08984e5adf537027ecb6de" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "ahash", "arrow", @@ -783,14 +782,13 @@ dependencies = [ "lazy_static", "sqlparser", "strum 0.25.0", - "strum_macros 0.25.1", + "strum_macros 0.25.2", ] [[package]] name = "datafusion-optimizer" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d685a100c66952aaadd0cbe766df46d1887d58fc8bcf3589e6387787f18492b" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "arrow", "async-trait", @@ -806,9 +804,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2c635da9b05b4b4c6c8d935f46fd99f9b6225f834091cf4e3c8a045b68beab" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "ahash", "arrow", @@ -841,7 +838,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "28.0.0" +version = "29.0.0" dependencies = [ "async-trait", "datafusion", @@ -860,7 +857,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.27", + "syn 2.0.28", "tokio", "url", "uuid", @@ -868,9 +865,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3ef8abf4dd84d3f20c910822b52779c035ab7f4f2d5e7125ede3bae618e9de8" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "arrow", "arrow-schema", @@ -882,9 +878,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "28.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c97d351bbd6bd6497e7c9606ddd3c00cd63e9d185d7ab96fc8a66cf3c449177" +version = "29.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" dependencies = [ "async-recursion", "chrono", @@ -943,9 +938,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", @@ -1065,7 +1060,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -1274,7 +1269,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -1534,9 +1529,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.10" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e6ab01971eb092ffe6a7d42f49f9ff42662f17604681e2843ad65077ba47dc" +checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" dependencies = [ "cc", "libc", @@ -1546,9 +1541,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" +checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" [[package]] name = "lock_api" @@ -1828,9 +1823,9 @@ dependencies = [ [[package]] name = "parquet" -version = "43.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7267a9607c3f955d4d0ac41b88a67cecc0d8d009173ad3da390699a6cb3750" +checksum = "49f9739b984380582bdb7749ae5b5d28839bce899212cf16465c1ac1f8b65d79" dependencies = [ "ahash", "arrow-array", @@ -1931,9 +1926,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" +checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" [[package]] name = "pin-utils" @@ -1960,7 +1955,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" dependencies = [ "proc-macro2", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2041,9 +2036,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb88ae05f306b4bfcde40ac4a51dc0b05936a9207a4b75b798c7729c4258a59" +checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" dependencies = [ "cfg-if", "indoc", @@ -2058,9 +2053,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554db24f0b3c180a9c0b1268f91287ab3f17c162e15b54caaae5a6b3773396b0" +checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" dependencies = [ "once_cell", "target-lexicon", @@ -2068,9 +2063,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "922ede8759e8600ad4da3195ae41259654b9c55da4f7eec84a0ccc7d067a70a4" +checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" dependencies = [ "libc", "pyo3-build-config", @@ -2078,9 +2073,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a5caec6a1dd355964a841fcbeeb1b89fe4146c87295573f94228911af3cc5a2" +checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2090,9 +2085,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.19.1" +version = "0.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0b78ccbb160db1556cdb6fd96c50334c5d4ec44dc5e0a968d0a1208fa0efa8b" +checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" dependencies = [ "proc-macro2", "quote", @@ -2165,9 +2160,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" dependencies = [ "aho-corasick", "memchr", @@ -2177,9 +2172,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.3" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" dependencies = [ "aho-corasick", "memchr", @@ -2281,11 +2276,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.4" +version = "0.38.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" dependencies = [ - "bitflags 2.3.3", + "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", @@ -2294,9 +2289,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.5" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79ea77c539259495ce8ca47f53e66ae0330a8819f67e23ac96ca02f50e7b7d36" +checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" dependencies = [ "log", "ring", @@ -2315,9 +2310,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.1" +version = "0.101.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +checksum = "261e9e0888cba427c3316e6322805653c9425240b6fd96cee7cb671ab70ab8d0" dependencies = [ "ring", "untrusted", @@ -2398,22 +2393,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.175" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.175" +version = "1.0.183" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2429,9 +2424,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.103" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa", "ryu", @@ -2447,7 +2442,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2545,6 +2540,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "spin" version = "0.5.2" @@ -2553,9 +2558,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.35.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca597d77c98894be1f965f2e4e2d2a61575d4998088e655476c73715c54b2b43" +checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87" dependencies = [ "log", "sqlparser_derive", @@ -2590,7 +2595,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros 0.25.1", + "strum_macros 0.25.2", ] [[package]] @@ -2608,22 +2613,22 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" +checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] name = "substrait" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac1ce8315086b127ca0abf162c62279550942bb26ebf7946fe17fe114446472" +checksum = "658f6cbbd29a250869b87e1bb5a4b42db534cacfc1c03284f2536cd36b6c1617" dependencies = [ "git2", "heck", @@ -2637,7 +2642,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.27", + "syn 2.0.28", "typify", "walkdir", ] @@ -2661,9 +2666,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.27" +version = "2.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60f673f44a8255b9c8c657daf66a596d435f2da81a555b06dc644d080ba45e0" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -2672,15 +2677,15 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.10" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2faeef5759ab89935255b1a4cd98e0baf99d1085e37d36599c625dac49ae8e" +checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "tempfile" -version = "3.7.0" +version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", "fastrand", @@ -2706,7 +2711,7 @@ checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2746,11 +2751,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.29.1" +version = "1.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" dependencies = [ - "autocfg", "backtrace", "bytes", "libc", @@ -2758,7 +2762,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", - "socket2", + "socket2 0.5.3", "tokio-macros", "windows-sys", ] @@ -2771,7 +2775,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2824,7 +2828,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", ] [[package]] @@ -2854,9 +2858,9 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.10.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" +checksum = "64cba322cb9b7bc6ca048de49e83918223f35e7a86311267013afff257004870" dependencies = [ "proc-macro2", "quote", @@ -2892,7 +2896,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.27", + "syn 2.0.28", "thiserror", "unicode-ident", ] @@ -2909,7 +2913,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.27", + "syn 2.0.28", "typify-impl", ] @@ -3043,7 +3047,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", "wasm-bindgen-shared", ] @@ -3077,7 +3081,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.27", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3267,9 +3271,9 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "332f188cc1bcf1fe1064b8c58d150f497e697f49774aa846f2dc949d9a25f236" +checksum = "f3b9c234616391070b0b173963ebc65a9195068e7ed3731c6edac2ec45ebe106" dependencies = [ "byteorder", "zerocopy-derive", @@ -3277,13 +3281,13 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.3.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" +checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.28", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8b09bc50c..169d3f5d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "28.0.0" +version = "29.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "28.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "28.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "28.0.0" } -datafusion-optimizer = { version = "28.0.0" } -datafusion-sql = { version = "28.0.0" } -datafusion-substrait = { version = "28.0.0" } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1", features = ["pyarrow", "avro"] } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1", features = ["pyarrow"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 199fb616e..078b8c841 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -242,6 +242,9 @@ impl DataTypeMap { ScalarValue::Decimal128(_, precision, scale) => { Ok(DataType::Decimal128(*precision, *scale)) } + ScalarValue::Decimal256(_, precision, scale) => { + Ok(DataType::Decimal256(*precision, *scale)) + } ScalarValue::Dictionary(data_type, scalar_type) => { // Call this function again to map the dictionary scalar_value to an Arrow type Ok(DataType::Dictionary( diff --git a/src/expr.rs b/src/expr.rs index d1022e905..ecf8fae32 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -30,7 +30,8 @@ use datafusion_expr::{ }, lit, utils::exprlist_to_fields, - Between, BinaryExpr, Case, Cast, Expr, GetIndexedField, Like, LogicalPlan, Operator, TryCast, + Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, LogicalPlan, + Operator, TryCast, }; use crate::common::data_type::{DataTypeMap, RexType}; @@ -213,7 +214,9 @@ impl PyExpr { fn __getitem__(&self, key: &str) -> PyResult { Ok(Expr::GetIndexedField(GetIndexedField::new( Box::new(self.expr.clone()), - ScalarValue::Utf8(Some(key.to_string())), + GetFieldAccess::NamedStructField { + name: ScalarValue::Utf8(Some(key.to_string())), + }, )) .into()) } @@ -309,6 +312,7 @@ impl PyExpr { ScalarValue::Float32(v) => v.into_py(py), ScalarValue::Float64(v) => v.into_py(py), ScalarValue::Decimal128(v, _, _) => v.into_py(py), + ScalarValue::Decimal256(_, _, _) => todo!(), ScalarValue::Int8(v) => v.into_py(py), ScalarValue::Int16(v) => v.into_py(py), ScalarValue::Int32(v) => v.into_py(py), @@ -577,6 +581,7 @@ impl PyExpr { | Operator::BitwiseXor | Operator::BitwiseAnd | Operator::BitwiseOr => DataTypeMap::map_from_arrow_type(&DataType::Binary), + Operator::AtArrow | Operator::ArrowAt => todo!(), }, Expr::Cast(Cast { expr: _, data_type }) => DataTypeMap::map_from_arrow_type(data_type), Expr::Literal(scalar_value) => DataTypeMap::map_from_scalar_value(scalar_value), diff --git a/src/expr/indexed_field.rs b/src/expr/indexed_field.rs index c98607712..e0dad6a4a 100644 --- a/src/expr/indexed_field.rs +++ b/src/expr/indexed_field.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::GetIndexedField; +use datafusion_expr::expr::{GetFieldAccess, GetIndexedField}; use pyo3::prelude::*; use std::fmt::{Display, Formatter}; @@ -47,7 +47,7 @@ impl Display for PyGetIndexedField { "GetIndexedField Expr: {:?} Key: {:?}", - &self.indexed_field.expr, &self.indexed_field.key + &self.indexed_field.expr, &self.indexed_field.field ) } } @@ -59,7 +59,10 @@ impl PyGetIndexedField { } fn key(&self) -> PyResult { - Ok(self.indexed_field.key.clone().into()) + match &self.indexed_field.field { + GetFieldAccess::NamedStructField { name, .. } => Ok(name.clone().into()), + _ => todo!(), + } } /// Get a String representation of this column diff --git a/src/udaf.rs b/src/udaf.rs index ae2e81a08..3b70aeb06 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -126,7 +126,7 @@ impl PyAggregateUDF { ) -> PyResult { let function = create_udaf( name, - input_type.0, + vec![input_type.0], Arc::new(return_type.0), parse_volatility(volatility)?, to_rust_accumulator(accumulator), From 217ede86961d3ffc356556ad75cbe2b514373048 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Thu, 17 Aug 2023 19:19:22 +0200 Subject: [PATCH 066/413] feat: add register_json (#458) --- datafusion/tests/test_sql.py | 53 +++++++++++++++++++++++++++++++++++- src/context.rs | 36 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/datafusion/tests/test_sql.py b/datafusion/tests/test_sql.py index 608bb1960..9d42a1f9d 100644 --- a/datafusion/tests/test_sql.py +++ b/datafusion/tests/test_sql.py @@ -14,12 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import gzip +import os import numpy as np import pyarrow as pa import pyarrow.dataset as ds import pytest -import gzip from datafusion import udf @@ -154,6 +155,56 @@ def test_register_dataset(ctx, tmp_path): assert result.to_pydict() == {"cnt": [100]} +def test_register_json(ctx, tmp_path): + path = os.path.dirname(os.path.abspath(__file__)) + test_data_path = os.path.join(path, "data_test_context", "data.json") + gzip_path = tmp_path / "data.json.gz" + + with open(test_data_path, "rb") as json_file: + with gzip.open(gzip_path, "wb") as gzipped_file: + gzipped_file.writelines(json_file) + + ctx.register_json("json", test_data_path) + ctx.register_json("json1", str(test_data_path)) + ctx.register_json( + "json2", + test_data_path, + schema_infer_max_records=10, + ) + ctx.register_json( + "json_gzip", + gzip_path, + file_extension="gz", + file_compression_type="gzip", + ) + + alternative_schema = pa.schema( + [ + ("some_int", pa.int16()), + ("some_bytes", pa.string()), + ("some_floats", pa.float32()), + ] + ) + ctx.register_json("json3", path, schema=alternative_schema) + + assert ctx.tables() == {"json", "json1", "json2", "json3", "json_gzip"} + + for table in ["json", "json1", "json2", "json_gzip"]: + result = ctx.sql(f'SELECT COUNT("B") AS cnt FROM {table}').collect() + result = pa.Table.from_batches(result) + assert result.to_pydict() == {"cnt": [3]} + + result = ctx.sql("SELECT * FROM json3").collect() + result = pa.Table.from_batches(result) + assert result.schema == alternative_schema + + with pytest.raises( + ValueError, + match="file_compression_type must one of: gzip, bz2, xz, zstd", + ): + ctx.register_json("json4", gzip_path, file_compression_type="rar") + + def test_execute(ctx, tmp_path): data = [1, 1, 2, 2, 3, 11, 12] diff --git a/src/context.rs b/src/context.rs index 1dca8a791..317ab785e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -509,6 +509,42 @@ impl PySessionContext { Ok(()) } + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (name, + path, + schema=None, + schema_infer_max_records=1000, + file_extension=".json", + table_partition_cols=vec![], + file_compression_type=None))] + fn register_json( + &mut self, + name: &str, + path: PathBuf, + schema: Option>, + schema_infer_max_records: usize, + file_extension: &str, + table_partition_cols: Vec<(String, String)>, + file_compression_type: Option, + py: Python, + ) -> PyResult<()> { + let path = path + .to_str() + .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; + + let mut options = NdJsonReadOptions::default() + .file_compression_type(parse_file_compression_type(file_compression_type)?) + .table_partition_cols(convert_table_partition_cols(table_partition_cols)?); + options.schema_infer_max_records = schema_infer_max_records; + options.file_extension = file_extension; + options.schema = schema.as_ref().map(|x| &x.0); + + let result = self.ctx.register_json(name, path, options); + wait_for_future(py, result).map_err(DataFusionError::from)?; + + Ok(()) + } + // Registers a PyArrow.Dataset fn register_dataset(&self, name: &str, dataset: &PyAny, py: Python) -> PyResult<()> { let table: Arc = Arc::new(Dataset::new(dataset, py)?); From 499f0458dcceea93d1512258808a805986b677e7 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Tue, 22 Aug 2023 15:44:42 +0200 Subject: [PATCH 067/413] feat: add basic compression configuration to write_parquet (#459) --- datafusion/tests/test_dataframe.py | 67 ++++++++++++++++++++++++++++++ src/dataframe.rs | 58 ++++++++++++++++++++++++-- 2 files changed, 122 insertions(+), 3 deletions(-) diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index 78cb50f12..ce7d89e7b 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -14,8 +14,10 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import os import pyarrow as pa +import pyarrow.parquet as pq import pytest from datafusion import functions as f @@ -645,3 +647,68 @@ def test_describe(df): "b": [3.0, 3.0, 5.0, 1.0, 4.0, 6.0, 5.0], "c": [3.0, 3.0, 7.0, 1.7320508075688772, 5.0, 8.0, 8.0], } + + +def test_write_parquet(df, tmp_path): + path = tmp_path + + df.write_parquet(str(path)) + result = pq.read_table(str(path)).to_pydict() + expected = df.to_pydict() + + assert result == expected + + +@pytest.mark.parametrize( + "compression, compression_level", + [("gzip", 6), ("brotli", 7), ("zstd", 15)], +) +def test_write_compressed_parquet( + df, tmp_path, compression, compression_level +): + path = tmp_path + + df.write_parquet( + str(path), compression=compression, compression_level=compression_level + ) + + # test that the actual compression scheme is the one written + for root, dirs, files in os.walk(path): + for file in files: + if file.endswith(".parquet"): + metadata = pq.ParquetFile(tmp_path / file).metadata.to_dict() + for row_group in metadata["row_groups"]: + for columns in row_group["columns"]: + assert columns["compression"].lower() == compression + + result = pq.read_table(str(path)).to_pydict() + expected = df.to_pydict() + + assert result == expected + + +@pytest.mark.parametrize( + "compression, compression_level", + [("gzip", 12), ("brotli", 15), ("zstd", 23), ("wrong", 12)], +) +def test_write_compressed_parquet_wrong_compression_level( + df, tmp_path, compression, compression_level +): + path = tmp_path + + with pytest.raises(ValueError): + df.write_parquet( + str(path), + compression=compression, + compression_level=compression_level, + ) + + +@pytest.mark.parametrize("compression", ["brotli", "zstd", "wrong"]) +def test_write_compressed_parquet_missing_compression_level( + df, tmp_path, compression +): + path = tmp_path + + with pytest.raises(ValueError): + df.write_parquet(str(path), compression=compression) diff --git a/src/dataframe.rs b/src/dataframe.rs index b8d8ddc3c..61a44484c 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -23,8 +23,10 @@ use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; use datafusion::dataframe::DataFrame; +use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; +use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; -use pyo3::exceptions::PyTypeError; +use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyTuple; use std::sync::Arc; @@ -308,8 +310,58 @@ impl PyDataFrame { } /// Write a `DataFrame` to a Parquet file. - fn write_parquet(&self, path: &str, py: Python) -> PyResult<()> { - wait_for_future(py, self.df.as_ref().clone().write_parquet(path, None))?; + #[pyo3(signature = ( + path, + compression="uncompressed", + compression_level=None + ))] + fn write_parquet( + &self, + path: &str, + compression: &str, + compression_level: Option, + py: Python, + ) -> PyResult<()> { + fn verify_compression_level(cl: Option) -> Result { + cl.ok_or(PyValueError::new_err("compression_level is not defined")) + } + + let compression_type = match compression.to_lowercase().as_str() { + "snappy" => Compression::SNAPPY, + "gzip" => Compression::GZIP( + GzipLevel::try_new(compression_level.unwrap_or(6)) + .map_err(|e| PyValueError::new_err(format!("{e}")))?, + ), + "brotli" => Compression::BROTLI( + BrotliLevel::try_new(verify_compression_level(compression_level)?) + .map_err(|e| PyValueError::new_err(format!("{e}")))?, + ), + "zstd" => Compression::ZSTD( + ZstdLevel::try_new(verify_compression_level(compression_level)? as i32) + .map_err(|e| PyValueError::new_err(format!("{e}")))?, + ), + "lz0" => Compression::LZO, + "lz4" => Compression::LZ4, + "lz4_raw" => Compression::LZ4_RAW, + "uncompressed" => Compression::UNCOMPRESSED, + _ => { + return Err(PyValueError::new_err(format!( + "Unrecognized compression type {compression}" + ))); + } + }; + + let writer_properties = WriterProperties::builder() + .set_compression(compression_type) + .build(); + + wait_for_future( + py, + self.df + .as_ref() + .clone() + .write_parquet(path, Option::from(writer_properties)), + )?; Ok(()) } From 9c643bf2ae63894952be69b5fa2ae145f160971a Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Tue, 22 Aug 2023 15:44:55 +0200 Subject: [PATCH 068/413] feat: add example of reading parquet from s3 (#460) --- examples/sql-parquet-s3.py | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 examples/sql-parquet-s3.py diff --git a/examples/sql-parquet-s3.py b/examples/sql-parquet-s3.py new file mode 100644 index 000000000..bd7da5e20 --- /dev/null +++ b/examples/sql-parquet-s3.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import datafusion +from datafusion.object_store import AmazonS3 + +region = "us-east-1" +bucket_name = "yellow-trips" + +s3 = AmazonS3( + bucket_name=bucket_name, + region=region, + access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), +) + +ctx = datafusion.SessionContext() +path = f"s3://{bucket_name}/" +ctx.register_object_store(path, s3) + +ctx.register_parquet("trips", path) + +df = ctx.sql("select count(passenger_count) from trips") +df.show() From e24dc75f2fe60efb5bc888fd70d2aede80027c25 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Tue, 22 Aug 2023 15:45:06 +0200 Subject: [PATCH 069/413] feat: add register_avro and read_table (#461) --- datafusion/tests/test_context.py | 10 ++++++++ datafusion/tests/test_sql.py | 26 ++++++++++++++++++++ src/context.rs | 41 ++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/datafusion/tests/test_context.py b/datafusion/tests/test_context.py index 55a324aec..97bff9bb9 100644 --- a/datafusion/tests/test_context.py +++ b/datafusion/tests/test_context.py @@ -214,6 +214,16 @@ def test_register_table(ctx, database): assert public.names() == {"csv", "csv1", "csv2", "csv3"} +def test_read_table(ctx, database): + default = ctx.catalog() + public = default.database("public") + assert public.names() == {"csv", "csv1", "csv2"} + + table = public.table("csv") + table_df = ctx.read_table(table) + table_df.show() + + def test_deregister_table(ctx, database): default = ctx.catalog() public = default.database("public") diff --git a/datafusion/tests/test_sql.py b/datafusion/tests/test_sql.py index 9d42a1f9d..19a2ad2cf 100644 --- a/datafusion/tests/test_sql.py +++ b/datafusion/tests/test_sql.py @@ -205,6 +205,32 @@ def test_register_json(ctx, tmp_path): ctx.register_json("json4", gzip_path, file_compression_type="rar") +def test_register_avro(ctx): + path = "testing/data/avro/alltypes_plain.avro" + ctx.register_avro("alltypes_plain", path) + result = ctx.sql( + "SELECT SUM(tinyint_col) as tinyint_sum FROM alltypes_plain" + ).collect() + result = pa.Table.from_batches(result).to_pydict() + assert result["tinyint_sum"][0] > 0 + + alternative_schema = pa.schema( + [ + pa.field("id", pa.int64()), + ] + ) + + ctx.register_avro( + "alltypes_plain_schema", + path, + schema=alternative_schema, + infinite=False, + ) + result = ctx.sql("SELECT * FROM alltypes_plain_schema").collect() + result = pa.Table.from_batches(result) + assert result.schema == alternative_schema + + def test_execute(ctx, tmp_path): data = [1, 1, 2, 2, 3, 11, 12] diff --git a/src/context.rs b/src/context.rs index 317ab785e..c7f89f2e6 100644 --- a/src/context.rs +++ b/src/context.rs @@ -545,6 +545,39 @@ impl PySessionContext { Ok(()) } + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (name, + path, + schema=None, + file_extension=".avro", + table_partition_cols=vec![], + infinite=false))] + fn register_avro( + &mut self, + name: &str, + path: PathBuf, + schema: Option>, + file_extension: &str, + table_partition_cols: Vec<(String, String)>, + infinite: bool, + py: Python, + ) -> PyResult<()> { + let path = path + .to_str() + .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; + + let mut options = AvroReadOptions::default() + .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) + .mark_infinite(infinite); + options.file_extension = file_extension; + options.schema = schema.as_ref().map(|x| &x.0); + + let result = self.ctx.register_avro(name, path, options); + wait_for_future(py, result).map_err(DataFusionError::from)?; + + Ok(()) + } + // Registers a PyArrow.Dataset fn register_dataset(&self, name: &str, dataset: &PyAny, py: Python) -> PyResult<()> { let table: Arc = Arc::new(Dataset::new(dataset, py)?); @@ -734,6 +767,14 @@ impl PySessionContext { Ok(PyDataFrame::new(df)) } + fn read_table(&self, table: &PyTable) -> PyResult { + let df = self + .ctx + .read_table(table.table()) + .map_err(DataFusionError::from)?; + Ok(PyDataFrame::new(df)) + } + fn __repr__(&self) -> PyResult { let config = self.ctx.copied_config(); let mut config_entries = config From bc62aafa34659a36cf188316e45740935ef333d5 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Wed, 6 Sep 2023 22:18:26 +0200 Subject: [PATCH 070/413] feat: add missing scalar math functions (#465) --- datafusion/tests/test_functions.py | 36 +++++++++++++++++++++++++++++- src/functions.rs | 32 ++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index c5ddea352..1727e7e5b 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import math import numpy as np import pyarrow as pa @@ -85,12 +86,15 @@ def test_math_functions(): ctx = SessionContext() # create a RecordBatch and a new DataFrame from it batch = pa.RecordBatch.from_arrays( - [pa.array([0.1, -0.7, 0.55])], names=["value"] + [pa.array([0.1, -0.7, 0.55]), pa.array([float("nan"), 0, 2.0])], + names=["value", "na_value"], ) df = ctx.create_dataframe([[batch]]) values = np.array([0.1, -0.7, 0.55]) + na_values = np.array([np.nan, 0, 2.0]) col_v = column("value") + col_nav = column("na_value") df = df.select( f.abs(col_v), f.sin(col_v), @@ -113,6 +117,20 @@ def test_math_functions(): f.sqrt(col_v), f.signum(col_v), f.trunc(col_v), + f.asinh(col_v), + f.acosh(col_v), + f.atanh(col_v), + f.cbrt(col_v), + f.cosh(col_v), + f.degrees(col_v), + f.gcd(literal(9), literal(3)), + f.lcm(literal(6), literal(4)), + f.nanvl(col_nav, literal(5)), + f.pi(), + f.radians(col_v), + f.sinh(col_v), + f.tanh(col_v), + f.factorial(literal(6)), ) batches = df.collect() assert len(batches) == 1 @@ -151,6 +169,22 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(18), np.sqrt(values)) np.testing.assert_array_almost_equal(result.column(19), np.sign(values)) np.testing.assert_array_almost_equal(result.column(20), np.trunc(values)) + np.testing.assert_array_almost_equal(result.column(21), np.arcsinh(values)) + np.testing.assert_array_almost_equal(result.column(22), np.arccosh(values)) + np.testing.assert_array_almost_equal(result.column(23), np.arctanh(values)) + np.testing.assert_array_almost_equal(result.column(24), np.cbrt(values)) + np.testing.assert_array_almost_equal(result.column(25), np.cosh(values)) + np.testing.assert_array_almost_equal(result.column(26), np.degrees(values)) + np.testing.assert_array_almost_equal(result.column(27), np.gcd(9, 3)) + np.testing.assert_array_almost_equal(result.column(28), np.lcm(6, 4)) + np.testing.assert_array_almost_equal( + result.column(29), np.where(np.isnan(na_values), 5, na_values) + ) + np.testing.assert_array_almost_equal(result.column(30), np.pi) + np.testing.assert_array_almost_equal(result.column(31), np.radians(values)) + np.testing.assert_array_almost_equal(result.column(32), np.sinh(values)) + np.testing.assert_array_almost_equal(result.column(33), np.tanh(values)) + np.testing.assert_array_almost_equal(result.column(34), math.factorial(6)) def test_string_functions(df): diff --git a/src/functions.rs b/src/functions.rs index 06d35ef3a..6037ce196 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -198,9 +198,12 @@ macro_rules! aggregate_function { scalar_function!(abs, Abs); scalar_function!(acos, Acos); +scalar_function!(acosh, Acosh); scalar_function!(ascii, Ascii, "Returns the numeric code of the first character of the argument. In UTF8 encoding, returns the Unicode code point of the character. In other multibyte encodings, the argument must be an ASCII character."); scalar_function!(asin, Asin); +scalar_function!(asinh, Asinh); scalar_function!(atan, Atan); +scalar_function!(atanh, Atanh); scalar_function!(atan2, Atan2); scalar_function!( bit_length, @@ -208,6 +211,7 @@ scalar_function!( "Returns number of bits in the string (8 times the octet_length)." ); scalar_function!(btrim, Btrim, "Removes the longest string containing only characters in characters (a space by default) from the start and end of string."); +scalar_function!(cbrt, Cbrt); scalar_function!(ceil, Ceil); scalar_function!( character_length, @@ -219,9 +223,14 @@ scalar_function!(char_length, CharacterLength); scalar_function!(chr, Chr, "Returns the character with the given code."); scalar_function!(coalesce, Coalesce); scalar_function!(cos, Cos); +scalar_function!(cosh, Cosh); +scalar_function!(degrees, Degrees); scalar_function!(exp, Exp); +scalar_function!(factorial, Factorial); scalar_function!(floor, Floor); +scalar_function!(gcd, Gcd); scalar_function!(initcap, InitCap, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); +scalar_function!(lcm, Lcm); scalar_function!(left, Left, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); scalar_function!(ln, Ln); scalar_function!(log, Log); @@ -235,9 +244,16 @@ scalar_function!( MD5, "Computes the MD5 hash of the argument, with the result written in hexadecimal." ); +scalar_function!( + nanvl, + Nanvl, + "Computes the MD5 hash of the argument, with the result written in hexadecimal." +); scalar_function!(octet_length, OctetLength, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); +scalar_function!(pi, Pi); scalar_function!(power, Power); scalar_function!(pow, Power); +scalar_function!(radians, Radians); scalar_function!(regexp_match, RegexpMatch); scalar_function!( regexp_replace, @@ -269,6 +285,7 @@ scalar_function!(sha384, SHA384); scalar_function!(sha512, SHA512); scalar_function!(signum, Signum); scalar_function!(sin, Sin); +scalar_function!(sinh, Sinh); scalar_function!( split_part, SplitPart, @@ -283,6 +300,7 @@ scalar_function!( scalar_function!(strpos, Strpos, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); scalar_function!(substr, Substr); scalar_function!(tan, Tan); +scalar_function!(tanh, Tanh); scalar_function!( to_hex, ToHex, @@ -343,6 +361,7 @@ aggregate_function!(var_samp, Variance); pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(abs))?; m.add_wrapped(wrap_pyfunction!(acos))?; + m.add_wrapped(wrap_pyfunction!(acosh))?; m.add_wrapped(wrap_pyfunction!(approx_distinct))?; m.add_wrapped(wrap_pyfunction!(alias))?; m.add_wrapped(wrap_pyfunction!(approx_median))?; @@ -353,11 +372,14 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(arrow_typeof))?; m.add_wrapped(wrap_pyfunction!(ascii))?; m.add_wrapped(wrap_pyfunction!(asin))?; + m.add_wrapped(wrap_pyfunction!(asinh))?; m.add_wrapped(wrap_pyfunction!(atan))?; + m.add_wrapped(wrap_pyfunction!(atanh))?; m.add_wrapped(wrap_pyfunction!(atan2))?; m.add_wrapped(wrap_pyfunction!(avg))?; m.add_wrapped(wrap_pyfunction!(bit_length))?; m.add_wrapped(wrap_pyfunction!(btrim))?; + m.add_wrapped(wrap_pyfunction!(cbrt))?; m.add_wrapped(wrap_pyfunction!(ceil))?; m.add_wrapped(wrap_pyfunction!(character_length))?; m.add_wrapped(wrap_pyfunction!(chr))?; @@ -369,6 +391,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(concat))?; m.add_wrapped(wrap_pyfunction!(corr))?; m.add_wrapped(wrap_pyfunction!(cos))?; + m.add_wrapped(wrap_pyfunction!(cosh))?; m.add_wrapped(wrap_pyfunction!(count))?; m.add_wrapped(wrap_pyfunction!(count_star))?; m.add_wrapped(wrap_pyfunction!(covar))?; @@ -376,6 +399,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(covar_samp))?; m.add_wrapped(wrap_pyfunction!(current_date))?; m.add_wrapped(wrap_pyfunction!(current_time))?; + m.add_wrapped(wrap_pyfunction!(degrees))?; m.add_wrapped(wrap_pyfunction!(date_bin))?; m.add_wrapped(wrap_pyfunction!(datepart))?; m.add_wrapped(wrap_pyfunction!(date_part))?; @@ -383,11 +407,14 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(date_trunc))?; m.add_wrapped(wrap_pyfunction!(digest))?; m.add_wrapped(wrap_pyfunction!(exp))?; + m.add_wrapped(wrap_pyfunction!(factorial))?; m.add_wrapped(wrap_pyfunction!(floor))?; m.add_wrapped(wrap_pyfunction!(from_unixtime))?; + m.add_wrapped(wrap_pyfunction!(gcd))?; m.add_wrapped(wrap_pyfunction!(grouping))?; m.add_wrapped(wrap_pyfunction!(in_list))?; m.add_wrapped(wrap_pyfunction!(initcap))?; + m.add_wrapped(wrap_pyfunction!(lcm))?; m.add_wrapped(wrap_pyfunction!(left))?; m.add_wrapped(wrap_pyfunction!(length))?; m.add_wrapped(wrap_pyfunction!(ln))?; @@ -403,12 +430,15 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(mean))?; m.add_wrapped(wrap_pyfunction!(median))?; m.add_wrapped(wrap_pyfunction!(min))?; + m.add_wrapped(wrap_pyfunction!(nanvl))?; m.add_wrapped(wrap_pyfunction!(now))?; m.add_wrapped(wrap_pyfunction!(nullif))?; m.add_wrapped(wrap_pyfunction!(octet_length))?; m.add_wrapped(wrap_pyfunction!(order_by))?; + m.add_wrapped(wrap_pyfunction!(pi))?; m.add_wrapped(wrap_pyfunction!(power))?; m.add_wrapped(wrap_pyfunction!(pow))?; + m.add_wrapped(wrap_pyfunction!(radians))?; m.add_wrapped(wrap_pyfunction!(random))?; m.add_wrapped(wrap_pyfunction!(regexp_match))?; m.add_wrapped(wrap_pyfunction!(regexp_replace))?; @@ -425,6 +455,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(sha512))?; m.add_wrapped(wrap_pyfunction!(signum))?; m.add_wrapped(wrap_pyfunction!(sin))?; + m.add_wrapped(wrap_pyfunction!(sinh))?; m.add_wrapped(wrap_pyfunction!(split_part))?; m.add_wrapped(wrap_pyfunction!(sqrt))?; m.add_wrapped(wrap_pyfunction!(starts_with))?; @@ -436,6 +467,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(substr))?; m.add_wrapped(wrap_pyfunction!(sum))?; m.add_wrapped(wrap_pyfunction!(tan))?; + m.add_wrapped(wrap_pyfunction!(tanh))?; m.add_wrapped(wrap_pyfunction!(to_hex))?; m.add_wrapped(wrap_pyfunction!(to_timestamp))?; m.add_wrapped(wrap_pyfunction!(to_timestamp_millis))?; From 944b1c98343aeae3dd6d2d82525a27068d5fb12d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Sep 2023 14:18:50 -0600 Subject: [PATCH 071/413] build(deps): bump arduino/setup-protoc from 1 to 2 (#452) Bumps [arduino/setup-protoc](https://github.com/arduino/setup-protoc) from 1 to 2. - [Release notes](https://github.com/arduino/setup-protoc/releases) - [Commits](https://github.com/arduino/setup-protoc/compare/v1...v2) --- updated-dependencies: - dependency-name: arduino/setup-protoc dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/build.yml | 4 ++-- .github/workflows/docs.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e72c4805a..199487d2c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: path: . - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v2 with: version: "3.x" repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -135,7 +135,7 @@ jobs: path: . - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v2 with: version: "3.x" repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index d9e7ad4ad..d538381db 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -38,7 +38,7 @@ jobs: python-version: "3.10" - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v2 with: version: '3.x' repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f672c8129..079fb4488 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -53,7 +53,7 @@ jobs: override: true - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v2 with: version: '3.x' repo-token: ${{ secrets.GITHUB_TOKEN }} From b4d383b5f85128f7f61e533c2f8d09f053189eb5 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 8 Sep 2023 06:37:20 -0700 Subject: [PATCH 072/413] Revert "build(deps): bump arduino/setup-protoc from 1 to 2 (#452)" (#474) This reverts commit 944b1c98343aeae3dd6d2d82525a27068d5fb12d. --- .github/workflows/build.yml | 4 ++-- .github/workflows/docs.yaml | 2 +- .github/workflows/test.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 199487d2c..e72c4805a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: path: . - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v1 with: version: "3.x" repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -135,7 +135,7 @@ jobs: path: . - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v1 with: version: "3.x" repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index d538381db..d9e7ad4ad 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -38,7 +38,7 @@ jobs: python-version: "3.10" - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v1 with: version: '3.x' repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 079fb4488..f672c8129 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -53,7 +53,7 @@ jobs: override: true - name: Install Protoc - uses: arduino/setup-protoc@v2 + uses: arduino/setup-protoc@v1 with: version: '3.x' repo-token: ${{ secrets.GITHUB_TOKEN }} From 0d7c19e44b61a1f269f1b835f529ad35aa76c6b6 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Mon, 11 Sep 2023 12:50:48 -0700 Subject: [PATCH 073/413] Minor: fix wrongly copied function description (#497) --- src/functions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions.rs b/src/functions.rs index 6037ce196..d9a7d6043 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -247,7 +247,7 @@ scalar_function!( scalar_function!( nanvl, Nanvl, - "Computes the MD5 hash of the argument, with the result written in hexadecimal." + "Returns x if x is not NaN otherwise returns y." ); scalar_function!(octet_length, OctetLength, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); scalar_function!(pi, Pi); From af4f75829a9aeaa9023ce742c58bbc6c50d5489c Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Mon, 11 Sep 2023 23:27:49 +0100 Subject: [PATCH 074/413] Upgrade to Datafusion 31.0.0 (#491) * Upgrade to Datafusion 30.0.0 * Upgrade to Datafusion 31.0.0-rc1 * Upgrade `object_store` to match Datafusion * Fix incompatibilities with Datafusion 31.0.0 * Add `schema` and `file_sort_order` to `read/register_parquet` * Upgrade to Datafusion 31.0.0 --- Cargo.lock | 474 ++++++++++++++++++++++++----------------------- Cargo.toml | 16 +- src/context.rs | 33 +++- src/dataframe.rs | 27 ++- 4 files changed, 300 insertions(+), 250 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 585e64cb5..d15f24bf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "gimli", ] @@ -38,9 +38,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8f9420f797f2d9e935edf629310eb938a0d839f984e25327f3c7eed22300c" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -83,9 +83,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.72" +version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "apache-avro" @@ -128,9 +128,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7104b9e9761613ae92fe770c741d6bbf1dbc791a0fe204400aebdd429875741" +checksum = "04a8801ebb147ad240b2d978d3ab9f73c9ccd4557ba6a03e7800496770ed10e0" dependencies = [ "ahash", "arrow-arith", @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38e597a8e8efb8ff52c50eaf8f4d85124ce3c1bf20fab82f476d73739d9ab1c2" +checksum = "895263144bd4a69751cbe6a34a53f26626e19770b313a9fa792c415cd0e78f11" dependencies = [ "arrow-array", "arrow-buffer", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a86d9c1473db72896bd2345ebb6b8ad75b8553ba390875c76708e8dc5c5492d" +checksum = "226fdc6c3a4ae154a74c24091d36a90b514f0ed7112f5b8322c1d8f354d8e20d" dependencies = [ "ahash", "arrow-buffer", @@ -183,19 +183,20 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234b3b1c8ed00c874bf95972030ac4def6f58e02ea5a7884314388307fb3669b" +checksum = "fc4843af4dd679c2f35b69c572874da8fde33be53eb549a5fb128e7a4b763510" dependencies = [ + "bytes", "half", "num", ] [[package]] name = "arrow-cast" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22f61168b853c7faea8cea23a2169fdff9c82fb10ae5e2c07ad1cab8f6884931" +checksum = "35e8b9990733a9b635f656efda3c9b8308c7a19695c9ec2c7046dd154f9b144b" dependencies = [ "arrow-array", "arrow-buffer", @@ -211,9 +212,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b545c114d9bf8569c84d2fbe2020ac4eea8db462c0a37d0b65f41a90d066fe" +checksum = "646fbb4e11dd0afb8083e883f53117713b8caadb4413b3c9e63e3f535da3683c" dependencies = [ "arrow-array", "arrow-buffer", @@ -230,9 +231,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6b6852635e7c43e5b242841c7470606ff0ee70eef323004cacc3ecedd33dd8f" +checksum = "da900f31ff01a0a84da0572209be72b2b6f980f3ea58803635de47913191c188" dependencies = [ "arrow-buffer", "arrow-schema", @@ -242,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66da9e16aecd9250af0ae9717ae8dd7ea0d8ca5a3e788fe3de9f4ee508da751" +checksum = "2707a8d7ee2d345d045283ece3ae43416175873483e5d96319c929da542a0b1f" dependencies = [ "arrow-array", "arrow-buffer", @@ -256,9 +257,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60ee0f9d8997f4be44a60ee5807443e396e025c23cf14d2b74ce56135cb04474" +checksum = "5d1b91a63c356d14eedc778b76d66a88f35ac8498426bb0799a769a49a74a8b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -276,9 +277,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcab05410e6b241442abdab6e1035177dc082bdb6f17049a4db49faed986d63" +checksum = "584325c91293abbca7aaaabf8da9fe303245d641f5f4a18a6058dc68009c7ebf" dependencies = [ "arrow-array", "arrow-buffer", @@ -291,9 +292,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91a847dd9eb0bacd7836ac63b3475c68b2210c2c96d0ec1b808237b973bd5d73" +checksum = "0e32afc1329f7b372463b21c6ca502b07cf237e1ed420d87706c1770bb0ebd38" dependencies = [ "ahash", "arrow-array", @@ -306,18 +307,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54df8c47918eb634c20e29286e69494fdc20cafa5173eb6dad49c7f6acece733" +checksum = "b104f5daa730f00fde22adc03a12aa5a2ae9ccbbf99cbd53d284119ddc90e03d" dependencies = [ "bitflags 2.4.0", ] [[package]] name = "arrow-select" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "941dbe481da043c4bd40c805a19ec2fc008846080c4953171b62bcad5ee5f7fb" +checksum = "73b3ca55356d1eae07cf48808d8c462cea674393ae6ad1e0b120f40b422eb2b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,9 +329,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "359b2cd9e071d5a3bcf44679f9d85830afebc5b9c98a08019a570a65ae933e0f" +checksum = "af1433ce02590cae68da0a18ed3a3ed868ffac2c6f24c533ddd2067f7ee04b4a" dependencies = [ "arrow-array", "arrow-buffer", @@ -344,9 +345,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" +checksum = "d495b6dc0184693324491a5ac05f559acc97bf937ab31d7a1c33dd0016be6d2b" dependencies = [ "bzip2", "flate2", @@ -362,24 +363,24 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" +checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "async-trait" -version = "0.1.72" +version = "0.1.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -399,9 +400,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" dependencies = [ "addr2line", "cc", @@ -414,9 +415,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -497,9 +498,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bzip2" @@ -524,9 +525,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "jobserver", "libc", @@ -540,15 +541,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", "serde", - "winapi", + "windows-targets", ] [[package]] @@ -675,9 +676,9 @@ dependencies = [ [[package]] name = "dashmap" -version = "5.5.0" +version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", "hashbrown 0.14.0", @@ -688,8 +689,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a4e4fc25698a14c90b34dda647ba10a5a966dc04b036d22e77fb1048663375d" dependencies = [ "ahash", "apache-avro", @@ -715,7 +717,6 @@ dependencies = [ "hashbrown 0.14.0", "indexmap 2.0.0", "itertools 0.11.0", - "lazy_static", "log", "num-traits", "num_cpus", @@ -725,7 +726,6 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand", - "smallvec", "sqlparser", "tempfile", "tokio", @@ -738,24 +738,35 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c23ad0229ea4a85bf76b236d8e75edf539881fdb02ce4e2394f9a76de6055206" dependencies = [ "apache-avro", "arrow", "arrow-array", + "async-compression", + "bytes", + "bzip2", "chrono", + "flate2", + "futures", "num_cpus", "object_store", "parquet", "pyo3", "sqlparser", + "tokio", + "tokio-util", + "xz2", + "zstd", ] [[package]] name = "datafusion-execution" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b37d2fc1a213baf34e0a57c85b8e6648f1a95152798fd6738163ee96c19203f" dependencies = [ "arrow", "dashmap", @@ -773,13 +784,13 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ea9844395f537730a145e5d87f61fecd37c2bc9d54e1dc89b35590d867345d" dependencies = [ "ahash", "arrow", "datafusion-common", - "lazy_static", "sqlparser", "strum 0.25.0", "strum_macros 0.25.2", @@ -787,8 +798,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8a30e0f79c5d59ba14d3d70f2500e87e0ff70236ad5e47f9444428f054fd2be" dependencies = [ "arrow", "async-trait", @@ -804,8 +816,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766c567082c9bbdcb784feec8fe40c7049cedaeb3a18d54f563f75fe0dc1932c" dependencies = [ "ahash", "arrow", @@ -823,7 +836,6 @@ dependencies = [ "hex", "indexmap 2.0.0", "itertools 0.11.0", - "lazy_static", "libc", "log", "md-5", @@ -838,7 +850,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "29.0.0" +version = "31.0.0" dependencies = [ "async-trait", "datafusion", @@ -857,7 +869,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.28", + "syn 2.0.32", "tokio", "url", "uuid", @@ -865,8 +877,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "811fd084cf2d78aa0c76b74320977c7084ad0383690612528b580795764b4dd0" dependencies = [ "arrow", "arrow-schema", @@ -878,8 +891,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=29.0.0-rc1#8265e99d05382fca57cc7399f8ee241966f4a1f5" +version = "31.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736201d7981b5ea91bea4245d0d33bd17dccfdb4a739350bbb990e09a57122ec" dependencies = [ "async-recursion", "chrono", @@ -911,9 +925,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" +checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" [[package]] name = "either" @@ -923,9 +937,9 @@ checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" [[package]] name = "encoding_rs" -version = "0.8.32" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" +checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ "cfg-if", ] @@ -938,9 +952,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -981,9 +995,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" +checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" dependencies = [ "crc32fast", "miniz_oxide", @@ -1060,7 +1074,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -1116,17 +1130,17 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "git2" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" +checksum = "12ef350ba88a33b4d524b1d1c79096c9ade5ef8c59395df0e60d1e1889414c0e" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "libc", "libgit2-sys", "log", @@ -1141,9 +1155,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.20" +version = "0.3.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97ec8491ebaf99c8eaa73058b045fe58073cd6be7f596ac993ced0b0a0c01049" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" dependencies = [ "bytes", "fnv", @@ -1212,6 +1226,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +dependencies = [ + "windows-sys", +] + [[package]] name = "http" version = "0.2.9" @@ -1242,9 +1265,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -1501,9 +1524,9 @@ dependencies = [ [[package]] name = "libgit2-sys" -version = "0.15.2+1.6.4" +version = "0.16.1+1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" +checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" dependencies = [ "cc", "libc", @@ -1519,9 +1542,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libmimalloc-sys" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ac0e912c8ef1b735e92369695618dc5b1819f5a7bf3f167301a3ba1cea515e" +checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d" dependencies = [ "cc", "libc", @@ -1541,9 +1564,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "lock_api" @@ -1557,9 +1580,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lz4" @@ -1603,9 +1626,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" [[package]] name = "memoffset" @@ -1618,9 +1641,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.37" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2894987a3459f3ffb755608bd82188f8ed00d0ae077f1edea29c068d639d98" +checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127" dependencies = [ "libmimalloc-sys", ] @@ -1673,9 +1696,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" dependencies = [ "autocfg", "num-integer", @@ -1684,9 +1707,9 @@ dependencies = [ [[package]] name = "num-complex" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" +checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" dependencies = [ "num-traits", ] @@ -1746,18 +1769,18 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c776db4f332b571958444982ff641d2531417a326ca368995073b639205d58" +checksum = "d359e231e5451f4f9fa889d56e3ce34f8724f1a61db2107739359717cf2bbf08" dependencies = [ "async-trait", "base64", @@ -1823,9 +1846,9 @@ dependencies = [ [[package]] name = "parquet" -version = "45.0.0" +version = "46.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f9739b984380582bdb7749ae5b5d28839bce899212cf16465c1ac1f8b65d79" +checksum = "1ad2cba786ae07da4d73371a88b9e0f9d3ffac1a9badc83922e0e15814f5c5fa" dependencies = [ "ahash", "arrow-array", @@ -1878,12 +1901,12 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap 2.0.0", ] [[package]] @@ -1926,9 +1949,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1950,12 +1973,12 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64d9ba0963cdcea2e1b2230fbae2bab30eb25a174be395c41e764bfb65dd62" +checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2112,9 +2135,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -2160,9 +2183,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" dependencies = [ "aho-corasick", "memchr", @@ -2172,9 +2195,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" dependencies = [ "aho-corasick", "memchr", @@ -2183,9 +2206,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regress" @@ -2199,9 +2222,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.18" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde824a14b7c14f85caff81225f411faacc04a2013f41670f41443742b1c1c55" +checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ "base64", "bytes", @@ -2276,9 +2299,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" dependencies = [ "bitflags 2.4.0", "errno", @@ -2289,9 +2312,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", @@ -2310,9 +2333,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.3" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261e9e0888cba427c3316e6322805653c9425240b6fd96cee7cb671ab70ab8d0" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -2341,9 +2364,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.12" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +checksum = "763f8cd0d4c71ed8389c90cb8100cba87e763bd01a8e614d4f0af97bcd50a161" dependencies = [ "dyn-clone", "schemars_derive", @@ -2353,9 +2376,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.12" +version = "0.8.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +checksum = "ec0f696e21e10fa546b7ffb1c9672c6de8fbc7a81acf59524386d8639bf12737" dependencies = [ "proc-macro2", "quote", @@ -2393,22 +2416,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2424,9 +2447,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.104" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" +checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" dependencies = [ "itoa", "ryu", @@ -2442,7 +2465,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2483,15 +2506,15 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] @@ -2542,9 +2565,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -2558,9 +2581,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eaa1e88e78d2c2460d78b7dc3f0c08dbb606ab4222f9aff36f420d36e307d87" +checksum = "37ae05a8250b968a3f7db93155a84d68b2e6cea1583949af5ca5b5170c76c075" dependencies = [ "log", "sqlparser_derive", @@ -2621,14 +2644,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] name = "substrait" -version = "0.12.4" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658f6cbbd29a250869b87e1bb5a4b42db534cacfc1c03284f2536cd36b6c1617" +checksum = "b3b705c631514962d48c0647e3ad2b4d93b7472f5da7bee2d1dc853bf4c61f19" dependencies = [ "git2", "heck", @@ -2642,7 +2665,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.28", + "syn 2.0.32", "typify", "walkdir", ] @@ -2666,9 +2689,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.28" +version = "2.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" dependencies = [ "proc-macro2", "quote", @@ -2683,9 +2706,9 @@ checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", @@ -2696,22 +2719,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.44" +version = "1.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2751,9 +2774,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.30.0" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3ce25f50619af8b0aec2eb23deebe84249e19e2ddd393a6e16e3300a6dadfd" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", "bytes", @@ -2762,7 +2785,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "windows-sys", ] @@ -2775,7 +2798,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2828,7 +2851,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] @@ -2896,7 +2919,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.28", + "syn 2.0.32", "thiserror", "unicode-ident", ] @@ -2913,7 +2936,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.28", + "syn 2.0.32", "typify-impl", ] @@ -2970,9 +2993,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -3003,9 +3026,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" dependencies = [ "same-file", "winapi-util", @@ -3047,7 +3070,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-shared", ] @@ -3081,7 +3104,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3094,9 +3117,9 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-streams" -version = "0.2.3" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbae3363c08332cadccd13b67db371814cd214c2524020932f0804b8cf7c078" +checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" dependencies = [ "futures-util", "js-sys", @@ -3115,34 +3138,22 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki-roots" -version = "0.22.6" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" -dependencies = [ - "webpki", -] +checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] @@ -3196,9 +3207,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -3211,53 +3222,54 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winreg" -version = "0.10.1" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "winapi", + "cfg-if", + "windows-sys", ] [[package]] @@ -3271,9 +3283,9 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3b9c234616391070b0b173963ebc65a9195068e7ed3731c6edac2ec45ebe106" +checksum = "20707b61725734c595e840fb3704378a0cd2b9c74cc9e6e20724838fc6a1e2f9" dependencies = [ "byteorder", "zerocopy-derive", @@ -3281,13 +3293,13 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f7f3a471f98d0a61c34322fbbfd10c384b07687f680d4119813713f72308d91" +checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.28", + "syn 2.0.32", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 169d3f5d4..eaa2602ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "29.0.0" +version = "31.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,19 +36,19 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1", features = ["pyarrow", "avro"] } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1", features = ["pyarrow"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion.git", rev = "29.0.0-rc1" } +datafusion = { version = "31.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "31.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "31.0.0" } +datafusion-optimizer = { version = "31.0.0" } +datafusion-sql = { version = "31.0.0" } +datafusion-substrait = { version = "31.0.0" } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.6.1", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.7.1" syn = "2.0.11" diff --git a/src/context.rs b/src/context.rs index c7f89f2e6..632b179a1 100644 --- a/src/context.rs +++ b/src/context.rs @@ -31,6 +31,7 @@ use crate::catalog::{PyCatalog, PyTable}; use crate::dataframe::PyDataFrame; use crate::dataset::Dataset; use crate::errors::{py_datafusion_err, DataFusionError}; +use crate::expr::PyExpr; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; @@ -41,7 +42,7 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::datasource::file_format::file_type::FileCompressionType; +use datafusion::common::FileCompressionType; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext}; @@ -445,7 +446,10 @@ impl PySessionContext { #[allow(clippy::too_many_arguments)] #[pyo3(signature = (name, path, table_partition_cols=vec![], parquet_pruning=true, - file_extension=".parquet"))] + file_extension=".parquet", + skip_metadata=true, + schema=None, + file_sort_order=None))] fn register_parquet( &mut self, name: &str, @@ -453,12 +457,23 @@ impl PySessionContext { table_partition_cols: Vec<(String, String)>, parquet_pruning: bool, file_extension: &str, + skip_metadata: bool, + schema: Option>, + file_sort_order: Option>>, py: Python, ) -> PyResult<()> { let mut options = ParquetReadOptions::default() .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) - .parquet_pruning(parquet_pruning); + .parquet_pruning(parquet_pruning) + .skip_metadata(skip_metadata); options.file_extension = file_extension; + options.schema = schema.as_ref().map(|x| &x.0); + options.file_sort_order = file_sort_order + .unwrap_or(vec![]) + .into_iter() + .map(|e| e.into_iter().map(|f| f.into()).collect()) + .collect(); + let result = self.ctx.register_parquet(name, path, options); wait_for_future(py, result).map_err(DataFusionError::from)?; Ok(()) @@ -722,7 +737,9 @@ impl PySessionContext { table_partition_cols=vec![], parquet_pruning=true, file_extension=".parquet", - skip_metadata=true))] + skip_metadata=true, + schema=None, + file_sort_order=None))] fn read_parquet( &self, path: &str, @@ -730,6 +747,8 @@ impl PySessionContext { parquet_pruning: bool, file_extension: &str, skip_metadata: bool, + schema: Option>, + file_sort_order: Option>>, py: Python, ) -> PyResult { let mut options = ParquetReadOptions::default() @@ -737,6 +756,12 @@ impl PySessionContext { .parquet_pruning(parquet_pruning) .skip_metadata(skip_metadata); options.file_extension = file_extension; + options.schema = schema.as_ref().map(|x| &x.0); + options.file_sort_order = file_sort_order + .unwrap_or(vec![]) + .into_iter() + .map(|e| e.into_iter().map(|f| f.into()).collect()) + .collect(); let result = self.ctx.read_parquet(path, options); let df = PyDataFrame::new(wait_for_future(py, result).map_err(DataFusionError::from)?); diff --git a/src/dataframe.rs b/src/dataframe.rs index 61a44484c..ebf06f3d2 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -22,7 +22,7 @@ use crate::{errors::DataFusionError, expr::PyExpr}; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; -use datafusion::dataframe::DataFrame; +use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; @@ -305,7 +305,13 @@ impl PyDataFrame { /// Write a `DataFrame` to a CSV file. fn write_csv(&self, path: &str, py: Python) -> PyResult<()> { - wait_for_future(py, self.df.as_ref().clone().write_csv(path))?; + wait_for_future( + py, + self.df + .as_ref() + .clone() + .write_csv(path, DataFrameWriteOptions::new(), None), + )?; Ok(()) } @@ -357,17 +363,24 @@ impl PyDataFrame { wait_for_future( py, - self.df - .as_ref() - .clone() - .write_parquet(path, Option::from(writer_properties)), + self.df.as_ref().clone().write_parquet( + path, + DataFrameWriteOptions::new(), + Option::from(writer_properties), + ), )?; Ok(()) } /// Executes a query and writes the results to a partitioned JSON file. fn write_json(&self, path: &str, py: Python) -> PyResult<()> { - wait_for_future(py, self.df.as_ref().clone().write_json(path))?; + wait_for_future( + py, + self.df + .as_ref() + .clone() + .write_json(path, DataFrameWriteOptions::new()), + )?; Ok(()) } From beabf2601038747e5b1a813114d6b22be83bf291 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Tue, 12 Sep 2023 15:20:11 +0100 Subject: [PATCH 075/413] Add `isnan` and `iszero` (#495) * Add `isnan` and `iszero` * Add test for `log` * Add missing newline! Co-authored-by: Liang-Chi Hsieh * Fix linting --------- Co-authored-by: Liang-Chi Hsieh --- datafusion/tests/test_functions.py | 10 ++++++++++ src/functions.rs | 4 ++++ 2 files changed, 14 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index 1727e7e5b..80795bf12 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -131,6 +131,9 @@ def test_math_functions(): f.sinh(col_v), f.tanh(col_v), f.factorial(literal(6)), + f.isnan(col_nav), + f.iszero(col_nav), + f.log(literal(3), col_v + literal(pa.scalar(1))), ) batches = df.collect() assert len(batches) == 1 @@ -185,6 +188,13 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(32), np.sinh(values)) np.testing.assert_array_almost_equal(result.column(33), np.tanh(values)) np.testing.assert_array_almost_equal(result.column(34), math.factorial(6)) + np.testing.assert_array_almost_equal( + result.column(35), np.isnan(na_values) + ) + np.testing.assert_array_almost_equal(result.column(36), na_values == 0) + np.testing.assert_array_almost_equal( + result.column(37), np.emath.logn(3, values + 1.0) + ) def test_string_functions(df): diff --git a/src/functions.rs b/src/functions.rs index d9a7d6043..79cc19b3c 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -230,6 +230,8 @@ scalar_function!(factorial, Factorial); scalar_function!(floor, Floor); scalar_function!(gcd, Gcd); scalar_function!(initcap, InitCap, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); +scalar_function!(isnan, Isnan); +scalar_function!(iszero, Iszero); scalar_function!(lcm, Lcm); scalar_function!(left, Left, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); scalar_function!(ln, Ln); @@ -414,6 +416,8 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(grouping))?; m.add_wrapped(wrap_pyfunction!(in_list))?; m.add_wrapped(wrap_pyfunction!(initcap))?; + m.add_wrapped(wrap_pyfunction!(isnan))?; + m.add_wrapped(wrap_pyfunction!(iszero))?; m.add_wrapped(wrap_pyfunction!(lcm))?; m.add_wrapped(wrap_pyfunction!(left))?; m.add_wrapped(wrap_pyfunction!(length))?; From a47712e17886d9202fed282963dc449470688fe0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 13 Sep 2023 08:22:01 -0600 Subject: [PATCH 076/413] Update CHANGELOG and run cargo update (#500) --- CHANGELOG.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.lock | 8 +++---- 2 files changed, 70 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2f2de219..dbc5a82e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,72 @@ # DataFusion Python Changelog +## [31.0.0](https://github.com/apache/arrow-datafusion-python/tree/31.0.0) (2023-09-12) + +[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/28.0.0...31.0.0) + +**Implemented enhancements:** + +- feat: add case function (#447) [#448](https://github.com/apache/arrow-datafusion-python/pull/448) (mesejo) +- feat: add compression options [#456](https://github.com/apache/arrow-datafusion-python/pull/456) (mesejo) +- feat: add register_json [#458](https://github.com/apache/arrow-datafusion-python/pull/458) (mesejo) +- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/arrow-datafusion-python/pull/459) (mesejo) +- feat: add example of reading parquet from s3 [#460](https://github.com/apache/arrow-datafusion-python/pull/460) (mesejo) +- feat: add register_avro and read_table [#461](https://github.com/apache/arrow-datafusion-python/pull/461) (mesejo) +- feat: add missing scalar math functions [#465](https://github.com/apache/arrow-datafusion-python/pull/465) (mesejo) + +**Documentation updates:** + +- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/arrow-datafusion-python/pull/455) (mesejo) + +**Merged pull requests:** + +- Build Linux aarch64 wheel [#443](https://github.com/apache/arrow-datafusion-python/pull/443) (gokselk) +- feat: add case function (#447) [#448](https://github.com/apache/arrow-datafusion-python/pull/448) (mesejo) +- enhancement(docs): Add user guide (#432) [#445](https://github.com/apache/arrow-datafusion-python/pull/445) (mesejo) +- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/arrow-datafusion-python/pull/455) (mesejo) +- feat: add compression options [#456](https://github.com/apache/arrow-datafusion-python/pull/456) (mesejo) +- Upgrade to DF 28.0.0-rc1 [#457](https://github.com/apache/arrow-datafusion-python/pull/457) (andygrove) +- feat: add register_json [#458](https://github.com/apache/arrow-datafusion-python/pull/458) (mesejo) +- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/arrow-datafusion-python/pull/459) (mesejo) +- feat: add example of reading parquet from s3 [#460](https://github.com/apache/arrow-datafusion-python/pull/460) (mesejo) +- feat: add register_avro and read_table [#461](https://github.com/apache/arrow-datafusion-python/pull/461) (mesejo) +- feat: add missing scalar math functions [#465](https://github.com/apache/arrow-datafusion-python/pull/465) (mesejo) +- build(deps): bump arduino/setup-protoc from 1 to 2 [#452](https://github.com/apache/arrow-datafusion-python/pull/452) (dependabot[bot]) +- Revert "build(deps): bump arduino/setup-protoc from 1 to 2 (#452)" [#474](https://github.com/apache/arrow-datafusion-python/pull/474) (viirya) +- Minor: fix wrongly copied function description [#497](https://github.com/apache/arrow-datafusion-python/pull/497) (viirya) +- Upgrade to Datafusion 31.0.0 [#491](https://github.com/apache/arrow-datafusion-python/pull/491) (judahrand) +- Add `isnan` and `iszero` [#495](https://github.com/apache/arrow-datafusion-python/pull/495) (judahrand) + +## 30.0.0 + +- Skipped due to a breaking change in DataFusion + +## 29.0.0 + +- Skipped + +## [28.0.0](https://github.com/apache/arrow-datafusion-python/tree/28.0.0) (2023-07-25) + +**Implemented enhancements:** + +- feat: expose offset in python API [#437](https://github.com/apache/arrow-datafusion-python/pull/437) (cpcloud) + +**Merged pull requests:** + +- File based input utils [#433](https://github.com/apache/arrow-datafusion-python/pull/433) (jdye64) +- Upgrade to 28.0.0-rc1 [#434](https://github.com/apache/arrow-datafusion-python/pull/434) (andygrove) +- Introduces utility for obtaining SqlTable information from a file like location [#398](https://github.com/apache/arrow-datafusion-python/pull/398) (jdye64) +- feat: expose offset in python API [#437](https://github.com/apache/arrow-datafusion-python/pull/437) (cpcloud) +- Use DataFusion 28 [#439](https://github.com/apache/arrow-datafusion-python/pull/439) (andygrove) + +## [27.0.0](https://github.com/apache/arrow-datafusion-python/tree/27.0.0) (2023-07-03) + +**Merged pull requests:** + +- LogicalPlan.to_variant() make public [#412](https://github.com/apache/arrow-datafusion-python/pull/412) (jdye64) +- Prepare 27.0.0 release [#423](https://github.com/apache/arrow-datafusion-python/pull/423) (andygrove) + ## [26.0.0](https://github.com/apache/arrow-datafusion-python/tree/26.0.0) (2023-06-11) [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/25.0.0...26.0.0) diff --git a/Cargo.lock b/Cargo.lock index d15f24bf8..c7113ffce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -345,9 +345,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d495b6dc0184693324491a5ac05f559acc97bf937ab31d7a1c33dd0016be6d2b" +checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" dependencies = [ "bzip2", "flate2", @@ -2333,9 +2333,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.4" +version = "0.101.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" +checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" dependencies = [ "ring", "untrusted", From 41d65d15283b2ca1b10a714a50167af8a62d3052 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 20 Sep 2023 05:14:16 -0600 Subject: [PATCH 077/413] Improve release process documentation (#505) --- dev/release/README.md | 47 ++++++++++++++++++++++++++++++- dev/release/rat_exclude_files.txt | 3 +- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index e85b288f4..6bd2c1eb2 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -179,7 +179,7 @@ When prompted for username, enter `__token__`. When prompted for a password, ent Download the source tarball created in the previous step, untar it, and run: ```bash -python3 -m build +maturin sdist ``` This will create a file named `dist/datafusion-0.7.0.tar.gz`. Upload this to testpypi: @@ -263,3 +263,48 @@ git checkout 0.8.0-rc1 git tag 0.8.0 git push apache 0.8.0 ``` + +### Add the release to Apache Reporter + +Add the release to https://reporter.apache.org/addrelease.html?arrow with a version name prefixed with `RS-DATAFUSION-PYTHON`, +for example `RS-DATAFUSION-PYTHON-31.0.0`. + +The release information is used to generate a template for a board report (see example +[here](https://github.com/apache/arrow/pull/14357)). + +### Delete old RCs and Releases + +See the ASF documentation on [when to archive](https://www.apache.org/legal/release-policy.html#when-to-archive) +for more information. + +#### Deleting old release candidates from `dev` svn + +Release candidates should be deleted once the release is published. + +Get a list of DataFusion release candidates: + +```bash +svn ls https://dist.apache.org/repos/dist/dev/arrow | grep datafusion-python +``` + +Delete a release candidate: + +```bash +svn delete -m "delete old DataFusion RC" https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-7.1.0-rc1/ +``` + +#### Deleting old releases from `release` svn + +Only the latest release should be available. Delete old releases after publishing the new release. + +Get a list of DataFusion releases: + +```bash +svn ls https://dist.apache.org/repos/dist/release/arrow | grep datafusion-python +``` + +Delete a release: + +```bash +svn delete -m "delete old DataFusion release" https://dist.apache.org/repos/dist/release/arrow/arrow-datafusion-python-7.0.0 +``` diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 6d0fee185..f65ddd06e 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -44,4 +44,5 @@ Cargo.lock */.git .github/* benchmarks/tpch/queries/q*.sql -benchmarks/tpch/create_tables.sql \ No newline at end of file +benchmarks/tpch/create_tables.sql +.cargo/config.toml \ No newline at end of file From 106786ab57cfe9b16f2b867fe6d53170bd14c841 Mon Sep 17 00:00:00 2001 From: zhenxing jiang Date: Wed, 20 Sep 2023 19:15:45 +0800 Subject: [PATCH 078/413] add Binary String Functions (#494) --- datafusion/tests/test_functions.py | 14 ++++++++++++++ src/functions.rs | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index 80795bf12..f1f64c30a 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -477,3 +477,17 @@ def test_case(df): assert result.column(0) == pa.array([10, 8, 8]) assert result.column(1) == pa.array(["Hola", "Mundo", "!!"]) assert result.column(2) == pa.array(["Hola", "Mundo", None]) + + +def test_binary_string_functions(df): + df = df.select( + f.encode(column("a"), literal("base64")), + f.decode(f.encode(column("a"), literal("base64")), literal("base64")), + ) + result = df.collect() + assert len(result) == 1 + result = result[0] + assert result.column(0) == pa.array(["SGVsbG8", "V29ybGQ", "IQ"]) + assert pa.array(result.column(1)).cast(pa.string()) == pa.array( + ["Hello", "World", "!"] + ) diff --git a/src/functions.rs b/src/functions.rs index 79cc19b3c..ef26240fe 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -332,6 +332,9 @@ scalar_function!(r#struct, Struct); // Use raw identifier since struct is a keyw scalar_function!(from_unixtime, FromUnixtime); scalar_function!(arrow_typeof, ArrowTypeof); scalar_function!(random, Random); +//Binary String Functions +scalar_function!(encode, Encode); +scalar_function!(decode, Decode); aggregate_function!(approx_distinct, ApproxDistinct); aggregate_function!(approx_median, ApproxMedian); @@ -486,5 +489,9 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(var_pop))?; m.add_wrapped(wrap_pyfunction!(var_samp))?; m.add_wrapped(wrap_pyfunction!(window))?; + + //Binary String Functions + m.add_wrapped(wrap_pyfunction!(encode))?; + m.add_wrapped(wrap_pyfunction!(decode))?; Ok(()) } From c574d684f5ac244bf2a8b813f67ce8642babf10f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 09:43:49 -0700 Subject: [PATCH 079/413] build(deps): bump mimalloc from 0.1.38 to 0.1.39 (#502) Bumps [mimalloc](https://github.com/purpleprotocol/mimalloc_rust) from 0.1.38 to 0.1.39. - [Release notes](https://github.com/purpleprotocol/mimalloc_rust/releases) - [Commits](https://github.com/purpleprotocol/mimalloc_rust/compare/v0.1.38...v0.1.39) --- updated-dependencies: - dependency-name: mimalloc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c7113ffce..8fb244715 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1542,9 +1542,9 @@ checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" [[package]] name = "libmimalloc-sys" -version = "0.1.34" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d" +checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" dependencies = [ "cc", "libc", @@ -1641,9 +1641,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.38" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127" +checksum = "fa01922b5ea280a911e323e4d2fd24b7fe5cc4042e0d2cda3c40775cdc4bdc9c" dependencies = [ "libmimalloc-sys", ] From 31241f8e3711edbf97ea0613f3dc03cee4ee8fb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 15:20:37 -0700 Subject: [PATCH 080/413] build(deps): bump syn from 2.0.32 to 2.0.35 (#503) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.32 to 2.0.35. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.32...2.0.35) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 42 +++++++++++++++++++++--------------------- Cargo.toml | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fb244715..a4b3fb769 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,7 +369,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -380,7 +380,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -869,7 +869,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.32", + "syn 2.0.35", "tokio", "url", "uuid", @@ -1074,7 +1074,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -1978,7 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -1989,9 +1989,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -2431,7 +2431,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -2465,7 +2465,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -2644,7 +2644,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -2665,7 +2665,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.32", + "syn 2.0.35", "typify", "walkdir", ] @@ -2689,9 +2689,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879" dependencies = [ "proc-macro2", "quote", @@ -2734,7 +2734,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -2798,7 +2798,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -2851,7 +2851,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] @@ -2919,7 +2919,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.32", + "syn 2.0.35", "thiserror", "unicode-ident", ] @@ -2936,7 +2936,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.32", + "syn 2.0.35", "typify-impl", ] @@ -3070,7 +3070,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", "wasm-bindgen-shared", ] @@ -3104,7 +3104,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3299,7 +3299,7 @@ checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.35", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index eaa2602ae..0b685ec74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ futures = "0.3" object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.7.1" -syn = "2.0.11" +syn = "2.0.35" url = "2.2" [build-dependencies] From 9ef0a57bd4733fb8314c9aa8ae5408a7411cb004 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 14:18:21 -0700 Subject: [PATCH 081/413] build(deps): bump syn from 2.0.35 to 2.0.37 (#506) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.35 to 2.0.37. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.35...2.0.37) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 38 +++++++++++++++++++------------------- Cargo.toml | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a4b3fb769..91910b5de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,7 +369,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -380,7 +380,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -869,7 +869,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.35", + "syn 2.0.37", "tokio", "url", "uuid", @@ -1074,7 +1074,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -1978,7 +1978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2431,7 +2431,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2465,7 +2465,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2644,7 +2644,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2665,7 +2665,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.35", + "syn 2.0.37", "typify", "walkdir", ] @@ -2689,9 +2689,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.35" +version = "2.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59bf04c28bee9043ed9ea1e41afc0552288d3aba9c6efdd78903b802926f4879" +checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" dependencies = [ "proc-macro2", "quote", @@ -2734,7 +2734,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2798,7 +2798,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2851,7 +2851,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] @@ -2919,7 +2919,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.35", + "syn 2.0.37", "thiserror", "unicode-ident", ] @@ -2936,7 +2936,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.35", + "syn 2.0.37", "typify-impl", ] @@ -3070,7 +3070,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", "wasm-bindgen-shared", ] @@ -3104,7 +3104,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3299,7 +3299,7 @@ checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.35", + "syn 2.0.37", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0b685ec74..6545e1c9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ futures = "0.3" object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.7.1" -syn = "2.0.35" +syn = "2.0.37" url = "2.2" [build-dependencies] From 8e430ab0102984c55ec715fd2151ff544732cc87 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 7 Oct 2023 16:17:59 -0600 Subject: [PATCH 082/413] Use latest DataFusion (#511) --- Cargo.lock | 487 +++++++++++++++++++++++++------------------- Cargo.toml | 12 +- src/context.rs | 11 +- src/expr/literal.rs | 2 +- 4 files changed, 290 insertions(+), 222 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 91910b5de..ee4eb3f64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,9 +38,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" +checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" dependencies = [ "memchr", ] @@ -89,11 +89,11 @@ checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" [[package]] name = "apache-avro" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7" +checksum = "ceb7c683b2f8f40970b70e39ff8be514c95b96fcb9c4af87e1ed2cb2e10801a0" dependencies = [ - "byteorder", + "bzip2", "crc32fast", "digest", "lazy_static", @@ -102,7 +102,7 @@ dependencies = [ "num-bigint", "quad-rand", "rand", - "regex", + "regex-lite", "serde", "serde_json", "snap", @@ -111,7 +111,8 @@ dependencies = [ "thiserror", "typed-builder", "uuid", - "zerocopy", + "xz2", + "zstd", ] [[package]] @@ -128,9 +129,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04a8801ebb147ad240b2d978d3ab9f73c9ccd4557ba6a03e7800496770ed10e0" +checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" dependencies = [ "ahash", "arrow-arith", @@ -151,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "895263144bd4a69751cbe6a34a53f26626e19770b313a9fa792c415cd0e78f11" +checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" dependencies = [ "arrow-array", "arrow-buffer", @@ -166,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "226fdc6c3a4ae154a74c24091d36a90b514f0ed7112f5b8322c1d8f354d8e20d" +checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" dependencies = [ "ahash", "arrow-buffer", @@ -177,15 +178,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "num", ] [[package]] name = "arrow-buffer" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc4843af4dd679c2f35b69c572874da8fde33be53eb549a5fb128e7a4b763510" +checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" dependencies = [ "bytes", "half", @@ -194,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e8b9990733a9b635f656efda3c9b8308c7a19695c9ec2c7046dd154f9b144b" +checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" dependencies = [ "arrow-array", "arrow-buffer", @@ -212,9 +213,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "646fbb4e11dd0afb8083e883f53117713b8caadb4413b3c9e63e3f535da3683c" +checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" dependencies = [ "arrow-array", "arrow-buffer", @@ -231,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da900f31ff01a0a84da0572209be72b2b6f980f3ea58803635de47913191c188" +checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" dependencies = [ "arrow-buffer", "arrow-schema", @@ -243,9 +244,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2707a8d7ee2d345d045283ece3ae43416175873483e5d96319c929da542a0b1f" +checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" dependencies = [ "arrow-array", "arrow-buffer", @@ -257,9 +258,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1b91a63c356d14eedc778b76d66a88f35ac8498426bb0799a769a49a74a8b4" +checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" dependencies = [ "arrow-array", "arrow-buffer", @@ -268,7 +269,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.0.0", + "indexmap 2.0.2", "lexical-core", "num", "serde", @@ -277,9 +278,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584325c91293abbca7aaaabf8da9fe303245d641f5f4a18a6058dc68009c7ebf" +checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" dependencies = [ "arrow-array", "arrow-buffer", @@ -292,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e32afc1329f7b372463b21c6ca502b07cf237e1ed420d87706c1770bb0ebd38" +checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" dependencies = [ "ahash", "arrow-array", @@ -302,24 +303,25 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.0", + "hashbrown 0.14.1", ] [[package]] name = "arrow-schema" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b104f5daa730f00fde22adc03a12aa5a2ae9ccbbf99cbd53d284119ddc90e03d" +checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" dependencies = [ "bitflags 2.4.0", ] [[package]] name = "arrow-select" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73b3ca55356d1eae07cf48808d8c462cea674393ae6ad1e0b120f40b422eb2b4" +checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" dependencies = [ + "ahash", "arrow-array", "arrow-buffer", "arrow-data", @@ -329,9 +331,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1433ce02590cae68da0a18ed3a3ed868ffac2c6f24c533ddd2067f7ee04b4a" +checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" dependencies = [ "arrow-array", "arrow-buffer", @@ -369,7 +371,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -380,7 +382,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -442,16 +444,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "digest", ] [[package]] @@ -465,9 +466,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.3.4" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -476,9 +477,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.4" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -486,15 +487,15 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" @@ -541,9 +542,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "defd4e7873dbddba6c7c91e199c7fcb946abc4a6a4ac3195400bcfb01b5de877" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", @@ -613,12 +614,31 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + [[package]] name = "cpufeatures" version = "0.2.9" @@ -655,9 +675,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.2.2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" dependencies = [ "csv-core", "itoa", @@ -667,13 +687,19 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" dependencies = [ "memchr", ] +[[package]] +name = "dary_heap" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" + [[package]] name = "dashmap" version = "5.5.3" @@ -681,7 +707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "lock_api", "once_cell", "parking_lot_core", @@ -690,8 +716,7 @@ dependencies = [ [[package]] name = "datafusion" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a4e4fc25698a14c90b34dda647ba10a5a966dc04b036d22e77fb1048663375d" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "ahash", "apache-avro", @@ -709,13 +734,14 @@ dependencies = [ "datafusion-expr", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-plan", "datafusion-sql", "flate2", "futures", "glob", "half", - "hashbrown 0.14.0", - "indexmap 2.0.0", + "hashbrown 0.14.1", + "indexmap 2.0.2", "itertools 0.11.0", "log", "num-traits", @@ -739,41 +765,35 @@ dependencies = [ [[package]] name = "datafusion-common" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c23ad0229ea4a85bf76b236d8e75edf539881fdb02ce4e2394f9a76de6055206" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ + "ahash", "apache-avro", "arrow", "arrow-array", - "async-compression", - "bytes", - "bzip2", + "arrow-buffer", + "arrow-schema", "chrono", - "flate2", - "futures", + "half", "num_cpus", "object_store", "parquet", "pyo3", "sqlparser", - "tokio", - "tokio-util", - "xz2", - "zstd", ] [[package]] name = "datafusion-execution" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b37d2fc1a213baf34e0a57c85b8e6648f1a95152798fd6738163ee96c19203f" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "arrow", + "chrono", "dashmap", "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "log", "object_store", "parking_lot", @@ -785,11 +805,11 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6ea9844395f537730a145e5d87f61fecd37c2bc9d54e1dc89b35590d867345d" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "ahash", "arrow", + "arrow-array", "datafusion-common", "sqlparser", "strum 0.25.0", @@ -799,8 +819,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8a30e0f79c5d59ba14d3d70f2500e87e0ff70236ad5e47f9444428f054fd2be" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "arrow", "async-trait", @@ -808,7 +827,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "itertools 0.11.0", "log", "regex-syntax", @@ -817,8 +836,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "766c567082c9bbdcb784feec8fe40c7049cedaeb3a18d54f563f75fe0dc1932c" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "ahash", "arrow", @@ -832,9 +850,9 @@ dependencies = [ "datafusion-common", "datafusion-expr", "half", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "hex", - "indexmap 2.0.0", + "indexmap 2.0.2", "itertools 0.11.0", "libc", "log", @@ -848,6 +866,36 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-physical-plan" +version = "31.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "futures", + "half", + "hashbrown 0.14.1", + "indexmap 2.0.2", + "itertools 0.11.0", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand", + "tokio", + "uuid", +] + [[package]] name = "datafusion-python" version = "31.0.0" @@ -869,7 +917,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.37", + "syn 2.0.38", "tokio", "url", "uuid", @@ -878,8 +926,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "811fd084cf2d78aa0c76b74320977c7084ad0383690612528b580795764b4dd0" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "arrow", "arrow-schema", @@ -892,8 +939,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "31.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736201d7981b5ea91bea4245d0d33bd17dccfdb4a739350bbb990e09a57122ec" +source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" dependencies = [ "async-recursion", "chrono", @@ -925,9 +971,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbfc4744c1b8f2a09adc0e55242f60b1af195d88596bd8700be74418c056c555" +checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" [[package]] name = "either" @@ -952,9 +998,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" dependencies = [ "errno-dragonfly", "libc", @@ -973,9 +1019,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fixedbitset" @@ -1074,7 +1120,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1136,9 +1182,9 @@ checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "git2" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ef350ba88a33b4d524b1d1c79096c9ade5ef8c59395df0e60d1e1889414c0e" +checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" dependencies = [ "bitflags 2.4.0", "libc", @@ -1200,9 +1246,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" dependencies = [ "ahash", "allocator-api2", @@ -1216,9 +1262,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" [[package]] name = "hex" @@ -1358,12 +1404,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.0" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown 0.14.1", ] [[package]] @@ -1498,27 +1544,31 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.147" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "libflate" -version = "1.4.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18" +checksum = "9f7d5654ae1795afc7ff76f4365c2c8791b0feb18e8996a96adad8ffd7c3b2bf" dependencies = [ "adler32", + "core2", "crc32fast", + "dary_heap", "libflate_lz77", ] [[package]] name = "libflate_lz77" -version = "1.2.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf" +checksum = "be5f52fb8c451576ec6b79d3f4deb327398bc05bbdbd99021a6e77a4c855d524" dependencies = [ + "core2", + "hashbrown 0.13.2", "rle-decode-fast", ] @@ -1536,9 +1586,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7012b1bbb0719e1097c47611d3898568c546d597c2e74d66f6087edd5233ff4" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" @@ -1564,9 +1614,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" +checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" [[package]] name = "lock_api" @@ -1617,18 +1667,19 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.6.3" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memoffset" @@ -1778,9 +1829,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d359e231e5451f4f9fa889d56e3ce34f8724f1a61db2107739359717cf2bbf08" +checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" dependencies = [ "async-trait", "base64", @@ -1789,7 +1840,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.10.5", + "itertools 0.11.0", "parking_lot", "percent-encoding", "quick-xml", @@ -1846,9 +1897,9 @@ dependencies = [ [[package]] name = "parquet" -version = "46.0.0" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad2cba786ae07da4d73371a88b9e0f9d3ffac1a9badc83922e0e15814f5c5fa" +checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" dependencies = [ "ahash", "arrow-array", @@ -1864,7 +1915,7 @@ dependencies = [ "chrono", "flate2", "futures", - "hashbrown 0.14.0", + "hashbrown 0.14.1", "lz4", "num", "num-bigint", @@ -1906,7 +1957,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.0", + "indexmap 2.0.2", ] [[package]] @@ -1978,7 +2029,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -1989,9 +2040,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.67" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" +checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" dependencies = [ "unicode-ident", ] @@ -2125,9 +2176,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", "serde", @@ -2183,9 +2234,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.5" +version = "1.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47" +checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" dependencies = [ "aho-corasick", "memchr", @@ -2195,15 +2246,21 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795" +checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f96ede7f386ba6e910092e7ccdc04176cface62abebea07ed6b46d870ed95ca2" + [[package]] name = "regex-syntax" version = "0.7.5" @@ -2222,9 +2279,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.20" +version = "0.11.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" +checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" dependencies = [ "base64", "bytes", @@ -2248,6 +2305,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "system-configuration", "tokio", "tokio-rustls", "tokio-util", @@ -2299,9 +2357,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.13" +version = "0.38.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662" +checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" dependencies = [ "bitflags 2.4.0", "errno", @@ -2333,9 +2391,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.5" +version = "0.101.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a27e3b59326c16e23d30aeb7a36a24cc0d29e71d68ff611cdfb4a01d013bed" +checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" dependencies = [ "ring", "untrusted", @@ -2364,9 +2422,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "763f8cd0d4c71ed8389c90cb8100cba87e763bd01a8e614d4f0af97bcd50a161" +checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" dependencies = [ "dyn-clone", "schemars_derive", @@ -2376,9 +2434,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.13" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0f696e21e10fa546b7ffb1c9672c6de8fbc7a81acf59524386d8639bf12737" +checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" dependencies = [ "proc-macro2", "quote", @@ -2404,9 +2462,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" +checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" [[package]] name = "seq-macro" @@ -2431,7 +2489,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2447,9 +2505,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -2465,7 +2523,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2486,7 +2544,7 @@ version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ - "indexmap 2.0.0", + "indexmap 2.0.2", "itoa", "ryu", "serde", @@ -2495,9 +2553,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.7" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479fb9d862239e610720565ca91403019f2f00410f1864c5aa7479b950a76ed8" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -2521,9 +2579,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "snafu" @@ -2581,9 +2639,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.37.0" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ae05a8250b968a3f7db93155a84d68b2e6cea1583949af5ca5b5170c76c075" +checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" dependencies = [ "log", "sqlparser_derive", @@ -2644,14 +2702,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] name = "substrait" -version = "0.13.2" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b705c631514962d48c0647e3ad2b4d93b7472f5da7bee2d1dc853bf4c61f19" +checksum = "27c3d276b85647003f434bfb7ce3ebaf4c47a711018af9d4350d25763535c239" dependencies = [ "git2", "heck", @@ -2665,7 +2723,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.37", + "syn 2.0.38", "typify", "walkdir", ] @@ -2689,15 +2747,36 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.37" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "target-lexicon" version = "0.12.11" @@ -2719,22 +2798,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2798,7 +2877,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2813,9 +2892,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -2851,7 +2930,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", ] [[package]] @@ -2881,20 +2960,29 @@ dependencies = [ [[package]] name = "typed-builder" -version = "0.14.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cba322cb9b7bc6ca048de49e83918223f35e7a86311267013afff257004870" +checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.38", ] [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" @@ -2919,7 +3007,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.37", + "syn 2.0.38", "thiserror", "unicode-ident", ] @@ -2936,7 +3024,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.37", + "syn 2.0.38", "typify-impl", ] @@ -2948,9 +3036,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -2969,9 +3057,9 @@ checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" +checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unindent" @@ -3070,7 +3158,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-shared", ] @@ -3104,7 +3192,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.37", + "syn 2.0.38", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3174,9 +3262,9 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" dependencies = [ "winapi", ] @@ -3281,27 +3369,6 @@ dependencies = [ "lzma-sys", ] -[[package]] -name = "zerocopy" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20707b61725734c595e840fb3704378a0cd2b9c74cc9e6e20724838fc6a1e2f9" -dependencies = [ - "byteorder", - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56097d5b91d711293a42be9289403896b68654625021732067eac7a4ca388a1f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.37", -] - [[package]] name = "zstd" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index 6545e1c9a..b1e432007 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "31.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "31.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "31.0.0" } -datafusion-optimizer = { version = "31.0.0" } -datafusion-sql = { version = "31.0.0" } -datafusion-substrait = { version = "31.0.0" } +datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83", features = ["pyarrow", "avro"] } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83", features = ["pyarrow"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } diff --git a/src/context.rs b/src/context.rs index 632b179a1..a0361acfc 100644 --- a/src/context.rs +++ b/src/context.rs @@ -42,10 +42,10 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::FileCompressionType; +use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; -use datafusion::execution::context::{SessionConfig, SessionContext, TaskContext}; +use datafusion::execution::context::{SessionConfig, SessionContext, SessionState, TaskContext}; use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; @@ -235,8 +235,9 @@ impl PySessionContext { RuntimeConfig::default() }; let runtime = Arc::new(RuntimeEnv::new(runtime_config)?); + let session_state = SessionState::new_with_config_rt(config, runtime); Ok(PySessionContext { - ctx: SessionContext::with_config_rt(config, runtime), + ctx: SessionContext::new_with_state(session_state), }) } @@ -469,7 +470,7 @@ impl PySessionContext { options.file_extension = file_extension; options.schema = schema.as_ref().map(|x| &x.0); options.file_sort_order = file_sort_order - .unwrap_or(vec![]) + .unwrap_or_default() .into_iter() .map(|e| e.into_iter().map(|f| f.into()).collect()) .collect(); @@ -758,7 +759,7 @@ impl PySessionContext { options.file_extension = file_extension; options.schema = schema.as_ref().map(|x| &x.0); options.file_sort_order = file_sort_order - .unwrap_or(vec![]) + .unwrap_or_default() .into_iter() .map(|e| e.into_iter().map(|f| f.into()).collect()) .collect(); diff --git a/src/expr/literal.rs b/src/expr/literal.rs index 076f89a66..bce987f76 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -50,7 +50,7 @@ macro_rules! extract_scalar_value { impl PyLiteral { /// Get the data type of this literal value fn data_type(&self) -> String { - format!("{}", self.value.get_datatype()) + format!("{}", self.value.data_type()) } pub fn value_f32(&self) -> PyResult> { From 4c7b14c6e4da99a7e52fddaf6fed9ae51ca2c66e Mon Sep 17 00:00:00 2001 From: zhenxing jiang Date: Thu, 12 Oct 2023 10:22:27 -0500 Subject: [PATCH 083/413] add bit_and,bit_or,bit_xor,bool_add,bool_or (#496) * add bit_and,bit_or,bit_xor,bool_add,bool_or * Update datafusion/tests/test_aggregation.py Co-authored-by: Liang-Chi Hsieh --------- Co-authored-by: Liang-Chi Hsieh --- datafusion/tests/test_aggregation.py | 38 ++++++++++++++++++++++++++-- src/functions.rs | 10 ++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/datafusion/tests/test_aggregation.py b/datafusion/tests/test_aggregation.py index 2c8c064b1..0a6c90c32 100644 --- a/datafusion/tests/test_aggregation.py +++ b/datafusion/tests/test_aggregation.py @@ -33,8 +33,9 @@ def df(): pa.array([1, 2, 3]), pa.array([4, 4, 6]), pa.array([9, 8, 5]), + pa.array([True, True, False]), ], - names=["a", "b", "c"], + names=["a", "b", "c", "d"], ) return ctx.create_dataframe([[batch]]) @@ -73,7 +74,7 @@ def test_built_in_aggregation(df): ], ) result = agg_df.collect()[0] - values_a, values_b, values_c = df.collect()[0] + values_a, values_b, values_c, values_d = df.collect()[0] assert result.column(0) == pa.array([2], type=pa.uint64()) assert result.column(1) == pa.array([4]) @@ -125,3 +126,36 @@ def test_built_in_aggregation(df): np.testing.assert_array_almost_equal( result.column(21), np.var(values_c, ddof=1) ) + + +def test_bit_add_or_xor(df): + + df = df.aggregate( + [], + [ + f.bit_and(column("a")), + f.bit_or(column("b")), + f.bit_xor(column("c")), + ], + ) + + result = df.collect() + result = result[0] + assert result.column(0) == pa.array([0]) + assert result.column(1) == pa.array([6]) + assert result.column(2) == pa.array([4]) + + +def test_bool_and_or(df): + + df = df.aggregate( + [], + [ + f.bool_and(column("d")), + f.bool_or(column("d")), + ], + ) + result = df.collect() + result = result[0] + assert result.column(0) == pa.array([False]) + assert result.column(1) == pa.array([True]) diff --git a/src/functions.rs b/src/functions.rs index ef26240fe..eed28154e 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -362,6 +362,11 @@ aggregate_function!(stddev_samp, Stddev); aggregate_function!(var, Variance); aggregate_function!(var_pop, VariancePop); aggregate_function!(var_samp, Variance); +aggregate_function!(bit_and, BitAnd); +aggregate_function!(bit_or, BitOr); +aggregate_function!(bit_xor, BitXor); +aggregate_function!(bool_and, BoolAnd); +aggregate_function!(bool_or, BoolOr); pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(abs))?; @@ -489,6 +494,11 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(var_pop))?; m.add_wrapped(wrap_pyfunction!(var_samp))?; m.add_wrapped(wrap_pyfunction!(window))?; + m.add_wrapped(wrap_pyfunction!(bit_and))?; + m.add_wrapped(wrap_pyfunction!(bit_or))?; + m.add_wrapped(wrap_pyfunction!(bit_xor))?; + m.add_wrapped(wrap_pyfunction!(bool_and))?; + m.add_wrapped(wrap_pyfunction!(bool_or))?; //Binary String Functions m.add_wrapped(wrap_pyfunction!(encode))?; From 804d0eb3160b0debe5beed77209ea497b0f026c7 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 12 Oct 2023 10:00:25 -0600 Subject: [PATCH 084/413] Use DataFusion 32 (#515) * use DataFusion 32 * update lock file --- Cargo.lock | 131 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 14 +++--- 2 files changed, 74 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ee4eb3f64..f739947f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -38,9 +38,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea5d730647d4fadd988536d06fecce94b7b4f2a7efdae548f1cf4b63205518ab" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] @@ -342,7 +342,7 @@ dependencies = [ "arrow-select", "num", "regex", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] @@ -715,8 +715,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7014432223f4d721cb9786cd88bb89e7464e0ba984d4a7f49db7787f5f268674" dependencies = [ "ahash", "apache-avro", @@ -764,8 +765,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3903ed8f102892f17b48efa437f3542159241d41c564f0d1e78efdc5e663aa" dependencies = [ "ahash", "apache-avro", @@ -784,8 +786,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780b73b2407050e53f51a9781868593f694102c59e622de9a8aafc0343c4f237" dependencies = [ "arrow", "chrono", @@ -804,8 +807,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24c382676338d8caba6c027ba0da47260f65ffedab38fda78f6d8043f607557c" dependencies = [ "ahash", "arrow", @@ -818,8 +822,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2904a432f795484fd45e29ded4537152adb60f636c05691db34fcd94c92c96" dependencies = [ "arrow", "async-trait", @@ -830,13 +835,14 @@ dependencies = [ "hashbrown 0.14.1", "itertools 0.11.0", "log", - "regex-syntax", + "regex-syntax 0.7.5", ] [[package]] name = "datafusion-physical-expr" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57b4968e9a998dc0476c4db7a82f280e2026b25f464e4aa0c3bb9807ee63ddfd" dependencies = [ "ahash", "arrow", @@ -868,8 +874,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd0d1fe54e37a47a2d58a1232c22786f2c28ad35805fdcd08f0253a8b0aaa90" dependencies = [ "ahash", "arrow", @@ -898,7 +905,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "31.0.0" +version = "32.0.0" dependencies = [ "async-trait", "datafusion", @@ -916,7 +923,7 @@ dependencies = [ "pyo3", "pyo3-build-config", "rand", - "regex-syntax", + "regex-syntax 0.7.5", "syn 2.0.38", "tokio", "url", @@ -925,8 +932,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b568d44c87ead99604d704f942e257c8a236ee1bbf890ee3e034ad659dcb2c21" dependencies = [ "arrow", "arrow-schema", @@ -938,8 +946,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "31.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=e23d34bae60bb2f9c496241e218bab795af3af83#e23d34bae60bb2f9c496241e218bab795af3af83" +version = "32.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2884dff8207774c1ea5f7b008d44b20e9723bd0b2e4b7dd6627390d8b526b50" dependencies = [ "async-recursion", "chrono", @@ -998,25 +1007,14 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "fastrand" version = "2.0.1" @@ -1456,9 +1454,9 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" dependencies = [ "libc", ] @@ -1614,9 +1612,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" @@ -1800,9 +1798,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", "libm", @@ -1865,9 +1863,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-float" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" dependencies = [ "num-traits", ] @@ -2040,9 +2038,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] @@ -2234,32 +2232,32 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.6" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebee201405406dbf528b8b672104ae6d6d63e6d118cb10e4d51abbc7b58044ff" +checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax", + "regex-syntax 0.8.1", ] [[package]] name = "regex-automata" -version = "0.3.9" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b23e92ee4318893fa3fe3e6fb365258efbfe6ac6ab30f090cdcbb7aa37efa9" +checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.8.1", ] [[package]] name = "regex-lite" -version = "0.1.0" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f96ede7f386ba6e910092e7ccdc04176cface62abebea07ed6b46d870ed95ca2" +checksum = "9a6ebcd15653947e6140f59a9811a06ed061d18a5c35dfca2e2e4c5525696878" [[package]] name = "regex-syntax" @@ -2267,6 +2265,12 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" +[[package]] +name = "regex-syntax" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" + [[package]] name = "regress" version = "0.6.0" @@ -2357,9 +2361,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.17" +version = "0.38.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" +checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" dependencies = [ "bitflags 2.4.0", "errno", @@ -2462,9 +2466,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad977052201c6de01a8ef2aa3378c4bd23217a056337d1d6da40468d267a4fb0" +checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "seq-macro" @@ -2853,9 +2857,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.32.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ "backtrace", "bytes", @@ -3390,11 +3394,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.8+zstd.1.5.5" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", - "libc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index b1e432007..8c86b5658 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "31.0.0" +version = "32.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,12 +36,12 @@ protoc = [ "datafusion-substrait/protoc" ] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83", features = ["pyarrow", "avro"] } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83", features = ["pyarrow"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "e23d34bae60bb2f9c496241e218bab795af3af83" } +datafusion = { version = "32.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "32.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "32.0.0" } +datafusion-optimizer = { version = "32.0.0" } +datafusion-sql = { version = "32.0.0" } +datafusion-substrait = { version = "32.0.0" } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } From a91188c51019a1f9c190a6596970914763415965 Mon Sep 17 00:00:00 2001 From: zhenxing jiang Date: Fri, 13 Oct 2023 17:18:55 -0500 Subject: [PATCH 085/413] add first_value last_value (#498) Co-authored-by: Andy Grove --- datafusion/tests/test_functions.py | 23 +++++++++++++++++++++++ src/functions.rs | 4 ++++ 2 files changed, 27 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index f1f64c30a..e504cc498 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -479,6 +479,29 @@ def test_case(df): assert result.column(2) == pa.array(["Hola", "Mundo", None]) +def test_first_last_value(df): + df = df.aggregate( + [], + [ + f.first_value(column("a")), + f.first_value(column("b")), + f.first_value(column("d")), + f.last_value(column("a")), + f.last_value(column("b")), + f.last_value(column("d")), + ], + ) + + result = df.collect() + result = result[0] + assert result.column(0) == pa.array(["Hello"]) + assert result.column(1) == pa.array([4]) + assert result.column(2) == pa.array([datetime(2022, 12, 31)]) + assert result.column(3) == pa.array(["!"]) + assert result.column(4) == pa.array([6]) + assert result.column(5) == pa.array([datetime(2020, 7, 2)]) + + def test_binary_string_functions(df): df = df.select( f.encode(column("a"), literal("base64")), diff --git a/src/functions.rs b/src/functions.rs index eed28154e..2f2f34ee0 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -362,6 +362,8 @@ aggregate_function!(stddev_samp, Stddev); aggregate_function!(var, Variance); aggregate_function!(var_pop, VariancePop); aggregate_function!(var_samp, Variance); +aggregate_function!(first_value, FirstValue); +aggregate_function!(last_value, LastValue); aggregate_function!(bit_and, BitAnd); aggregate_function!(bit_or, BitOr); aggregate_function!(bit_xor, BitXor); @@ -494,6 +496,8 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(var_pop))?; m.add_wrapped(wrap_pyfunction!(var_samp))?; m.add_wrapped(wrap_pyfunction!(window))?; + m.add_wrapped(wrap_pyfunction!(first_value))?; + m.add_wrapped(wrap_pyfunction!(last_value))?; m.add_wrapped(wrap_pyfunction!(bit_and))?; m.add_wrapped(wrap_pyfunction!(bit_or))?; m.add_wrapped(wrap_pyfunction!(bit_xor))?; From 484ed1108736d4cfd3073ace3743c511a4764bb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Oct 2023 14:37:43 -0700 Subject: [PATCH 086/413] build(deps): bump regex-syntax from 0.7.5 to 0.8.1 (#517) Bumps [regex-syntax](https://github.com/rust-lang/regex) from 0.7.5 to 0.8.1. - [Release notes](https://github.com/rust-lang/regex/releases) - [Changelog](https://github.com/rust-lang/regex/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-lang/regex/compare/regex-syntax-0.7.5...regex-syntax-0.8.1) --- updated-dependencies: - dependency-name: regex-syntax dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f739947f0..cb49f1aa3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,7 +923,7 @@ dependencies = [ "pyo3", "pyo3-build-config", "rand", - "regex-syntax 0.7.5", + "regex-syntax 0.8.1", "syn 2.0.38", "tokio", "url", diff --git a/Cargo.toml b/Cargo.toml index 8c86b5658..556450cd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ async-trait = "0.1" futures = "0.3" object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" -regex-syntax = "0.7.1" +regex-syntax = "0.8.1" syn = "2.0.37" url = "2.2" From c4675b76926dbc605735d26fc54dd6e9bd67b83e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Oct 2023 16:18:32 -0700 Subject: [PATCH 087/413] build(deps): bump pyo3-build-config from 0.19.2 to 0.20.0 (#516) Bumps [pyo3-build-config](https://github.com/pyo3/pyo3) from 0.19.2 to 0.20.0. - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/compare/v0.19.2...v0.20.0) --- updated-dependencies: - dependency-name: pyo3-build-config dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 16 +++++++++++++--- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb49f1aa3..76ea5784c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -921,7 +921,7 @@ dependencies = [ "prost", "prost-types", "pyo3", - "pyo3-build-config", + "pyo3-build-config 0.20.0", "rand", "regex-syntax 0.8.1", "syn 2.0.38", @@ -2117,7 +2117,7 @@ dependencies = [ "libc", "memoffset", "parking_lot", - "pyo3-build-config", + "pyo3-build-config 0.19.2", "pyo3-ffi", "pyo3-macros", "unindent", @@ -2133,6 +2133,16 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "pyo3-build-config" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +dependencies = [ + "once_cell", + "target-lexicon", +] + [[package]] name = "pyo3-ffi" version = "0.19.2" @@ -2140,7 +2150,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" dependencies = [ "libc", - "pyo3-build-config", + "pyo3-build-config 0.19.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 556450cd1..7c1a57bd3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ syn = "2.0.37" url = "2.2" [build-dependencies] -pyo3-build-config = "0.19.0" +pyo3-build-config = "0.20.0" [lib] name = "datafusion_python" From 5ec45ddd5f3b44a3b39d591e3aa3c6eb8880c5ee Mon Sep 17 00:00:00 2001 From: zhenxing jiang Date: Sun, 15 Oct 2023 11:05:57 -0500 Subject: [PATCH 088/413] add regr_* functions (#499) Co-authored-by: Andy Grove --- datafusion/tests/test_functions.py | 22 ++++++++++++++++++++++ src/functions.rs | 18 ++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index e504cc498..be2a2f1f5 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -479,6 +479,28 @@ def test_case(df): assert result.column(2) == pa.array(["Hola", "Mundo", None]) +def test_regr_funcs(df): + # test case base on + # https://github.com/apache/arrow-datafusion/blob/d1361d56b9a9e0c165d3d71a8df6795d2a5f51dd/datafusion/core/tests/sqllogictests/test_files/aggregate.slt#L2330 + ctx = SessionContext() + result = ctx.sql( + "select regr_slope(1,1), regr_intercept(1,1), " + "regr_count(1,1), regr_r2(1,1), regr_avgx(1,1), " + "regr_avgy(1,1), regr_sxx(1,1), regr_syy(1,1), " + "regr_sxy(1,1);" + ).collect() + + assert result[0].column(0) == pa.array([None], type=pa.float64()) + assert result[0].column(1) == pa.array([None], type=pa.float64()) + assert result[0].column(2) == pa.array([1], type=pa.float64()) + assert result[0].column(3) == pa.array([None], type=pa.float64()) + assert result[0].column(4) == pa.array([1], type=pa.float64()) + assert result[0].column(5) == pa.array([1], type=pa.float64()) + assert result[0].column(6) == pa.array([0], type=pa.float64()) + assert result[0].column(7) == pa.array([0], type=pa.float64()) + assert result[0].column(8) == pa.array([0], type=pa.float64()) + + def test_first_last_value(df): df = df.aggregate( [], diff --git a/src/functions.rs b/src/functions.rs index 2f2f34ee0..e509aff71 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -362,6 +362,15 @@ aggregate_function!(stddev_samp, Stddev); aggregate_function!(var, Variance); aggregate_function!(var_pop, VariancePop); aggregate_function!(var_samp, Variance); +aggregate_function!(regr_avgx, RegrAvgx); +aggregate_function!(regr_avgy, RegrAvgy); +aggregate_function!(regr_count, RegrCount); +aggregate_function!(regr_intercept, RegrIntercept); +aggregate_function!(regr_r2, RegrR2); +aggregate_function!(regr_slope, RegrSlope); +aggregate_function!(regr_sxx, RegrSXX); +aggregate_function!(regr_sxy, RegrSXY); +aggregate_function!(regr_syy, RegrSYY); aggregate_function!(first_value, FirstValue); aggregate_function!(last_value, LastValue); aggregate_function!(bit_and, BitAnd); @@ -496,6 +505,15 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(var_pop))?; m.add_wrapped(wrap_pyfunction!(var_samp))?; m.add_wrapped(wrap_pyfunction!(window))?; + m.add_wrapped(wrap_pyfunction!(regr_avgx))?; + m.add_wrapped(wrap_pyfunction!(regr_avgy))?; + m.add_wrapped(wrap_pyfunction!(regr_count))?; + m.add_wrapped(wrap_pyfunction!(regr_intercept))?; + m.add_wrapped(wrap_pyfunction!(regr_r2))?; + m.add_wrapped(wrap_pyfunction!(regr_slope))?; + m.add_wrapped(wrap_pyfunction!(regr_sxx))?; + m.add_wrapped(wrap_pyfunction!(regr_sxy))?; + m.add_wrapped(wrap_pyfunction!(regr_syy))?; m.add_wrapped(wrap_pyfunction!(first_value))?; m.add_wrapped(wrap_pyfunction!(last_value))?; m.add_wrapped(wrap_pyfunction!(bit_and))?; From 399fa758ccb1dc785929123350144902a9b6c502 Mon Sep 17 00:00:00 2001 From: Dan Lovell Date: Tue, 17 Oct 2023 16:59:41 -0400 Subject: [PATCH 089/413] feat: expose PyWindowFrame (#509) * feat: expose PyWindowFrame * fix: PyWindowFrame: return Err instead of panicking * test: test PyWindowFrame creation --- datafusion/__init__.py | 2 + datafusion/tests/test_dataframe.py | 41 ++++++++++- src/functions.rs | 20 +++++- src/lib.rs | 2 + src/udaf.rs | 30 +++++++- src/window_frame.rs | 110 +++++++++++++++++++++++++++++ 6 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 src/window_frame.rs diff --git a/datafusion/__init__.py b/datafusion/__init__.py index bb1beacd9..4a495b468 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -33,6 +33,7 @@ SessionConfig, RuntimeConfig, ScalarUDF, + WindowFrame, ) from .common import ( @@ -98,6 +99,7 @@ "Expr", "AggregateUDF", "ScalarUDF", + "WindowFrame", "column", "literal", "TableScan", diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index ce7d89e7b..c9b0f076f 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -21,7 +21,14 @@ import pytest from datafusion import functions as f -from datafusion import DataFrame, SessionContext, column, literal, udf +from datafusion import ( + DataFrame, + SessionContext, + WindowFrame, + column, + literal, + udf, +) @pytest.fixture @@ -304,6 +311,38 @@ def test_window_functions(df): assert table.sort_by("a").to_pydict() == expected +@pytest.mark.parametrize( + ("units", "start_bound", "end_bound"), + [ + (units, start_bound, end_bound) + for units in ("rows", "range") + for start_bound in (None, 0, 1) + for end_bound in (None, 0, 1) + ] + + [ + ("groups", 0, 0), + ], +) +def test_valid_window_frame(units, start_bound, end_bound): + WindowFrame(units, start_bound, end_bound) + + +@pytest.mark.parametrize( + ("units", "start_bound", "end_bound"), + [ + ("invalid-units", 0, None), + ("invalid-units", None, 0), + ("invalid-units", None, None), + ("groups", None, 0), + ("groups", 0, None), + ("groups", None, None), + ], +) +def test_invalid_window_frame(units, start_bound, end_bound): + with pytest.raises(RuntimeError): + WindowFrame(units, start_bound, end_bound) + + def test_get_dataframe(tmp_path): ctx = SessionContext() diff --git a/src/functions.rs b/src/functions.rs index e509aff71..42203d7b4 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -17,9 +17,12 @@ use pyo3::{prelude::*, wrap_pyfunction}; +use crate::context::PySessionContext; use crate::errors::DataFusionError; use crate::expr::conditional_expr::PyCaseBuilder; use crate::expr::PyExpr; +use crate::window_frame::PyWindowFrame; +use datafusion::execution::FunctionRegistry; use datafusion_common::Column; use datafusion_expr::expr::Alias; use datafusion_expr::{ @@ -27,7 +30,7 @@ use datafusion_expr::{ expr::{AggregateFunction, ScalarFunction, Sort, WindowFunction}, lit, window_function::find_df_window_func, - BuiltinScalarFunction, Expr, WindowFrame, + BuiltinScalarFunction, Expr, }; #[pyfunction] @@ -130,13 +133,24 @@ fn window( args: Vec, partition_by: Option>, order_by: Option>, + window_frame: Option, + ctx: Option, ) -> PyResult { - let fun = find_df_window_func(name); + let fun = find_df_window_func(name).or_else(|| { + ctx.and_then(|ctx| { + ctx.ctx + .udaf(name) + .map(|fun| datafusion_expr::WindowFunction::AggregateUDF(fun)) + .ok() + }) + }); if fun.is_none() { return Err(DataFusionError::Common("window function not found".to_string()).into()); } let fun = fun.unwrap(); - let window_frame = WindowFrame::new(order_by.is_some()); + let window_frame = window_frame + .unwrap_or_else(|| PyWindowFrame::new("rows", None, Some(0)).unwrap()) + .into(); Ok(PyExpr { expr: datafusion_expr::Expr::WindowFunction(WindowFunction { fun, diff --git a/src/lib.rs b/src/lib.rs index 2512aefa4..b9bd5766e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,6 +54,7 @@ mod udaf; #[allow(clippy::borrow_deref_ref)] mod udf; pub mod utils; +mod window_frame; #[cfg(feature = "mimalloc")] #[global_allocator] @@ -83,6 +84,7 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/udaf.rs b/src/udaf.rs index 3b70aeb06..6450f03fe 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use pyo3::{prelude::*, types::PyTuple}; +use pyo3::{prelude::*, types::PyBool, types::PyTuple}; use datafusion::arrow::array::{Array, ArrayRef}; use datafusion::arrow::datatypes::DataType; @@ -93,6 +93,34 @@ impl Accumulator for RustAccumulator { fn size(&self) -> usize { std::mem::size_of_val(self) } + + fn retract_batch(&mut self, values: &[ArrayRef]) -> Result<()> { + Python::with_gil(|py| { + // 1. cast args to Pyarrow array + let py_args = values + .iter() + .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) + .collect::>(); + let py_args = PyTuple::new(py, py_args); + + // 2. call function + self.accum + .as_ref(py) + .call_method1("retract_batch", py_args) + .map_err(|e| DataFusionError::Execution(format!("{e}")))?; + + Ok(()) + }) + } + + fn supports_retract_batch(&self) -> bool { + Python::with_gil(|py| { + let x: Result<&PyAny, PyErr> = + self.accum.as_ref(py).call_method0("supports_retract_batch"); + let x: &PyAny = x.unwrap_or(PyBool::new(py, false)); + x.extract().unwrap_or(false) + }) + } } pub fn to_rust_accumulator(accum: PyObject) -> AccumulatorFactoryFunction { diff --git a/src/window_frame.rs b/src/window_frame.rs new file mode 100644 index 000000000..b8f414e6a --- /dev/null +++ b/src/window_frame.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_expr::window_frame::{WindowFrame, WindowFrameBound, WindowFrameUnits}; +use pyo3::prelude::*; +use std::fmt::{Display, Formatter}; + +use crate::errors::py_datafusion_err; + +#[pyclass(name = "WindowFrame", module = "datafusion", subclass)] +#[derive(Clone)] +pub struct PyWindowFrame { + frame: WindowFrame, +} + +impl From for WindowFrame { + fn from(frame: PyWindowFrame) -> Self { + frame.frame + } +} + +impl From for PyWindowFrame { + fn from(frame: WindowFrame) -> PyWindowFrame { + PyWindowFrame { frame } + } +} + +impl Display for PyWindowFrame { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "OVER ({} BETWEEN {} AND {})", + self.frame.units, self.frame.start_bound, self.frame.end_bound + ) + } +} + +#[pymethods] +impl PyWindowFrame { + #[new(unit, start_bound, end_bound)] + pub fn new(units: &str, start_bound: Option, end_bound: Option) -> PyResult { + let units = units.to_ascii_lowercase(); + let units = match units.as_str() { + "rows" => WindowFrameUnits::Rows, + "range" => WindowFrameUnits::Range, + "groups" => WindowFrameUnits::Groups, + _ => { + return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + units, + )))); + } + }; + let start_bound = match start_bound { + Some(start_bound) => { + WindowFrameBound::Preceding(ScalarValue::UInt64(Some(start_bound))) + } + None => match units { + WindowFrameUnits::Range => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), + WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), + WindowFrameUnits::Groups => { + return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + units, + )))); + } + }, + }; + let end_bound = match end_bound { + Some(end_bound) => WindowFrameBound::Following(ScalarValue::UInt64(Some(end_bound))), + None => match units { + WindowFrameUnits::Rows => WindowFrameBound::Following(ScalarValue::UInt64(None)), + WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)), + WindowFrameUnits::Groups => { + return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + units, + )))); + } + }, + }; + Ok(PyWindowFrame { + frame: WindowFrame { + units, + start_bound, + end_bound, + }, + }) + } + + /// Get a String representation of this window frame + fn __repr__(&self) -> String { + format!("{}", self) + } +} From c2768d8f9d558f3a5e042f1e934c02f945ede797 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 18 Oct 2023 11:40:26 -0400 Subject: [PATCH 090/413] Add random missing bindings (#522) --- src/expr/aggregate.rs | 49 +++++++++++++++++++++++++++++++++++++++++++ src/expr/join.rs | 14 +++++++++++++ src/expr/sort.rs | 4 ++++ 3 files changed, 67 insertions(+) diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index c3de9673a..5ebf8c6cf 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -16,12 +16,15 @@ // under the License. use datafusion_common::DataFusionError; +use datafusion_expr::expr::{AggregateFunction, AggregateUDF, Alias}; use datafusion_expr::logical_plan::Aggregate; +use datafusion_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; use super::logical_node::LogicalNode; use crate::common::df_schema::PyDFSchema; +use crate::errors::py_type_err; use crate::expr::PyExpr; use crate::sql::logical::PyLogicalPlan; @@ -84,6 +87,24 @@ impl PyAggregate { .collect()) } + /// Returns the inner Aggregate Expr(s) + pub fn agg_expressions(&self) -> PyResult> { + Ok(self + .aggregate + .aggr_expr + .iter() + .map(|e| PyExpr::from(e.clone())) + .collect()) + } + + pub fn agg_func_name(&self, expr: PyExpr) -> PyResult { + Self::_agg_func_name(&expr.expr) + } + + pub fn aggregation_arguments(&self, expr: PyExpr) -> PyResult> { + self._aggregation_arguments(&expr.expr) + } + // Retrieves the input `LogicalPlan` to this `Aggregate` node fn input(&self) -> PyResult> { Ok(Self::inputs(self)) @@ -99,6 +120,34 @@ impl PyAggregate { } } +impl PyAggregate { + #[allow(clippy::only_used_in_recursion)] + fn _aggregation_arguments(&self, expr: &Expr) -> PyResult> { + match expr { + // TODO: This Alias logic seems to be returning some strange results that we should investigate + Expr::Alias(Alias { expr, .. }) => self._aggregation_arguments(expr.as_ref()), + Expr::AggregateFunction(AggregateFunction { fun: _, args, .. }) + | Expr::AggregateUDF(AggregateUDF { fun: _, args, .. }) => { + Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()) + } + _ => Err(py_type_err( + "Encountered a non Aggregate type in aggregation_arguments", + )), + } + } + + fn _agg_func_name(expr: &Expr) -> PyResult { + match expr { + Expr::Alias(Alias { expr, .. }) => Self::_agg_func_name(expr.as_ref()), + Expr::AggregateFunction(AggregateFunction { fun, .. }) => Ok(fun.to_string()), + Expr::AggregateUDF(AggregateUDF { fun, .. }) => Ok(fun.name.clone()), + _ => Err(py_type_err( + "Encountered a non Aggregate type in agg_func_name", + )), + } + } +} + impl LogicalNode for PyAggregate { fn inputs(&self) -> Vec { vec![PyLogicalPlan::from((*self.aggregate.input).clone())] diff --git a/src/expr/join.rs b/src/expr/join.rs index 801662962..a53ddd3ba 100644 --- a/src/expr/join.rs +++ b/src/expr/join.rs @@ -46,6 +46,10 @@ impl PyJoinType { pub fn is_outer(&self) -> bool { self.join_type.is_outer() } + + fn __repr__(&self) -> PyResult { + Ok(format!("{}", self.join_type)) + } } impl Display for PyJoinType { @@ -72,6 +76,16 @@ impl From for JoinConstraint { } } +#[pymethods] +impl PyJoinConstraint { + fn __repr__(&self) -> PyResult { + match self.join_constraint { + JoinConstraint::On => Ok("On".to_string()), + JoinConstraint::Using => Ok("Using".to_string()), + } + } +} + #[pyclass(name = "Join", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyJoin { diff --git a/src/expr/sort.rs b/src/expr/sort.rs index 8843c638d..f9f9e5899 100644 --- a/src/expr/sort.rs +++ b/src/expr/sort.rs @@ -72,6 +72,10 @@ impl PySort { .collect()) } + fn get_fetch_val(&self) -> PyResult> { + Ok(self.sort.fetch) + } + /// Retrieves the input `LogicalPlan` to this `Sort` node fn input(&self) -> PyResult> { Ok(Self::inputs(self)) From 59140f29e524684ee2250260308b40af932a97c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 12:29:57 -0700 Subject: [PATCH 091/413] build(deps): bump rustix from 0.38.18 to 0.38.19 (#523) Bumps [rustix](https://github.com/bytecodealliance/rustix) from 0.38.18 to 0.38.19. - [Release notes](https://github.com/bytecodealliance/rustix/releases) - [Commits](https://github.com/bytecodealliance/rustix/compare/v0.38.18...v0.38.19) --- updated-dependencies: - dependency-name: rustix dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 76ea5784c..0495b7e1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2371,9 +2371,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ "bitflags 2.4.0", "errno", From 501acfff1b82cda7e4892a57deb3ab2cb83ea2b4 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sat, 21 Oct 2023 08:47:04 -0400 Subject: [PATCH 092/413] Allow for multiple input files per table instead of a single file (#519) --- datafusion/input/location.py | 10 +++++----- datafusion/tests/test_input.py | 2 +- src/common/schema.rs | 16 ++++++++-------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/datafusion/input/location.py b/datafusion/input/location.py index efbc82f23..939c7f415 100644 --- a/datafusion/input/location.py +++ b/datafusion/input/location.py @@ -16,6 +16,7 @@ # under the License. import os +import glob from typing import Any from datafusion.common import DataTypeMap, SqlTable @@ -41,14 +42,12 @@ def build_table( format = extension.lstrip(".").lower() num_rows = 0 # Total number of rows in the file. Used for statistics columns = [] - if format == "parquet": import pyarrow.parquet as pq # Read the Parquet metadata metadata = pq.read_metadata(input_file) num_rows = metadata.num_rows - # Iterate through the schema and build the SqlTable for col in metadata.schema: columns.append( @@ -57,7 +56,6 @@ def build_table( DataTypeMap.from_parquet_type_str(col.physical_type), ) ) - elif format == "csv": import csv @@ -73,7 +71,6 @@ def build_table( print(header_row) for _ in reader: num_rows += 1 - # TODO: Need to actually consume this row into resonable columns raise RuntimeError( "TODO: Currently unable to support CSV input files." @@ -84,4 +81,7 @@ def build_table( Only Parquet and CSV." ) - return SqlTable(table_name, columns, num_rows, input_file) + # Input could possibly be multiple files. Create a list if so + input_files = glob.glob(input_file) + + return SqlTable(table_name, columns, num_rows, input_files) diff --git a/datafusion/tests/test_input.py b/datafusion/tests/test_input.py index 1e2ef4166..5b1decf26 100644 --- a/datafusion/tests/test_input.py +++ b/datafusion/tests/test_input.py @@ -30,4 +30,4 @@ def test_location_input(): tbl = location_input.build_table(input_file, table_name) assert "blog" == tbl.name assert 3 == len(tbl.columns) - assert "blogs.parquet" in tbl.filepath + assert "blogs.parquet" in tbl.filepaths[0] diff --git a/src/common/schema.rs b/src/common/schema.rs index a003d0ca1..77b0ce2ba 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -56,7 +56,7 @@ pub struct SqlTable { #[pyo3(get, set)] pub statistics: SqlStatistics, #[pyo3(get, set)] - pub filepath: Option, + pub filepaths: Option>, } #[pymethods] @@ -66,7 +66,7 @@ impl SqlTable { table_name: String, columns: Vec<(String, DataTypeMap)>, row_count: f64, - filepath: Option, + filepaths: Option>, ) -> Self { Self { name: table_name, @@ -76,7 +76,7 @@ impl SqlTable { indexes: Vec::new(), constraints: Vec::new(), statistics: SqlStatistics::new(row_count), - filepath, + filepaths, } } } @@ -124,7 +124,7 @@ impl SqlSchema { pub struct SqlTableSource { schema: SchemaRef, statistics: Option, - filepath: Option, + filepaths: Option>, } impl SqlTableSource { @@ -132,12 +132,12 @@ impl SqlTableSource { pub fn new( schema: SchemaRef, statistics: Option, - filepath: Option, + filepaths: Option>, ) -> Self { Self { schema, statistics, - filepath, + filepaths, } } @@ -148,8 +148,8 @@ impl SqlTableSource { /// Access optional filepath associated with this table source #[allow(dead_code)] - pub fn filepath(&self) -> Option<&String> { - self.filepath.as_ref() + pub fn filepaths(&self) -> Option<&Vec> { + self.filepaths.as_ref() } } From c6a7af590191c7c71e805df826e5ce8535a6f1b9 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sat, 21 Oct 2023 13:21:07 -0400 Subject: [PATCH 093/413] Add support for window function bindings (#521) * Add support for window function bindings * Refactor PyWindow to be moved to Expr submodule * Adjust Expr module imports, moved WindowFrame to expr * Update src/expr/window.rs --------- Co-authored-by: Andy Grove --- Cargo.lock | 117 +++++++++------- datafusion/__init__.py | 4 +- src/common/data_type.rs | 18 +++ src/expr.rs | 13 ++ src/expr/window.rs | 294 ++++++++++++++++++++++++++++++++++++++++ src/functions.rs | 2 +- src/lib.rs | 2 - src/sql/logical.rs | 8 +- src/window_frame.rs | 110 --------------- 9 files changed, 403 insertions(+), 165 deletions(-) create mode 100644 src/expr/window.rs delete mode 100644 src/window_frame.rs diff --git a/Cargo.lock b/Cargo.lock index 0495b7e1d..4ae3ec1f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,12 +107,12 @@ dependencies = [ "serde_json", "snap", "strum 0.25.0", - "strum_macros 0.25.2", + "strum_macros 0.25.3", "thiserror", "typed-builder", "uuid", "xz2", - "zstd", + "zstd 0.12.4", ] [[package]] @@ -312,7 +312,7 @@ version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", ] [[package]] @@ -347,9 +347,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb42b2197bf15ccb092b62c74515dbd8b86d0effd934795f6687c93b6e679a2c" +checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" dependencies = [ "bzip2", "flate2", @@ -359,8 +359,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd", - "zstd-safe", + "zstd 0.13.0", + "zstd-safe 7.0.0", ] [[package]] @@ -376,9 +376,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -429,9 +429,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "blake2" @@ -760,7 +760,7 @@ dependencies = [ "url", "uuid", "xz2", - "zstd", + "zstd 0.12.4", ] [[package]] @@ -817,7 +817,7 @@ dependencies = [ "datafusion-common", "sqlparser", "strum 0.25.0", - "strum_macros 0.25.2", + "strum_macros 0.25.3", ] [[package]] @@ -923,7 +923,7 @@ dependencies = [ "pyo3", "pyo3-build-config 0.20.0", "rand", - "regex-syntax 0.8.1", + "regex-syntax 0.8.2", "syn 2.0.38", "tokio", "url", @@ -1039,9 +1039,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" dependencies = [ "crc32fast", "miniz_oxide", @@ -1184,7 +1184,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "libc", "libgit2-sys", "log", @@ -1359,16 +1359,16 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.57" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "windows", + "windows-core", ] [[package]] @@ -1924,7 +1924,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd", + "zstd 0.12.4", ] [[package]] @@ -2242,32 +2242,32 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.0" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.1", + "regex-syntax 0.8.2", ] [[package]] name = "regex-automata" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.1", + "regex-syntax 0.8.2", ] [[package]] name = "regex-lite" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a6ebcd15653947e6140f59a9811a06ed061d18a5c35dfca2e2e4c5525696878" +checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" [[package]] name = "regex-syntax" @@ -2277,9 +2277,9 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "regex-syntax" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d84fdd47036b038fc80dd333d10b6aab10d5d31f4a366e20014def75328d33" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "regress" @@ -2375,7 +2375,7 @@ version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags 2.4.0", + "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", @@ -2488,18 +2488,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" dependencies = [ "proc-macro2", "quote", @@ -2690,7 +2690,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros 0.25.2", + "strum_macros 0.25.3", ] [[package]] @@ -2708,9 +2708,9 @@ dependencies = [ [[package]] name = "strum_macros" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8d03b598d3d0fff69bf533ee3ef19b8eeb342729596df84bcc7e1f96ec4059" +checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ "heck", "proc-macro2", @@ -2926,11 +2926,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" dependencies = [ - "cfg-if", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2938,9 +2937,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -2949,9 +2948,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] @@ -3290,10 +3289,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows" -version = "0.48.0" +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] @@ -3389,7 +3388,16 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ - "zstd-safe", + "zstd-safe 6.0.6", +] + +[[package]] +name = "zstd" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +dependencies = [ + "zstd-safe 7.0.0", ] [[package]] @@ -3402,6 +3410,15 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/datafusion/__init__.py b/datafusion/__init__.py index 4a495b468..c854f3f9d 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -33,7 +33,6 @@ SessionConfig, RuntimeConfig, ScalarUDF, - WindowFrame, ) from .common import ( @@ -86,6 +85,8 @@ DropTable, Repartition, Partitioning, + Window, + WindowFrame, ) __version__ = importlib_metadata.version(__name__) @@ -99,6 +100,7 @@ "Expr", "AggregateUDF", "ScalarUDF", + "Window", "WindowFrame", "column", "literal", diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 078b8c841..405a5632d 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -329,6 +329,7 @@ impl DataTypeMap { } "float" => Ok(DataType::Float32), "double" => Ok(DataType::Float64), + "byte_array" => Ok(DataType::Utf8), _ => Err(PyValueError::new_err(format!( "Unable to determine Arrow Data Type from Parquet String type: {:?}", parquet_str_type @@ -604,13 +605,30 @@ impl PyDataType { /// is presented as a String rather than an actual DataType. This function is used to /// convert that String to a DataType for the Python side to use. pub fn py_map_from_arrow_type_str(arrow_str_type: String) -> PyResult { + // Certain string types contain "metadata" that should be trimmed here. Ex: "datetime64[ns, Europe/Berlin]" + let arrow_str_type = match arrow_str_type.find('[') { + Some(index) => arrow_str_type[0..index].to_string(), + None => arrow_str_type, // Return early if ',' is not found. + }; + let arrow_dtype = match arrow_str_type.to_lowercase().as_str() { + "bool" => Ok(DataType::Boolean), "boolean" => Ok(DataType::Boolean), + "uint8" => Ok(DataType::UInt8), + "uint16" => Ok(DataType::UInt16), + "uint32" => Ok(DataType::UInt32), + "uint64" => Ok(DataType::UInt64), + "int8" => Ok(DataType::Int8), + "int16" => Ok(DataType::Int16), "int32" => Ok(DataType::Int32), "int64" => Ok(DataType::Int64), "float" => Ok(DataType::Float32), "double" => Ok(DataType::Float64), + "float16" => Ok(DataType::Float16), + "float32" => Ok(DataType::Float32), "float64" => Ok(DataType::Float64), + "datetime64" => Ok(DataType::Date64), + "object" => Ok(DataType::Utf8), _ => Err(PyValueError::new_err(format!( "Unable to determine Arrow Data Type from Arrow String type: {:?}", arrow_str_type diff --git a/src/expr.rs b/src/expr.rs index ecf8fae32..e502edced 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -92,6 +92,7 @@ pub mod subquery; pub mod subquery_alias; pub mod table_scan; pub mod union; +pub mod window; /// A PyExpr that can be used on a DataFrame #[pyclass(name = "Expr", module = "datafusion.expr", subclass)] @@ -112,6 +113,11 @@ impl From for PyExpr { } } +/// Convert a list of DataFusion Expr to PyExpr +pub fn py_expr_list(expr: &[Expr]) -> PyResult> { + Ok(expr.iter().map(|e| PyExpr::from(e.clone())).collect()) +} + #[pymethods] impl PyExpr { /// Return the specific expression @@ -542,6 +548,10 @@ impl PyExpr { // appear in projections) so we just delegate to the contained expression instead Self::expr_to_field(expr, input_plan) } + Expr::Wildcard => { + // Since * could be any of the valid column names just return the first one + Ok(input_plan.schema().field(0).clone()) + } _ => { let fields = exprlist_to_fields(&[expr.clone()], input_plan).map_err(PyErr::from)?; @@ -652,5 +662,8 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/src/expr/window.rs b/src/expr/window.rs new file mode 100644 index 000000000..6583c97af --- /dev/null +++ b/src/expr/window.rs @@ -0,0 +1,294 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_expr::expr::WindowFunction; +use datafusion_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; +use pyo3::prelude::*; +use std::fmt::{self, Display, Formatter}; + +use crate::common::df_schema::PyDFSchema; +use crate::errors::py_type_err; +use crate::expr::logical_node::LogicalNode; +use crate::expr::PyExpr; +use crate::sql::logical::PyLogicalPlan; + +use super::py_expr_list; + +use crate::errors::py_datafusion_err; + +#[pyclass(name = "Window", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyWindow { + window: Window, +} + +#[pyclass(name = "WindowFrame", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyWindowFrame { + window_frame: WindowFrame, +} + +impl From for WindowFrame { + fn from(window_frame: PyWindowFrame) -> Self { + window_frame.window_frame + } +} + +impl From for PyWindowFrame { + fn from(window_frame: WindowFrame) -> PyWindowFrame { + PyWindowFrame { window_frame } + } +} + +#[pyclass(name = "WindowFrameBound", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyWindowFrameBound { + frame_bound: WindowFrameBound, +} + +impl From for Window { + fn from(window: PyWindow) -> Window { + window.window + } +} + +impl From for PyWindow { + fn from(window: Window) -> PyWindow { + PyWindow { window } + } +} + +impl From for PyWindowFrameBound { + fn from(frame_bound: WindowFrameBound) -> Self { + PyWindowFrameBound { frame_bound } + } +} + +impl Display for PyWindow { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Over\n + Window Expr: {:?} + Schema: {:?}", + &self.window.window_expr, &self.window.schema + ) + } +} + +impl Display for PyWindowFrame { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!( + f, + "OVER ({} BETWEEN {} AND {})", + self.window_frame.units, self.window_frame.start_bound, self.window_frame.end_bound + ) + } +} + +#[pymethods] +impl PyWindow { + /// Returns the schema of the Window + pub fn schema(&self) -> PyResult { + Ok(self.window.schema.as_ref().clone().into()) + } + + /// Returns window expressions + pub fn get_window_expr(&self) -> PyResult> { + py_expr_list(&self.window.window_expr) + } + + /// Returns order by columns in a window function expression + pub fn get_sort_exprs(&self, expr: PyExpr) -> PyResult> { + match expr.expr.unalias() { + Expr::WindowFunction(WindowFunction { order_by, .. }) => py_expr_list(&order_by), + other => Err(not_window_function_err(other)), + } + } + + /// Return partition by columns in a window function expression + pub fn get_partition_exprs(&self, expr: PyExpr) -> PyResult> { + match expr.expr.unalias() { + Expr::WindowFunction(WindowFunction { partition_by, .. }) => { + py_expr_list(&partition_by) + } + other => Err(not_window_function_err(other)), + } + } + + /// Return input args for window function + pub fn get_args(&self, expr: PyExpr) -> PyResult> { + match expr.expr.unalias() { + Expr::WindowFunction(WindowFunction { args, .. }) => py_expr_list(&args), + other => Err(not_window_function_err(other)), + } + } + + /// Return window function name + pub fn window_func_name(&self, expr: PyExpr) -> PyResult { + match expr.expr.unalias() { + Expr::WindowFunction(WindowFunction { fun, .. }) => Ok(fun.to_string()), + other => Err(not_window_function_err(other)), + } + } + + /// Returns a Pywindow frame for a given window function expression + pub fn get_frame(&self, expr: PyExpr) -> Option { + match expr.expr.unalias() { + Expr::WindowFunction(WindowFunction { window_frame, .. }) => Some(window_frame.into()), + _ => None, + } + } +} + +fn not_window_function_err(expr: Expr) -> PyErr { + py_type_err(format!( + "Provided {} Expr {:?} is not a WindowFunction type", + expr.variant_name(), + expr + )) +} + +#[pymethods] +impl PyWindowFrame { + #[new(unit, start_bound, end_bound)] + pub fn new(units: &str, start_bound: Option, end_bound: Option) -> PyResult { + let units = units.to_ascii_lowercase(); + let units = match units.as_str() { + "rows" => WindowFrameUnits::Rows, + "range" => WindowFrameUnits::Range, + "groups" => WindowFrameUnits::Groups, + _ => { + return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + units, + )))); + } + }; + let start_bound = match start_bound { + Some(start_bound) => { + WindowFrameBound::Preceding(ScalarValue::UInt64(Some(start_bound))) + } + None => match units { + WindowFrameUnits::Range => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), + WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), + WindowFrameUnits::Groups => { + return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + units, + )))); + } + }, + }; + let end_bound = match end_bound { + Some(end_bound) => WindowFrameBound::Following(ScalarValue::UInt64(Some(end_bound))), + None => match units { + WindowFrameUnits::Rows => WindowFrameBound::Following(ScalarValue::UInt64(None)), + WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)), + WindowFrameUnits::Groups => { + return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + units, + )))); + } + }, + }; + Ok(PyWindowFrame { + window_frame: WindowFrame { + units, + start_bound, + end_bound, + }, + }) + } + + /// Returns the window frame units for the bounds + pub fn get_frame_units(&self) -> PyResult { + Ok(self.window_frame.units.to_string()) + } + /// Returns starting bound + pub fn get_lower_bound(&self) -> PyResult { + Ok(self.window_frame.start_bound.clone().into()) + } + /// Returns end bound + pub fn get_upper_bound(&self) -> PyResult { + Ok(self.window_frame.end_bound.clone().into()) + } + + /// Get a String representation of this window frame + fn __repr__(&self) -> String { + format!("{}", self) + } +} + +#[pymethods] +impl PyWindowFrameBound { + /// Returns if the frame bound is current row + pub fn is_current_row(&self) -> bool { + matches!(self.frame_bound, WindowFrameBound::CurrentRow) + } + + /// Returns if the frame bound is preceding + pub fn is_preceding(&self) -> bool { + matches!(self.frame_bound, WindowFrameBound::Preceding(_)) + } + + /// Returns if the frame bound is following + pub fn is_following(&self) -> bool { + matches!(self.frame_bound, WindowFrameBound::Following(_)) + } + /// Returns the offset of the window frame + pub fn get_offset(&self) -> PyResult> { + match &self.frame_bound { + WindowFrameBound::Preceding(val) | WindowFrameBound::Following(val) => match val { + x if x.is_null() => Ok(None), + ScalarValue::UInt64(v) => Ok(*v), + // The cast below is only safe because window bounds cannot be negative + ScalarValue::Int64(v) => Ok(v.map(|n| n as u64)), + ScalarValue::Utf8(Some(s)) => match s.parse::() { + Ok(s) => Ok(Some(s)), + Err(_e) => Err(DataFusionError::Plan(format!( + "Unable to parse u64 from Utf8 value '{s}'" + )) + .into()), + }, + ref x => { + Err(DataFusionError::Plan(format!("Unexpected window frame bound: {x}")).into()) + } + }, + WindowFrameBound::CurrentRow => Ok(None), + } + } + /// Returns if the frame bound is unbounded + pub fn is_unbounded(&self) -> PyResult { + match &self.frame_bound { + WindowFrameBound::Preceding(v) | WindowFrameBound::Following(v) => Ok(v.is_null()), + WindowFrameBound::CurrentRow => Ok(false), + } + } +} + +impl LogicalNode for PyWindow { + fn inputs(&self) -> Vec { + vec![self.window.input.as_ref().clone().into()] + } + + fn to_variant(&self, py: Python) -> PyResult { + Ok(self.clone().into_py(py)) + } +} diff --git a/src/functions.rs b/src/functions.rs index 42203d7b4..be903609c 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -20,8 +20,8 @@ use pyo3::{prelude::*, wrap_pyfunction}; use crate::context::PySessionContext; use crate::errors::DataFusionError; use crate::expr::conditional_expr::PyCaseBuilder; +use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; -use crate::window_frame::PyWindowFrame; use datafusion::execution::FunctionRegistry; use datafusion_common::Column; use datafusion_expr::expr::Alias; diff --git a/src/lib.rs b/src/lib.rs index b9bd5766e..2512aefa4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,7 +54,6 @@ mod udaf; #[allow(clippy::borrow_deref_ref)] mod udf; pub mod utils; -mod window_frame; #[cfg(feature = "mimalloc")] #[global_allocator] @@ -84,7 +83,6 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 2183155bf..3aa8a699f 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -20,17 +20,20 @@ use std::sync::Arc; use crate::errors::py_unsupported_variant_err; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; +use crate::expr::cross_join::PyCrossJoin; use crate::expr::distinct::PyDistinct; use crate::expr::empty_relation::PyEmptyRelation; use crate::expr::explain::PyExplain; use crate::expr::extension::PyExtension; use crate::expr::filter::PyFilter; +use crate::expr::join::PyJoin; use crate::expr::limit::PyLimit; use crate::expr::projection::PyProjection; use crate::expr::sort::PySort; use crate::expr::subquery::PySubquery; use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; +use crate::expr::window::PyWindow; use datafusion_expr::LogicalPlan; use pyo3::prelude::*; @@ -62,17 +65,20 @@ impl PyLogicalPlan { Python::with_gil(|_| match self.plan.as_ref() { LogicalPlan::Aggregate(plan) => PyAggregate::from(plan.clone()).to_variant(py), LogicalPlan::Analyze(plan) => PyAnalyze::from(plan.clone()).to_variant(py), + LogicalPlan::CrossJoin(plan) => PyCrossJoin::from(plan.clone()).to_variant(py), + LogicalPlan::Distinct(plan) => PyDistinct::from(plan.clone()).to_variant(py), LogicalPlan::EmptyRelation(plan) => PyEmptyRelation::from(plan.clone()).to_variant(py), LogicalPlan::Explain(plan) => PyExplain::from(plan.clone()).to_variant(py), LogicalPlan::Extension(plan) => PyExtension::from(plan.clone()).to_variant(py), - LogicalPlan::Distinct(plan) => PyDistinct::from(plan.clone()).to_variant(py), LogicalPlan::Filter(plan) => PyFilter::from(plan.clone()).to_variant(py), + LogicalPlan::Join(plan) => PyJoin::from(plan.clone()).to_variant(py), LogicalPlan::Limit(plan) => PyLimit::from(plan.clone()).to_variant(py), LogicalPlan::Projection(plan) => PyProjection::from(plan.clone()).to_variant(py), LogicalPlan::Sort(plan) => PySort::from(plan.clone()).to_variant(py), LogicalPlan::TableScan(plan) => PyTableScan::from(plan.clone()).to_variant(py), LogicalPlan::Subquery(plan) => PySubquery::from(plan.clone()).to_variant(py), LogicalPlan::SubqueryAlias(plan) => PySubqueryAlias::from(plan.clone()).to_variant(py), + LogicalPlan::Window(plan) => PyWindow::from(plan.clone()).to_variant(py), other => Err(py_unsupported_variant_err(format!( "Cannot convert this plan to a LogicalNode: {:?}", other diff --git a/src/window_frame.rs b/src/window_frame.rs deleted file mode 100644 index b8f414e6a..000000000 --- a/src/window_frame.rs +++ /dev/null @@ -1,110 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion_common::{DataFusionError, ScalarValue}; -use datafusion_expr::window_frame::{WindowFrame, WindowFrameBound, WindowFrameUnits}; -use pyo3::prelude::*; -use std::fmt::{Display, Formatter}; - -use crate::errors::py_datafusion_err; - -#[pyclass(name = "WindowFrame", module = "datafusion", subclass)] -#[derive(Clone)] -pub struct PyWindowFrame { - frame: WindowFrame, -} - -impl From for WindowFrame { - fn from(frame: PyWindowFrame) -> Self { - frame.frame - } -} - -impl From for PyWindowFrame { - fn from(frame: WindowFrame) -> PyWindowFrame { - PyWindowFrame { frame } - } -} - -impl Display for PyWindowFrame { - fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!( - f, - "OVER ({} BETWEEN {} AND {})", - self.frame.units, self.frame.start_bound, self.frame.end_bound - ) - } -} - -#[pymethods] -impl PyWindowFrame { - #[new(unit, start_bound, end_bound)] - pub fn new(units: &str, start_bound: Option, end_bound: Option) -> PyResult { - let units = units.to_ascii_lowercase(); - let units = match units.as_str() { - "rows" => WindowFrameUnits::Rows, - "range" => WindowFrameUnits::Range, - "groups" => WindowFrameUnits::Groups, - _ => { - return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - units, - )))); - } - }; - let start_bound = match start_bound { - Some(start_bound) => { - WindowFrameBound::Preceding(ScalarValue::UInt64(Some(start_bound))) - } - None => match units { - WindowFrameUnits::Range => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), - WindowFrameUnits::Groups => { - return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - units, - )))); - } - }, - }; - let end_bound = match end_bound { - Some(end_bound) => WindowFrameBound::Following(ScalarValue::UInt64(Some(end_bound))), - None => match units { - WindowFrameUnits::Rows => WindowFrameBound::Following(ScalarValue::UInt64(None)), - WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)), - WindowFrameUnits::Groups => { - return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - units, - )))); - } - }, - }; - Ok(PyWindowFrame { - frame: WindowFrame { - units, - start_bound, - end_bound, - }, - }) - } - - /// Get a String representation of this window frame - fn __repr__(&self) -> String { - format!("{}", self) - } -} From d7fcea29833a43d2af56d1486e46448ab4261258 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 21 Oct 2023 11:21:32 -0600 Subject: [PATCH 094/413] small clippy fix (#524) --- src/functions.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/functions.rs b/src/functions.rs index be903609c..b8c825552 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -140,7 +140,7 @@ fn window( ctx.and_then(|ctx| { ctx.ctx .udaf(name) - .map(|fun| datafusion_expr::WindowFunction::AggregateUDF(fun)) + .map(datafusion_expr::WindowFunction::AggregateUDF) .ok() }) }); From fc3c24b52e8bfa1e170fb9f3708fe014e41b3e9e Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 21 Oct 2023 13:10:30 -0600 Subject: [PATCH 095/413] Prepare 32.0.0 Release (#525) * small clippy fix * update changelog * cargo update --- CHANGELOG.md | 38 ++++++++++++++-- Cargo.lock | 122 +++++++++++++++++++++++---------------------------- 2 files changed, 90 insertions(+), 70 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbc5a82e6..2890de4f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,40 @@ # DataFusion Python Changelog +## [32.0.0](https://github.com/apache/arrow-datafusion-python/tree/32.0.0) (2023-10-21) + +**Implemented enhancements:** + +- feat: expose PyWindowFrame [#509](https://github.com/apache/arrow-datafusion-python/pull/509) (dlovell) +- add Binary String Functions;encode,decode [#494](https://github.com/apache/arrow-datafusion-python/pull/494) (jiangzhx) +- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/arrow-datafusion-python/pull/496) (jiangzhx) +- add first_value last_value [#498](https://github.com/apache/arrow-datafusion-python/pull/498) (jiangzhx) +- add regr\_\* functions [#499](https://github.com/apache/arrow-datafusion-python/pull/499) (jiangzhx) +- Add random missing bindings [#522](https://github.com/apache/arrow-datafusion-python/pull/522) (jdye64) +- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/arrow-datafusion-python/pull/519) (jdye64) +- Add support for window function bindings [#521](https://github.com/apache/arrow-datafusion-python/pull/521) (jdye64) + +**Merged pull requests:** + +- Prepare 31.0.0 release [#500](https://github.com/apache/arrow-datafusion-python/pull/500) (andygrove) +- Improve release process documentation [#505](https://github.com/apache/arrow-datafusion-python/pull/505) (andygrove) +- add Binary String Functions;encode,decode [#494](https://github.com/apache/arrow-datafusion-python/pull/494) (jiangzhx) +- build(deps): bump mimalloc from 0.1.38 to 0.1.39 [#502](https://github.com/apache/arrow-datafusion-python/pull/502) (dependabot[bot]) +- build(deps): bump syn from 2.0.32 to 2.0.35 [#503](https://github.com/apache/arrow-datafusion-python/pull/503) (dependabot[bot]) +- build(deps): bump syn from 2.0.35 to 2.0.37 [#506](https://github.com/apache/arrow-datafusion-python/pull/506) (dependabot[bot]) +- Use latest DataFusion [#511](https://github.com/apache/arrow-datafusion-python/pull/511) (andygrove) +- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/arrow-datafusion-python/pull/496) (jiangzhx) +- use DataFusion 32 [#515](https://github.com/apache/arrow-datafusion-python/pull/515) (andygrove) +- add first_value last_value [#498](https://github.com/apache/arrow-datafusion-python/pull/498) (jiangzhx) +- build(deps): bump regex-syntax from 0.7.5 to 0.8.1 [#517](https://github.com/apache/arrow-datafusion-python/pull/517) (dependabot[bot]) +- build(deps): bump pyo3-build-config from 0.19.2 to 0.20.0 [#516](https://github.com/apache/arrow-datafusion-python/pull/516) (dependabot[bot]) +- add regr\_\* functions [#499](https://github.com/apache/arrow-datafusion-python/pull/499) (jiangzhx) +- Add random missing bindings [#522](https://github.com/apache/arrow-datafusion-python/pull/522) (jdye64) +- build(deps): bump rustix from 0.38.18 to 0.38.19 [#523](https://github.com/apache/arrow-datafusion-python/pull/523) (dependabot[bot]) +- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/arrow-datafusion-python/pull/519) (jdye64) +- Add support for window function bindings [#521](https://github.com/apache/arrow-datafusion-python/pull/521) (jdye64) +- Small clippy fix [#524](https://github.com/apache/arrow-datafusion-python/pull/524) (andygrove) + ## [31.0.0](https://github.com/apache/arrow-datafusion-python/tree/31.0.0) (2023-09-12) [Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/28.0.0...31.0.0) @@ -445,7 +479,3 @@ - Why is pandas a requirement? [\#24](https://github.com/datafusion-contrib/datafusion-python/issues/24) - Unable to build [\#18](https://github.com/datafusion-contrib/datafusion-python/issues/18) - Setup CI against multiple Python version [\#6](https://github.com/datafusion-contrib/datafusion-python/issues/6) - -\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ - -\* _This Changelog was automatically generated by [github_changelog_generator](https://github.com/github-changelog-generator/github-changelog-generator)_ diff --git a/Cargo.lock b/Cargo.lock index 4ae3ec1f9..3887ac10f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,8 +106,8 @@ dependencies = [ "serde", "serde_json", "snap", - "strum 0.25.0", - "strum_macros 0.25.3", + "strum", + "strum_macros", "thiserror", "typed-builder", "uuid", @@ -178,7 +178,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "num", ] @@ -303,7 +303,7 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -577,12 +577,12 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.0.1" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab77dbd8adecaf3f0db40581631b995f312a8a5ae3aa9993188bb8f23d83a5b" +checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum 0.24.1", - "strum_macros 0.24.3", + "strum", + "strum_macros", "unicode-width", ] @@ -641,9 +641,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" dependencies = [ "libc", ] @@ -707,7 +707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lock_api", "once_cell", "parking_lot_core", @@ -741,7 +741,7 @@ dependencies = [ "futures", "glob", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "indexmap 2.0.2", "itertools 0.11.0", "log", @@ -796,7 +796,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "log", "object_store", "parking_lot", @@ -816,8 +816,8 @@ dependencies = [ "arrow-array", "datafusion-common", "sqlparser", - "strum 0.25.0", - "strum_macros 0.25.3", + "strum", + "strum_macros", ] [[package]] @@ -832,7 +832,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "itertools 0.11.0", "log", "regex-syntax 0.7.5", @@ -856,7 +856,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "hex", "indexmap 2.0.2", "itertools 0.11.0", @@ -891,7 +891,7 @@ dependencies = [ "datafusion-physical-expr", "futures", "half", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "indexmap 2.0.2", "itertools 0.11.0", "log", @@ -1244,9 +1244,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ "ahash", "allocator-api2", @@ -1336,7 +1336,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -1407,7 +1407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -1618,9 +1618,9 @@ checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -1882,13 +1882,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.8" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.4.1", "smallvec", "windows-targets", ] @@ -1913,7 +1913,7 @@ dependencies = [ "chrono", "flate2", "futures", - "hashbrown 0.14.1", + "hashbrown 0.14.2", "lz4", "num", "num-bigint", @@ -2240,6 +2240,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "regex" version = "1.10.2" @@ -2371,9 +2380,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" dependencies = [ "bitflags 2.4.1", "errno", @@ -2627,9 +2636,9 @@ checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" [[package]] name = "socket2" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" dependencies = [ "libc", "winapi", @@ -2637,9 +2646,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", "windows-sys", @@ -2678,32 +2687,13 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" -[[package]] -name = "strum" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" - [[package]] name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros 0.25.3", -] - -[[package]] -name = "strum_macros" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 1.0.109", + "strum_macros", ] [[package]] @@ -2793,9 +2783,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.11" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0e916b1148c8e263850e1ebcbd046f333e0683c724876bb0da63ea4373dc8a" +checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" [[package]] name = "tempfile" @@ -2805,25 +2795,25 @@ checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", + "redox_syscall 0.3.5", "rustix", "windows-sys", ] [[package]] name = "thiserror" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1177e8c6d7ede7afde3585fd2513e611227efd6481bd78d2e82ba1ce16557ed4" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.49" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", @@ -2878,7 +2868,7 @@ dependencies = [ "num_cpus", "parking_lot", "pin-project-lite", - "socket2 0.5.4", + "socket2 0.5.5", "tokio-macros", "windows-sys", ] @@ -2926,9 +2916,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -3105,9 +3095,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ "getrandom", "serde", From aaaeeb1a97c0b65a01c752c8359969743eb8745f Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Mon, 23 Oct 2023 15:50:11 -0400 Subject: [PATCH 096/413] First pass at getting architectured builds working (#350) * First pass at getting architectured builds working * Fix RAT failures * Build add builds on windows/macOS * Refactor artifacts upload * Remove windows/macOS builds, include all py versions * Rename conda package to avoid collision * Handle licensing * Update recipes to reflect recent changes * Build for linux-aarch64 * Add licenses to files * Add zlib to host deps * Fixes to build/overlinking errors * Always include c compiler in build deps --- .github/workflows/conda.yml | 94 +++++++++++++++++++++++++++---------- conda/recipes/bld.bat | 26 ++++++++++ conda/recipes/build.sh | 84 +++++++++++++++++++++++++++++++++ conda/recipes/meta.yaml | 31 ++++++++---- 4 files changed, 202 insertions(+), 33 deletions(-) create mode 100644 conda/recipes/bld.bat create mode 100644 conda/recipes/build.sh diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 9853230de..16dfd7a4e 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -1,7 +1,20 @@ name: Build conda nightly -on: [push, pull_request] +on: + push: + branches: + - main + pull_request: + paths: + - Cargo.toml + - Cargo.lock + - pyproject.toml + - conda/recipes/** + - .github/workflows/conda.yml + schedule: + - cron: '0 0 * * 0' -# Cancel any already running instances of this build +# When this workflow is queued, automatically cancel any previous running +# or pending jobs from the same branch concurrency: group: conda-${{ github.head_ref }} cancel-in-progress: true @@ -13,9 +26,44 @@ defaults: jobs: conda: - name: Build (and optionally upload) the conda nightly + name: "Build conda nightlies (python: ${{ matrix.python }}, arch: ${{ matrix.arch }})" runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python: ["3.8", "3.9", "3.10"] + arch: ["linux-64", "linux-aarch64"] steps: + - name: Manage disk space + if: matrix.arch == 'linux-aarch64' + run: | + sudo mkdir -p /opt/empty_dir || true + for d in \ + /opt/ghc \ + /opt/hostedtoolcache \ + /usr/lib/jvm \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift \ + ; do + sudo rsync --stats -a --delete /opt/empty_dir/ $d || true + done + sudo apt-get purge -y -f firefox \ + google-chrome-stable \ + microsoft-edge-stable + sudo apt-get autoremove -y >& /dev/null + sudo apt-get autoclean -y >& /dev/null + sudo docker image prune --all --force + df -h + - name: Create swapfile + if: matrix.arch == 'linux-aarch64' + run: | + sudo fallocate -l 10GiB /swapfile || true + sudo chmod 600 /swapfile || true + sudo mkswap /swapfile || true + sudo swapon /swapfile || true - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -24,38 +72,34 @@ jobs: with: miniforge-variant: Mambaforge use-mamba: true - python-version: "3.10" + python-version: "3.8" channel-priority: strict - name: Install dependencies run: | - mamba install boa conda-verify + mamba install -c conda-forge boa conda-verify which python pip list mamba list - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - components: clippy, rustfmt - - name: Build conda package + - name: Build conda packages run: | # suffix for nightly package versions export VERSION_SUFFIX=a`date +%y%m%d` conda mambabuild conda/recipes \ + --python ${{ matrix.python }} \ + --variants "{target_platform: [${{ matrix.arch }}]}" \ + --error-overlinking \ + --no-test \ --no-anaconda-upload \ - --output-folder . - # - name: Upload conda package - # if: | - # github.event_name == 'push' - # && github.repository == 'apache/arrow-datafusion-python' - # env: - # ANACONDA_API_TOKEN: ${{ secrets.DASK_CONDA_TOKEN }} - # LABEL: ${{ github.ref == 'refs/heads/datafusion-sql-planner' && 'dev_datafusion' || 'dev' }} - # run: | - # # install anaconda for upload - # mamba install anaconda-client - - # anaconda upload --label $LABEL linux-64/*.tar.bz2 + --output-folder packages + - name: Test conda packages + if: matrix.arch == 'linux-64' # can only test native platform packages + run: | + conda mambabuild --test packages/${{ matrix.arch }}/*.tar.bz2 + - name: Upload conda packages as artifacts + uses: actions/upload-artifact@v3 + with: + name: "conda nightlies (python - ${{ matrix.python }}, arch - ${{ matrix.arch }})" + # need to install all conda channel metadata to properly install locally + path: packages/ diff --git a/conda/recipes/bld.bat b/conda/recipes/bld.bat new file mode 100644 index 000000000..fc3ac88b3 --- /dev/null +++ b/conda/recipes/bld.bat @@ -0,0 +1,26 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +maturin build -vv -j %CPU_COUNT% --release --strip --manylinux off --interpreter=%PYTHON% + +FOR /F "delims=" %%i IN ('dir /s /b target\wheels\*.whl') DO set datafusion_wheel=%%i + +%PYTHON% -m pip install --no-deps %datafusion_wheel% -vv + +cargo-bundle-licenses --format yaml --output THIRDPARTY.yml diff --git a/conda/recipes/build.sh b/conda/recipes/build.sh new file mode 100644 index 000000000..af640e6ca --- /dev/null +++ b/conda/recipes/build.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -ex + +# See https://github.com/conda-forge/rust-feedstock/blob/master/recipe/build.sh for cc env explanation +if [ "$c_compiler" = gcc ] ; then + case "$target_platform" in + linux-64) rust_env_arch=X86_64_UNKNOWN_LINUX_GNU ;; + linux-aarch64) rust_env_arch=AARCH64_UNKNOWN_LINUX_GNU ;; + linux-ppc64le) rust_env_arch=POWERPC64LE_UNKNOWN_LINUX_GNU ;; + *) echo "unknown target_platform $target_platform" ; exit 1 ;; + esac + + export CARGO_TARGET_${rust_env_arch}_LINKER=$CC +fi + +declare -a _xtra_maturin_args + +mkdir -p $SRC_DIR/.cargo + +if [ "$target_platform" = "osx-64" ] ; then + cat <> $SRC_DIR/.cargo/config +[target.x86_64-apple-darwin] +linker = "$CC" +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +EOF + + _xtra_maturin_args+=(--target=x86_64-apple-darwin) + +elif [ "$target_platform" = "osx-arm64" ] ; then + cat <> $SRC_DIR/.cargo/config +# Required for intermediate codegen stuff +[target.x86_64-apple-darwin] +linker = "$CC_FOR_BUILD" + +# Required for final binary artifacts for target +[target.aarch64-apple-darwin] +linker = "$CC" +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +EOF + _xtra_maturin_args+=(--target=aarch64-apple-darwin) + + # This variable must be set to the directory containing the target's libpython DSO + export PYO3_CROSS_LIB_DIR=$PREFIX/lib + + # xref: https://github.com/PyO3/pyo3/commit/7beb2720 + export PYO3_PYTHON_VERSION=${PY_VER} + + # xref: https://github.com/conda-forge/python-feedstock/issues/621 + sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/os-patch.py + sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/platform-patch.py +fi + +maturin build -vv -j "${CPU_COUNT}" --release --strip --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}" + +"${PYTHON}" -m pip install $SRC_DIR/target/wheels/datafusion*.whl --no-deps -vv + +cargo-bundle-licenses --format yaml --output THIRDPARTY.yml diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index e2bb8bee3..113e7a441 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -29,18 +29,27 @@ source: git_url: ../.. build: - noarch: python - script: {{ PYTHON }} -m pip install . -vv - number: 0 + number: {{ GIT_DESCRIBE_NUMBER }} + string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} requirements: - host: - - python >=3.7 + build: + - python # [build_platform != target_platform] + - cross-python_{{ target_platform }} # [build_platform != target_platform] + - zlib # [build_platform != target_platform] + - {{ compiler('c') }} + - {{ compiler('rust') }} + - cargo-bundle-licenses - maturin >=0.15,<0.16 - libprotobuf =3 + host: + - python + - maturin >=0.15,<0.16 - pip + - zlib + - xz # [linux64] run: - - python >=3.7 + - python - pyarrow >=11.0.0 test: @@ -55,5 +64,11 @@ about: home: https://arrow.apache.org/datafusion license: Apache-2.0 license_family: APACHE - license_file: LICENSE.txt - summary: Apache Arrow DataFusion Python Bindings + license_file: + - LICENSE.txt + - THIRDPARTY.yml + description: | + DataFusion is an extensible query execution framework, written in Rust, + that uses Apache Arrow as its in-memory format. + doc_url: https://arrow.apache.org/datafusion + dev_url: https://github.com/apache/arrow-datafusion From da6c183ebb673b27808ff35b1b9fd8d577a203c0 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Thu, 26 Oct 2023 14:24:43 -0400 Subject: [PATCH 097/413] Remove libprotobuf dep (#527) * Remove libprotobuf as a build time dependency of Conda * Add substrait feature which enables building with substrait features * Add datafusion._internal/substrait to the pyproject.toml file so substrait will be built for pytests when using pip * Rename feature to just substrait * Add protoc since libprotobuf was removed from conda requirements file * Refactor substrait module being initialized as a function * Reintroduce libprotobuf as build dependency for conda build --- Cargo.toml | 3 ++- pyproject.toml | 1 + src/lib.rs | 13 ++++++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7c1a57bd3..06834b92f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ include = ["/src", "/datafusion", "/LICENSE.txt", "pyproject.toml", "Cargo.toml" [features] default = ["mimalloc"] protoc = [ "datafusion-substrait/protoc" ] +substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } @@ -41,7 +42,7 @@ datafusion-common = { version = "32.0.0", features = ["pyarrow"] } datafusion-expr = { version = "32.0.0" } datafusion-optimizer = { version = "32.0.0" } datafusion-sql = { version = "32.0.0" } -datafusion-substrait = { version = "32.0.0" } +datafusion-substrait = { version = "32.0.0", optional = true } prost = "0.11" prost-types = "0.11" uuid = { version = "1.3", features = ["v4"] } diff --git a/pyproject.toml b/pyproject.toml index 4fdc4586f..d35360519 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,3 +62,4 @@ include = [ exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true +features = ["substrait"] diff --git a/src/lib.rs b/src/lib.rs index 2512aefa4..413b2a429 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,6 +25,8 @@ pub use datafusion_common; pub use datafusion_expr; pub use datafusion_optimizer; pub use datafusion_sql; + +#[cfg(feature = "substrait")] pub use datafusion_substrait; #[allow(clippy::borrow_deref_ref)] @@ -48,6 +50,8 @@ mod pyarrow_filter_expression; mod record_batch; pub mod sql; pub mod store; + +#[cfg(feature = "substrait")] pub mod substrait; #[allow(clippy::borrow_deref_ref)] mod udaf; @@ -108,9 +112,16 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { m.add_submodule(store)?; // Register substrait as a submodule + #[cfg(feature = "substrait")] + setup_substrait_module(py, m)?; + + Ok(()) +} + +#[cfg(feature = "substrait")] +fn setup_substrait_module(py: Python, m: &PyModule) -> PyResult<()> { let substrait = PyModule::new(py, "substrait")?; substrait::init_module(substrait)?; m.add_submodule(substrait)?; - Ok(()) } From e4942b897472ec2f0c59682db4954d0aa2d5c185 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 6 Nov 2023 11:18:18 -0700 Subject: [PATCH 098/413] Upgrade to DataFusion 33.0.0-rc1 (#530) * fix issues * fix TODO * clippy * use df 33.0.0-rc1 --- Cargo.lock | 562 ++++++++++++++++++++-------------------- Cargo.toml | 20 +- src/common/data_type.rs | 2 +- src/dataset_exec.rs | 7 +- src/expr.rs | 2 +- src/expr/window.rs | 7 +- src/substrait.rs | 2 +- src/udaf.rs | 3 +- src/udf.rs | 3 +- 9 files changed, 299 insertions(+), 309 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3887ac10f..9cf592d29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,15 +25,16 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "const-random", "getrandom", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -129,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fab9e93ba8ce88a37d5a30dce4b9913b75413dc1ac56cb5d72e5a840543f829" +checksum = "edb738d83750ec705808f6d44046d165e6bb8623f64e29a4d53fcb136ab22dfb" dependencies = [ "ahash", "arrow-arith", @@ -152,9 +153,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1d4e368e87ad9ee64f28b9577a3834ce10fe2703a26b28417d485bbbdff956" +checksum = "c5c3d17fc5b006e7beeaebfb1d2edfc92398b981f82d9744130437909b72a468" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d02efa7253ede102d45a4e802a129e83bcc3f49884cab795b1ac223918e4318d" +checksum = "55705ada5cdde4cb0f202ffa6aa756637e33fea30e13d8d0d0fd6a24ffcee1e3" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +185,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda119225204141138cb0541c692fbfef0e875ba01bfdeaed09e9d354f9d6195" +checksum = "a722f90a09b94f295ab7102542e97199d3500128843446ef63e410ad546c5333" dependencies = [ "bytes", "half", @@ -195,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d825d51b9968868d50bc5af92388754056796dbc62a4e25307d588a1fc84dee" +checksum = "af01fc1a06f6f2baf31a04776156d47f9f31ca5939fe6d00cd7a059f95a46ff1" dependencies = [ "arrow-array", "arrow-buffer", @@ -213,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43ef855dc6b126dc197f43e061d4de46b9d4c033aa51c2587657f7508242cef1" +checksum = "83cbbfde86f9ecd3f875c42a73d8aeab3d95149cd80129b18d09e039ecf5391b" dependencies = [ "arrow-array", "arrow-buffer", @@ -232,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475a4c3699c8b4095ca61cecf15da6f67841847a5f5aac983ccb9a377d02f73a" +checksum = "d0a547195e607e625e7fafa1a7269b8df1a4a612c919efd9b26bd86e74538f3a" dependencies = [ "arrow-buffer", "arrow-schema", @@ -244,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1248005c8ac549f869b7a840859d942bf62471479c1a2d82659d453eebcd166a" +checksum = "e36bf091502ab7e37775ff448413ef1ffff28ff93789acb669fffdd51b394d51" dependencies = [ "arrow-array", "arrow-buffer", @@ -258,9 +259,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03d7e3b04dd688ccec354fe449aed56b831679f03e44ee2c1cfc4045067b69c" +checksum = "7ac346bc84846ab425ab3c8c7b6721db90643bc218939677ed7e071ccbfb919d" dependencies = [ "arrow-array", "arrow-buffer", @@ -269,7 +270,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.0.2", + "indexmap 2.1.0", "lexical-core", "num", "serde", @@ -278,9 +279,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b87aa408ea6a6300e49eb2eba0c032c88ed9dc19e0a9948489c55efdca71f4" +checksum = "4502123d2397319f3a13688432bc678c61cb1582f2daa01253186da650bf5841" dependencies = [ "arrow-array", "arrow-buffer", @@ -293,9 +294,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114a348ab581e7c9b6908fcab23cb39ff9f060eb19e72b13f8fb8eaa37f65d22" +checksum = "249fc5a07906ab3f3536a6e9f118ec2883fbcde398a97a5ba70053f0276abda4" dependencies = [ "ahash", "arrow-array", @@ -308,18 +309,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d1d179c117b158853e0101bfbed5615e86fe97ee356b4af901f1c5001e1ce4b" +checksum = "9d7a8c3f97f5ef6abd862155a6f39aaba36b029322462d72bbcfa69782a50614" dependencies = [ "bitflags 2.4.1", ] [[package]] name = "arrow-select" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5c71e003202e67e9db139e5278c79f5520bb79922261dfe140e4637ee8b6108" +checksum = "f868f4a5001429e20f7c1994b5cd1aa68b82e3db8cf96c559cdb56dc8be21410" dependencies = [ "ahash", "arrow-array", @@ -331,9 +332,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cebbb282d6b9244895f4a9a912e55e57bce112554c7fa91fcec5459cb421ab" +checksum = "a27fdf8fc70040a2dee78af2e217479cb5b263bd7ab8711c7999e74056eb688a" dependencies = [ "arrow-array", "arrow-buffer", @@ -342,7 +343,7 @@ dependencies = [ "arrow-select", "num", "regex", - "regex-syntax 0.7.5", + "regex-syntax", ] [[package]] @@ -371,7 +372,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -382,7 +383,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -417,9 +418,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.4" +version = "0.21.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" [[package]] name = "bitflags" @@ -477,9 +478,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" +checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -555,9 +556,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1369bc6b9e9a7dfdae2055f6ec151fe9c554a9d23d357c0237cee2e25eaabb7" +checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" dependencies = [ "chrono", "chrono-tz-build", @@ -566,9 +567,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f5ebdc942f57ed96d560a6d1a459bae5851102a25d5bf89dc04ae453e31ecf" +checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" dependencies = [ "parse-zoneinfo", "phf", @@ -588,23 +589,21 @@ dependencies = [ [[package]] name = "const-random" -version = "0.1.15" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368a7a772ead6ce7e1de82bfb04c485f3db8ec744f72925af5735e29a22cc18e" +checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" dependencies = [ "const-random-macro", - "proc-macro-hack", ] [[package]] name = "const-random-macro" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7d6ab3c3a2282db210df5f02c4dab6e0a7057af0fb7ebd4070f30fe05c0ddb" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ "getrandom", "once_cell", - "proc-macro-hack", "tiny-keccak", ] @@ -641,9 +640,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fbc60abd742b35f2492f808e1abbb83d45f72db402e14c55057edc9c7b1e9e4" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -715,9 +714,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7014432223f4d721cb9786cd88bb89e7464e0ba984d4a7f49db7787f5f268674" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "ahash", "apache-avro", @@ -742,15 +740,14 @@ dependencies = [ "glob", "half", "hashbrown 0.14.2", - "indexmap 2.0.2", - "itertools 0.11.0", + "indexmap 2.1.0", + "itertools", "log", "num-traits", "num_cpus", "object_store", "parking_lot", "parquet", - "percent-encoding", "pin-project-lite", "rand", "sqlparser", @@ -760,14 +757,13 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.12.4", + "zstd 0.13.0", ] [[package]] name = "datafusion-common" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3903ed8f102892f17b48efa437f3542159241d41c564f0d1e78efdc5e663aa" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "ahash", "apache-avro", @@ -786,9 +782,8 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "780b73b2407050e53f51a9781868593f694102c59e622de9a8aafc0343c4f237" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "arrow", "chrono", @@ -807,9 +802,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24c382676338d8caba6c027ba0da47260f65ffedab38fda78f6d8043f607557c" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "ahash", "arrow", @@ -822,9 +816,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f2904a432f795484fd45e29ded4537152adb60f636c05691db34fcd94c92c96" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "arrow", "async-trait", @@ -833,21 +826,21 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.2", - "itertools 0.11.0", + "itertools", "log", - "regex-syntax 0.7.5", + "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57b4968e9a998dc0476c4db7a82f280e2026b25f464e4aa0c3bb9807ee63ddfd" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "base64", "blake2", @@ -858,8 +851,8 @@ dependencies = [ "half", "hashbrown 0.14.2", "hex", - "indexmap 2.0.2", - "itertools 0.11.0", + "indexmap 2.1.0", + "itertools", "libc", "log", "md-5", @@ -874,9 +867,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd0d1fe54e37a47a2d58a1232c22786f2c28ad35805fdcd08f0253a8b0aaa90" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "ahash", "arrow", @@ -892,8 +884,8 @@ dependencies = [ "futures", "half", "hashbrown 0.14.2", - "indexmap 2.0.2", - "itertools 0.11.0", + "indexmap 2.1.0", + "itertools", "log", "once_cell", "parking_lot", @@ -905,7 +897,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "32.0.0" +version = "33.0.0" dependencies = [ "async-trait", "datafusion", @@ -921,10 +913,10 @@ dependencies = [ "prost", "prost-types", "pyo3", - "pyo3-build-config 0.20.0", + "pyo3-build-config", "rand", - "regex-syntax 0.8.2", - "syn 2.0.38", + "regex-syntax", + "syn 2.0.39", "tokio", "url", "uuid", @@ -932,9 +924,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b568d44c87ead99604d704f942e257c8a236ee1bbf890ee3e034ad659dcb2c21" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "arrow", "arrow-schema", @@ -946,14 +937,13 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "32.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2884dff8207774c1ea5f7b008d44b20e9723bd0b2e4b7dd6627390d8b526b50" +version = "33.0.0" +source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" dependencies = [ "async-recursion", "chrono", "datafusion", - "itertools 0.11.0", + "itertools", "object_store", "prost", "prost-types", @@ -980,9 +970,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d2f3407d9a573d666de4b5bdf10569d73ca9478087346697dcbae6244bfbcd" +checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" [[package]] name = "either" @@ -1064,9 +1054,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" dependencies = [ "futures-channel", "futures-core", @@ -1079,9 +1069,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -1089,15 +1079,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" dependencies = [ "futures-core", "futures-task", @@ -1106,38 +1096,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -1345,9 +1335,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d78e1e73ec14cf7375674f74d7dde185c8206fd9dea6fb6295e8a98098aaa97" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", "http", @@ -1402,9 +1392,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", "hashbrown 0.14.2", @@ -1412,9 +1402,9 @@ dependencies = [ [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "integer-encoding" @@ -1424,18 +1414,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b29a3cd74f0f4598934efe3aeba42bae0eb4680554128851ebbecb02af14e6" - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itertools" @@ -1463,9 +1444,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" +checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" dependencies = [ "wasm-bindgen", ] @@ -1542,9 +1523,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libflate" @@ -1633,23 +1614,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] -name = "lz4" -version = "1.24.0" +name = "lz4_flex" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" dependencies = [ - "libc", - "lz4-sys", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" -dependencies = [ - "cc", - "libc", + "twox-hash", ] [[package]] @@ -1714,9 +1684,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" dependencies = [ "libc", "wasi", @@ -1838,13 +1808,13 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.11.0", + "itertools", "parking_lot", "percent-encoding", "quick-xml", "rand", "reqwest", - "ring", + "ring 0.16.20", "rustls-pemfile", "serde", "serde_json", @@ -1888,16 +1858,16 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", "windows-targets", ] [[package]] name = "parquet" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0463cc3b256d5f50408c49a4be3a16674f4c8ceef60941709620a062b1f6bf4d" +checksum = "239229e6a668ab50c61de3dce61cf0fa1069345f7aa0f4c934491f92205a4945" dependencies = [ "ahash", "arrow-array", @@ -1914,7 +1884,7 @@ dependencies = [ "flate2", "futures", "hashbrown 0.14.2", - "lz4", + "lz4_flex", "num", "num-bigint", "object_store", @@ -1924,7 +1894,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.12.4", + "zstd 0.13.0", ] [[package]] @@ -1955,7 +1925,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.2", + "indexmap 2.1.0", ] [[package]] @@ -2027,15 +1997,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.38", + "syn 2.0.39", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.20+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" - [[package]] name = "proc-macro2" version = "1.0.69" @@ -2047,9 +2011,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" +checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" dependencies = [ "bytes", "prost-derive", @@ -2057,16 +2021,16 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" +checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck", - "itertools 0.10.5", - "lazy_static", + "itertools", "log", "multimap", + "once_cell", "petgraph", "prost", "prost-types", @@ -2077,22 +2041,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" +checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] name = "prost-types" -version = "0.11.9" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" +checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" dependencies = [ "prost", ] @@ -2108,31 +2072,21 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e681a6cfdc4adcc93b4d3cf993749a4552018ee0a9b65fc0ccfad74352c72a38" +checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", "parking_lot", - "pyo3-build-config 0.19.2", + "pyo3-build-config", "pyo3-ffi", "pyo3-macros", "unindent", ] -[[package]] -name = "pyo3-build-config" -version = "0.19.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076c73d0bc438f7a4ef6fdd0c3bb4732149136abd952b110ac93e4edb13a6ba5" -dependencies = [ - "once_cell", - "target-lexicon", -] - [[package]] name = "pyo3-build-config" version = "0.20.0" @@ -2145,35 +2099,36 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53cee42e77ebe256066ba8aa77eff722b3bb91f3419177cf4cd0f304d3284d9" +checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" dependencies = [ "libc", - "pyo3-build-config 0.19.2", + "pyo3-build-config", ] [[package]] name = "pyo3-macros" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfeb4c99597e136528c6dd7d5e3de5434d1ceaf487436a3f03b2d56b6fc9efd1" +checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] name = "pyo3-macros-backend" -version = "0.19.2" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947dc12175c254889edc0c02e399476c2f652b4b9ebd123aa655c224de259536" +checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" dependencies = [ + "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.39", ] [[package]] @@ -2231,15 +2186,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "redox_syscall" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.4.1" @@ -2258,7 +2204,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] @@ -2269,7 +2215,7 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] @@ -2278,12 +2224,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - [[package]] name = "regex-syntax" version = "0.8.2" @@ -2292,9 +2232,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "regress" -version = "0.6.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a9ecfa0cb04d0b04dddb99b8ccf4f66bc8dfd23df694b398570bd8ae3a50fb" +checksum = "4ed9969cad8051328011596bf549629f1b800cf1731e7964b1eef8dfc480d2c2" dependencies = [ "hashbrown 0.13.2", "memchr", @@ -2351,12 +2291,26 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", - "untrusted", + "spin 0.5.2", + "untrusted 0.7.1", "web-sys", "winapi", ] +[[package]] +name = "ring" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +dependencies = [ + "cc", + "getrandom", + "libc", + "spin 0.9.8", + "untrusted 0.9.0", + "windows-sys", +] + [[package]] name = "rle-decode-fast" version = "1.0.3" @@ -2380,9 +2334,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.20" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce50cb2e16c2903e30d1cbccfd8387a74b9d4c938b6a4c5ec6cc7556f7a8a0" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ "bitflags 2.4.1", "errno", @@ -2393,12 +2347,12 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.7" +version = "0.21.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" +checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" dependencies = [ "log", - "ring", + "ring 0.17.5", "rustls-webpki", "sct", ] @@ -2414,12 +2368,12 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c7d5dece342910d9ba34d259310cae3e0154b873b35408b787b59bce53d34fe" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -2475,12 +2429,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sct" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring", - "untrusted", + "ring 0.17.5", + "untrusted 0.9.0", ] [[package]] @@ -2497,22 +2451,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2528,9 +2482,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -2546,7 +2500,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2563,11 +2517,11 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ - "indexmap 2.0.2", + "indexmap 2.1.0", "itoa", "ryu", "serde", @@ -2660,11 +2614,17 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "sqlparser" -version = "0.38.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272b7bb0a225320170c99901b4b5fb3a4384e255a7f2cc228f61e2ba3893e75" +checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" dependencies = [ "log", "sqlparser_derive", @@ -2706,14 +2666,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "substrait" -version = "0.15.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c3d276b85647003f434bfb7ce3ebaf4c47a711018af9d4350d25763535c239" +checksum = "49fd0bc602d396b95761b348a68ef6190e4cde28a5981e82b8449855db1d321d" dependencies = [ "git2", "heck", @@ -2727,7 +2687,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.38", + "syn 2.0.39", "typify", "walkdir", ] @@ -2751,9 +2711,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -2789,13 +2749,13 @@ checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", + "redox_syscall", "rustix", "windows-sys", ] @@ -2817,7 +2777,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2881,7 +2841,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2896,9 +2856,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -2933,7 +2893,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2978,7 +2938,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2989,9 +2949,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" -version = "0.0.13" +version = "0.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be9bb640c0eece20cac2028ebbc2ca1a3d17e3b1ddd98540309c309ed178d158" +checksum = "c2e3b707a653e2915a2fc2c4ee96a3d30b9554b9435eb4cc8b5c6c74bbdd3044" dependencies = [ "typify-impl", "typify-macro", @@ -2999,9 +2959,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.13" +version = "0.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8d9ecedde2fd77e975c38eeb9ca40b34ad0247b2259c6e6bbd2a8d6cc2444f" +checksum = "9d9c752192779f666e4c868672dee56a652b82c08032c7e9d23f6a845b282298" dependencies = [ "heck", "log", @@ -3010,16 +2970,16 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.38", + "syn 2.0.39", "thiserror", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.0.13" +version = "0.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08942cd65d458d2da15777a649cb6400cb545f17964f1ca965583f22e9cc3a9" +checksum = "a14defd554507e72a2bb93cd081c8b374cfed43b3d986b141ad3839d9fd6986b" dependencies = [ "proc-macro2", "quote", @@ -3027,7 +2987,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.38", + "syn 2.0.39", "typify-impl", ] @@ -3066,9 +3026,9 @@ checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "unsafe-libyaml" @@ -3082,6 +3042,12 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.4.1" @@ -3142,9 +3108,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" +checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3152,24 +3118,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" +checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.37" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02dbc21516f9f1f04f187958890d7e6026df8d16540b7ad9492bc34a67cea03" +checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" dependencies = [ "cfg-if", "js-sys", @@ -3179,9 +3145,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" +checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3189,22 +3155,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.87" +version = "0.2.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" +checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" [[package]] name = "wasm-streams" @@ -3221,9 +3187,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.64" +version = "0.3.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" dependencies = [ "js-sys", "wasm-bindgen", @@ -3372,6 +3338,26 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "zerocopy" +version = "0.7.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "zstd" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index 06834b92f..610fa9d58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "32.0.0" +version = "33.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -36,15 +36,15 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" -pyo3 = { version = "0.19", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "32.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "32.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "32.0.0" } -datafusion-optimizer = { version = "32.0.0" } -datafusion-sql = { version = "32.0.0" } -datafusion-substrait = { version = "32.0.0", optional = true } -prost = "0.11" -prost-types = "0.11" +pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } +datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", features = ["pyarrow", "avro"] } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", features = ["pyarrow"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" } +datafusion-sql = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" } +datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", optional = true } +prost = "0.12" +prost-types = "0.12" uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false } async-trait = "0.1" diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 405a5632d..9719226a2 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -288,7 +288,7 @@ impl DataTypeMap { ScalarValue::IntervalMonthDayNano(..) => { Ok(DataType::Interval(IntervalUnit::MonthDayNano)) } - ScalarValue::List(_val, field_ref) => Ok(DataType::List(field_ref.to_owned())), + ScalarValue::List(arr) => Ok(arr.data_type().to_owned()), ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())), ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)), ScalarValue::Fixedsizelist(_, field_ref, size) => { diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 686bbf74c..0266ad9e3 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -133,13 +133,14 @@ impl DatasetExec { .downcast() .map_err(PyErr::from)?; + let projected_statistics = Statistics::new_unknown(&schema); Ok(DatasetExec { dataset: dataset.into(), schema, fragments: fragments.into(), columns, filter_expr, - projected_statistics: Default::default(), + projected_statistics, }) } } @@ -236,8 +237,8 @@ impl ExecutionPlan for DatasetExec { }) } - fn statistics(&self) -> Statistics { - self.projected_statistics.clone() + fn statistics(&self) -> DFResult { + Ok(self.projected_statistics.clone()) } } diff --git a/src/expr.rs b/src/expr.rs index e502edced..97c168e69 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -332,7 +332,7 @@ impl PyExpr { ScalarValue::Binary(v) => v.clone().into_py(py), ScalarValue::FixedSizeBinary(_, _) => todo!(), ScalarValue::LargeBinary(v) => v.clone().into_py(py), - ScalarValue::List(_, _) => todo!(), + ScalarValue::List(_) => todo!(), ScalarValue::Date32(v) => v.into_py(py), ScalarValue::Date64(v) => v.into_py(py), ScalarValue::Time32Second(v) => v.into_py(py), diff --git a/src/expr/window.rs b/src/expr/window.rs index 6583c97af..3a5760c15 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -166,9 +166,10 @@ fn not_window_function_err(expr: Expr) -> PyErr { #[pymethods] impl PyWindowFrame { - #[new(unit, start_bound, end_bound)] - pub fn new(units: &str, start_bound: Option, end_bound: Option) -> PyResult { - let units = units.to_ascii_lowercase(); + #[new] + #[pyo3(signature=(unit, start_bound, end_bound))] + pub fn new(unit: &str, start_bound: Option, end_bound: Option) -> PyResult { + let units = unit.to_ascii_lowercase(); let units = match units.as_str() { "rows" => WindowFrameUnits::Rows, "range" => WindowFrameUnits::Range, diff --git a/src/substrait.rs b/src/substrait.rs index d027ad21f..73606fdfa 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -134,7 +134,7 @@ impl PySubstraitConsumer { plan: PyPlan, py: Python, ) -> PyResult { - let result = consumer::from_substrait_plan(&mut ctx.ctx, &plan.plan); + let result = consumer::from_substrait_plan(&ctx.ctx, &plan.plan); let logical_plan = wait_for_future(py, result).map_err(DataFusionError::from)?; Ok(PyLogicalPlan::new(logical_plan)) } diff --git a/src/udaf.rs b/src/udaf.rs index 6450f03fe..596ed6904 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -143,7 +143,8 @@ pub struct PyAggregateUDF { #[pymethods] impl PyAggregateUDF { - #[new(name, accumulator, input_type, return_type, state_type, volatility)] + #[new] + #[pyo3(signature=(name, accumulator, input_type, return_type, state_type, volatility))] fn new( name: &str, accumulator: PyObject, diff --git a/src/udf.rs b/src/udf.rs index 417e5f327..bba8ae551 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -68,7 +68,8 @@ pub struct PyScalarUDF { #[pymethods] impl PyScalarUDF { - #[new(name, func, input_types, return_type, volatility)] + #[new] + #[pyo3(signature=(name, func, input_types, return_type, volatility))] fn new( name: &str, func: PyObject, From d1a7505a72400d8f69b63dbad6123eccaef58366 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 16 Nov 2023 11:11:15 -0700 Subject: [PATCH 099/413] Prepare 33.0.0 Release (#533) * Use latest DF 33 branch * use DF 33.0.0-rc2 * use 33.0.0 release * changelog --- CHANGELOG.md | 7 ++ Cargo.lock | 209 +++++++++++++++++++++++++-------------------------- Cargo.toml | 12 +-- src/expr.rs | 11 +-- 4 files changed, 118 insertions(+), 121 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2890de4f9..d875cf13d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,13 @@ # DataFusion Python Changelog +## [33.0.0](https://github.com/apache/arrow-datafusion-python/tree/33.0.0) (2023-11-16) + +**Merged pull requests:** + +- First pass at getting architectured builds working [#350](https://github.com/apache/arrow-datafusion-python/pull/350) (charlesbluca) +- Remove libprotobuf dep [#527](https://github.com/apache/arrow-datafusion-python/pull/527) (jdye64) + ## [32.0.0](https://github.com/apache/arrow-datafusion-python/tree/32.0.0) (2023-10-21) **Implemented enhancements:** diff --git a/Cargo.lock b/Cargo.lock index 9cf592d29..c2d34115d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb738d83750ec705808f6d44046d165e6bb8623f64e29a4d53fcb136ab22dfb" +checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378" dependencies = [ "ahash", "arrow-arith", @@ -153,9 +153,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c3d17fc5b006e7beeaebfb1d2edfc92398b981f82d9744130437909b72a468" +checksum = "ef983914f477d4278b068f13b3224b7d19eb2b807ac9048544d3bfebdf2554c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55705ada5cdde4cb0f202ffa6aa756637e33fea30e13d8d0d0fd6a24ffcee1e3" +checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b" dependencies = [ "ahash", "arrow-buffer", @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a722f90a09b94f295ab7102542e97199d3500128843446ef63e410ad546c5333" +checksum = "55512d988c6fbd76e514fd3ff537ac50b0a675da5a245e4fdad77ecfd654205f" dependencies = [ "bytes", "half", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af01fc1a06f6f2baf31a04776156d47f9f31ca5939fe6d00cd7a059f95a46ff1" +checksum = "655ee51a2156ba5375931ce21c1b2494b1d9260e6dcdc6d4db9060c37dc3325b" dependencies = [ "arrow-array", "arrow-buffer", @@ -214,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83cbbfde86f9ecd3f875c42a73d8aeab3d95149cd80129b18d09e039ecf5391b" +checksum = "258bb689997ad5b6660b3ce3638bd6b383d668ec555ed41ad7c6559cbb2e4f91" dependencies = [ "arrow-array", "arrow-buffer", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a547195e607e625e7fafa1a7269b8df1a4a612c919efd9b26bd86e74538f3a" +checksum = "6dc2b9fec74763427e2e5575b8cc31ce96ba4c9b4eb05ce40e0616d9fad12461" dependencies = [ "arrow-buffer", "arrow-schema", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36bf091502ab7e37775ff448413ef1ffff28ff93789acb669fffdd51b394d51" +checksum = "6eaa6ab203cc6d89b7eaa1ac781c1dfeef325454c5d5a0419017f95e6bafc03c" dependencies = [ "arrow-array", "arrow-buffer", @@ -259,9 +259,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ac346bc84846ab425ab3c8c7b6721db90643bc218939677ed7e071ccbfb919d" +checksum = "fb64e30d9b73f66fdc5c52d5f4cf69bbf03d62f64ffeafa0715590a5320baed7" dependencies = [ "arrow-array", "arrow-buffer", @@ -270,7 +270,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.1.0", + "indexmap", "lexical-core", "num", "serde", @@ -279,9 +279,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4502123d2397319f3a13688432bc678c61cb1582f2daa01253186da650bf5841" +checksum = "f9a818951c0d11c428dda03e908175969c262629dd20bd0850bd6c7a8c3bfe48" dependencies = [ "arrow-array", "arrow-buffer", @@ -294,9 +294,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "249fc5a07906ab3f3536a6e9f118ec2883fbcde398a97a5ba70053f0276abda4" +checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3" dependencies = [ "ahash", "arrow-array", @@ -309,18 +309,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d7a8c3f97f5ef6abd862155a6f39aaba36b029322462d72bbcfa69782a50614" +checksum = "aaf4d737bba93da59f16129bec21e087aed0be84ff840e74146d4703879436cb" dependencies = [ "bitflags 2.4.1", ] [[package]] name = "arrow-select" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f868f4a5001429e20f7c1994b5cd1aa68b82e3db8cf96c559cdb56dc8be21410" +checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a" dependencies = [ "ahash", "arrow-array", @@ -332,9 +332,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27fdf8fc70040a2dee78af2e217479cb5b263bd7ab8711c7999e74056eb688a" +checksum = "b15aed5624bb23da09142f58502b59c23f5bea607393298bb81dab1ce60fc769" dependencies = [ "arrow-array", "arrow-buffer", @@ -348,9 +348,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" +checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" dependencies = [ "bzip2", "flate2", @@ -715,7 +715,8 @@ dependencies = [ [[package]] name = "datafusion" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "676796427e638d85e9eadf13765705212be60b34f8fc5d3934d95184c63ca1b4" dependencies = [ "ahash", "apache-avro", @@ -740,7 +741,7 @@ dependencies = [ "glob", "half", "hashbrown 0.14.2", - "indexmap 2.1.0", + "indexmap", "itertools", "log", "num-traits", @@ -763,7 +764,8 @@ dependencies = [ [[package]] name = "datafusion-common" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e23b3d21a6531259d291bd20ce59282ea794bda1018b0a1e278c13cd52e50c" dependencies = [ "ahash", "apache-avro", @@ -783,7 +785,8 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4de1fd0d8db0f2b8e4f4121bfa1c7c09d3a5c08a0a65c2229cd849eb65cff855" dependencies = [ "arrow", "chrono", @@ -803,7 +806,8 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18e227fe88bf6730cab378d0cd8fc4c6b2ea42bc7e414a8ea9feba7225932735" dependencies = [ "ahash", "arrow", @@ -817,7 +821,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6648e62ea7605b9bfcd87fdc9d67e579c3b9ac563a87734ae5fe6d79ee4547" dependencies = [ "arrow", "async-trait", @@ -834,7 +839,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f32b8574add16a32411a9b3fb3844ac1fc09ab4e7be289f86fd56d620e4f2508" dependencies = [ "ahash", "arrow", @@ -851,7 +857,7 @@ dependencies = [ "half", "hashbrown 0.14.2", "hex", - "indexmap 2.1.0", + "indexmap", "itertools", "libc", "log", @@ -868,7 +874,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796abd77d5bfecd9e5275a99daf0ec45f5b3a793ec431349ce8211a67826fd22" dependencies = [ "ahash", "arrow", @@ -884,7 +891,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.2", - "indexmap 2.1.0", + "indexmap", "itertools", "log", "once_cell", @@ -925,7 +932,8 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced70b8a5648ba7b95c61fc512183c33287ffe2c9f22ffe22700619d7d48c84f" dependencies = [ "arrow", "arrow-schema", @@ -938,7 +946,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" version = "33.0.0" -source = "git+https://github.com/apache/arrow-datafusion?rev=33.0.0-rc1#262f08778b8ec231d96792c01fc3e051640eb5d4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f8fe74a4541eccbd096aa9fa3e734f1387a831613c9a3cb4f06bf7283ec77b7" dependencies = [ "async-recursion", "chrono", @@ -997,9 +1006,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys", @@ -1153,9 +1162,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if", "libc", @@ -1189,9 +1198,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -1199,7 +1208,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap", "slab", "tokio", "tokio-util", @@ -1217,12 +1226,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.13.2" @@ -1271,9 +1274,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -1380,16 +1383,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.1.0" @@ -1593,9 +1586,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" @@ -1865,9 +1858,9 @@ dependencies = [ [[package]] name = "parquet" -version = "48.0.0" +version = "48.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239229e6a668ab50c61de3dce61cf0fa1069345f7aa0f4c934491f92205a4945" +checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597" dependencies = [ "ahash", "arrow-array", @@ -1925,7 +1918,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.1.0", + "indexmap", ] [[package]] @@ -2011,9 +2004,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +checksum = "5a5a410fc7882af66deb8d01d01737353cf3ad6204c408177ba494291a626312" dependencies = [ "bytes", "prost-derive", @@ -2021,9 +2014,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" +checksum = "1fa3d084c8704911bfefb2771be2f9b6c5c0da7343a71e0021ee3c665cada738" dependencies = [ "bytes", "heck", @@ -2041,9 +2034,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +checksum = "065717a5dfaca4a83d2fe57db3487b311365200000551d7a364e715dbf4346bc" dependencies = [ "anyhow", "itertools", @@ -2054,9 +2047,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +checksum = "8339f32236f590281e2f6368276441394fcd1b2133b549cc895d0ae80f2f9a52" dependencies = [ "prost", ] @@ -2334,9 +2327,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "9ad981d6c340a49cdc40a1028d9c6084ec7e9fa33fcb839cab656a267071e234" dependencies = [ "bitflags 2.4.1", "errno", @@ -2347,9 +2340,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.8" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring 0.17.5", @@ -2359,9 +2352,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64", ] @@ -2399,9 +2392,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f7b0ce13155372a76ee2e1c5ffba1fe61ede73fbea5630d61eee6fac4929c0c" +checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" dependencies = [ "dyn-clone", "schemars_derive", @@ -2411,9 +2404,9 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e85e2a16b12bdb763244c69ab79363d71db2b4b918a2def53f80b02e0574b13c" +checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" dependencies = [ "proc-macro2", "quote", @@ -2451,18 +2444,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.190" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", @@ -2521,7 +2514,7 @@ version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ - "indexmap 2.1.0", + "indexmap", "itoa", "ryu", "serde", @@ -2556,9 +2549,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "snafu" @@ -2671,9 +2664,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.18.1" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49fd0bc602d396b95761b348a68ef6190e4cde28a5981e82b8449855db1d321d" +checksum = "7299fc531294d189834eeaf7928482f311c0ada2cf0007948989cf75d0228183" dependencies = [ "git2", "heck", @@ -2817,9 +2810,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -2835,9 +2828,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", @@ -3340,18 +3333,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.25" +version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd369a67c0edfef15010f980c3cbe45d7f651deac2cd67ce097cd801de16557" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.25" +version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2f140bda219a26ccc0cdb03dba58af72590c53b22642577d88a927bc5c87d6b" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 610fa9d58..f16f74ae8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,12 +37,12 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", features = ["pyarrow", "avro"] } -datafusion-common = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", features = ["pyarrow"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" } -datafusion-sql = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1" } -datafusion-substrait = { git = "https://github.com/apache/arrow-datafusion", rev = "33.0.0-rc1", optional = true } +datafusion = { version = "33.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "33.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "33.0.0" } +datafusion-optimizer = { version = "33.0.0" } +datafusion-sql = { version = "33.0.0" } +datafusion-substrait = { version = "33.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.3", features = ["v4"] } diff --git a/src/expr.rs b/src/expr.rs index 97c168e69..bbab8bf41 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -265,9 +265,7 @@ impl PyExpr { pub fn rex_type(&self) -> PyResult { Ok(match self.expr { Expr::Alias(..) => RexType::Alias, - Expr::Column(..) | Expr::QualifiedWildcard { .. } | Expr::GetIndexedField { .. } => { - RexType::Reference - } + Expr::Column(..) | Expr::GetIndexedField { .. } => RexType::Reference, Expr::ScalarVariable(..) | Expr::Literal(..) => RexType::Literal, Expr::BinaryExpr { .. } | Expr::Not(..) @@ -286,7 +284,7 @@ impl PyExpr { | Expr::WindowFunction { .. } | Expr::AggregateUDF { .. } | Expr::InList { .. } - | Expr::Wildcard + | Expr::Wildcard { .. } | Expr::ScalarUDF { .. } | Expr::Exists { .. } | Expr::InSubquery { .. } @@ -463,8 +461,7 @@ impl PyExpr { // Currently un-support/implemented Expr types for Rex Call operations Expr::GroupingSet(..) | Expr::OuterReferenceColumn(_, _) - | Expr::Wildcard - | Expr::QualifiedWildcard { .. } + | Expr::Wildcard { .. } | Expr::ScalarSubquery(..) | Expr::Placeholder { .. } | Expr::Exists { .. } => Err(py_runtime_err(format!( @@ -548,7 +545,7 @@ impl PyExpr { // appear in projections) so we just delegate to the contained expression instead Self::expr_to_field(expr, input_plan) } - Expr::Wildcard => { + Expr::Wildcard { .. } => { // Since * could be any of the valid column names just return the first one Ok(input_plan.schema().field(0).clone()) } From eaa71311977e1673d962235cee15691ef148fc91 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sun, 3 Dec 2023 07:43:23 -0500 Subject: [PATCH 100/413] Adjust visibility of crate private members & Functions (#537) --- src/catalog.rs | 10 +++--- src/common/df_field.rs | 2 +- src/context.rs | 77 ++++++++++++++++++++++-------------------- src/expr.rs | 1 + src/lib.rs | 2 +- src/substrait.rs | 12 +++---- src/utils.rs | 10 +++++- 7 files changed, 63 insertions(+), 51 deletions(-) diff --git a/src/catalog.rs b/src/catalog.rs index 94faea067..ba7e22255 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -30,18 +30,18 @@ use datafusion::{ }; #[pyclass(name = "Catalog", module = "datafusion", subclass)] -pub(crate) struct PyCatalog { - catalog: Arc, +pub struct PyCatalog { + pub catalog: Arc, } #[pyclass(name = "Database", module = "datafusion", subclass)] -pub(crate) struct PyDatabase { - database: Arc, +pub struct PyDatabase { + pub database: Arc, } #[pyclass(name = "Table", module = "datafusion", subclass)] pub struct PyTable { - table: Arc, + pub table: Arc, } impl PyCatalog { diff --git a/src/common/df_field.rs b/src/common/df_field.rs index 703af0aa2..68c05361f 100644 --- a/src/common/df_field.rs +++ b/src/common/df_field.rs @@ -27,7 +27,7 @@ use super::data_type::PyDataType; #[pyclass(name = "DFField", module = "datafusion.common", subclass)] #[derive(Debug, Clone)] pub struct PyDFField { - field: DFField, + pub field: DFField, } impl From for DFField { diff --git a/src/context.rs b/src/context.rs index a0361acfc..fb7ca8385 100644 --- a/src/context.rs +++ b/src/context.rs @@ -60,8 +60,8 @@ use tokio::task::JoinHandle; /// Configuration options for a SessionContext #[pyclass(name = "SessionConfig", module = "datafusion", subclass)] #[derive(Clone, Default)] -pub(crate) struct PySessionConfig { - pub(crate) config: SessionConfig, +pub struct PySessionConfig { + pub config: SessionConfig, } impl From for PySessionConfig { @@ -153,8 +153,8 @@ impl PySessionConfig { /// Runtime options for a SessionContext #[pyclass(name = "RuntimeConfig", module = "datafusion", subclass)] #[derive(Clone)] -pub(crate) struct PyRuntimeConfig { - pub(crate) config: RuntimeConfig, +pub struct PyRuntimeConfig { + pub config: RuntimeConfig, } #[pymethods] @@ -215,15 +215,18 @@ impl PyRuntimeConfig { /// multi-threaded execution engine to perform the execution. #[pyclass(name = "SessionContext", module = "datafusion", subclass)] #[derive(Clone)] -pub(crate) struct PySessionContext { - pub(crate) ctx: SessionContext, +pub struct PySessionContext { + pub ctx: SessionContext, } #[pymethods] impl PySessionContext { #[pyo3(signature = (config=None, runtime=None))] #[new] - fn new(config: Option, runtime: Option) -> PyResult { + pub fn new( + config: Option, + runtime: Option, + ) -> PyResult { let config = if let Some(c) = config { c.config } else { @@ -242,7 +245,7 @@ impl PySessionContext { } /// Register a an object store with the given name - fn register_object_store( + pub fn register_object_store( &mut self, scheme: &str, store: &PyAny, @@ -276,13 +279,13 @@ impl PySessionContext { } /// Returns a PyDataFrame whose plan corresponds to the SQL statement. - fn sql(&mut self, query: &str, py: Python) -> PyResult { + pub fn sql(&mut self, query: &str, py: Python) -> PyResult { let result = self.ctx.sql(query); let df = wait_for_future(py, result).map_err(DataFusionError::from)?; Ok(PyDataFrame::new(df)) } - fn create_dataframe( + pub fn create_dataframe( &mut self, partitions: PyArrowType>>, name: Option<&str>, @@ -314,13 +317,13 @@ impl PySessionContext { } /// Create a DataFrame from an existing logical plan - fn create_dataframe_from_logical_plan(&mut self, plan: PyLogicalPlan) -> PyDataFrame { + pub fn create_dataframe_from_logical_plan(&mut self, plan: PyLogicalPlan) -> PyDataFrame { PyDataFrame::new(DataFrame::new(self.ctx.state(), plan.plan.as_ref().clone())) } /// Construct datafusion dataframe from Python list #[allow(clippy::wrong_self_convention)] - fn from_pylist( + pub fn from_pylist( &mut self, data: PyObject, name: Option<&str>, @@ -340,7 +343,7 @@ impl PySessionContext { /// Construct datafusion dataframe from Python dictionary #[allow(clippy::wrong_self_convention)] - fn from_pydict( + pub fn from_pydict( &mut self, data: PyObject, name: Option<&str>, @@ -360,7 +363,7 @@ impl PySessionContext { /// Construct datafusion dataframe from Arrow Table #[allow(clippy::wrong_self_convention)] - fn from_arrow_table( + pub fn from_arrow_table( &mut self, data: PyObject, name: Option<&str>, @@ -381,7 +384,7 @@ impl PySessionContext { /// Construct datafusion dataframe from pandas #[allow(clippy::wrong_self_convention)] - fn from_pandas( + pub fn from_pandas( &mut self, data: PyObject, name: Option<&str>, @@ -401,7 +404,7 @@ impl PySessionContext { /// Construct datafusion dataframe from polars #[allow(clippy::wrong_self_convention)] - fn from_polars( + pub fn from_polars( &mut self, data: PyObject, name: Option<&str>, @@ -417,21 +420,21 @@ impl PySessionContext { }) } - fn register_table(&mut self, name: &str, table: &PyTable) -> PyResult<()> { + pub fn register_table(&mut self, name: &str, table: &PyTable) -> PyResult<()> { self.ctx .register_table(name, table.table()) .map_err(DataFusionError::from)?; Ok(()) } - fn deregister_table(&mut self, name: &str) -> PyResult<()> { + pub fn deregister_table(&mut self, name: &str) -> PyResult<()> { self.ctx .deregister_table(name) .map_err(DataFusionError::from)?; Ok(()) } - fn register_record_batches( + pub fn register_record_batches( &mut self, name: &str, partitions: PyArrowType>>, @@ -451,7 +454,7 @@ impl PySessionContext { skip_metadata=true, schema=None, file_sort_order=None))] - fn register_parquet( + pub fn register_parquet( &mut self, name: &str, path: &str, @@ -489,7 +492,7 @@ impl PySessionContext { schema_infer_max_records=1000, file_extension=".csv", file_compression_type=None))] - fn register_csv( + pub fn register_csv( &mut self, name: &str, path: PathBuf, @@ -533,7 +536,7 @@ impl PySessionContext { file_extension=".json", table_partition_cols=vec![], file_compression_type=None))] - fn register_json( + pub fn register_json( &mut self, name: &str, path: PathBuf, @@ -568,7 +571,7 @@ impl PySessionContext { file_extension=".avro", table_partition_cols=vec![], infinite=false))] - fn register_avro( + pub fn register_avro( &mut self, name: &str, path: PathBuf, @@ -595,7 +598,7 @@ impl PySessionContext { } // Registers a PyArrow.Dataset - fn register_dataset(&self, name: &str, dataset: &PyAny, py: Python) -> PyResult<()> { + pub fn register_dataset(&self, name: &str, dataset: &PyAny, py: Python) -> PyResult<()> { let table: Arc = Arc::new(Dataset::new(dataset, py)?); self.ctx @@ -605,18 +608,18 @@ impl PySessionContext { Ok(()) } - fn register_udf(&mut self, udf: PyScalarUDF) -> PyResult<()> { + pub fn register_udf(&mut self, udf: PyScalarUDF) -> PyResult<()> { self.ctx.register_udf(udf.function); Ok(()) } - fn register_udaf(&mut self, udaf: PyAggregateUDF) -> PyResult<()> { + pub fn register_udaf(&mut self, udaf: PyAggregateUDF) -> PyResult<()> { self.ctx.register_udaf(udaf.function); Ok(()) } #[pyo3(signature = (name="datafusion"))] - fn catalog(&self, name: &str) -> PyResult { + pub fn catalog(&self, name: &str) -> PyResult { match self.ctx.catalog(name) { Some(catalog) => Ok(PyCatalog::new(catalog)), None => Err(PyKeyError::new_err(format!( @@ -626,31 +629,31 @@ impl PySessionContext { } } - fn tables(&self) -> HashSet { + pub fn tables(&self) -> HashSet { #[allow(deprecated)] self.ctx.tables().unwrap() } - fn table(&self, name: &str, py: Python) -> PyResult { + pub fn table(&self, name: &str, py: Python) -> PyResult { let x = wait_for_future(py, self.ctx.table(name)).map_err(DataFusionError::from)?; Ok(PyDataFrame::new(x)) } - fn table_exist(&self, name: &str) -> PyResult { + pub fn table_exist(&self, name: &str) -> PyResult { Ok(self.ctx.table_exist(name)?) } - fn empty_table(&self) -> PyResult { + pub fn empty_table(&self) -> PyResult { Ok(PyDataFrame::new(self.ctx.read_empty()?)) } - fn session_id(&self) -> String { + pub fn session_id(&self) -> String { self.ctx.session_id() } #[allow(clippy::too_many_arguments)] #[pyo3(signature = (path, schema=None, schema_infer_max_records=1000, file_extension=".json", table_partition_cols=vec![], file_compression_type=None))] - fn read_json( + pub fn read_json( &mut self, path: PathBuf, schema: Option>, @@ -689,7 +692,7 @@ impl PySessionContext { file_extension=".csv", table_partition_cols=vec![], file_compression_type=None))] - fn read_csv( + pub fn read_csv( &self, path: PathBuf, schema: Option>, @@ -741,7 +744,7 @@ impl PySessionContext { skip_metadata=true, schema=None, file_sort_order=None))] - fn read_parquet( + pub fn read_parquet( &self, path: &str, table_partition_cols: Vec<(String, String)>, @@ -771,7 +774,7 @@ impl PySessionContext { #[allow(clippy::too_many_arguments)] #[pyo3(signature = (path, schema=None, table_partition_cols=vec![], file_extension=".avro"))] - fn read_avro( + pub fn read_avro( &self, path: &str, schema: Option>, @@ -793,7 +796,7 @@ impl PySessionContext { Ok(PyDataFrame::new(df)) } - fn read_table(&self, table: &PyTable) -> PyResult { + pub fn read_table(&self, table: &PyTable) -> PyResult { let df = self .ctx .read_table(table.table()) diff --git a/src/expr.rs b/src/expr.rs index bbab8bf41..3875fb381 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -128,6 +128,7 @@ impl PyExpr { Expr::ScalarVariable(data_type, variables) => { Ok(PyScalarVariable::new(data_type, variables).into_py(py)) } + Expr::Like(value) => Ok(PyLike::from(value.clone()).into_py(py)), Expr::Literal(value) => Ok(PyLiteral::from(value.clone()).into_py(py)), Expr::BinaryExpr(expr) => Ok(PyBinaryExpr::from(expr.clone()).into_py(py)), Expr::Not(expr) => Ok(PyNot::new(*expr.clone()).into_py(py)), diff --git a/src/lib.rs b/src/lib.rs index 413b2a429..5e57db9cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,7 +35,7 @@ pub mod common; #[allow(clippy::borrow_deref_ref)] mod config; #[allow(clippy::borrow_deref_ref)] -mod context; +pub mod context; #[allow(clippy::borrow_deref_ref)] mod dataframe; mod dataset; diff --git a/src/substrait.rs b/src/substrait.rs index 73606fdfa..ff83f6f79 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -29,8 +29,8 @@ use prost::Message; #[pyclass(name = "plan", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] -pub(crate) struct PyPlan { - pub(crate) plan: Plan, +pub struct PyPlan { + pub plan: Plan, } #[pymethods] @@ -61,7 +61,7 @@ impl From for PyPlan { /// to a valid `LogicalPlan` instance. #[pyclass(name = "serde", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] -pub(crate) struct PySubstraitSerializer; +pub struct PySubstraitSerializer; #[pymethods] impl PySubstraitSerializer { @@ -107,7 +107,7 @@ impl PySubstraitSerializer { #[pyclass(name = "producer", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] -pub(crate) struct PySubstraitProducer; +pub struct PySubstraitProducer; #[pymethods] impl PySubstraitProducer { @@ -123,7 +123,7 @@ impl PySubstraitProducer { #[pyclass(name = "consumer", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] -pub(crate) struct PySubstraitConsumer; +pub struct PySubstraitConsumer; #[pymethods] impl PySubstraitConsumer { @@ -140,7 +140,7 @@ impl PySubstraitConsumer { } } -pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { +pub fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/utils.rs b/src/utils.rs index 427a8a064..c5965bd2f 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -25,7 +25,15 @@ use tokio::runtime::Runtime; /// Utility to get the Tokio Runtime from Python pub(crate) fn get_tokio_runtime(py: Python) -> PyRef { let datafusion = py.import("datafusion._internal").unwrap(); - datafusion.getattr("runtime").unwrap().extract().unwrap() + let tmp = datafusion.getattr("runtime").unwrap(); + match tmp.extract::>() { + Ok(runtime) => runtime, + Err(_e) => { + let rt = TokioRuntime(tokio::runtime::Runtime::new().unwrap()); + let obj: &PyAny = Py::new(py, rt).unwrap().into_ref(py); + obj.extract().unwrap() + } + } } /// Utility to collect rust futures with GIL released From 85366d6a54802a633c7e4b58699eb72821d2b29b Mon Sep 17 00:00:00 2001 From: ray-andrew <76701804+ray-andrew@users.noreply.github.com> Date: Sun, 3 Dec 2023 20:43:56 +0800 Subject: [PATCH 101/413] Update json.rst (#538) --- docs/source/user-guide/io/json.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/io/json.rst b/docs/source/user-guide/io/json.rst index 5949a0380..1ee065c44 100644 --- a/docs/source/user-guide/io/json.rst +++ b/docs/source/user-guide/io/json.rst @@ -26,4 +26,4 @@ When it comes to reading a JSON file, using :meth:`.SessionContext.read_json` is from datafusion import SessionContext ctx = SessionContext() - df = ctx.read_avro("file.json") \ No newline at end of file + df = ctx.read_json("file.json") From 0b2962a7a5369669adbd335a889373379dd8bb6a Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sun, 3 Dec 2023 07:44:53 -0500 Subject: [PATCH 102/413] Enable mimalloc local_dynamic_tls feature (#540) --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index f16f74ae8..ba4216839 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ datafusion-substrait = { version = "33.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.3", features = ["v4"] } -mimalloc = { version = "0.1", optional = true, default-features = false } +mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] } From cce468021e369df23c524cb314404303d4677bd5 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Sun, 3 Dec 2023 07:45:30 -0500 Subject: [PATCH 103/413] Enable substrait feature to be built by default in CI, for nightlies and releases (#544) --- .github/workflows/build.yml | 4 ++-- conda/recipes/bld.bat | 2 +- conda/recipes/build.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e72c4805a..968bdc1f0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package - run: maturin build --release --strip + run: maturin build --release --strip --features substrait - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -141,7 +141,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package - run: maturin build --release --strip --target aarch64-apple-darwin + run: maturin build --release --strip --target aarch64-apple-darwin --features substrait - name: List Mac wheels run: find target/wheels/ diff --git a/conda/recipes/bld.bat b/conda/recipes/bld.bat index fc3ac88b3..90626a637 100644 --- a/conda/recipes/bld.bat +++ b/conda/recipes/bld.bat @@ -17,7 +17,7 @@ # under the License. # -maturin build -vv -j %CPU_COUNT% --release --strip --manylinux off --interpreter=%PYTHON% +maturin build -vv -j %CPU_COUNT% --release --strip --features substrait --manylinux off --interpreter=%PYTHON% FOR /F "delims=" %%i IN ('dir /s /b target\wheels\*.whl') DO set datafusion_wheel=%%i diff --git a/conda/recipes/build.sh b/conda/recipes/build.sh index af640e6ca..259894313 100644 --- a/conda/recipes/build.sh +++ b/conda/recipes/build.sh @@ -77,7 +77,7 @@ EOF sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/platform-patch.py fi -maturin build -vv -j "${CPU_COUNT}" --release --strip --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}" +maturin build -vv -j "${CPU_COUNT}" --release --strip --features substrait --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}" "${PYTHON}" -m pip install $SRC_DIR/target/wheels/datafusion*.whl --no-deps -vv From 2889de097dbc6002030c54726dd8fd1915df4a76 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 28 Dec 2023 13:09:38 -0700 Subject: [PATCH 104/413] Prepare 34.0.0 Release (#550) --- CHANGELOG.md | 9 + Cargo.lock | 558 ++++++++++++++++++++----------------- Cargo.toml | 16 +- src/common/data_type.rs | 8 +- src/common/schema.rs | 2 +- src/context.rs | 2 +- src/dataframe.rs | 2 +- src/dataset_exec.rs | 2 +- src/expr.rs | 17 +- src/expr/aggregate.rs | 14 +- src/expr/aggregate_expr.rs | 4 +- src/expr/distinct.rs | 27 +- src/functions.rs | 19 +- src/udaf.rs | 2 +- src/udf.rs | 2 +- 15 files changed, 382 insertions(+), 302 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d875cf13d..4af70d568 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,15 @@ # DataFusion Python Changelog +## [34.0.0](https://github.com/apache/arrow-datafusion-python/tree/34.0.0) (2023-12-28) + +**Merged pull requests:** + +- Adjust visibility of crate private members & Functions [#537](https://github.com/apache/arrow-datafusion-python/pull/537) (jdye64) +- Update json.rst [#538](https://github.com/apache/arrow-datafusion-python/pull/538) (ray-andrew) +- Enable mimalloc local_dynamic_tls feature [#540](https://github.com/apache/arrow-datafusion-python/pull/540) (jdye64) +- Enable substrait feature to be built by default in CI, for nightlies … [#544](https://github.com/apache/arrow-datafusion-python/pull/544) (jdye64) + ## [33.0.0](https://github.com/apache/arrow-datafusion-python/tree/33.0.0) (2023-11-16) **Merged pull requests:** diff --git a/Cargo.lock b/Cargo.lock index c2d34115d..db776d5bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8919668503a4f2d8b6da96fa7c16e93046bfb3412ffcfa1e5dc7d2e3adcb378" +checksum = "5bc25126d18a012146a888a0298f2c22e1150327bd2765fc76d710a556b2d614" dependencies = [ "ahash", "arrow-arith", @@ -153,9 +153,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef983914f477d4278b068f13b3224b7d19eb2b807ac9048544d3bfebdf2554c4" +checksum = "34ccd45e217ffa6e53bbb0080990e77113bdd4e91ddb84e97b77649810bcf1a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -168,9 +168,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6eaf89041fa5937940ae390294ece29e1db584f46d995608d6e5fe65a2e0e9b" +checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d" dependencies = [ "ahash", "arrow-buffer", @@ -179,15 +179,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "num", ] [[package]] name = "arrow-buffer" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55512d988c6fbd76e514fd3ff537ac50b0a675da5a245e4fdad77ecfd654205f" +checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c" dependencies = [ "bytes", "half", @@ -196,15 +196,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "655ee51a2156ba5375931ce21c1b2494b1d9260e6dcdc6d4db9060c37dc3325b" +checksum = "5dc0368ed618d509636c1e3cc20db1281148190a78f43519487b2daf07b63b4a" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "base64", "chrono", "comfy-table", "half", @@ -214,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258bb689997ad5b6660b3ce3638bd6b383d668ec555ed41ad7c6559cbb2e4f91" +checksum = "2e09aa6246a1d6459b3f14baeaa49606cfdbca34435c46320e14054d244987ca" dependencies = [ "arrow-array", "arrow-buffer", @@ -233,9 +234,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dc2b9fec74763427e2e5575b8cc31ce96ba4c9b4eb05ce40e0616d9fad12461" +checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634" dependencies = [ "arrow-buffer", "arrow-schema", @@ -245,9 +246,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eaa6ab203cc6d89b7eaa1ac781c1dfeef325454c5d5a0419017f95e6bafc03c" +checksum = "79a43d6808411886b8c7d4f6f7dd477029c1e77ffffffb7923555cc6579639cd" dependencies = [ "arrow-array", "arrow-buffer", @@ -259,9 +260,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb64e30d9b73f66fdc5c52d5f4cf69bbf03d62f64ffeafa0715590a5320baed7" +checksum = "d82565c91fd627922ebfe2810ee4e8346841b6f9361b87505a9acea38b614fee" dependencies = [ "arrow-array", "arrow-buffer", @@ -279,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a818951c0d11c428dda03e908175969c262629dd20bd0850bd6c7a8c3bfe48" +checksum = "9b23b0e53c0db57c6749997fd343d4c0354c994be7eca67152dd2bdb9a3e1bb4" dependencies = [ "arrow-array", "arrow-buffer", @@ -294,9 +295,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d664318bc05f930559fc088888f0f7174d3c5bc888c0f4f9ae8f23aa398ba3" +checksum = "361249898d2d6d4a6eeb7484be6ac74977e48da12a4dd81a708d620cc558117a" dependencies = [ "ahash", "arrow-array", @@ -304,23 +305,23 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.2", + "hashbrown 0.14.3", ] [[package]] name = "arrow-schema" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf4d737bba93da59f16129bec21e087aed0be84ff840e74146d4703879436cb" +checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167" dependencies = [ "bitflags 2.4.1", ] [[package]] name = "arrow-select" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374c4c3b812ecc2118727b892252a4a4308f87a8aca1dbf09f3ce4bc578e668a" +checksum = "4f6208466590960efc1d2a7172bc4ff18a67d6e25c529381d7f96ddaf0dc4036" dependencies = [ "ahash", "arrow-array", @@ -332,9 +333,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15aed5624bb23da09142f58502b59c23f5bea607393298bb81dab1ce60fc769" +checksum = "a4a48149c63c11c9ff571e50ab8f017d2a7cb71037a882b42f6354ed2da9acc7" dependencies = [ "arrow-array", "arrow-buffer", @@ -372,7 +373,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -383,7 +384,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -551,7 +552,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -615,9 +616,9 @@ checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -625,9 +626,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "core2" @@ -706,7 +707,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "lock_api", "once_cell", "parking_lot_core", @@ -714,9 +715,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "676796427e638d85e9eadf13765705212be60b34f8fc5d3934d95184c63ca1b4" +checksum = "193fd1e7628278d0641c5122860f9a7fd6a1d77d055838d12f55d15bbe28d4d0" dependencies = [ "ahash", "apache-avro", @@ -740,9 +741,9 @@ dependencies = [ "futures", "glob", "half", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "indexmap", - "itertools", + "itertools 0.12.0", "log", "num-traits", "num_cpus", @@ -763,9 +764,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e23b3d21a6531259d291bd20ce59282ea794bda1018b0a1e278c13cd52e50c" +checksum = "548bc49c4a489e3de474813831ea556dc9d368f9ed8d867b1493da42e8e9f613" dependencies = [ "ahash", "apache-avro", @@ -775,6 +776,7 @@ dependencies = [ "arrow-schema", "chrono", "half", + "libc", "num_cpus", "object_store", "parquet", @@ -784,9 +786,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4de1fd0d8db0f2b8e4f4121bfa1c7c09d3a5c08a0a65c2229cd849eb65cff855" +checksum = "ecc865657ffcf4da5ff08bdc6436a9a833bc0aa96c3254c8d18ab8a0ad4e437d" dependencies = [ "arrow", "chrono", @@ -794,7 +796,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "log", "object_store", "parking_lot", @@ -805,14 +807,15 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e227fe88bf6730cab378d0cd8fc4c6b2ea42bc7e414a8ea9feba7225932735" +checksum = "33c473f72d8d81a532e63f6e562ed66dd9209dfd8e433d9712abd42444ee161e" dependencies = [ "ahash", "arrow", "arrow-array", "datafusion-common", + "paste", "sqlparser", "strum", "strum_macros", @@ -820,9 +823,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6648e62ea7605b9bfcd87fdc9d67e579c3b9ac563a87734ae5fe6d79ee4547" +checksum = "cb6218318001d2f6783b7fffa17592318f65f26609d7aab605a3dd0c7c2e2618" dependencies = [ "arrow", "async-trait", @@ -830,17 +833,17 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.2", - "itertools", + "hashbrown 0.14.3", + "itertools 0.12.0", "log", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f32b8574add16a32411a9b3fb3844ac1fc09ab4e7be289f86fd56d620e4f2508" +checksum = "9e1ca7e35ca22f9dc506c2375b92054b03ccf91afe25c0a90b395a1473a09735" dependencies = [ "ahash", "arrow", @@ -855,11 +858,10 @@ dependencies = [ "datafusion-common", "datafusion-expr", "half", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "hex", "indexmap", - "itertools", - "libc", + "itertools 0.12.0", "log", "md-5", "paste", @@ -873,9 +875,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796abd77d5bfecd9e5275a99daf0ec45f5b3a793ec431349ce8211a67826fd22" +checksum = "ddde97adefcca3a55257c646ffee2a95b6cac66f74d1146a6e3a6dbb37830631" dependencies = [ "ahash", "arrow", @@ -890,9 +892,9 @@ dependencies = [ "datafusion-physical-expr", "futures", "half", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "indexmap", - "itertools", + "itertools 0.12.0", "log", "once_cell", "parking_lot", @@ -904,7 +906,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "33.0.0" +version = "34.0.0" dependencies = [ "async-trait", "datafusion", @@ -923,7 +925,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.39", + "syn 2.0.41", "tokio", "url", "uuid", @@ -931,9 +933,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced70b8a5648ba7b95c61fc512183c33287ffe2c9f22ffe22700619d7d48c84f" +checksum = "a60d9d6460a64fddb8663db41da97e6b8b0bf79da42f997ebe81722731eaf0e5" dependencies = [ "arrow", "arrow-schema", @@ -945,14 +947,14 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "33.0.0" +version = "34.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f8fe74a4541eccbd096aa9fa3e734f1387a831613c9a3cb4f06bf7283ec77b7" +checksum = "2243428e8df8b2c5ce2c9e3af588312ae89c1b073c52de5693ec44ba29674547" dependencies = [ "async-recursion", "chrono", "datafusion", - "itertools", + "itertools 0.12.0", "object_store", "prost", "prost-types", @@ -1006,12 +1008,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1054,9 +1056,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" dependencies = [ "percent-encoding", ] @@ -1117,7 +1119,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -1173,9 +1175,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" +checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "git2" @@ -1237,9 +1239,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ "ahash", "allocator-api2", @@ -1265,11 +1267,11 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "home" -version = "0.5.5" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -1285,9 +1287,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", "http", @@ -1375,9 +1377,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ "unicode-bidi", "unicode-normalization", @@ -1390,7 +1392,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.2", + "hashbrown 0.14.3", ] [[package]] @@ -1420,11 +1422,20 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +dependencies = [ + "either", +] + [[package]] name = "itoa" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "jobserver" @@ -1437,9 +1448,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8" +checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" dependencies = [ "wasm-bindgen", ] @@ -1516,9 +1527,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.150" +version = "0.2.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" [[package]] name = "libflate" @@ -1586,9 +1597,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" [[package]] name = "lock_api" @@ -1677,13 +1688,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.9" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" dependencies = [ "libc", "wasi", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1790,9 +1801,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" +checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" dependencies = [ "async-trait", "base64", @@ -1801,13 +1812,13 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools", + "itertools 0.11.0", "parking_lot", "percent-encoding", "quick-xml", "rand", "reqwest", - "ring 0.16.20", + "ring", "rustls-pemfile", "serde", "serde_json", @@ -1820,9 +1831,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "ordered-float" @@ -1853,14 +1864,14 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] name = "parquet" -version = "48.0.1" +version = "49.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bfe55df96e3f02f11bf197ae37d91bb79801631f82f6195dd196ef521df3597" +checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4" dependencies = [ "ahash", "arrow-array", @@ -1876,7 +1887,7 @@ dependencies = [ "chrono", "flate2", "futures", - "hashbrown 0.14.2", + "hashbrown 0.14.3", "lz4_flex", "num", "num-bigint", @@ -1907,9 +1918,9 @@ checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "percent-encoding" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" @@ -1990,23 +2001,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] name = "proc-macro2" -version = "1.0.69" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5a410fc7882af66deb8d01d01737353cf3ad6204c408177ba494291a626312" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", "prost-derive", @@ -2014,13 +2025,13 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa3d084c8704911bfefb2771be2f9b6c5c0da7343a71e0021ee3c665cada738" +checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -2034,22 +2045,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065717a5dfaca4a83d2fe57db3487b311365200000551d7a364e715dbf4346bc" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" dependencies = [ "anyhow", - "itertools", + "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] name = "prost-types" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8339f32236f590281e2f6368276441394fcd1b2133b549cc895d0ae80f2f9a52" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" dependencies = [ "prost", ] @@ -2109,7 +2120,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2121,7 +2132,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2132,9 +2143,9 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quick-xml" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" dependencies = [ "memchr", "serde", @@ -2277,31 +2288,16 @@ dependencies = [ [[package]] name = "ring" -version = "0.16.20" +version = "0.17.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin 0.5.2", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - -[[package]] -name = "ring" -version = "0.17.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" +checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" dependencies = [ "cc", "getrandom", "libc", - "spin 0.9.8", - "untrusted 0.9.0", - "windows-sys", + "spin", + "untrusted", + "windows-sys 0.48.0", ] [[package]] @@ -2327,25 +2323,25 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.24" +version = "0.38.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ad981d6c340a49cdc40a1028d9c6084ec7e9fa33fcb839cab656a267071e234" +checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.21.9" +version = "0.21.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" +checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.5", + "ring", "rustls-webpki", "sct", ] @@ -2365,8 +2361,8 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.5", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -2377,9 +2373,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.15" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" +checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" [[package]] name = "same-file" @@ -2426,8 +2422,8 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.5", - "untrusted 0.9.0", + "ring", + "untrusted", ] [[package]] @@ -2444,22 +2440,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.192" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.192" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2493,7 +2489,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2577,9 +2573,9 @@ dependencies = [ [[package]] name = "snap" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9f0ab6ef7eb7353d9119c170a436d1bf248eea575ac42d19d12f4e34130831" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" @@ -2598,15 +2594,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spin" version = "0.9.8" @@ -2615,9 +2605,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.39.0" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" +checksum = "7c80afe31cdb649e56c0d9bb5503be9166600d68a852c38dd445636d126858e5" dependencies = [ "log", "sqlparser_derive", @@ -2625,9 +2615,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55fe75cb4a364c7f7ae06c7dbbc8d84bddd85d6cdf9975963c3935bc1991761e" +checksum = "3e9c2e1dde0efa87003e7923d94a90f46e3274ad1649f51de96812be561f041f" dependencies = [ "proc-macro2", "quote", @@ -2659,14 +2649,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] name = "substrait" -version = "0.19.0" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7299fc531294d189834eeaf7928482f311c0ada2cf0007948989cf75d0228183" +checksum = "bdbe404d89610b54be889fb6540bcc6a9e234d0991cde0a8a9e341559768aa75" dependencies = [ "git2", "heck", @@ -2680,7 +2670,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.39", + "syn 2.0.41", "typify", "walkdir", ] @@ -2704,9 +2694,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.39" +version = "2.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" +checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" dependencies = [ "proc-macro2", "quote", @@ -2750,27 +2740,27 @@ dependencies = [ "fastrand", "redox_syscall", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2810,9 +2800,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.34.0" +version = "1.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" +checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" dependencies = [ "backtrace", "bytes", @@ -2823,7 +2813,7 @@ dependencies = [ "pin-project-lite", "socket2 0.5.5", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2834,7 +2824,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2886,7 +2876,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2900,9 +2890,9 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" @@ -2931,7 +2921,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] @@ -2942,9 +2932,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" -version = "0.0.14" +version = "0.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e3b707a653e2915a2fc2c4ee96a3d30b9554b9435eb4cc8b5c6c74bbdd3044" +checksum = "63ed4d717aa95e598e2f9183376b060e95669ef8f444701ea6afb990fde1cf69" dependencies = [ "typify-impl", "typify-macro", @@ -2952,9 +2942,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.14" +version = "0.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d9c752192779f666e4c868672dee56a652b82c08032c7e9d23f6a845b282298" +checksum = "89057244dfade7c58af9e62beccbcbeb7a7e7701697a33b06dbe0b7331fb79cf" dependencies = [ "heck", "log", @@ -2963,16 +2953,16 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.39", + "syn 2.0.41", "thiserror", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.0.14" +version = "0.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a14defd554507e72a2bb93cd081c8b374cfed43b3d986b141ad3839d9fd6986b" +checksum = "2ddade397f5957d2cd7fb27f905a9a569db20e8e1e3ea589edce40be07b92825" dependencies = [ "proc-macro2", "quote", @@ -2980,15 +2970,15 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.39", + "syn 2.0.41", "typify-impl", ] [[package]] name = "unicode-bidi" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" +checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" [[package]] name = "unicode-ident" @@ -3025,15 +3015,9 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "unsafe-libyaml" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" - -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" [[package]] name = "untrusted" @@ -3043,9 +3027,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", "idna", @@ -3054,9 +3038,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.5.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" +checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" dependencies = [ "getrandom", "serde", @@ -3101,9 +3085,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce" +checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3111,24 +3095,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217" +checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02" +checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" dependencies = [ "cfg-if", "js-sys", @@ -3138,9 +3122,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2" +checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3148,22 +3132,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" +checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.88" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b" +checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" [[package]] name = "wasm-streams" @@ -3180,9 +3164,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.65" +version = "0.3.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85" +checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" dependencies = [ "js-sys", "wasm-bindgen", @@ -3190,9 +3174,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.2" +version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" [[package]] name = "which" @@ -3243,7 +3227,7 @@ version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -3252,7 +3236,16 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets", + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.0", ] [[package]] @@ -3261,13 +3254,28 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +dependencies = [ + "windows_aarch64_gnullvm 0.52.0", + "windows_aarch64_msvc 0.52.0", + "windows_i686_gnu 0.52.0", + "windows_i686_msvc 0.52.0", + "windows_x86_64_gnu 0.52.0", + "windows_x86_64_gnullvm 0.52.0", + "windows_x86_64_msvc 0.52.0", ] [[package]] @@ -3276,42 +3284,84 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" + [[package]] name = "winreg" version = "0.50.0" @@ -3319,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3333,22 +3383,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.26" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" +checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.26" +version = "0.7.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" +checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.39", + "syn 2.0.41", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ba4216839..08f566d2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "33.0.0" +version = "34.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -37,19 +37,19 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "33.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "33.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "33.0.0" } -datafusion-optimizer = { version = "33.0.0" } -datafusion-sql = { version = "33.0.0" } -datafusion-substrait = { version = "33.0.0", optional = true } +datafusion = { version = "34.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "34.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "34.0.0" } +datafusion-optimizer = { version = "34.0.0" } +datafusion-sql = { version = "34.0.0" } +datafusion-substrait = { version = "34.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.7.0", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.8.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" syn = "2.0.37" diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 9719226a2..03248cc5b 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -291,9 +291,13 @@ impl DataTypeMap { ScalarValue::List(arr) => Ok(arr.data_type().to_owned()), ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())), ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)), - ScalarValue::Fixedsizelist(_, field_ref, size) => { - Ok(DataType::FixedSizeList(field_ref.to_owned(), *size)) + ScalarValue::FixedSizeList(_array_ref) => { + // The FieldRef was removed from ScalarValue::FixedSizeList in + // https://github.com/apache/arrow-datafusion/pull/8221, so we can no + // longer convert back to a DataType here + todo!() } + ScalarValue::LargeList(_) => todo!(), ScalarValue::DurationSecond(_) => Ok(DataType::Duration(TimeUnit::Second)), ScalarValue::DurationMillisecond(_) => Ok(DataType::Duration(TimeUnit::Millisecond)), ScalarValue::DurationMicrosecond(_) => Ok(DataType::Duration(TimeUnit::Microsecond)), diff --git a/src/common/schema.rs b/src/common/schema.rs index 77b0ce2ba..00113a510 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -21,7 +21,7 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource}; use pyo3::prelude::*; -use datafusion_optimizer::utils::split_conjunction; +use datafusion_expr::utils::split_conjunction; use super::{data_type::DataTypeMap, function::SqlFunction}; diff --git a/src/context.rs b/src/context.rs index fb7ca8385..63c48f007 100644 --- a/src/context.rs +++ b/src/context.rs @@ -377,7 +377,7 @@ impl PySessionContext { // Because create_dataframe() expects a vector of vectors of record batches // here we need to wrap the vector of record batches in an additional vector let batches = table.extract::>>(py)?; - let list_of_batches = PyArrowType::try_from(vec![batches.0])?; + let list_of_batches = PyArrowType::from(vec![batches.0]); self.create_dataframe(list_of_batches, name, py) }) } diff --git a/src/dataframe.rs b/src/dataframe.rs index ebf06f3d2..1e8790990 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -36,7 +36,7 @@ use std::sync::Arc; /// The actual execution of a plan runs natively on Rust and Arrow on a multi-threaded environment. #[pyclass(name = "DataFrame", module = "datafusion", subclass)] #[derive(Clone)] -pub(crate) struct PyDataFrame { +pub struct PyDataFrame { df: Arc, } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 0266ad9e3..89f73a93d 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -38,8 +38,8 @@ use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, Statistics, }; +use datafusion_expr::utils::conjunction; use datafusion_expr::Expr; -use datafusion_optimizer::utils::conjunction; use crate::errors::DataFusionError; use crate::pyarrow_filter_expression::PyArrowFilterExpression; diff --git a/src/expr.rs b/src/expr.rs index 3875fb381..dbb56ee99 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -24,10 +24,7 @@ use datafusion::scalar::ScalarValue; use datafusion_common::DFField; use datafusion_expr::{ col, - expr::{ - AggregateFunction, AggregateUDF, InList, InSubquery, ScalarFunction, ScalarUDF, Sort, - WindowFunction, - }, + expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, lit, utils::exprlist_to_fields, Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, LogicalPlan, @@ -283,10 +280,8 @@ impl PyExpr { | Expr::ScalarFunction { .. } | Expr::AggregateFunction { .. } | Expr::WindowFunction { .. } - | Expr::AggregateUDF { .. } | Expr::InList { .. } | Expr::Wildcard { .. } - | Expr::ScalarUDF { .. } | Expr::Exists { .. } | Expr::InSubquery { .. } | Expr::GroupingSet(..) @@ -351,7 +346,8 @@ impl PyExpr { ScalarValue::DurationMillisecond(v) => v.into_py(py), ScalarValue::Struct(_, _) => todo!(), ScalarValue::Dictionary(_, _) => todo!(), - ScalarValue::Fixedsizelist(_, _, _) => todo!(), + ScalarValue::FixedSizeList(_) => todo!(), + ScalarValue::LargeList(_) => todo!(), }), _ => Err(py_type_err(format!( "Non Expr::Literal encountered in types: {:?}", @@ -391,9 +387,7 @@ impl PyExpr { // Expr variants containing a collection of Expr(s) for operands Expr::AggregateFunction(AggregateFunction { args, .. }) - | Expr::AggregateUDF(AggregateUDF { args, .. }) | Expr::ScalarFunction(ScalarFunction { args, .. }) - | Expr::ScalarUDF(ScalarUDF { args, .. }) | Expr::WindowFunction(WindowFunction { args, .. }) => { Ok(args.iter().map(|arg| PyExpr::from(arg.clone())).collect()) } @@ -480,8 +474,9 @@ impl PyExpr { op, right: _, }) => format!("{op}"), - Expr::ScalarFunction(ScalarFunction { fun, args: _ }) => format!("{fun}"), - Expr::ScalarUDF(ScalarUDF { fun, .. }) => fun.name.clone(), + Expr::ScalarFunction(ScalarFunction { func_def, args: _ }) => { + func_def.name().to_string() + } Expr::Cast { .. } => "cast".to_string(), Expr::Between { .. } => "between".to_string(), Expr::Case { .. } => "case".to_string(), diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index 5ebf8c6cf..626d92c79 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -16,7 +16,7 @@ // under the License. use datafusion_common::DataFusionError; -use datafusion_expr::expr::{AggregateFunction, AggregateUDF, Alias}; +use datafusion_expr::expr::{AggregateFunction, Alias}; use datafusion_expr::logical_plan::Aggregate; use datafusion_expr::Expr; use pyo3::prelude::*; @@ -126,10 +126,9 @@ impl PyAggregate { match expr { // TODO: This Alias logic seems to be returning some strange results that we should investigate Expr::Alias(Alias { expr, .. }) => self._aggregation_arguments(expr.as_ref()), - Expr::AggregateFunction(AggregateFunction { fun: _, args, .. }) - | Expr::AggregateUDF(AggregateUDF { fun: _, args, .. }) => { - Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()) - } + Expr::AggregateFunction(AggregateFunction { + func_def: _, args, .. + }) => Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()), _ => Err(py_type_err( "Encountered a non Aggregate type in aggregation_arguments", )), @@ -139,8 +138,9 @@ impl PyAggregate { fn _agg_func_name(expr: &Expr) -> PyResult { match expr { Expr::Alias(Alias { expr, .. }) => Self::_agg_func_name(expr.as_ref()), - Expr::AggregateFunction(AggregateFunction { fun, .. }) => Ok(fun.to_string()), - Expr::AggregateUDF(AggregateUDF { fun, .. }) => Ok(fun.name.clone()), + Expr::AggregateFunction(AggregateFunction { func_def, .. }) => { + Ok(func_def.name().to_owned()) + } _ => Err(py_type_err( "Encountered a non Aggregate type in agg_func_name", )), diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs index 180105180..04ec29a15 100644 --- a/src/expr/aggregate_expr.rs +++ b/src/expr/aggregate_expr.rs @@ -41,7 +41,7 @@ impl From for PyAggregateFunction { impl Display for PyAggregateFunction { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { let args: Vec = self.aggr.args.iter().map(|expr| expr.to_string()).collect(); - write!(f, "{}({})", self.aggr.fun, args.join(", ")) + write!(f, "{}({})", self.aggr.func_def.name(), args.join(", ")) } } @@ -49,7 +49,7 @@ impl Display for PyAggregateFunction { impl PyAggregateFunction { /// Get the aggregate type, such as "MIN", or "MAX" fn aggregate_type(&self) -> String { - format!("{}", self.aggr.fun) + self.aggr.func_def.name().to_string() } /// is this a distinct aggregate such as `COUNT(DISTINCT expr)` diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs index 681ae953b..5d3a0b459 100644 --- a/src/expr/distinct.rs +++ b/src/expr/distinct.rs @@ -44,12 +44,22 @@ impl From for PyDistinct { impl Display for PyDistinct { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "Distinct + match &self.distinct { + Distinct::All(input) => write!( + f, + "Distinct ALL \nInput: {:?}", - self.distinct.input, - ) + input, + ), + Distinct::On(distinct_on) => { + write!( + f, + "Distinct ON + \nInput: {:?}", + distinct_on.input, + ) + } + } } } @@ -71,7 +81,12 @@ impl PyDistinct { impl LogicalNode for PyDistinct { fn inputs(&self) -> Vec { - vec![PyLogicalPlan::from((*self.distinct.input).clone())] + match &self.distinct { + Distinct::All(input) => vec![PyLogicalPlan::from(input.as_ref().clone())], + Distinct::On(distinct_on) => { + vec![PyLogicalPlan::from(distinct_on.input.as_ref().clone())] + } + } } fn to_variant(&self, py: Python) -> PyResult { diff --git a/src/functions.rs b/src/functions.rs index b8c825552..d1f3e807b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -23,11 +23,11 @@ use crate::expr::conditional_expr::PyCaseBuilder; use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; use datafusion::execution::FunctionRegistry; -use datafusion_common::Column; +use datafusion_common::{Column, TableReference}; use datafusion_expr::expr::Alias; use datafusion_expr::{ aggregate_function, - expr::{AggregateFunction, ScalarFunction, Sort, WindowFunction}, + expr::{AggregateFunction, AggregateFunctionDefinition, ScalarFunction, Sort, WindowFunction}, lit, window_function::find_df_window_func, BuiltinScalarFunction, Expr, @@ -88,8 +88,9 @@ fn order_by(expr: PyExpr, asc: Option, nulls_first: Option) -> PyRes /// Creates a new Alias Expr #[pyfunction] fn alias(expr: PyExpr, name: &str) -> PyResult { + let relation: Option = None; Ok(PyExpr { - expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, name)), + expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, relation, name)), }) } @@ -109,7 +110,9 @@ fn col(name: &str) -> PyResult { fn count_star() -> PyResult { Ok(PyExpr { expr: Expr::AggregateFunction(AggregateFunction { - fun: aggregate_function::AggregateFunction::Count, + func_def: datafusion_expr::expr::AggregateFunctionDefinition::BuiltIn( + aggregate_function::AggregateFunction::Count, + ), args: vec![lit(1)], distinct: false, filter: None, @@ -181,7 +184,9 @@ macro_rules! scalar_function { #[pyo3(signature = (*args))] fn $NAME(args: Vec) -> PyExpr { let expr = datafusion_expr::Expr::ScalarFunction(ScalarFunction { - fun: BuiltinScalarFunction::$FUNC, + func_def: datafusion_expr::ScalarFunctionDefinition::BuiltIn( + BuiltinScalarFunction::$FUNC, + ), args: args.into_iter().map(|e| e.into()).collect(), }); expr.into() @@ -199,7 +204,9 @@ macro_rules! aggregate_function { #[pyo3(signature = (*args, distinct=false))] fn $NAME(args: Vec, distinct: bool) -> PyExpr { let expr = datafusion_expr::Expr::AggregateFunction(AggregateFunction { - fun: datafusion_expr::aggregate_function::AggregateFunction::$FUNC, + func_def: AggregateFunctionDefinition::BuiltIn( + datafusion_expr::aggregate_function::AggregateFunction::$FUNC, + ), args: args.into_iter().map(|e| e.into()).collect(), distinct, filter: None, diff --git a/src/udaf.rs b/src/udaf.rs index 596ed6904..5c43b6710 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -172,6 +172,6 @@ impl PyAggregateUDF { } fn __repr__(&self) -> PyResult { - Ok(format!("AggregateUDF({})", self.function.name)) + Ok(format!("AggregateUDF({})", self.function.name())) } } diff --git a/src/udf.rs b/src/udf.rs index bba8ae551..af17b57e8 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -95,6 +95,6 @@ impl PyScalarUDF { } fn __repr__(&self) -> PyResult { - Ok(format!("ScalarUDF({})", self.function.name)) + Ok(format!("ScalarUDF({})", self.function.name())) } } From 76d7fcffdd9d8664a003b76754033ebad4a15847 Mon Sep 17 00:00:00 2001 From: Dan Lovell Date: Thu, 28 Dec 2023 15:10:46 -0500 Subject: [PATCH 105/413] feat: udaf: enable multiple column input (#546) --- datafusion/__init__.py | 2 ++ src/udaf.rs | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/datafusion/__init__.py b/datafusion/__init__.py index c854f3f9d..df53b396a 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -213,6 +213,8 @@ def udaf(accum, input_type, return_type, state_type, volatility, name=None): ) if name is None: name = accum.__qualname__.lower() + if isinstance(input_type, pa.lib.DataType): + input_type = [input_type] return AggregateUDF( name=name, accumulator=accum, diff --git a/src/udaf.rs b/src/udaf.rs index 5c43b6710..0e7a8deab 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -148,14 +148,14 @@ impl PyAggregateUDF { fn new( name: &str, accumulator: PyObject, - input_type: PyArrowType, + input_type: PyArrowType>, return_type: PyArrowType, state_type: PyArrowType>, volatility: &str, ) -> PyResult { let function = create_udaf( name, - vec![input_type.0], + input_type.0, Arc::new(return_type.0), parse_volatility(volatility)?, to_rust_accumulator(accumulator), From b22f82f3055941dc3599c9a18458a2de163ff4c0 Mon Sep 17 00:00:00 2001 From: Chih Wang Date: Fri, 29 Dec 2023 04:11:31 +0800 Subject: [PATCH 106/413] Add missing array functions (#551) * Add array_append, array_concat and array_cat * Add tests for array functions array_append, array_concat and array_cat * Add array_dims and list_dims * Add tests for array_dims and list_dims * Add array_element, array_extract, list_element and list_extract * Add tests for array_element, array_extract, list_element and list_extract * Add array_length and list_length --- datafusion/tests/test_functions.py | 64 ++++++++++++++++++++++++++++++ src/functions.rs | 27 +++++++++++++ 2 files changed, 91 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index be2a2f1f5..d0514f892 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -25,6 +25,8 @@ from datafusion import functions as f from datafusion import literal +np.seterr(invalid="ignore") + @pytest.fixture def df(): @@ -197,6 +199,68 @@ def test_math_functions(): ) +def test_array_functions(): + data = [[1.0, 2.0, 3.0], [4.0, 5.0], [6.0]] + ctx = SessionContext() + batch = pa.RecordBatch.from_arrays( + [np.array(data, dtype=object)], names=["arr"] + ) + df = ctx.create_dataframe([[batch]]) + + col = column("arr") + test_items = [ + [ + f.array_append(col, literal(99.0)), + lambda: [np.append(arr, 99.0) for arr in data], + ], + [ + f.array_concat(col, col), + lambda: [np.concatenate([arr, arr]) for arr in data], + ], + [ + f.array_cat(col, col), + lambda: [np.concatenate([arr, arr]) for arr in data], + ], + [ + f.array_dims(col), + lambda: [[len(r)] for r in data], + ], + [ + f.list_dims(col), + lambda: [[len(r)] for r in data], + ], + [ + f.array_element(col, literal(1)), + lambda: [r[0] for r in data], + ], + [ + f.array_extract(col, literal(1)), + lambda: [r[0] for r in data], + ], + [ + f.list_element(col, literal(1)), + lambda: [r[0] for r in data], + ], + [ + f.list_extract(col, literal(1)), + lambda: [r[0] for r in data], + ], + [ + f.array_length(col), + lambda: [len(r) for r in data], + ], + [ + f.list_length(col), + lambda: [len(r) for r in data], + ], + ] + + for stmt, py_expr in test_items: + query_result = df.select(stmt).collect()[0].column(0).tolist() + for a, b in zip(query_result, py_expr()): + np.testing.assert_array_almost_equal(a, b) + + def test_string_functions(df): df = df.select( f.ascii(column("a")), diff --git a/src/functions.rs b/src/functions.rs index d1f3e807b..3dc5322aa 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -357,6 +357,19 @@ scalar_function!(random, Random); scalar_function!(encode, Encode); scalar_function!(decode, Decode); +// Array Functions +scalar_function!(array_append, ArrayAppend); +scalar_function!(array_concat, ArrayConcat); +scalar_function!(array_cat, ArrayConcat); +scalar_function!(array_dims, ArrayDims); +scalar_function!(list_dims, ArrayDims); +scalar_function!(array_element, ArrayElement); +scalar_function!(array_extract, ArrayElement); +scalar_function!(list_element, ArrayElement); +scalar_function!(list_extract, ArrayElement); +scalar_function!(array_length, ArrayLength); +scalar_function!(list_length, ArrayLength); + aggregate_function!(approx_distinct, ApproxDistinct); aggregate_function!(approx_median, ApproxMedian); aggregate_function!(approx_percentile_cont, ApproxPercentileCont); @@ -546,5 +559,19 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { //Binary String Functions m.add_wrapped(wrap_pyfunction!(encode))?; m.add_wrapped(wrap_pyfunction!(decode))?; + + // Array Functions + m.add_wrapped(wrap_pyfunction!(array_append))?; + m.add_wrapped(wrap_pyfunction!(array_concat))?; + m.add_wrapped(wrap_pyfunction!(array_cat))?; + m.add_wrapped(wrap_pyfunction!(array_dims))?; + m.add_wrapped(wrap_pyfunction!(list_dims))?; + m.add_wrapped(wrap_pyfunction!(array_element))?; + m.add_wrapped(wrap_pyfunction!(array_extract))?; + m.add_wrapped(wrap_pyfunction!(list_element))?; + m.add_wrapped(wrap_pyfunction!(list_extract))?; + m.add_wrapped(wrap_pyfunction!(array_length))?; + m.add_wrapped(wrap_pyfunction!(list_length))?; + Ok(()) } From 967bbffb93123df0109d82f3a4b36ca0528d4fdb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 20:04:11 -0800 Subject: [PATCH 107/413] build(deps): bump syn from 2.0.41 to 2.0.43 (#559) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.41 to 2.0.43. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.41...2.0.43) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 46 +++++++++++++++++++++++----------------------- Cargo.toml | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db776d5bd..c8b82d61e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,7 +373,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -384,7 +384,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -925,7 +925,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.41", + "syn 2.0.43", "tokio", "url", "uuid", @@ -1119,7 +1119,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2001,7 +2001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2053,7 +2053,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2120,7 +2120,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2132,7 +2132,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2455,7 +2455,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2489,7 +2489,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2649,7 +2649,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2670,7 +2670,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.41", + "syn 2.0.43", "typify", "walkdir", ] @@ -2694,9 +2694,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.41" +version = "2.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c8b28c477cc3bf0e7966561e3460130e1255f7a1cf71931075f1c5e7a7e269" +checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" dependencies = [ "proc-macro2", "quote", @@ -2760,7 +2760,7 @@ checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2824,7 +2824,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2876,7 +2876,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2921,7 +2921,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] @@ -2953,7 +2953,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.41", + "syn 2.0.43", "thiserror", "unicode-ident", ] @@ -2970,7 +2970,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.41", + "syn 2.0.43", "typify-impl", ] @@ -3104,7 +3104,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", "wasm-bindgen-shared", ] @@ -3138,7 +3138,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3398,7 +3398,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.41", + "syn 2.0.43", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 08f566d2d..0c4e3853d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ futures = "0.3" object_store = { version = "0.8.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" -syn = "2.0.37" +syn = "2.0.43" url = "2.2" [build-dependencies] From 3f8e862508194e1db8e6f893bf5c67cab457fef0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 20:04:26 -0800 Subject: [PATCH 108/413] build(deps): bump tokio from 1.35.0 to 1.35.1 (#558) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.35.0 to 1.35.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.35.0...tokio-1.35.1) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c8b82d61e..640d1c56d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2800,9 +2800,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.0" +version = "1.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d45b238a16291a4e1584e61820b8ae57d696cc5015c459c229ccc6990cc1c" +checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 0c4e3853d..d0a1b8b1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ protoc = [ "datafusion-substrait/protoc" ] substrait = ["dep:datafusion-substrait"] [dependencies] -tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] } +tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } datafusion = { version = "34.0.0", features = ["pyarrow", "avro"] } From 767d43e05d9e398df713bfc0545635f3b048ca17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 21:15:41 -0800 Subject: [PATCH 109/413] build(deps): bump async-trait from 0.1.74 to 0.1.77 (#556) Bumps [async-trait](https://github.com/dtolnay/async-trait) from 0.1.74 to 0.1.77. - [Release notes](https://github.com/dtolnay/async-trait/releases) - [Commits](https://github.com/dtolnay/async-trait/commits) --- updated-dependencies: - dependency-name: async-trait dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 640d1c56d..e3bad1a38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -373,18 +373,18 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] name = "async-trait" -version = "0.1.74" +version = "0.1.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -925,7 +925,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.43", + "syn 2.0.48", "tokio", "url", "uuid", @@ -1119,7 +1119,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2001,14 +2001,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] name = "proc-macro2" -version = "1.0.70" +version = "1.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" +checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" dependencies = [ "unicode-ident", ] @@ -2053,7 +2053,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2120,7 +2120,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2132,7 +2132,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2153,9 +2153,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.33" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] @@ -2455,7 +2455,7 @@ checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2489,7 +2489,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2649,7 +2649,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2670,7 +2670,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.43", + "syn 2.0.48", "typify", "walkdir", ] @@ -2694,9 +2694,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.43" +version = "2.0.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee659fb5f3d355364e1f3e5bc10fb82068efbf824a1e9d1c9504244a6469ad53" +checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" dependencies = [ "proc-macro2", "quote", @@ -2760,7 +2760,7 @@ checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2824,7 +2824,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2876,7 +2876,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2921,7 +2921,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] @@ -2953,7 +2953,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.43", + "syn 2.0.48", "thiserror", "unicode-ident", ] @@ -2970,7 +2970,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.43", + "syn 2.0.48", "typify-impl", ] @@ -3104,7 +3104,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", "wasm-bindgen-shared", ] @@ -3138,7 +3138,7 @@ checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3398,7 +3398,7 @@ checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.43", + "syn 2.0.48", ] [[package]] From 7de4316b70dd1b2aeed68bfee293d95a2cc384b5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 23:09:01 -0800 Subject: [PATCH 110/413] build(deps): bump pyo3 from 0.20.0 to 0.20.2 (#557) Bumps [pyo3](https://github.com/pyo3/pyo3) from 0.20.0 to 0.20.2. - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/compare/v0.20.0...v0.20.2) --- updated-dependencies: - dependency-name: pyo3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3bad1a38..e2ed3d3f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2076,9 +2076,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e8453b658fe480c3e70c8ed4e3d3ec33eb74988bd186561b0cc66b85c3bc4b" +checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" dependencies = [ "cfg-if", "indoc", @@ -2093,9 +2093,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a96fe70b176a89cff78f2fa7b3c930081e163d5379b4dcdf993e3ae29ca662e5" +checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" dependencies = [ "once_cell", "target-lexicon", @@ -2103,9 +2103,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "214929900fd25e6604661ed9cf349727c8920d47deff196c4e28165a6ef2a96b" +checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" dependencies = [ "libc", "pyo3-build-config", @@ -2113,9 +2113,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dac53072f717aa1bfa4db832b39de8c875b7c7af4f4a6fe93cdbf9264cf8383b" +checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2125,9 +2125,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7774b5a8282bd4f25f803b1f0d945120be959a36c72e08e7cd031c792fdfd424" +checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" dependencies = [ "heck", "proc-macro2", From bef6cb66599588c096dae59ddfd707053e5741cd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 1 Feb 2024 05:59:02 -0700 Subject: [PATCH 111/413] Prepare 35.0.0-rc1 (#564) --- CHANGELOG.md | 9 + Cargo.lock | 442 +++++++++++++++++++---------------- Cargo.toml | 16 +- datafusion/tests/test_sql.py | 3 +- src/common/data_type.rs | 1 + src/context.rs | 7 +- src/expr/window.rs | 6 +- src/functions.rs | 11 +- 8 files changed, 274 insertions(+), 221 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4af70d568..d5e3abbe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,15 @@ # DataFusion Python Changelog +## [35.0.0](https://github.com/apache/arrow-datafusion-python/tree/35.0.0) (2024-01-20) + +**Merged pull requests:** + +- build(deps): bump syn from 2.0.41 to 2.0.43 [#559](https://github.com/apache/arrow-datafusion-python/pull/559) (dependabot[bot]) +- build(deps): bump tokio from 1.35.0 to 1.35.1 [#558](https://github.com/apache/arrow-datafusion-python/pull/558) (dependabot[bot]) +- build(deps): bump async-trait from 0.1.74 to 0.1.77 [#556](https://github.com/apache/arrow-datafusion-python/pull/556) (dependabot[bot]) +- build(deps): bump pyo3 from 0.20.0 to 0.20.2 [#557](https://github.com/apache/arrow-datafusion-python/pull/557) (dependabot[bot]) + ## [34.0.0](https://github.com/apache/arrow-datafusion-python/tree/34.0.0) (2023-12-28) **Merged pull requests:** diff --git a/Cargo.lock b/Cargo.lock index e2ed3d3f4..e6aa6050a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" dependencies = [ "cfg-if", "const-random", @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.75" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" [[package]] name = "apache-avro" @@ -130,11 +130,10 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc25126d18a012146a888a0298f2c22e1150327bd2765fc76d710a556b2d614" +checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" dependencies = [ - "ahash", "arrow-arith", "arrow-array", "arrow-buffer", @@ -153,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ccd45e217ffa6e53bbb0080990e77113bdd4e91ddb84e97b77649810bcf1a7" +checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" dependencies = [ "arrow-array", "arrow-buffer", @@ -168,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d" +checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ "ahash", "arrow-buffer", @@ -185,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c" +checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" dependencies = [ "bytes", "half", @@ -196,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc0368ed618d509636c1e3cc20db1281148190a78f43519487b2daf07b63b4a" +checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" dependencies = [ "arrow-array", "arrow-buffer", @@ -215,9 +214,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e09aa6246a1d6459b3f14baeaa49606cfdbca34435c46320e14054d244987ca" +checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" dependencies = [ "arrow-array", "arrow-buffer", @@ -234,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634" +checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" dependencies = [ "arrow-buffer", "arrow-schema", @@ -246,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a43d6808411886b8c7d4f6f7dd477029c1e77ffffffb7923555cc6579639cd" +checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" dependencies = [ "arrow-array", "arrow-buffer", @@ -256,13 +255,14 @@ dependencies = [ "arrow-data", "arrow-schema", "flatbuffers", + "lz4_flex", ] [[package]] name = "arrow-json" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82565c91fd627922ebfe2810ee4e8346841b6f9361b87505a9acea38b614fee" +checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b23b0e53c0db57c6749997fd343d4c0354c994be7eca67152dd2bdb9a3e1bb4" +checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" dependencies = [ "arrow-array", "arrow-buffer", @@ -295,9 +295,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "361249898d2d6d4a6eeb7484be6ac74977e48da12a4dd81a708d620cc558117a" +checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ "ahash", "arrow-array", @@ -310,18 +310,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167" +checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", ] [[package]] name = "arrow-select" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f6208466590960efc1d2a7172bc4ff18a67d6e25c529381d7f96ddaf0dc4036" +checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ "ahash", "arrow-array", @@ -333,9 +333,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a48149c63c11c9ff571e50ab8f017d2a7cb71037a882b42f6354ed2da9acc7" +checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -349,9 +349,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc2d0cfb2a7388d34f590e76686704c494ed7aaceed62ee1ba35cbf363abc2a5" +checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" dependencies = [ "bzip2", "flate2", @@ -419,9 +419,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.5" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" [[package]] name = "bitflags" @@ -431,9 +431,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "blake2" @@ -557,9 +557,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e23185c0e21df6ed832a12e2bda87c7d1def6842881fb634a8511ced741b0d76" +checksum = "91d7b79e99bfaa0d47da0687c43aa3b7381938a62ad3a6498599039321f660b7" dependencies = [ "chrono", "chrono-tz-build", @@ -641,9 +641,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] @@ -715,14 +715,15 @@ dependencies = [ [[package]] name = "datafusion" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193fd1e7628278d0641c5122860f9a7fd6a1d77d055838d12f55d15bbe28d4d0" +checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" dependencies = [ "ahash", "apache-avro", "arrow", "arrow-array", + "arrow-ipc", "arrow-schema", "async-compression", "async-trait", @@ -764,9 +765,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "548bc49c4a489e3de474813831ea556dc9d368f9ed8d867b1493da42e8e9f613" +checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" dependencies = [ "ahash", "apache-avro", @@ -786,9 +787,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc865657ffcf4da5ff08bdc6436a9a833bc0aa96c3254c8d18ab8a0ad4e437d" +checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" dependencies = [ "arrow", "chrono", @@ -807,9 +808,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c473f72d8d81a532e63f6e562ed66dd9209dfd8e433d9712abd42444ee161e" +checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" dependencies = [ "ahash", "arrow", @@ -823,9 +824,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6218318001d2f6783b7fffa17592318f65f26609d7aab605a3dd0c7c2e2618" +checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" dependencies = [ "arrow", "async-trait", @@ -841,9 +842,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1ca7e35ca22f9dc506c2375b92054b03ccf91afe25c0a90b395a1473a09735" +checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" dependencies = [ "ahash", "arrow", @@ -875,9 +876,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddde97adefcca3a55257c646ffee2a95b6cac66f74d1146a6e3a6dbb37830631" +checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" dependencies = [ "ahash", "arrow", @@ -906,7 +907,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "34.0.0" +version = "35.0.0" dependencies = [ "async-trait", "datafusion", @@ -933,9 +934,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60d9d6460a64fddb8663db41da97e6b8b0bf79da42f997ebe81722731eaf0e5" +checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" dependencies = [ "arrow", "arrow-schema", @@ -947,9 +948,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "34.0.0" +version = "35.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2243428e8df8b2c5ce2c9e3af588312ae89c1b073c52de5693ec44ba29674547" +checksum = "dad6bef77af3d8a697ae63ffbcb5aa66b74cd08ea93a31e2e757da75b2f1452f" dependencies = [ "async-recursion", "chrono", @@ -959,7 +960,6 @@ dependencies = [ "prost", "prost-types", "substrait", - "tokio", ] [[package]] @@ -1065,9 +1065,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -1080,9 +1080,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -1090,15 +1090,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1107,15 +1107,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1124,21 +1124,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1164,9 +1164,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" +checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" dependencies = [ "cfg-if", "libc", @@ -1185,7 +1185,7 @@ version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "libc", "libgit2-sys", "log", @@ -1200,9 +1200,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.22" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -1255,9 +1255,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" +checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" [[package]] name = "hex" @@ -1316,9 +1316,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" dependencies = [ "bytes", "futures-channel", @@ -1331,7 +1331,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2", "tokio", "tower-service", "tracing", @@ -1354,9 +1354,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1448,9 +1448,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee9c64da59eae3b50095c18d3e74f8b73c0b86d2792824ff01bbce68ba229ca" +checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" dependencies = [ "wasm-bindgen", ] @@ -1527,9 +1527,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libflate" @@ -1585,9 +1585,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.12" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97137b25e321a73eef1418d1d5d2eda4d77e12813f8e6dead84bc52c5870a7b" +checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" dependencies = [ "cc", "libc", @@ -1597,9 +1597,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4cd1a83af159aa67994778be9070f0ae1bd732942279cabb14f86f986a21456" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "lock_api" @@ -1619,9 +1619,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "lz4_flex" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" dependencies = [ "twox-hash", ] @@ -1649,9 +1649,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" @@ -1792,18 +1792,18 @@ dependencies = [ [[package]] name = "object" -version = "0.32.1" +version = "0.32.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2524735495ea1268be33d200e1ee97455096a0846295a21548cd2f3541de7050" +checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" dependencies = [ "async-trait", "base64", @@ -1812,14 +1812,14 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.11.0", + "itertools 0.12.0", "parking_lot", "percent-encoding", "quick-xml", "rand", "reqwest", "ring", - "rustls-pemfile", + "rustls-pemfile 2.0.0", "serde", "serde_json", "snafu", @@ -1835,6 +1835,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "ordered-float" version = "2.10.1" @@ -1869,9 +1875,9 @@ dependencies = [ [[package]] name = "parquet" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4" +checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ "ahash", "arrow-array", @@ -1887,6 +1893,7 @@ dependencies = [ "chrono", "flate2", "futures", + "half", "hashbrown 0.14.3", "lz4_flex", "num", @@ -1984,9 +1991,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" [[package]] name = "ppv-lite86" @@ -1996,9 +2003,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", "syn 2.0.48", @@ -2246,9 +2253,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.22" +version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046cd98826c46c2ac8ddecae268eb5c2e58628688a5fc7a2643704a73faba95b" +checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ "base64", "bytes", @@ -2268,7 +2275,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile", + "rustls-native-certs", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -2282,7 +2290,6 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", "winreg", ] @@ -2323,11 +2330,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", @@ -2346,6 +2353,18 @@ dependencies = [ "sct", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -2355,6 +2374,22 @@ dependencies = [ "base64", ] +[[package]] +name = "rustls-pemfile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -2386,6 +2421,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "schemars" version = "0.8.16" @@ -2426,11 +2470,34 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" [[package]] name = "seq-macro" @@ -2440,18 +2507,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" +checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.193" +version = "1.0.195" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" +checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" dependencies = [ "proc-macro2", "quote", @@ -2471,9 +2538,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" dependencies = [ "itoa", "ryu", @@ -2506,9 +2573,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.27" +version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" +checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ "indexmap", "itoa", @@ -2545,9 +2612,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.2" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" [[package]] name = "snafu" @@ -2577,16 +2644,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" -[[package]] -name = "socket2" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "socket2" version = "0.5.5" @@ -2605,9 +2662,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.40.0" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c80afe31cdb649e56c0d9bb5503be9166600d68a852c38dd445636d126858e5" +checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" dependencies = [ "log", "sqlparser_derive", @@ -2615,13 +2672,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e9c2e1dde0efa87003e7923d94a90f46e3274ad1649f51de96812be561f041f" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] @@ -2654,9 +2711,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.20.3" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdbe404d89610b54be889fb6540bcc6a9e234d0991cde0a8a9e341559768aa75" +checksum = "5478fbd0313a9b0915a1c0e7ebf15b5fed7d7c6dd7229b4f5e32ce75b10f256a" dependencies = [ "git2", "heck", @@ -2726,37 +2783,37 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.12" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c39fd04924ca3a864207c66fc2cd7d22d7c016007f9ce846cbb9326331930a" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11c217e1416d6f036b870f14e0413d480dbf28edbee1f877abaf0206af43bb7" +checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.51" +version = "1.0.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01742297787513b79cf8e29d1056ede1313e2420b7b3b15d0a768b4921f549df" +checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" dependencies = [ "proc-macro2", "quote", @@ -2809,9 +2866,8 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", "pin-project-lite", - "socket2 0.5.5", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] @@ -2976,9 +3032,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" @@ -3038,9 +3094,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", "serde", @@ -3085,9 +3141,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed0d4f68a3015cc185aff4db9506a015f4b96f95303897bfa23f846db54064e" +checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3095,9 +3151,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b56f625e64f3a1084ded111c4d5f477df9f8c92df113852fa5a374dbda78826" +checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" dependencies = [ "bumpalo", "log", @@ -3110,9 +3166,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac36a15a220124ac510204aec1c3e5db8a22ab06fd6706d881dc6149f8ed9a12" +checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" dependencies = [ "cfg-if", "js-sys", @@ -3122,9 +3178,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0162dbf37223cd2afce98f3d0785506dcb8d266223983e4b5b525859e6e182b2" +checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3132,9 +3188,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0eb82fcb7930ae6219a7ecfd55b217f5f0893484b7a13022ebb2b2bf20b5283" +checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", @@ -3145,9 +3201,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.89" +version = "0.2.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab9b36309365056cd639da3134bf87fa8f3d86008abf99e612384a6eecd459f" +checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" [[package]] name = "wasm-streams" @@ -3164,20 +3220,14 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.66" +version = "0.3.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c24a44ec86bb68fbecd1b3efed7e85ea5621b39b35ef2766b66cd984f8010f" +checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "webpki-roots" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" - [[package]] name = "which" version = "4.4.2" @@ -3223,11 +3273,11 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.51.1" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -3383,18 +3433,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c4061bedbb353041c12f413700357bec76df2c7e2ca8e4df8bac24c6bf68e3d" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.31" +version = "0.7.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3c129550b3e6de3fd0ba67ba5c81818f9805e58b8d7fee80a3a59d2c9fc601a" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index d0a1b8b1b..6c47484b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "34.0.0" +version = "35.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -37,19 +37,19 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "34.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "34.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "34.0.0" } -datafusion-optimizer = { version = "34.0.0" } -datafusion-sql = { version = "34.0.0" } -datafusion-substrait = { version = "34.0.0", optional = true } +datafusion = { version = "35.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "35.0.0", features = ["pyarrow"] } +datafusion-expr = "35.0.0" +datafusion-optimizer = "35.0.0" +datafusion-sql = "35.0.0" +datafusion-substrait = { version = "35.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.8.0", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.9.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" syn = "2.0.43" diff --git a/datafusion/tests/test_sql.py b/datafusion/tests/test_sql.py index 19a2ad2cf..0c6b26727 100644 --- a/datafusion/tests/test_sql.py +++ b/datafusion/tests/test_sql.py @@ -195,7 +195,7 @@ def test_register_json(ctx, tmp_path): assert result.to_pydict() == {"cnt": [3]} result = ctx.sql("SELECT * FROM json3").collect() - result = pa.Table.from_batches(result) + result = pa.Table.from_batches(result, alternative_schema) assert result.schema == alternative_schema with pytest.raises( @@ -224,7 +224,6 @@ def test_register_avro(ctx): "alltypes_plain_schema", path, schema=alternative_schema, - infinite=False, ) result = ctx.sql("SELECT * FROM alltypes_plain_schema").collect() result = pa.Table.from_batches(result) diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 03248cc5b..6059768e3 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::arrow::array::Array; use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; use datafusion_common::{DataFusionError, ScalarValue}; use pyo3::{exceptions::PyValueError, prelude::*}; diff --git a/src/context.rs b/src/context.rs index 63c48f007..9053e4f2e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -569,8 +569,7 @@ impl PySessionContext { path, schema=None, file_extension=".avro", - table_partition_cols=vec![], - infinite=false))] + table_partition_cols=vec![]))] pub fn register_avro( &mut self, name: &str, @@ -578,7 +577,6 @@ impl PySessionContext { schema: Option>, file_extension: &str, table_partition_cols: Vec<(String, String)>, - infinite: bool, py: Python, ) -> PyResult<()> { let path = path @@ -586,8 +584,7 @@ impl PySessionContext { .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; let mut options = AvroReadOptions::default() - .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) - .mark_infinite(infinite); + .table_partition_cols(convert_table_partition_cols(table_partition_cols)?); options.file_extension = file_extension; options.schema = schema.as_ref().map(|x| &x.0); diff --git a/src/expr/window.rs b/src/expr/window.rs index 3a5760c15..786651194 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -210,11 +210,7 @@ impl PyWindowFrame { }, }; Ok(PyWindowFrame { - window_frame: WindowFrame { - units, - start_bound, - end_bound, - }, + window_frame: WindowFrame::new_bounds(units, start_bound, end_bound), }) } diff --git a/src/functions.rs b/src/functions.rs index 3dc5322aa..045e7e01e 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -27,10 +27,11 @@ use datafusion_common::{Column, TableReference}; use datafusion_expr::expr::Alias; use datafusion_expr::{ aggregate_function, - expr::{AggregateFunction, AggregateFunctionDefinition, ScalarFunction, Sort, WindowFunction}, - lit, - window_function::find_df_window_func, - BuiltinScalarFunction, Expr, + expr::{ + find_df_window_func, AggregateFunction, AggregateFunctionDefinition, ScalarFunction, Sort, + WindowFunction, + }, + lit, BuiltinScalarFunction, Expr, WindowFunctionDefinition, }; #[pyfunction] @@ -143,7 +144,7 @@ fn window( ctx.and_then(|ctx| { ctx.ctx .udaf(name) - .map(datafusion_expr::WindowFunction::AggregateUDF) + .map(WindowFunctionDefinition::AggregateUDF) .ok() }) }); From 5296c0cfcf8e6fcb654d5935252469bf04f929e9 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 10 Feb 2024 10:23:30 -0700 Subject: [PATCH 112/413] Allow PyDataFrame to be used from other projects (#582) * Allow PyDataFrame to be used from other projects * revert --- src/context.rs | 4 ++-- src/lib.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/context.rs b/src/context.rs index 9053e4f2e..f34fbce81 100644 --- a/src/context.rs +++ b/src/context.rs @@ -842,7 +842,7 @@ impl PySessionContext { } } -fn convert_table_partition_cols( +pub fn convert_table_partition_cols( table_partition_cols: Vec<(String, String)>, ) -> Result, DataFusionError> { table_partition_cols @@ -856,7 +856,7 @@ fn convert_table_partition_cols( .collect::, _>>() } -fn parse_file_compression_type( +pub fn parse_file_compression_type( file_compression_type: Option, ) -> Result { FileCompressionType::from_str(&*file_compression_type.unwrap_or("".to_string()).as_str()) diff --git a/src/lib.rs b/src/lib.rs index 5e57db9cf..49c325a53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -37,7 +37,7 @@ mod config; #[allow(clippy::borrow_deref_ref)] pub mod context; #[allow(clippy::borrow_deref_ref)] -mod dataframe; +pub mod dataframe; mod dataset; mod dataset_exec; pub mod errors; From 476ca228cfcf75012a73932c4fb1fb53271c1e8a Mon Sep 17 00:00:00 2001 From: Dejan Simic <10134699+simicd@users.noreply.github.com> Date: Sun, 11 Feb 2024 20:34:04 +0100 Subject: [PATCH 113/413] docs: Add ASF attribution (#580) * Add ASF attribution and update docs README * Fix formatting * Fix unit test * Undo changes in functions & tests and requirements.txt --- docs/README.md | 45 ++++++++++++++++-------- docs/requirements.txt | 2 +- docs/source/_templates/docs-sidebar.html | 2 +- docs/source/_templates/layout.html | 21 +++++++++++ docs/source/conf.py | 6 ++-- 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/docs/README.md b/docs/README.md index 85278588e..8cb101d92 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,15 +28,31 @@ when changes are merged to the main branch. It's recommended to install build dependencies and build the documentation inside a Python `venv`. -```bash -python -m pip install -r requirements-310.txt -``` +To prepare building the documentation run the following on the root level of the project: + +1. Set up virtual environment if it was not already created + ```bash + python3 -m venv venv + ``` +1. Activate virtual environment + ```bash + source venv/bin/activate + ``` +1. Install Datafusion's Python dependencies + ```bash + pip install -r requirements-310.txt + ``` +1. Install documentation dependencies + ```bash + pip install -r docs/requirements.txt + ``` ## Build & Preview Run the provided script to build the HTML pages. ```bash +cd docs ./build.sh ``` @@ -50,14 +66,15 @@ firefox build/html/index.html ## Release Process -The documentation is served through the -[arrow-site](https://github.com/apache/arrow-site/) repo. To release a new -version of the docs, follow these steps: - -1. Run `./build.sh` inside `docs` folder to generate the docs website inside the `build/html` folder. -2. Clone the arrow-site repo -3. Checkout to the `asf-site` branch (NOT `master`) -4. Copy build artifacts into `arrow-site` repo's `datafusion` folder with a command such as - - `cp -rT ./build/html/ ../../arrow-site/datafusion/` (doesn't work on mac) - - `rsync -avzr ./build/html/ ../../arrow-site/datafusion/` -5. Commit changes in `arrow-site` and send a PR. +This documentation is hosted at https://arrow.apache.org/datafusion-python/ + +When the PR is merged to the `main` branch of the DataFusion +repository, a [github workflow](https://github.com/apache/arrow-datafusion-python/blob/main/.github/workflows/docs.yaml) which: + +1. Builds the html content +2. Pushes the html content to the [`asf-site`](https://github.com/apache/arrow-datafusion-python/tree/asf-site) branch in this repository. + +The Apache Software Foundation provides https://arrow.apache.org/, +which serves content based on the configuration in +[.asf.yaml](https://github.com/apache/arrow-datafusion-python/blob/main/.asf.yaml), +which specifies the target as https://arrow.apache.org/datafusion-python/. \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt index eb7f5b7bb..67f1ec6ac 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -sphinx==5.3.0 +sphinx pydata-sphinx-theme==0.8.0 myst-parser maturin diff --git a/docs/source/_templates/docs-sidebar.html b/docs/source/_templates/docs-sidebar.html index bc2bf0092..6541b7713 100644 --- a/docs/source/_templates/docs-sidebar.html +++ b/docs/source/_templates/docs-sidebar.html @@ -1,6 +1,6 @@ - +
diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html index a9d0f30bc..9f7880049 100644 --- a/docs/source/_templates/layout.html +++ b/docs/source/_templates/layout.html @@ -3,3 +3,24 @@ {# Silence the navbar #} {% block docs_navbar %} {% endblock %} + + +{% block footer %} + +
+
+ {% for footer_item in theme_footer_items %} +
+ {% include footer_item %} +
+ {% endfor %} +
+

Apache Arrow DataFusion, Arrow DataFusion, Apache, the Apache feather logo, and the Apache Arrow DataFusion project logo

+

are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries.

+
+
+
+ +{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py index 0822e0ab6..a6beee5ba 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -33,9 +33,9 @@ # -- Project information ----------------------------------------------------- -project = "Arrow DataFusion" -copyright = "2022, Apache Software Foundation" -author = "Arrow DataFusion Authors" +project = "Apache Arrow DataFusion" +copyright = "2019-2024, Apache Software Foundation" +author = "Apache Software Foundation" # -- General configuration --------------------------------------------------- From 697ca2c4d8a5b02cddcf5108edcae26cd76fe823 Mon Sep 17 00:00:00 2001 From: Chih Wang Date: Mon, 12 Feb 2024 23:26:49 +0800 Subject: [PATCH 114/413] Add array functions (#560) * Add array_has, array_has_all and array_has_any * Add array_position, array_indexof, list_position and list_indexof * Add array_to_string, array_join, list_to_string and list_join * Add array_ndims and list_ndims * Add array_push_back, list_append and list_push_back * Add array_prepend, array_push_front, list_prepend and list_push_front * Add array_pop_back and array_pop_front * Add array_positions and list_positions * Add array_remove, list_remove, array_remove_n, list_remove_n, array_remove_all and list_remove_all * Add array_repeat * Add array_replace, list_replace, array_replace_n, list_replace_n, array_replace_all, list_replace_all * Add array_slice and list_slice --- datafusion/tests/test_functions.py | 211 ++++++++++++++++++++++++++++- src/functions.rs | 78 +++++++++++ 2 files changed, 286 insertions(+), 3 deletions(-) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index d0514f892..7e772589b 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -200,19 +200,62 @@ def test_math_functions(): def test_array_functions(): - data = [[1.0, 2.0, 3.0], [4.0, 5.0], [6.0]] + data = [[1.0, 2.0, 3.0, 3.0], [4.0, 5.0, 3.0], [6.0]] ctx = SessionContext() batch = pa.RecordBatch.from_arrays( [np.array(data, dtype=object)], names=["arr"] ) df = ctx.create_dataframe([[batch]]) + def py_indexof(arr, v): + try: + return arr.index(v) + 1 + except ValueError: + return np.nan + + def py_arr_remove(arr, v, n=None): + new_arr = arr[:] + found = 0 + while found != n: + try: + new_arr.remove(v) + found += 1 + except ValueError: + break + + return new_arr + + def py_arr_replace(arr, from_, to, n=None): + new_arr = arr[:] + found = 0 + while found != n: + try: + idx = new_arr.index(from_) + new_arr[idx] = to + found += 1 + except ValueError: + break + + return new_arr + col = column("arr") test_items = [ [ f.array_append(col, literal(99.0)), lambda: [np.append(arr, 99.0) for arr in data], ], + [ + f.array_push_back(col, literal(99.0)), + lambda: [np.append(arr, 99.0) for arr in data], + ], + [ + f.list_append(col, literal(99.0)), + lambda: [np.append(arr, 99.0) for arr in data], + ], + [ + f.list_push_back(col, literal(99.0)), + lambda: [np.append(arr, 99.0) for arr in data], + ], [ f.array_concat(col, col), lambda: [np.concatenate([arr, arr]) for arr in data], @@ -253,12 +296,174 @@ def test_array_functions(): f.list_length(col), lambda: [len(r) for r in data], ], + [ + f.array_has(col, literal(1.0)), + lambda: [1.0 in r for r in data], + ], + [ + f.array_has_all( + col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) + ), + lambda: [np.all([v in r for v in [1.0, 3.0, 5.0]]) for r in data], + ], + [ + f.array_has_any( + col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) + ), + lambda: [np.any([v in r for v in [1.0, 3.0, 5.0]]) for r in data], + ], + [ + f.array_position(col, literal(1.0)), + lambda: [py_indexof(r, 1.0) for r in data], + ], + [ + f.array_indexof(col, literal(1.0)), + lambda: [py_indexof(r, 1.0) for r in data], + ], + [ + f.list_position(col, literal(1.0)), + lambda: [py_indexof(r, 1.0) for r in data], + ], + [ + f.list_indexof(col, literal(1.0)), + lambda: [py_indexof(r, 1.0) for r in data], + ], + [ + f.array_positions(col, literal(1.0)), + lambda: [ + [i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data + ], + ], + [ + f.list_positions(col, literal(1.0)), + lambda: [ + [i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data + ], + ], + [ + f.array_ndims(col), + lambda: [np.array(r).ndim for r in data], + ], + [ + f.list_ndims(col), + lambda: [np.array(r).ndim for r in data], + ], + [ + f.array_prepend(literal(99.0), col), + lambda: [np.insert(arr, 0, 99.0) for arr in data], + ], + [ + f.array_push_front(literal(99.0), col), + lambda: [np.insert(arr, 0, 99.0) for arr in data], + ], + [ + f.list_prepend(literal(99.0), col), + lambda: [np.insert(arr, 0, 99.0) for arr in data], + ], + [ + f.list_push_front(literal(99.0), col), + lambda: [np.insert(arr, 0, 99.0) for arr in data], + ], + [ + f.array_pop_back(col), + lambda: [arr[:-1] for arr in data], + ], + [ + f.array_pop_front(col), + lambda: [arr[1:] for arr in data], + ], + [ + f.array_remove(col, literal(3.0)), + lambda: [py_arr_remove(arr, 3.0, 1) for arr in data], + ], + [ + f.list_remove(col, literal(3.0)), + lambda: [py_arr_remove(arr, 3.0, 1) for arr in data], + ], + [ + f.array_remove_n(col, literal(3.0), literal(2)), + lambda: [py_arr_remove(arr, 3.0, 2) for arr in data], + ], + [ + f.list_remove_n(col, literal(3.0), literal(2)), + lambda: [py_arr_remove(arr, 3.0, 2) for arr in data], + ], + [ + f.array_remove_all(col, literal(3.0)), + lambda: [py_arr_remove(arr, 3.0) for arr in data], + ], + [ + f.list_remove_all(col, literal(3.0)), + lambda: [py_arr_remove(arr, 3.0) for arr in data], + ], + [ + f.array_repeat(col, literal(2)), + lambda: [[arr] * 2 for arr in data], + ], + [ + f.array_replace(col, literal(3.0), literal(4.0)), + lambda: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], + ], + [ + f.list_replace(col, literal(3.0), literal(4.0)), + lambda: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], + ], + [ + f.array_replace_n(col, literal(3.0), literal(4.0), literal(1)), + lambda: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], + ], + [ + f.list_replace_n(col, literal(3.0), literal(4.0), literal(2)), + lambda: [py_arr_replace(arr, 3.0, 4.0, 2) for arr in data], + ], + [ + f.array_replace_all(col, literal(3.0), literal(4.0)), + lambda: [py_arr_replace(arr, 3.0, 4.0) for arr in data], + ], + [ + f.list_replace_all(col, literal(3.0), literal(4.0)), + lambda: [py_arr_replace(arr, 3.0, 4.0) for arr in data], + ], + [ + f.array_slice(col, literal(2), literal(4)), + lambda: [arr[1:4] for arr in data], + ], + [ + f.list_slice(col, literal(-1), literal(2)), + lambda: [arr[-1:2] for arr in data], + ], ] for stmt, py_expr in test_items: - query_result = df.select(stmt).collect()[0].column(0).tolist() + query_result = df.select(stmt).collect()[0].column(0) + for a, b in zip(query_result, py_expr()): + np.testing.assert_array_almost_equal( + np.array(a.as_py(), dtype=float), np.array(b, dtype=float) + ) + + obj_test_items = [ + [ + f.array_to_string(col, literal(",")), + lambda: [",".join([str(int(v)) for v in r]) for r in data], + ], + [ + f.array_join(col, literal(",")), + lambda: [",".join([str(int(v)) for v in r]) for r in data], + ], + [ + f.list_to_string(col, literal(",")), + lambda: [",".join([str(int(v)) for v in r]) for r in data], + ], + [ + f.list_join(col, literal(",")), + lambda: [",".join([str(int(v)) for v in r]) for r in data], + ], + ] + + for stmt, py_expr in obj_test_items: + query_result = np.array(df.select(stmt).collect()[0].column(0)) for a, b in zip(query_result, py_expr()): - np.testing.assert_array_almost_equal(a, b) + assert a == b def test_string_functions(df): diff --git a/src/functions.rs b/src/functions.rs index 045e7e01e..bb204c35e 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -360,6 +360,9 @@ scalar_function!(decode, Decode); // Array Functions scalar_function!(array_append, ArrayAppend); +scalar_function!(array_push_back, ArrayAppend); +scalar_function!(list_append, ArrayAppend); +scalar_function!(list_push_back, ArrayAppend); scalar_function!(array_concat, ArrayConcat); scalar_function!(array_cat, ArrayConcat); scalar_function!(array_dims, ArrayDims); @@ -370,6 +373,42 @@ scalar_function!(list_element, ArrayElement); scalar_function!(list_extract, ArrayElement); scalar_function!(array_length, ArrayLength); scalar_function!(list_length, ArrayLength); +scalar_function!(array_has, ArrayHas); +scalar_function!(array_has_all, ArrayHasAll); +scalar_function!(array_has_any, ArrayHasAny); +scalar_function!(array_position, ArrayPosition); +scalar_function!(array_indexof, ArrayPosition); +scalar_function!(list_position, ArrayPosition); +scalar_function!(list_indexof, ArrayPosition); +scalar_function!(array_positions, ArrayPositions); +scalar_function!(list_positions, ArrayPositions); +scalar_function!(array_to_string, ArrayToString); +scalar_function!(array_join, ArrayToString); +scalar_function!(list_to_string, ArrayToString); +scalar_function!(list_join, ArrayToString); +scalar_function!(array_ndims, ArrayNdims); +scalar_function!(list_ndims, ArrayNdims); +scalar_function!(array_prepend, ArrayPrepend); +scalar_function!(array_push_front, ArrayPrepend); +scalar_function!(list_prepend, ArrayPrepend); +scalar_function!(list_push_front, ArrayPrepend); +scalar_function!(array_pop_back, ArrayPopBack); +scalar_function!(array_pop_front, ArrayPopFront); +scalar_function!(array_remove, ArrayRemove); +scalar_function!(list_remove, ArrayRemove); +scalar_function!(array_remove_n, ArrayRemoveN); +scalar_function!(list_remove_n, ArrayRemoveN); +scalar_function!(array_remove_all, ArrayRemoveAll); +scalar_function!(list_remove_all, ArrayRemoveAll); +scalar_function!(array_repeat, ArrayRepeat); +scalar_function!(array_replace, ArrayReplace); +scalar_function!(list_replace, ArrayReplace); +scalar_function!(array_replace_n, ArrayReplaceN); +scalar_function!(list_replace_n, ArrayReplaceN); +scalar_function!(array_replace_all, ArrayReplaceAll); +scalar_function!(list_replace_all, ArrayReplaceAll); +scalar_function!(array_slice, ArraySlice); +scalar_function!(list_slice, ArraySlice); aggregate_function!(approx_distinct, ApproxDistinct); aggregate_function!(approx_median, ApproxMedian); @@ -563,6 +602,9 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { // Array Functions m.add_wrapped(wrap_pyfunction!(array_append))?; + m.add_wrapped(wrap_pyfunction!(array_push_back))?; + m.add_wrapped(wrap_pyfunction!(list_append))?; + m.add_wrapped(wrap_pyfunction!(list_push_back))?; m.add_wrapped(wrap_pyfunction!(array_concat))?; m.add_wrapped(wrap_pyfunction!(array_cat))?; m.add_wrapped(wrap_pyfunction!(array_dims))?; @@ -573,6 +615,42 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(list_extract))?; m.add_wrapped(wrap_pyfunction!(array_length))?; m.add_wrapped(wrap_pyfunction!(list_length))?; + m.add_wrapped(wrap_pyfunction!(array_has))?; + m.add_wrapped(wrap_pyfunction!(array_has_all))?; + m.add_wrapped(wrap_pyfunction!(array_has_any))?; + m.add_wrapped(wrap_pyfunction!(array_position))?; + m.add_wrapped(wrap_pyfunction!(array_indexof))?; + m.add_wrapped(wrap_pyfunction!(list_position))?; + m.add_wrapped(wrap_pyfunction!(list_indexof))?; + m.add_wrapped(wrap_pyfunction!(array_positions))?; + m.add_wrapped(wrap_pyfunction!(list_positions))?; + m.add_wrapped(wrap_pyfunction!(array_to_string))?; + m.add_wrapped(wrap_pyfunction!(array_join))?; + m.add_wrapped(wrap_pyfunction!(list_to_string))?; + m.add_wrapped(wrap_pyfunction!(list_join))?; + m.add_wrapped(wrap_pyfunction!(array_ndims))?; + m.add_wrapped(wrap_pyfunction!(list_ndims))?; + m.add_wrapped(wrap_pyfunction!(array_prepend))?; + m.add_wrapped(wrap_pyfunction!(array_push_front))?; + m.add_wrapped(wrap_pyfunction!(list_prepend))?; + m.add_wrapped(wrap_pyfunction!(list_push_front))?; + m.add_wrapped(wrap_pyfunction!(array_pop_back))?; + m.add_wrapped(wrap_pyfunction!(array_pop_front))?; + m.add_wrapped(wrap_pyfunction!(array_remove))?; + m.add_wrapped(wrap_pyfunction!(list_remove))?; + m.add_wrapped(wrap_pyfunction!(array_remove_n))?; + m.add_wrapped(wrap_pyfunction!(list_remove_n))?; + m.add_wrapped(wrap_pyfunction!(array_remove_all))?; + m.add_wrapped(wrap_pyfunction!(list_remove_all))?; + m.add_wrapped(wrap_pyfunction!(array_repeat))?; + m.add_wrapped(wrap_pyfunction!(array_replace))?; + m.add_wrapped(wrap_pyfunction!(list_replace))?; + m.add_wrapped(wrap_pyfunction!(array_replace_n))?; + m.add_wrapped(wrap_pyfunction!(list_replace_n))?; + m.add_wrapped(wrap_pyfunction!(array_replace_all))?; + m.add_wrapped(wrap_pyfunction!(list_replace_all))?; + m.add_wrapped(wrap_pyfunction!(array_slice))?; + m.add_wrapped(wrap_pyfunction!(list_slice))?; Ok(()) } From 27a926433eb0f5a79386fd6705ea3302df4c8b66 Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Sun, 25 Feb 2024 03:11:00 +0300 Subject: [PATCH 115/413] feat: Add `flatten` array function (#562) * Updated functions.rs, test_functions.py - Flatten * Updated functions.rs - Formatting * Updated test_functions.py - Converted test array to a literal for testing * Updated test_functions.py - Linting --- datafusion/tests/test_functions.py | 10 ++++++++++ src/functions.rs | 2 ++ 2 files changed, 12 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index 7e772589b..eb37692b2 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -238,6 +238,15 @@ def py_arr_replace(arr, from_, to, n=None): return new_arr + def py_flatten(arr): + result = [] + for elem in arr: + if isinstance(elem, list): + result.extend(py_flatten(elem)) + else: + result.append(elem) + return result + col = column("arr") test_items = [ [ @@ -432,6 +441,7 @@ def py_arr_replace(arr, from_, to, n=None): f.list_slice(col, literal(-1), literal(2)), lambda: [arr[-1:2] for arr in data], ], + [f.flatten(literal(data)), lambda: [py_flatten(data)]], ] for stmt, py_expr in test_items: diff --git a/src/functions.rs b/src/functions.rs index bb204c35e..5a558de0e 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -409,6 +409,7 @@ scalar_function!(array_replace_all, ArrayReplaceAll); scalar_function!(list_replace_all, ArrayReplaceAll); scalar_function!(array_slice, ArraySlice); scalar_function!(list_slice, ArraySlice); +scalar_function!(flatten, Flatten); aggregate_function!(approx_distinct, ApproxDistinct); aggregate_function!(approx_median, ApproxMedian); @@ -651,6 +652,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(list_replace_all))?; m.add_wrapped(wrap_pyfunction!(array_slice))?; m.add_wrapped(wrap_pyfunction!(list_slice))?; + m.add_wrapped(wrap_pyfunction!(flatten))?; Ok(()) } From dd4ed977138b98749664bf63bb1e843559fd3923 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 2 Mar 2024 16:57:04 -0700 Subject: [PATCH 116/413] Prepare 36.0.0 Release (#599) --- CHANGELOG.md | 17 ++++++ Cargo.lock | 131 +++++++++++++++++++++++++++------------ Cargo.toml | 15 ++--- src/common/data_type.rs | 12 +++- src/expr.rs | 132 ++++++++++++++++++++++++++-------------- src/functions.rs | 53 +++++++++++++--- src/udaf.rs | 4 +- 7 files changed, 258 insertions(+), 106 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5e3abbe2..0a05429f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,23 @@ # DataFusion Python Changelog +## [36.0.0](https://github.com/apache/arrow-datafusion-python/tree/36.0.0) (2024-03-02) + +**Implemented enhancements:** + +- feat: Add `flatten` array function [#562](https://github.com/apache/arrow-datafusion-python/pull/562) (mobley-trent) + +**Documentation updates:** + +- docs: Add ASF attribution [#580](https://github.com/apache/arrow-datafusion-python/pull/580) (simicd) + +**Merged pull requests:** + +- Allow PyDataFrame to be used from other projects [#582](https://github.com/apache/arrow-datafusion-python/pull/582) (andygrove) +- docs: Add ASF attribution [#580](https://github.com/apache/arrow-datafusion-python/pull/580) (simicd) +- Add array functions [#560](https://github.com/apache/arrow-datafusion-python/pull/560) (ongchi) +- feat: Add `flatten` array function [#562](https://github.com/apache/arrow-datafusion-python/pull/562) (mobley-trent) + ## [35.0.0](https://github.com/apache/arrow-datafusion-python/tree/35.0.0) (2024-01-20) **Merged pull requests:** diff --git a/Cargo.lock b/Cargo.lock index e6aa6050a..31e7bad01 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,8 +107,8 @@ dependencies = [ "serde", "serde_json", "snap", - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.3", "thiserror", "typed-builder", "uuid", @@ -544,15 +544,15 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", "serde", - "windows-targets 0.48.5", + "windows-targets 0.52.0", ] [[package]] @@ -583,8 +583,8 @@ version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ - "strum", - "strum_macros", + "strum 0.25.0", + "strum_macros 0.25.3", "unicode-width", ] @@ -715,9 +715,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4328f5467f76d890fe3f924362dbc3a838c6a733f762b32d87f9e0b7bef5fb49" +checksum = "b2b360b692bf6c6d6e6b6dbaf41a3be0020daeceac0f406aed54c75331e50dbb" dependencies = [ "ahash", "apache-avro", @@ -734,6 +734,8 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions", + "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-plan", @@ -765,9 +767,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29a7752143b446db4a2cccd9a6517293c6b97e8c39e520ca43ccd07135a4f7e" +checksum = "37f343ccc298f440e25aa38ff82678291a7acc24061c7370ba6c0ff5cc811412" dependencies = [ "ahash", "apache-avro", @@ -787,9 +789,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d447650af16e138c31237f53ddaef6dd4f92f0e2d3f2f35d190e16c214ca496" +checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" dependencies = [ "arrow", "chrono", @@ -808,9 +810,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8d19598e48a498850fb79f97a9719b1f95e7deb64a7a06f93f313e8fa1d524b" +checksum = "e204d89909e678846b6a95f156aafc1ee5b36cb6c9e37ec2e1449b078a38c818" dependencies = [ "ahash", "arrow", @@ -818,15 +820,44 @@ dependencies = [ "datafusion-common", "paste", "sqlparser", - "strum", - "strum_macros", + "strum 0.26.1", + "strum_macros 0.26.1", +] + +[[package]] +name = "datafusion-functions" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98f1c73f7801b2b8ba2297b3ad78ffcf6c1fc6b8171f502987eb9ad5cb244ee7" +dependencies = [ + "arrow", + "base64", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hex", + "log", +] + +[[package]] +name = "datafusion-functions-array" +version = "36.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42d16a0ddf2c991526f6ffe2f47a72c6da0b7354d6c32411dd20631fe2e38937" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "log", + "paste", ] [[package]] name = "datafusion-optimizer" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b7feb0391f1fc75575acb95b74bfd276903dc37a5409fcebe160bc7ddff2010" +checksum = "5ae27e07bf1f04d327be5c2a293470879801ab5535204dc3b16b062fda195496" dependencies = [ "arrow", "async-trait", @@ -842,9 +873,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e911bca609c89a54e8f014777449d8290327414d3e10c57a3e3c2122e38878d0" +checksum = "dde620cd9ef76a3bca9c754fb68854bd2349c49f55baf97e08001f9e967f6d6b" dependencies = [ "ahash", "arrow", @@ -852,11 +883,13 @@ dependencies = [ "arrow-buffer", "arrow-ord", "arrow-schema", + "arrow-string", "base64", "blake2", "blake3", "chrono", "datafusion-common", + "datafusion-execution", "datafusion-expr", "half", "hashbrown 0.14.3", @@ -876,9 +909,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b546b8a02e9c2ab35ac6420d511f12a4701950c1eb2e568c122b4fefb0be3" +checksum = "9a4c75fba9ea99d64b2246cbd2fcae2e6fc973e6616b1015237a616036506dd4" dependencies = [ "ahash", "arrow", @@ -907,12 +940,13 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "35.0.0" +version = "36.0.0" dependencies = [ "async-trait", "datafusion", "datafusion-common", "datafusion-expr", + "datafusion-functions-array", "datafusion-optimizer", "datafusion-sql", "datafusion-substrait", @@ -934,9 +968,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d18d36f260bbbd63aafdb55339213a23d540d3419810575850ef0a798a6b768" +checksum = "21474a95c3a62d113599d21b439fa15091b538bac06bd20be0bb2e7d22903c09" dependencies = [ "arrow", "arrow-schema", @@ -948,9 +982,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "35.0.0" +version = "36.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad6bef77af3d8a697ae63ffbcb5aa66b74cd08ea93a31e2e757da75b2f1452f" +checksum = "aab89c01ef66a59ec92d2360db63893224b4f7e085e2ee6351e0bb77f88931f0" dependencies = [ "async-recursion", "chrono", @@ -1181,9 +1215,9 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "git2" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" +checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" dependencies = [ "bitflags 2.4.2", "libc", @@ -1557,9 +1591,9 @@ dependencies = [ [[package]] name = "libgit2-sys" -version = "0.16.1+1.7.1" +version = "0.16.2+1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2a2bb3680b094add03bb3732ec520ece34da31a8cd2d633d1389d0f0fb60d0c" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" dependencies = [ "cc", "libc", @@ -2662,9 +2696,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.41.0" +version = "0.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc2c25a6c66789625ef164b4c7d2e548d627902280c13710d33da8222169964" +checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" dependencies = [ "log", "sqlparser_derive", @@ -2692,8 +2726,14 @@ name = "strum" version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" + +[[package]] +name = "strum" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "723b93e8addf9aa965ebe2d11da6d7540fa2283fcea14b3371ff055f7ba13f5f" dependencies = [ - "strum_macros", + "strum_macros 0.26.1", ] [[package]] @@ -2709,11 +2749,24 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "strum_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.48", +] + [[package]] name = "substrait" -version = "0.22.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5478fbd0313a9b0915a1c0e7ebf15b5fed7d7c6dd7229b4f5e32ce75b10f256a" +checksum = "2c8ffb7a3e7505bb835513e77ebfe67d359e57d684a5972323e3bdefbecc1f25" dependencies = [ "git2", "heck", @@ -2857,9 +2910,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 6c47484b3..f8c6bf9f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "35.0.0" +version = "36.0.0" homepage = "https://github.com/apache/arrow-datafusion-python" repository = "https://github.com/apache/arrow-datafusion-python" authors = ["Apache Arrow "] @@ -37,12 +37,13 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "35.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "35.0.0", features = ["pyarrow"] } -datafusion-expr = "35.0.0" -datafusion-optimizer = "35.0.0" -datafusion-sql = "35.0.0" -datafusion-substrait = { version = "35.0.0", optional = true } +datafusion = { version = "36.0.0", features = ["pyarrow", "avro"] } +datafusion-common = { version = "36.0.0", features = ["pyarrow"] } +datafusion-expr = "36.0.0" +datafusion-functions-array = "36.0.0" +datafusion-optimizer = "36.0.0" +datafusion-sql = "36.0.0" +datafusion-substrait = { version = "36.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.3", features = ["v4"] } diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 6059768e3..d3203fdcd 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -290,15 +290,21 @@ impl DataTypeMap { Ok(DataType::Interval(IntervalUnit::MonthDayNano)) } ScalarValue::List(arr) => Ok(arr.data_type().to_owned()), - ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())), + ScalarValue::Struct(_fields) => Err(py_datafusion_err( + DataFusionError::NotImplemented("ScalarValue::Struct".to_string()), + )), ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)), ScalarValue::FixedSizeList(_array_ref) => { // The FieldRef was removed from ScalarValue::FixedSizeList in // https://github.com/apache/arrow-datafusion/pull/8221, so we can no // longer convert back to a DataType here - todo!() + Err(py_datafusion_err(DataFusionError::NotImplemented( + "ScalarValue::FixedSizeList".to_string(), + ))) } - ScalarValue::LargeList(_) => todo!(), + ScalarValue::LargeList(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( + "ScalarValue::LargeList".to_string(), + ))), ScalarValue::DurationSecond(_) => Ok(DataType::Duration(TimeUnit::Second)), ScalarValue::DurationMillisecond(_) => Ok(DataType::Duration(TimeUnit::Millisecond)), ScalarValue::DurationMicrosecond(_) => Ok(DataType::Duration(TimeUnit::Microsecond)), diff --git a/src/expr.rs b/src/expr.rs index dbb56ee99..c0e7019f0 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -32,7 +32,7 @@ use datafusion_expr::{ }; use crate::common::data_type::{DataTypeMap, RexType}; -use crate::errors::{py_runtime_err, py_type_err, DataFusionError}; +use crate::errors::{py_datafusion_err, py_runtime_err, py_type_err, DataFusionError}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; @@ -292,6 +292,7 @@ impl PyExpr { | Expr::IsNotFalse(..) | Expr::Placeholder { .. } | Expr::OuterReferenceColumn(_, _) + | Expr::Unnest(_) | Expr::IsNotUnknown(_) => RexType::Call, Expr::ScalarSubquery(..) => RexType::ScalarSubquery, }) @@ -306,49 +307,81 @@ impl PyExpr { /// Extracts the Expr value into a PyObject that can be shared with Python pub fn python_value(&self, py: Python) -> PyResult { match &self.expr { - Expr::Literal(scalar_value) => Ok(match scalar_value { - ScalarValue::Null => todo!(), - ScalarValue::Boolean(v) => v.into_py(py), - ScalarValue::Float32(v) => v.into_py(py), - ScalarValue::Float64(v) => v.into_py(py), - ScalarValue::Decimal128(v, _, _) => v.into_py(py), - ScalarValue::Decimal256(_, _, _) => todo!(), - ScalarValue::Int8(v) => v.into_py(py), - ScalarValue::Int16(v) => v.into_py(py), - ScalarValue::Int32(v) => v.into_py(py), - ScalarValue::Int64(v) => v.into_py(py), - ScalarValue::UInt8(v) => v.into_py(py), - ScalarValue::UInt16(v) => v.into_py(py), - ScalarValue::UInt32(v) => v.into_py(py), - ScalarValue::UInt64(v) => v.into_py(py), - ScalarValue::Utf8(v) => v.clone().into_py(py), - ScalarValue::LargeUtf8(v) => v.clone().into_py(py), - ScalarValue::Binary(v) => v.clone().into_py(py), - ScalarValue::FixedSizeBinary(_, _) => todo!(), - ScalarValue::LargeBinary(v) => v.clone().into_py(py), - ScalarValue::List(_) => todo!(), - ScalarValue::Date32(v) => v.into_py(py), - ScalarValue::Date64(v) => v.into_py(py), - ScalarValue::Time32Second(v) => v.into_py(py), - ScalarValue::Time32Millisecond(v) => v.into_py(py), - ScalarValue::Time64Microsecond(v) => v.into_py(py), - ScalarValue::Time64Nanosecond(v) => v.into_py(py), - ScalarValue::TimestampSecond(v, _) => v.into_py(py), - ScalarValue::TimestampMillisecond(v, _) => v.into_py(py), - ScalarValue::TimestampMicrosecond(v, _) => v.into_py(py), - ScalarValue::TimestampNanosecond(v, _) => v.into_py(py), - ScalarValue::IntervalYearMonth(v) => v.into_py(py), - ScalarValue::IntervalDayTime(v) => v.into_py(py), - ScalarValue::IntervalMonthDayNano(v) => v.into_py(py), - ScalarValue::DurationSecond(v) => v.into_py(py), - ScalarValue::DurationMicrosecond(v) => v.into_py(py), - ScalarValue::DurationNanosecond(v) => v.into_py(py), - ScalarValue::DurationMillisecond(v) => v.into_py(py), - ScalarValue::Struct(_, _) => todo!(), - ScalarValue::Dictionary(_, _) => todo!(), - ScalarValue::FixedSizeList(_) => todo!(), - ScalarValue::LargeList(_) => todo!(), - }), + Expr::Literal(scalar_value) => match scalar_value { + ScalarValue::Null => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::Null".to_string(), + ), + )), + ScalarValue::Boolean(v) => Ok(v.into_py(py)), + ScalarValue::Float32(v) => Ok(v.into_py(py)), + ScalarValue::Float64(v) => Ok(v.into_py(py)), + ScalarValue::Decimal128(v, _, _) => Ok(v.into_py(py)), + ScalarValue::Decimal256(_, _, _) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::Decimal256".to_string(), + ), + )), + ScalarValue::Int8(v) => Ok(v.into_py(py)), + ScalarValue::Int16(v) => Ok(v.into_py(py)), + ScalarValue::Int32(v) => Ok(v.into_py(py)), + ScalarValue::Int64(v) => Ok(v.into_py(py)), + ScalarValue::UInt8(v) => Ok(v.into_py(py)), + ScalarValue::UInt16(v) => Ok(v.into_py(py)), + ScalarValue::UInt32(v) => Ok(v.into_py(py)), + ScalarValue::UInt64(v) => Ok(v.into_py(py)), + ScalarValue::Utf8(v) => Ok(v.clone().into_py(py)), + ScalarValue::LargeUtf8(v) => Ok(v.clone().into_py(py)), + ScalarValue::Binary(v) => Ok(v.clone().into_py(py)), + ScalarValue::FixedSizeBinary(_, _) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::FixedSizeBinary".to_string(), + ), + )), + ScalarValue::LargeBinary(v) => Ok(v.clone().into_py(py)), + ScalarValue::List(_) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::List".to_string(), + ), + )), + ScalarValue::Date32(v) => Ok(v.into_py(py)), + ScalarValue::Date64(v) => Ok(v.into_py(py)), + ScalarValue::Time32Second(v) => Ok(v.into_py(py)), + ScalarValue::Time32Millisecond(v) => Ok(v.into_py(py)), + ScalarValue::Time64Microsecond(v) => Ok(v.into_py(py)), + ScalarValue::Time64Nanosecond(v) => Ok(v.into_py(py)), + ScalarValue::TimestampSecond(v, _) => Ok(v.into_py(py)), + ScalarValue::TimestampMillisecond(v, _) => Ok(v.into_py(py)), + ScalarValue::TimestampMicrosecond(v, _) => Ok(v.into_py(py)), + ScalarValue::TimestampNanosecond(v, _) => Ok(v.into_py(py)), + ScalarValue::IntervalYearMonth(v) => Ok(v.into_py(py)), + ScalarValue::IntervalDayTime(v) => Ok(v.into_py(py)), + ScalarValue::IntervalMonthDayNano(v) => Ok(v.into_py(py)), + ScalarValue::DurationSecond(v) => Ok(v.into_py(py)), + ScalarValue::DurationMicrosecond(v) => Ok(v.into_py(py)), + ScalarValue::DurationNanosecond(v) => Ok(v.into_py(py)), + ScalarValue::DurationMillisecond(v) => Ok(v.into_py(py)), + ScalarValue::Struct(_) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::Struct".to_string(), + ), + )), + ScalarValue::Dictionary(_, _) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::Dictionary".to_string(), + ), + )), + ScalarValue::FixedSizeList(_) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::FixedSizeList".to_string(), + ), + )), + ScalarValue::LargeList(_) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::LargeList".to_string(), + ), + )), + }, _ => Err(py_type_err(format!( "Non Expr::Literal encountered in types: {:?}", &self.expr @@ -455,6 +488,7 @@ impl PyExpr { // Currently un-support/implemented Expr types for Rex Call operations Expr::GroupingSet(..) + | Expr::Unnest(_) | Expr::OuterReferenceColumn(_, _) | Expr::Wildcard { .. } | Expr::ScalarSubquery(..) @@ -573,7 +607,11 @@ impl PyExpr { | Operator::RegexMatch | Operator::RegexIMatch | Operator::RegexNotMatch - | Operator::RegexNotIMatch => DataTypeMap::map_from_arrow_type(&DataType::Boolean), + | Operator::RegexNotIMatch + | Operator::LikeMatch + | Operator::ILikeMatch + | Operator::NotLikeMatch + | Operator::NotILikeMatch => DataTypeMap::map_from_arrow_type(&DataType::Boolean), Operator::Plus | Operator::Minus | Operator::Multiply | Operator::Modulo => { DataTypeMap::map_from_arrow_type(&DataType::Int64) } @@ -584,7 +622,9 @@ impl PyExpr { | Operator::BitwiseXor | Operator::BitwiseAnd | Operator::BitwiseOr => DataTypeMap::map_from_arrow_type(&DataType::Binary), - Operator::AtArrow | Operator::ArrowAt => todo!(), + Operator::AtArrow | Operator::ArrowAt => { + Err(py_type_err(format!("Unsupported expr: ${op}"))) + } }, Expr::Cast(Cast { expr: _, data_type }) => DataTypeMap::map_from_arrow_type(data_type), Expr::Literal(scalar_value) => DataTypeMap::map_from_scalar_value(scalar_value), diff --git a/src/functions.rs b/src/functions.rs index 5a558de0e..757fb31d1 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -23,6 +23,7 @@ use crate::expr::conditional_expr::PyCaseBuilder; use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; use datafusion::execution::FunctionRegistry; +use datafusion::functions; use datafusion_common::{Column, TableReference}; use datafusion_expr::expr::Alias; use datafusion_expr::{ @@ -34,6 +35,49 @@ use datafusion_expr::{ lit, BuiltinScalarFunction, Expr, WindowFunctionDefinition, }; +#[pyfunction] +pub fn isnan(expr: PyExpr) -> PyExpr { + functions::expr_fn::isnan(expr.into()).into() +} + +#[pyfunction] +pub fn nullif(expr1: PyExpr, expr2: PyExpr) -> PyExpr { + functions::expr_fn::nullif(expr1.into(), expr2.into()).into() +} + +#[pyfunction] +pub fn encode(input: PyExpr, encoding: PyExpr) -> PyExpr { + functions::expr_fn::encode(input.into(), encoding.into()).into() +} + +#[pyfunction] +pub fn decode(input: PyExpr, encoding: PyExpr) -> PyExpr { + functions::expr_fn::decode(input.into(), encoding.into()).into() +} + +#[pyfunction] +pub fn array_to_string(expr: PyExpr, delim: PyExpr) -> PyExpr { + datafusion_functions_array::expr_fn::array_to_string(expr.into(), delim.into()).into() +} + +#[pyfunction] +pub fn array_join(expr: PyExpr, delim: PyExpr) -> PyExpr { + // alias for array_to_string + array_to_string(expr, delim) +} + +#[pyfunction] +pub fn list_to_string(expr: PyExpr, delim: PyExpr) -> PyExpr { + // alias for array_to_string + array_to_string(expr, delim) +} + +#[pyfunction] +pub fn list_join(expr: PyExpr, delim: PyExpr) -> PyExpr { + // alias for array_to_string + array_to_string(expr, delim) +} + #[pyfunction] fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { datafusion_expr::in_list( @@ -252,7 +296,6 @@ scalar_function!(factorial, Factorial); scalar_function!(floor, Floor); scalar_function!(gcd, Gcd); scalar_function!(initcap, InitCap, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); -scalar_function!(isnan, Isnan); scalar_function!(iszero, Iszero); scalar_function!(lcm, Lcm); scalar_function!(left, Left, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); @@ -348,15 +391,11 @@ scalar_function!(trunc, Trunc); scalar_function!(upper, Upper, "Converts the string to all upper case."); scalar_function!(make_array, MakeArray); scalar_function!(array, MakeArray); -scalar_function!(nullif, NullIf); scalar_function!(uuid, Uuid); scalar_function!(r#struct, Struct); // Use raw identifier since struct is a keyword scalar_function!(from_unixtime, FromUnixtime); scalar_function!(arrow_typeof, ArrowTypeof); scalar_function!(random, Random); -//Binary String Functions -scalar_function!(encode, Encode); -scalar_function!(decode, Decode); // Array Functions scalar_function!(array_append, ArrayAppend); @@ -382,10 +421,6 @@ scalar_function!(list_position, ArrayPosition); scalar_function!(list_indexof, ArrayPosition); scalar_function!(array_positions, ArrayPositions); scalar_function!(list_positions, ArrayPositions); -scalar_function!(array_to_string, ArrayToString); -scalar_function!(array_join, ArrayToString); -scalar_function!(list_to_string, ArrayToString); -scalar_function!(list_join, ArrayToString); scalar_function!(array_ndims, ArrayNdims); scalar_function!(list_ndims, ArrayNdims); scalar_function!(array_prepend, ArrayPrepend); diff --git a/src/udaf.rs b/src/udaf.rs index 0e7a8deab..9aea761cd 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -41,12 +41,12 @@ impl RustAccumulator { } impl Accumulator for RustAccumulator { - fn state(&self) -> Result> { + fn state(&mut self) -> Result> { Python::with_gil(|py| self.accum.as_ref(py).call_method0("state")?.extract()) .map_err(|e| DataFusionError::Execution(format!("{e}"))) } - fn evaluate(&self) -> Result { + fn evaluate(&mut self) -> Result { Python::with_gil(|py| self.accum.as_ref(py).call_method0("evaluate")?.extract()) .map_err(|e| DataFusionError::Execution(format!("{e}"))) } From 3a82be08c458358a3c07587c2b4d9ffbaf646ca2 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sun, 3 Mar 2024 11:24:38 -0700 Subject: [PATCH 117/413] Python3.11 support (#603) --- .github/workflows/conda.yml | 2 +- .github/workflows/test.yaml | 16 +- requirements-311.txt | 327 +++++++++++++++++++----------------- 3 files changed, 184 insertions(+), 161 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 16dfd7a4e..f8474565d 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - python: ["3.8", "3.9", "3.10"] + python: ["3.8", "3.9", "3.10", "3.11"] arch: ["linux-64", "linux-aarch64"] steps: - name: Manage disk space diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f672c8129..371f07979 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -34,10 +34,9 @@ jobs: matrix: python-version: - "3.10" + - "3.11" toolchain: - "stable" - # we are not that much eager in walking on the edge yet - # - nightly # build stable for only 3.7 include: - python-version: "3.7" @@ -83,6 +82,13 @@ jobs: command: clippy args: --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure + - name: Create Virtualenv (3.7) + if: ${{ matrix.python-version == '3.7' }} + run: | + python -m venv venv + source venv/bin/activate + pip install -r requirements-37.txt + - name: Create Virtualenv (3.10) if: ${{ matrix.python-version == '3.10' }} run: | @@ -90,12 +96,12 @@ jobs: source venv/bin/activate pip install -r requirements-310.txt - - name: Create Virtualenv (3.7) - if: ${{ matrix.python-version == '3.7' }} + - name: Create Virtualenv (3.11) + if: ${{ matrix.python-version == '3.11' }} run: | python -m venv venv source venv/bin/activate - pip install -r requirements-37.txt + pip install -r requirements-311.txt - name: Run Python Linters if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} diff --git a/requirements-311.txt b/requirements-311.txt index 9161e3f5b..c5ae2ef79 100644 --- a/requirements-311.txt +++ b/requirements-311.txt @@ -4,95 +4,93 @@ # # pip-compile --generate-hashes --output-file=requirements-311.txt # -black==23.3.0 \ - --hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \ - --hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \ - --hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \ - --hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \ - --hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \ - --hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \ - --hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \ - --hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \ - --hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \ - --hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \ - --hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \ - --hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \ - --hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \ - --hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \ - --hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \ - --hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \ - --hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \ - --hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \ - --hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \ - --hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \ - --hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \ - --hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \ - --hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \ - --hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \ - --hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3 +black==24.2.0 \ + --hash=sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8 \ + --hash=sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8 \ + --hash=sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd \ + --hash=sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9 \ + --hash=sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31 \ + --hash=sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92 \ + --hash=sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f \ + --hash=sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29 \ + --hash=sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4 \ + --hash=sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693 \ + --hash=sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218 \ + --hash=sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a \ + --hash=sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23 \ + --hash=sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0 \ + --hash=sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982 \ + --hash=sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894 \ + --hash=sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540 \ + --hash=sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430 \ + --hash=sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b \ + --hash=sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2 \ + --hash=sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6 \ + --hash=sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d # via -r requirements.in -click==8.1.3 \ - --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ - --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via black -flake8==6.0.0 \ - --hash=sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7 \ - --hash=sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181 +flake8==7.0.0 \ + --hash=sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132 \ + --hash=sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3 # via -r requirements.in iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via pytest -isort==5.12.0 \ - --hash=sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504 \ - --hash=sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6 +isort==5.13.2 \ + --hash=sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109 \ + --hash=sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6 # via -r requirements.in -maturin==0.14.17 \ - --hash=sha256:0308d4124311c92ec5c08600f93385a5ff16f7764410585261d1ca0aed739764 \ - --hash=sha256:2684302ded1559a26635ec45e91ae7913aeb386a8b658d5c2eb13ff410df7930 \ - --hash=sha256:3bf173adac3e9f84a84123ec0b5b4b28b2de79446a49da2b4549016614025b83 \ - --hash=sha256:64dd1b017eab2f9c938d833522bb505506908d824388927011dbdfff26ccea2c \ - --hash=sha256:683e1127b54b9fbdbd661ce3b27b073bd5d7c51d7e980870f4caffdb40f364b0 \ - --hash=sha256:7fef3950a34c0b5c1806637169fca12f89b7440d7608832bf7765b34f394b06a \ - --hash=sha256:8076993118089abb19e7d2312b138afea418dccea60be4198ab9f348618b47df \ - --hash=sha256:a7f06984bd3ffa4ab7be67f7352353a486f262d7a0dbd4dc305944e0b09c8f35 \ - --hash=sha256:abfd98529cfab59deb59639962d93df72121013d5151a5f01c66cda19d9c88a9 \ - --hash=sha256:bb8f8ac375d022c5de5da3efed79e712129de13ebdb9f668b96bf5df0673bc03 \ - --hash=sha256:cf1429a7854ecd4830bb42407d30ff0625c3f6480fa7b1e90da2896ba7a1b2fc \ - --hash=sha256:dcd44c2fa66d79497e2f4361654ed0d1c2ef1c58139469e8e0083409f948eea4 \ - --hash=sha256:fb4e3311e8ce707843235fbe8748a05a3ae166c3efd6d2aa335b53dfc2bd3b88 +maturin==1.4.0 \ + --hash=sha256:01473dc30aed8f2cee3572b3e99e3ea75bf09c84b028bf6077f7643a189699c8 \ + --hash=sha256:076970a73da7fa3648204a584cd347b899c1ea67f8124b212bccd06728e63ed9 \ + --hash=sha256:16239a7648ef17976585353e381840c18e650d352576ed9545abca407d65e534 \ + --hash=sha256:2979175a7eee837dc3a6931980b37ddc86b9ced54d600856668fc074ca2530ef \ + --hash=sha256:77428c043d585f038f4b056c4d617e00a8027b49598ab6d065b8f6b9b9b8d144 \ + --hash=sha256:b4b2f006db1e92687c814576029157dcc2d97b5750fd35fd4f3aabb97e36444f \ + --hash=sha256:b84bee85620e1b7b662a7af71289f7f6c23df8269e42c0f76882676dfc9c733f \ + --hash=sha256:e2c1b157397ef3721b9c2f3f24d9a5a60bd84322aac13b4dd0704a80448741b0 \ + --hash=sha256:e669ba5984c15e29b8545b295ba6738974180b44f47f5d9e75569a5ce6b8add5 \ + --hash=sha256:ed12e1768094a7adeafc3a74ebdb8dc2201fa64c4e7e31f14cfc70378bf93790 \ + --hash=sha256:f8eded83abdb30b2b6ae6d32c80b8192bdd8bcfec0ebfacee6ac02434aa499d6 \ + --hash=sha256:ff95a4494d9e57b6e74d4d7f8a9a2ee8ed29bd7f0e61855656ad959a432c0efc \ + --hash=sha256:ffe4e967080ceb83c156e73a37d3974b30cad01c376a86dc39a76a0c6bccf9b0 # via -r requirements.in mccabe==0.7.0 \ --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e # via flake8 -mypy==1.2.0 \ - --hash=sha256:023fe9e618182ca6317ae89833ba422c411469156b690fde6a315ad10695a521 \ - --hash=sha256:031fc69c9a7e12bcc5660b74122ed84b3f1c505e762cc4296884096c6d8ee140 \ - --hash=sha256:2de7babe398cb7a85ac7f1fd5c42f396c215ab3eff731b4d761d68d0f6a80f48 \ - --hash=sha256:2e93a8a553e0394b26c4ca683923b85a69f7ccdc0139e6acd1354cc884fe0128 \ - --hash=sha256:390bc685ec209ada4e9d35068ac6988c60160b2b703072d2850457b62499e336 \ - --hash=sha256:3a2d219775a120581a0ae8ca392b31f238d452729adbcb6892fa89688cb8306a \ - --hash=sha256:3efde4af6f2d3ccf58ae825495dbb8d74abd6d176ee686ce2ab19bd025273f41 \ - --hash=sha256:4a99fe1768925e4a139aace8f3fb66db3576ee1c30b9c0f70f744ead7e329c9f \ - --hash=sha256:4b41412df69ec06ab141808d12e0bf2823717b1c363bd77b4c0820feaa37249e \ - --hash=sha256:4c8d8c6b80aa4a1689f2a179d31d86ae1367ea4a12855cc13aa3ba24bb36b2d8 \ - --hash=sha256:4d19f1a239d59f10fdc31263d48b7937c585810288376671eaf75380b074f238 \ - --hash=sha256:4e4a682b3f2489d218751981639cffc4e281d548f9d517addfd5a2917ac78119 \ - --hash=sha256:695c45cea7e8abb6f088a34a6034b1d273122e5530aeebb9c09626cea6dca4cb \ - --hash=sha256:701189408b460a2ff42b984e6bd45c3f41f0ac9f5f58b8873bbedc511900086d \ - --hash=sha256:70894c5345bea98321a2fe84df35f43ee7bb0feec117a71420c60459fc3e1eed \ - --hash=sha256:8293a216e902ac12779eb7a08f2bc39ec6c878d7c6025aa59464e0c4c16f7eb9 \ - --hash=sha256:8d26b513225ffd3eacece727f4387bdce6469192ef029ca9dd469940158bc89e \ - --hash=sha256:a197ad3a774f8e74f21e428f0de7f60ad26a8d23437b69638aac2764d1e06a6a \ - --hash=sha256:bea55fc25b96c53affab852ad94bf111a3083bc1d8b0c76a61dd101d8a388cf5 \ - --hash=sha256:c9a084bce1061e55cdc0493a2ad890375af359c766b8ac311ac8120d3a472950 \ - --hash=sha256:d0e9464a0af6715852267bf29c9553e4555b61f5904a4fc538547a4d67617937 \ - --hash=sha256:d8e9187bfcd5ffedbe87403195e1fc340189a68463903c39e2b63307c9fa0394 \ - --hash=sha256:eaeaa0888b7f3ccb7bcd40b50497ca30923dba14f385bde4af78fac713d6d6f6 \ - --hash=sha256:f46af8d162f3d470d8ffc997aaf7a269996d205f9d746124a179d3abe05ac602 \ - --hash=sha256:f70a40410d774ae23fcb4afbbeca652905a04de7948eaf0b1789c8d1426b72d1 \ - --hash=sha256:fe91be1c51c90e2afe6827601ca14353bbf3953f343c2129fa1e247d55fd95ba +mypy==1.8.0 \ + --hash=sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6 \ + --hash=sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d \ + --hash=sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02 \ + --hash=sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d \ + --hash=sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3 \ + --hash=sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3 \ + --hash=sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3 \ + --hash=sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66 \ + --hash=sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259 \ + --hash=sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835 \ + --hash=sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd \ + --hash=sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d \ + --hash=sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8 \ + --hash=sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07 \ + --hash=sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b \ + --hash=sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e \ + --hash=sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6 \ + --hash=sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae \ + --hash=sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9 \ + --hash=sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d \ + --hash=sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a \ + --hash=sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592 \ + --hash=sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218 \ + --hash=sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817 \ + --hash=sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4 \ + --hash=sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410 \ + --hash=sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55 # via -r requirements.in mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ @@ -100,100 +98,119 @@ mypy-extensions==1.0.0 \ # via # black # mypy -numpy==1.24.2 \ - --hash=sha256:003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22 \ - --hash=sha256:150947adbdfeceec4e5926d956a06865c1c690f2fd902efede4ca6fe2e657c3f \ - --hash=sha256:2620e8592136e073bd12ee4536149380695fbe9ebeae845b81237f986479ffc9 \ - --hash=sha256:2eabd64ddb96a1239791da78fa5f4e1693ae2dadc82a76bc76a14cbb2b966e96 \ - --hash=sha256:4173bde9fa2a005c2c6e2ea8ac1618e2ed2c1c6ec8a7657237854d42094123a0 \ - --hash=sha256:4199e7cfc307a778f72d293372736223e39ec9ac096ff0a2e64853b866a8e18a \ - --hash=sha256:4cecaed30dc14123020f77b03601559fff3e6cd0c048f8b5289f4eeabb0eb281 \ - --hash=sha256:557d42778a6869c2162deb40ad82612645e21d79e11c1dc62c6e82a2220ffb04 \ - --hash=sha256:63e45511ee4d9d976637d11e6c9864eae50e12dc9598f531c035265991910468 \ - --hash=sha256:6524630f71631be2dabe0c541e7675db82651eb998496bbe16bc4f77f0772253 \ - --hash=sha256:76807b4063f0002c8532cfeac47a3068a69561e9c8715efdad3c642eb27c0756 \ - --hash=sha256:7de8fdde0003f4294655aa5d5f0a89c26b9f22c0a58790c38fae1ed392d44a5a \ - --hash=sha256:889b2cc88b837d86eda1b17008ebeb679d82875022200c6e8e4ce6cf549b7acb \ - --hash=sha256:92011118955724465fb6853def593cf397b4a1367495e0b59a7e69d40c4eb71d \ - --hash=sha256:97cf27e51fa078078c649a51d7ade3c92d9e709ba2bfb97493007103c741f1d0 \ - --hash=sha256:9a23f8440561a633204a67fb44617ce2a299beecf3295f0d13c495518908e910 \ - --hash=sha256:a51725a815a6188c662fb66fb32077709a9ca38053f0274640293a14fdd22978 \ - --hash=sha256:a77d3e1163a7770164404607b7ba3967fb49b24782a6ef85d9b5f54126cc39e5 \ - --hash=sha256:adbdce121896fd3a17a77ab0b0b5eedf05a9834a18699db6829a64e1dfccca7f \ - --hash=sha256:c29e6bd0ec49a44d7690ecb623a8eac5ab8a923bce0bea6293953992edf3a76a \ - --hash=sha256:c72a6b2f4af1adfe193f7beb91ddf708ff867a3f977ef2ec53c0ffb8283ab9f5 \ - --hash=sha256:d0a2db9d20117bf523dde15858398e7c0858aadca7c0f088ac0d6edd360e9ad2 \ - --hash=sha256:e3ab5d32784e843fc0dd3ab6dcafc67ef806e6b6828dc6af2f689be0eb4d781d \ - --hash=sha256:e428c4fbfa085f947b536706a2fc349245d7baa8334f0c5723c56a10595f9b95 \ - --hash=sha256:e8d2859428712785e8a8b7d2b3ef0a1d1565892367b32f915c4a4df44d0e64f5 \ - --hash=sha256:eef70b4fc1e872ebddc38cddacc87c19a3709c0e3e5d20bf3954c147b1dd941d \ - --hash=sha256:f64bb98ac59b3ea3bf74b02f13836eb2e24e48e0ab0145bbda646295769bd780 \ - --hash=sha256:f9006288bcf4895917d02583cf3411f98631275bc67cce355a7f39f8c14338fa +numpy==1.26.4 \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via # -r requirements.in # pyarrow -packaging==23.1 \ - --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \ - --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f +packaging==23.2 \ + --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ + --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 # via # black # pytest -pathspec==0.11.1 \ - --hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \ - --hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293 +pathspec==0.12.1 \ + --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ + --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 # via black -platformdirs==3.2.0 \ - --hash=sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08 \ - --hash=sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e +platformdirs==4.2.0 \ + --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ + --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 # via black -pluggy==1.0.0 \ - --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ - --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 +pluggy==1.4.0 \ + --hash=sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 \ + --hash=sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be # via pytest -pyarrow==11.0.0 \ - --hash=sha256:1cbcfcbb0e74b4d94f0b7dde447b835a01bc1d16510edb8bb7d6224b9bf5bafc \ - --hash=sha256:25aa11c443b934078bfd60ed63e4e2d42461682b5ac10f67275ea21e60e6042c \ - --hash=sha256:2d53ba72917fdb71e3584ffc23ee4fcc487218f8ff29dd6df3a34c5c48fe8c06 \ - --hash=sha256:2d942c690ff24a08b07cb3df818f542a90e4d359381fbff71b8f2aea5bf58841 \ - --hash=sha256:2f51dc7ca940fdf17893227edb46b6784d37522ce08d21afc56466898cb213b2 \ - --hash=sha256:362a7c881b32dc6b0eccf83411a97acba2774c10edcec715ccaab5ebf3bb0835 \ - --hash=sha256:3e99be85973592051e46412accea31828da324531a060bd4585046a74ba45854 \ - --hash=sha256:40bb42afa1053c35c749befbe72f6429b7b5f45710e85059cdd534553ebcf4f2 \ - --hash=sha256:410624da0708c37e6a27eba321a72f29d277091c8f8d23f72c92bada4092eb5e \ - --hash=sha256:41a1451dd895c0b2964b83d91019e46f15b5564c7ecd5dcb812dadd3f05acc97 \ - --hash=sha256:5461c57dbdb211a632a48facb9b39bbeb8a7905ec95d768078525283caef5f6d \ - --hash=sha256:69309be84dcc36422574d19c7d3a30a7ea43804f12552356d1ab2a82a713c418 \ - --hash=sha256:7c28b5f248e08dea3b3e0c828b91945f431f4202f1a9fe84d1012a761324e1ba \ - --hash=sha256:8f40be0d7381112a398b93c45a7e69f60261e7b0269cc324e9f739ce272f4f70 \ - --hash=sha256:a37bc81f6c9435da3c9c1e767324ac3064ffbe110c4e460660c43e144be4ed85 \ - --hash=sha256:aaee8f79d2a120bf3e032d6d64ad20b3af6f56241b0ffc38d201aebfee879d00 \ - --hash=sha256:ad42bb24fc44c48f74f0d8c72a9af16ba9a01a2ccda5739a517aa860fa7e3d56 \ - --hash=sha256:ad7c53def8dbbc810282ad308cc46a523ec81e653e60a91c609c2233ae407689 \ - --hash=sha256:becc2344be80e5dce4e1b80b7c650d2fc2061b9eb339045035a1baa34d5b8f1c \ - --hash=sha256:caad867121f182d0d3e1a0d36f197df604655d0b466f1bc9bafa903aa95083e4 \ - --hash=sha256:ccbf29a0dadfcdd97632b4f7cca20a966bb552853ba254e874c66934931b9841 \ - --hash=sha256:da93340fbf6f4e2a62815064383605b7ffa3e9eeb320ec839995b1660d69f89b \ - --hash=sha256:e217d001e6389b20a6759392a5ec49d670757af80101ee6b5f2c8ff0172e02ca \ - --hash=sha256:f010ce497ca1b0f17a8243df3048055c0d18dcadbcc70895d5baf8921f753de5 \ - --hash=sha256:f12932e5a6feb5c58192209af1d2607d488cb1d404fbc038ac12ada60327fa34 +pyarrow==15.0.0 \ + --hash=sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4 \ + --hash=sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970 \ + --hash=sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555 \ + --hash=sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b \ + --hash=sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d \ + --hash=sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2 \ + --hash=sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99 \ + --hash=sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac \ + --hash=sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6 \ + --hash=sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5 \ + --hash=sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f \ + --hash=sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3 \ + --hash=sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb \ + --hash=sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642 \ + --hash=sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2 \ + --hash=sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f \ + --hash=sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929 \ + --hash=sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e \ + --hash=sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83 \ + --hash=sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340 \ + --hash=sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540 \ + --hash=sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed \ + --hash=sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351 \ + --hash=sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177 \ + --hash=sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4 \ + --hash=sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7 \ + --hash=sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593 \ + --hash=sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7 \ + --hash=sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31 \ + --hash=sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b \ + --hash=sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e \ + --hash=sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6 \ + --hash=sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4 \ + --hash=sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40 \ + --hash=sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5 \ + --hash=sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19 # via -r requirements.in -pycodestyle==2.10.0 \ - --hash=sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053 \ - --hash=sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610 +pycodestyle==2.11.1 \ + --hash=sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f \ + --hash=sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67 # via flake8 -pyflakes==3.0.1 \ - --hash=sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf \ - --hash=sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd +pyflakes==3.2.0 \ + --hash=sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f \ + --hash=sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a # via flake8 -pytest==7.3.1 \ - --hash=sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362 \ - --hash=sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3 +pytest==8.0.2 \ + --hash=sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd \ + --hash=sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096 # via -r requirements.in toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f # via -r requirements.in -typing-extensions==4.5.0 \ - --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ - --hash=sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4 +typing-extensions==4.10.0 \ + --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \ + --hash=sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb # via mypy From 76ecf56f2f48d032a3a215cd0df504b97cd61de3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 4 Mar 2024 07:38:39 -0700 Subject: [PATCH 118/413] Switch to Ruff for Python linting (#529) --- .github/workflows/build.yml | 16 ++ .github/workflows/test.yaml | 7 - .pre-commit-config.yaml | 21 +- README.md | 21 +- benchmarks/db-benchmark/groupby-datafusion.py | 16 +- benchmarks/db-benchmark/join-datafusion.py | 24 +-- benchmarks/tpch/tpch.py | 4 +- ci/scripts/python_lint.sh | 22 +++ datafusion/__init__.py | 4 +- datafusion/cudf.py | 4 +- datafusion/input/base.py | 8 +- datafusion/input/location.py | 4 +- datafusion/pandas.py | 4 +- datafusion/polars.py | 12 +- datafusion/tests/generic.py | 12 +- datafusion/tests/test_aggregation.py | 34 +--- datafusion/tests/test_config.py | 5 +- datafusion/tests/test_context.py | 12 +- datafusion/tests/test_dataframe.py | 54 ++---- datafusion/tests/test_functions.py | 101 +++------- datafusion/tests/test_input.py | 4 +- datafusion/tests/test_sql.py | 28 +-- datafusion/tests/test_substrait.py | 18 +- dev/release/check-rat-report.py | 4 +- dev/release/generate-changelog.py | 8 +- examples/sql-on-polars.py | 4 +- examples/sql-using-python-udaf.py | 8 +- examples/substrait.py | 8 +- requirements-310.txt | 181 ++++++------------ requirements-311.txt | 83 ++------ requirements.in | 4 +- 31 files changed, 239 insertions(+), 496 deletions(-) create mode 100755 ci/scripts/python_lint.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 968bdc1f0..b553749ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,6 +24,22 @@ on: branches: ["branch-*"] jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff + # Update output format to enable automatic inline annotations. + - name: Run Ruff + run: ruff check --output-format=github datafusion + generate-license: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 371f07979..291120bf8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -103,13 +103,6 @@ jobs: source venv/bin/activate pip install -r requirements-311.txt - - name: Run Python Linters - if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} - run: | - source venv/bin/activate - flake8 --exclude venv,benchmarks/db-benchmark --ignore=E501,W503 - black --line-length 79 --diff --check . - - name: Run tests env: RUST_BACKTRACE: 1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 39049bf49..8509fae2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,21 +20,14 @@ repos: rev: v1.6.23 hooks: - id: actionlint-docker - - repo: https://github.com/psf/black - rev: 22.3.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.3.0 hooks: - - id: black - files: datafusion/.* - # Explicitly specify the pyproject.toml at the repo root, not per-project. - args: ["--config", "pyproject.toml", "--line-length", "79", "--diff", "--check", "."] - - repo: https://github.com/PyCQA/flake8 - rev: 5.0.4 - hooks: - - id: flake8 - files: datafusion/.*$ - types: [file] - types_or: [python] - additional_dependencies: ["flake8-force"] + # Run the linter. + - id: ruff + # Run the formatter. + - id: ruff-format - repo: local hooks: - id: rust-fmt diff --git a/README.md b/README.md index a682f73d5..a2e3efd4e 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ source venv/bin/activate # update pip itself if necessary python -m pip install -U pip # install dependencies (for Python 3.8+) -python -m pip install -r requirements-310.txt +python -m pip install -r requirements.in ``` The tests rely on test data in git submodules. @@ -222,12 +222,27 @@ python -m pytest ### Running & Installing pre-commit hooks -arrow-datafusion-python takes advantage of [pre-commit](https://pre-commit.com/) to assist developers with code linting to help reduce the number of commits that ultimately fail in CI due to linter errors. Using the pre-commit hooks is optional for the developer but certainly helpful for keeping PRs clean and concise. +arrow-datafusion-python takes advantage of [pre-commit](https://pre-commit.com/) to assist developers with code linting to help reduce +the number of commits that ultimately fail in CI due to linter errors. Using the pre-commit hooks is optional for the +developer but certainly helpful for keeping PRs clean and concise. -Our pre-commit hooks can be installed by running `pre-commit install`, which will install the configurations in your ARROW_DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit, failing to complete the commit if an offending lint is found allowing you to make changes locally before pushing. +Our pre-commit hooks can be installed by running `pre-commit install`, which will install the configurations in +your ARROW_DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit, failing to complete +the commit if an offending lint is found allowing you to make changes locally before pushing. The pre-commit hooks can also be run adhoc without installing them by simply running `pre-commit run --all-files` +## Running linters without using pre-commit + +There are scripts in `ci/scripts` for running Rust and Python linters. + +```shell +./ci/scripts/python_lint.sh +./ci/scripts/rust_clippy.sh +./ci/scripts/rust_fmt.sh +./ci/scripts/rust_toml_fmt.sh +``` + ## How to update dependencies To change test dependencies, change the `requirements.in` and run diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 2c35259e8..3a4399f7d 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -79,17 +79,13 @@ def execute(df): data = pacsv.read_csv( src_grp, - convert_options=pacsv.ConvertOptions( - auto_dict_encode=True, column_types=schema - ), + convert_options=pacsv.ConvertOptions(auto_dict_encode=True, column_types=schema), ) print("dataset loaded") # create a session context with explicit runtime and config settings runtime = ( - RuntimeConfig() - .with_disk_manager_os() - .with_fair_spill_pool(64 * 1024 * 1024 * 1024) + RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(64 * 1024 * 1024 * 1024) ) config = ( SessionConfig() @@ -116,9 +112,7 @@ def execute(df): if sql: df = ctx.sql("SELECT id1, SUM(v1) AS v1 FROM x GROUP BY id1") else: - df = ctx.table("x").aggregate( - [f.col("id1")], [f.sum(f.col("v1")).alias("v1")] - ) + df = ctx.table("x").aggregate([f.col("id1")], [f.sum(f.col("v1")).alias("v1")]) ans = execute(df) shape = ans_shape(ans) @@ -197,9 +191,7 @@ def execute(df): gc.collect() t_start = timeit.default_timer() if sql: - df = ctx.sql( - "SELECT id3, SUM(v1) AS v1, AVG(v3) AS v3 FROM x GROUP BY id3" - ) + df = ctx.sql("SELECT id3, SUM(v1) AS v1, AVG(v3) AS v3 FROM x GROUP BY id3") else: df = ctx.table("x").aggregate( [f.col("id3")], diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py index 602cee697..4d59c7dc2 100755 --- a/benchmarks/db-benchmark/join-datafusion.py +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -152,11 +152,7 @@ def ans_shape(batches): print(f"q2: {t}") t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) -chk = ( - df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) - .collect()[0] - .column(0)[0] -) +chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start m = memory_usage() write_log( @@ -193,11 +189,7 @@ def ans_shape(batches): print(f"q3: {t}") t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) -chk = ( - df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) - .collect()[0] - .column(0)[0] -) +chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start m = memory_usage() write_log( @@ -234,11 +226,7 @@ def ans_shape(batches): print(f"q4: {t}") t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) -chk = ( - df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) - .collect()[0] - .column(0)[0] -) +chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start m = memory_usage() write_log( @@ -275,11 +263,7 @@ def ans_shape(batches): print(f"q5: {t}") t_start = timeit.default_timer() df = ctx.create_dataframe([ans]) -chk = ( - df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]) - .collect()[0] - .column(0)[0] -) +chk = df.aggregate([], [f.sum(col("v1")), f.sum(col("v2"))]).collect()[0].column(0)[0] chkt = timeit.default_timer() - t_start m = memory_usage() write_log( diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py index ea830a1ff..7f104a4cb 100644 --- a/benchmarks/tpch/tpch.py +++ b/benchmarks/tpch/tpch.py @@ -83,9 +83,7 @@ def bench(data_path, query_path): time_millis = (end - start) * 1000 total_time_millis += time_millis print("q{},{}".format(query, round(time_millis, 1))) - results.write( - "q{},{}\n".format(query, round(time_millis, 1)) - ) + results.write("q{},{}\n".format(query, round(time_millis, 1))) results.flush() except Exception as e: print("query", query, "failed", e) diff --git a/ci/scripts/python_lint.sh b/ci/scripts/python_lint.sh new file mode 100755 index 000000000..3f7310ba7 --- /dev/null +++ b/ci/scripts/python_lint.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -ex +ruff format datafusion +ruff check datafusion \ No newline at end of file diff --git a/datafusion/__init__.py b/datafusion/__init__.py index df53b396a..cfb673613 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -208,9 +208,7 @@ def udaf(accum, input_type, return_type, state_type, volatility, name=None): Create a new User Defined Aggregate Function """ if not issubclass(accum, Accumulator): - raise TypeError( - "`accum` must implement the abstract base class Accumulator" - ) + raise TypeError("`accum` must implement the abstract base class Accumulator") if name is None: name = accum.__qualname__.lower() if isinstance(input_type, pa.lib.DataType): diff --git a/datafusion/cudf.py b/datafusion/cudf.py index e39daea31..d8bc8e6d0 100644 --- a/datafusion/cudf.py +++ b/datafusion/cudf.py @@ -68,9 +68,7 @@ def to_cudf_df(self, plan): elif isinstance(node, TableScan): return cudf.read_parquet(self.parquet_tables[node.table_name()]) else: - raise Exception( - "unsupported logical operator: {}".format(type(node)) - ) + raise Exception("unsupported logical operator: {}".format(type(node))) def create_schema(self, schema_name: str, **kwargs): logger.debug(f"Creating schema: {schema_name}") diff --git a/datafusion/input/base.py b/datafusion/input/base.py index fb1207896..efcaf7697 100644 --- a/datafusion/input/base.py +++ b/datafusion/input/base.py @@ -31,13 +31,9 @@ class BaseInputSource(ABC): """ @abstractmethod - def is_correct_input( - self, input_item: Any, table_name: str, **kwargs - ) -> bool: + def is_correct_input(self, input_item: Any, table_name: str, **kwargs) -> bool: pass @abstractmethod - def build_table( - self, input_item: Any, table_name: str, **kwarg - ) -> SqlTable: + def build_table(self, input_item: Any, table_name: str, **kwarg) -> SqlTable: pass diff --git a/datafusion/input/location.py b/datafusion/input/location.py index 939c7f415..16e632d1b 100644 --- a/datafusion/input/location.py +++ b/datafusion/input/location.py @@ -72,9 +72,7 @@ def build_table( for _ in reader: num_rows += 1 # TODO: Need to actually consume this row into resonable columns - raise RuntimeError( - "TODO: Currently unable to support CSV input files." - ) + raise RuntimeError("TODO: Currently unable to support CSV input files.") else: raise RuntimeError( f"Input of format: `{format}` is currently not supported.\ diff --git a/datafusion/pandas.py b/datafusion/pandas.py index c2da83ff6..cf1fa6f7b 100644 --- a/datafusion/pandas.py +++ b/datafusion/pandas.py @@ -64,9 +64,7 @@ def to_pandas_df(self, plan): elif isinstance(node, TableScan): return pd.read_parquet(self.parquet_tables[node.table_name()]) else: - raise Exception( - "unsupported logical operator: {}".format(type(node)) - ) + raise Exception("unsupported logical operator: {}".format(type(node))) def create_schema(self, schema_name: str, **kwargs): logger.debug(f"Creating schema: {schema_name}") diff --git a/datafusion/polars.py b/datafusion/polars.py index e4eb966fc..ac5e26e3e 100644 --- a/datafusion/polars.py +++ b/datafusion/polars.py @@ -51,9 +51,7 @@ def to_polars_df(self, plan): args = [self.to_polars_expr(expr) for expr in node.projections()] return inputs[0].select(*args) elif isinstance(node, Aggregate): - groupby_expr = [ - self.to_polars_expr(expr) for expr in node.group_by_exprs() - ] + groupby_expr = [self.to_polars_expr(expr) for expr in node.group_by_exprs()] aggs = [] for expr in node.aggregate_exprs(): expr = expr.to_variant() @@ -67,17 +65,13 @@ def to_polars_df(self, plan): ) ) else: - raise Exception( - "Unsupported aggregate function {}".format(expr) - ) + raise Exception("Unsupported aggregate function {}".format(expr)) df = inputs[0].groupby(groupby_expr).agg(aggs) return df elif isinstance(node, TableScan): return polars.read_parquet(self.parquet_tables[node.table_name()]) else: - raise Exception( - "unsupported logical operator: {}".format(type(node)) - ) + raise Exception("unsupported logical operator: {}".format(type(node))) def create_schema(self, schema_name: str, **kwargs): logger.debug(f"Creating schema: {schema_name}") diff --git a/datafusion/tests/generic.py b/datafusion/tests/generic.py index 1f984a40a..07399792c 100644 --- a/datafusion/tests/generic.py +++ b/datafusion/tests/generic.py @@ -50,9 +50,7 @@ def data_datetime(f): datetime.datetime.now() - datetime.timedelta(days=1), datetime.datetime.now() + datetime.timedelta(days=1), ] - return pa.array( - data, type=pa.timestamp(f), mask=np.array([False, True, False]) - ) + return pa.array(data, type=pa.timestamp(f), mask=np.array([False, True, False])) def data_date32(): @@ -61,9 +59,7 @@ def data_date32(): datetime.date(1980, 1, 1), datetime.date(2030, 1, 1), ] - return pa.array( - data, type=pa.date32(), mask=np.array([False, True, False]) - ) + return pa.array(data, type=pa.date32(), mask=np.array([False, True, False])) def data_timedelta(f): @@ -72,9 +68,7 @@ def data_timedelta(f): datetime.timedelta(days=1), datetime.timedelta(seconds=1), ] - return pa.array( - data, type=pa.duration(f), mask=np.array([False, True, False]) - ) + return pa.array(data, type=pa.duration(f), mask=np.array([False, True, False])) def data_binary_other(): diff --git a/datafusion/tests/test_aggregation.py b/datafusion/tests/test_aggregation.py index 0a6c90c32..99a470b6b 100644 --- a/datafusion/tests/test_aggregation.py +++ b/datafusion/tests/test_aggregation.py @@ -81,9 +81,7 @@ def test_built_in_aggregation(df): assert result.column(2) == pa.array([4]) assert result.column(3) == pa.array([6]) assert result.column(4) == pa.array([[4, 4, 6]]) - np.testing.assert_array_almost_equal( - result.column(5), np.average(values_a) - ) + np.testing.assert_array_almost_equal(result.column(5), np.average(values_a)) np.testing.assert_array_almost_equal( result.column(6), np.corrcoef(values_a, values_b)[0][1] ) @@ -101,35 +99,20 @@ def test_built_in_aggregation(df): ) np.testing.assert_array_almost_equal(result.column(11), np.max(values_a)) np.testing.assert_array_almost_equal(result.column(12), np.mean(values_b)) - np.testing.assert_array_almost_equal( - result.column(13), np.median(values_b) - ) + np.testing.assert_array_almost_equal(result.column(13), np.median(values_b)) np.testing.assert_array_almost_equal(result.column(14), np.min(values_a)) np.testing.assert_array_almost_equal( result.column(15), np.sum(values_b.to_pylist()) ) - np.testing.assert_array_almost_equal( - result.column(16), np.std(values_a, ddof=1) - ) - np.testing.assert_array_almost_equal( - result.column(17), np.std(values_b, ddof=0) - ) - np.testing.assert_array_almost_equal( - result.column(18), np.std(values_c, ddof=1) - ) - np.testing.assert_array_almost_equal( - result.column(19), np.var(values_a, ddof=1) - ) - np.testing.assert_array_almost_equal( - result.column(20), np.var(values_b, ddof=0) - ) - np.testing.assert_array_almost_equal( - result.column(21), np.var(values_c, ddof=1) - ) + np.testing.assert_array_almost_equal(result.column(16), np.std(values_a, ddof=1)) + np.testing.assert_array_almost_equal(result.column(17), np.std(values_b, ddof=0)) + np.testing.assert_array_almost_equal(result.column(18), np.std(values_c, ddof=1)) + np.testing.assert_array_almost_equal(result.column(19), np.var(values_a, ddof=1)) + np.testing.assert_array_almost_equal(result.column(20), np.var(values_b, ddof=0)) + np.testing.assert_array_almost_equal(result.column(21), np.var(values_c, ddof=1)) def test_bit_add_or_xor(df): - df = df.aggregate( [], [ @@ -147,7 +130,6 @@ def test_bit_add_or_xor(df): def test_bool_and_or(df): - df = df.aggregate( [], [ diff --git a/datafusion/tests/test_config.py b/datafusion/tests/test_config.py index 960e72c94..12d9fc3ff 100644 --- a/datafusion/tests/test_config.py +++ b/datafusion/tests/test_config.py @@ -35,10 +35,7 @@ def test_get_then_set(config): def test_get_all(config): config_dict = config.get_all() - assert ( - config_dict["datafusion.catalog.create_default_catalog_and_schema"] - == "true" - ) + assert config_dict["datafusion.catalog.create_default_catalog_and_schema"] == "true" def test_get_invalid_config(config): diff --git a/datafusion/tests/test_context.py b/datafusion/tests/test_context.py index 97bff9bb9..d48bdd929 100644 --- a/datafusion/tests/test_context.py +++ b/datafusion/tests/test_context.py @@ -36,9 +36,7 @@ def test_create_context_no_args(): def test_create_context_with_all_valid_args(): - runtime = ( - RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) - ) + runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) @@ -357,9 +355,7 @@ def test_read_json_compressed(ctx, tmp_path): with gzip.open(gzip_path, "wb") as gzipped_file: gzipped_file.writelines(csv_file) - df = ctx.read_json( - gzip_path, file_extension=".gz", file_compression_type="gz" - ) + df = ctx.read_json(gzip_path, file_extension=".gz", file_compression_type="gz") result = df.collect() assert result[0].column(0) == pa.array(["a", "b", "c"]) @@ -381,9 +377,7 @@ def test_read_csv_compressed(ctx, tmp_path): with gzip.open(gzip_path, "wb") as gzipped_file: gzipped_file.writelines(csv_file) - csv_df = ctx.read_csv( - gzip_path, file_extension=".gz", file_compression_type="gz" - ) + csv_df = ctx.read_csv(gzip_path, file_extension=".gz", file_compression_type="gz") csv_df.select(column("c1")).show() diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index c9b0f076f..9944abeef 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -151,9 +151,7 @@ def test_with_column(df): def test_with_column_renamed(df): - df = df.with_column("c", column("a") + column("b")).with_column_renamed( - "c", "sum" - ) + df = df.with_column("c", column("a") + column("b")).with_column_renamed("c", "sum") result = df.collect()[0] @@ -218,9 +216,7 @@ def test_distinct(): [pa.array([1, 2, 3]), pa.array([4, 5, 6])], names=["a", "b"], ) - df_b = ctx.create_dataframe([[batch]]).sort( - column("a").sort(ascending=True) - ) + df_b = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) assert df_a.collect() == df_b.collect() @@ -251,9 +247,7 @@ def test_window_functions(df): "cume_dist", ), f.alias( - f.window( - "ntile", [literal(2)], order_by=[f.order_by(column("c"))] - ), + f.window("ntile", [literal(2)], order_by=[f.order_by(column("c"))]), "ntile", ), f.alias( @@ -261,9 +255,7 @@ def test_window_functions(df): "previous", ), f.alias( - f.window( - "lead", [column("b")], order_by=[f.order_by(column("b"))] - ), + f.window("lead", [column("b")], order_by=[f.order_by(column("b"))]), "next", ), f.alias( @@ -275,9 +267,7 @@ def test_window_functions(df): "first_value", ), f.alias( - f.window( - "last_value", [column("b")], order_by=[f.order_by(column("b"))] - ), + f.window("last_value", [column("b")], order_by=[f.order_by(column("b"))]), "last_value", ), f.alias( @@ -418,12 +408,14 @@ def test_optimized_logical_plan(aggregate_df): def test_execution_plan(aggregate_df): plan = aggregate_df.execution_plan() - expected = "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[SUM(test.c2)]\n" # noqa: E501 + expected = ( + "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[SUM(test.c2)]\n" # noqa: E501 + ) assert expected == plan.display() # Check the number of partitions is as expected. - assert type(plan.partition_count) is int + assert isinstance(plan.partition_count, int) expected = ( "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n" @@ -477,9 +469,7 @@ def test_intersect(): [pa.array([3]), pa.array([6])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort( - column("a").sort(ascending=True) - ) + df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) df_a_i_b = df_a.intersect(df_b).sort(column("a").sort(ascending=True)) @@ -505,9 +495,7 @@ def test_except_all(): [pa.array([1, 2]), pa.array([4, 5])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort( - column("a").sort(ascending=True) - ) + df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) df_a_e_b = df_a.except_all(df_b).sort(column("a").sort(ascending=True)) @@ -542,9 +530,7 @@ def test_union(ctx): [pa.array([1, 2, 3, 3, 4, 5]), pa.array([4, 5, 6, 6, 7, 8])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort( - column("a").sort(ascending=True) - ) + df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) df_a_u_b = df_a.union(df_b).sort(column("a").sort(ascending=True)) @@ -568,9 +554,7 @@ def test_union_distinct(ctx): [pa.array([1, 2, 3, 4, 5]), pa.array([4, 5, 6, 7, 8])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort( - column("a").sort(ascending=True) - ) + df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) df_a_u_b = df_a.union(df_b, True).sort(column("a").sort(ascending=True)) @@ -650,7 +634,7 @@ def test_empty_to_arrow_table(df): def test_to_pylist(df): # Convert datafusion dataframe to Python list pylist = df.to_pylist() - assert type(pylist) == list + assert isinstance(pylist, list) assert pylist == [ {"a": 1, "b": 4, "c": 8}, {"a": 2, "b": 5, "c": 5}, @@ -661,7 +645,7 @@ def test_to_pylist(df): def test_to_pydict(df): # Convert datafusion dataframe to Python dictionary pydict = df.to_pydict() - assert type(pydict) == dict + assert isinstance(pydict, dict) assert pydict == {"a": [1, 2, 3], "b": [4, 5, 6], "c": [8, 5, 8]} @@ -702,9 +686,7 @@ def test_write_parquet(df, tmp_path): "compression, compression_level", [("gzip", 6), ("brotli", 7), ("zstd", 15)], ) -def test_write_compressed_parquet( - df, tmp_path, compression, compression_level -): +def test_write_compressed_parquet(df, tmp_path, compression, compression_level): path = tmp_path df.write_parquet( @@ -744,9 +726,7 @@ def test_write_compressed_parquet_wrong_compression_level( @pytest.mark.parametrize("compression", ["brotli", "zstd", "wrong"]) -def test_write_compressed_parquet_missing_compression_level( - df, tmp_path, compression -): +def test_write_compressed_parquet_missing_compression_level(df, tmp_path, compression): path = tmp_path with pytest.raises(ValueError): diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index eb37692b2..1a18fd823 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -74,9 +74,7 @@ def test_lit_arith(df): """ Test literals with arithmetic operations """ - df = df.select( - literal(1) + column("b"), f.concat(column("a"), literal("!")) - ) + df = df.select(literal(1) + column("b"), f.concat(column("a"), literal("!"))) result = df.collect() assert len(result) == 1 result = result[0] @@ -148,28 +146,16 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(4), np.arcsin(values)) np.testing.assert_array_almost_equal(result.column(5), np.arccos(values)) np.testing.assert_array_almost_equal(result.column(6), np.exp(values)) - np.testing.assert_array_almost_equal( - result.column(7), np.log(values + 1.0) - ) - np.testing.assert_array_almost_equal( - result.column(8), np.log2(values + 1.0) - ) - np.testing.assert_array_almost_equal( - result.column(9), np.log10(values + 1.0) - ) + np.testing.assert_array_almost_equal(result.column(7), np.log(values + 1.0)) + np.testing.assert_array_almost_equal(result.column(8), np.log2(values + 1.0)) + np.testing.assert_array_almost_equal(result.column(9), np.log10(values + 1.0)) np.testing.assert_array_less(result.column(10), np.ones_like(values)) np.testing.assert_array_almost_equal(result.column(11), np.arctan(values)) - np.testing.assert_array_almost_equal( - result.column(12), np.arctan2(values, 1.1) - ) + np.testing.assert_array_almost_equal(result.column(12), np.arctan2(values, 1.1)) np.testing.assert_array_almost_equal(result.column(13), np.ceil(values)) np.testing.assert_array_almost_equal(result.column(14), np.floor(values)) - np.testing.assert_array_almost_equal( - result.column(15), np.power(values, 3) - ) - np.testing.assert_array_almost_equal( - result.column(16), np.power(values, 4) - ) + np.testing.assert_array_almost_equal(result.column(15), np.power(values, 3)) + np.testing.assert_array_almost_equal(result.column(16), np.power(values, 4)) np.testing.assert_array_almost_equal(result.column(17), np.round(values)) np.testing.assert_array_almost_equal(result.column(18), np.sqrt(values)) np.testing.assert_array_almost_equal(result.column(19), np.sign(values)) @@ -190,9 +176,7 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(32), np.sinh(values)) np.testing.assert_array_almost_equal(result.column(33), np.tanh(values)) np.testing.assert_array_almost_equal(result.column(34), math.factorial(6)) - np.testing.assert_array_almost_equal( - result.column(35), np.isnan(na_values) - ) + np.testing.assert_array_almost_equal(result.column(35), np.isnan(na_values)) np.testing.assert_array_almost_equal(result.column(36), na_values == 0) np.testing.assert_array_almost_equal( result.column(37), np.emath.logn(3, values + 1.0) @@ -202,9 +186,7 @@ def test_math_functions(): def test_array_functions(): data = [[1.0, 2.0, 3.0, 3.0], [4.0, 5.0, 3.0], [6.0]] ctx = SessionContext() - batch = pa.RecordBatch.from_arrays( - [np.array(data, dtype=object)], names=["arr"] - ) + batch = pa.RecordBatch.from_arrays([np.array(data, dtype=object)], names=["arr"]) df = ctx.create_dataframe([[batch]]) def py_indexof(arr, v): @@ -310,15 +292,11 @@ def py_flatten(arr): lambda: [1.0 in r for r in data], ], [ - f.array_has_all( - col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) - ), + f.array_has_all(col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]])), lambda: [np.all([v in r for v in [1.0, 3.0, 5.0]]) for r in data], ], [ - f.array_has_any( - col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) - ), + f.array_has_any(col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]])), lambda: [np.any([v in r for v in [1.0, 3.0, 5.0]]) for r in data], ], [ @@ -339,15 +317,11 @@ def py_flatten(arr): ], [ f.array_positions(col, literal(1.0)), - lambda: [ - [i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data - ], + lambda: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], ], [ f.list_positions(col, literal(1.0)), - lambda: [ - [i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data - ], + lambda: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], ], [ f.array_ndims(col), @@ -588,18 +562,9 @@ def test_hash_functions(df): ) assert result.column(2) == pa.array( [ - b( - "185F8DB32271FE25F561A6FC938B2E26" - "4306EC304EDA518007D1764826381969" - ), - b( - "78AE647DC5544D227130A0682A51E30B" - "C7777FBB6D8A8F17007463A3ECD1D524" - ), - b( - "BB7208BC9B5D7C04F1236A82A0093A5E" - "33F40423D5BA8D4266F7092C3BA43B62" - ), + b("185F8DB32271FE25F561A6FC938B2E26" "4306EC304EDA518007D1764826381969"), + b("78AE647DC5544D227130A0682A51E30B" "C7777FBB6D8A8F17007463A3ECD1D524"), + b("BB7208BC9B5D7C04F1236A82A0093A5E" "33F40423D5BA8D4266F7092C3BA43B62"), ] ) assert result.column(3) == pa.array( @@ -645,34 +610,16 @@ def test_hash_functions(df): ) assert result.column(5) == pa.array( [ - b( - "F73A5FBF881F89B814871F46E26AD3FA" - "37CB2921C5E8561618639015B3CCBB71" - ), - b( - "B792A0383FB9E7A189EC150686579532" - "854E44B71AC394831DAED169BA85CCC5" - ), - b( - "27988A0E51812297C77A433F63523334" - "6AEE29A829DCF4F46E0F58F402C6CFCB" - ), + b("F73A5FBF881F89B814871F46E26AD3FA" "37CB2921C5E8561618639015B3CCBB71"), + b("B792A0383FB9E7A189EC150686579532" "854E44B71AC394831DAED169BA85CCC5"), + b("27988A0E51812297C77A433F63523334" "6AEE29A829DCF4F46E0F58F402C6CFCB"), ] ) assert result.column(6) == pa.array( [ - b( - "FBC2B0516EE8744D293B980779178A35" - "08850FDCFE965985782C39601B65794F" - ), - b( - "BF73D18575A736E4037D45F9E316085B" - "86C19BE6363DE6AA789E13DEAACC1C4E" - ), - b( - "C8D11B9F7237E4034ADBCD2005735F9B" - "C4C597C75AD89F4492BEC8F77D15F7EB" - ), + b("FBC2B0516EE8744D293B980779178A35" "08850FDCFE965985782C39601B65794F"), + b("BF73D18575A736E4037D45F9E316085B" "86C19BE6363DE6AA789E13DEAACC1C4E"), + b("C8D11B9F7237E4034ADBCD2005735F9B" "C4C597C75AD89F4492BEC8F77D15F7EB"), ] ) assert result.column(7) == result.column(1) # SHA-224 @@ -738,9 +685,7 @@ def test_temporal_functions(df): def test_case(df): df = df.select( - f.case(column("b")) - .when(literal(4), literal(10)) - .otherwise(literal(8)), + f.case(column("b")).when(literal(4), literal(10)).otherwise(literal(8)), f.case(column("a")) .when(literal("Hello"), literal("Hola")) .when(literal("World"), literal("Mundo")) diff --git a/datafusion/tests/test_input.py b/datafusion/tests/test_input.py index 5b1decf26..fb53d86e5 100644 --- a/datafusion/tests/test_input.py +++ b/datafusion/tests/test_input.py @@ -23,9 +23,7 @@ def test_location_input(): location_input = LocationInputPlugin() cwd = os.getcwd() - input_file = ( - cwd + "/testing/data/parquet/generated_simple_numerics/blogs.parquet" - ) + input_file = cwd + "/testing/data/parquet/generated_simple_numerics/blogs.parquet" table_name = "blog" tbl = location_input.build_table(input_file, table_name) assert "blog" == tbl.name diff --git a/datafusion/tests/test_sql.py b/datafusion/tests/test_sql.py index 0c6b26727..9d4c8f677 100644 --- a/datafusion/tests/test_sql.py +++ b/datafusion/tests/test_sql.py @@ -86,9 +86,7 @@ def test_register_csv(ctx, tmp_path): result = pa.Table.from_batches(result) assert result.schema == alternative_schema - with pytest.raises( - ValueError, match="Delimiter must be a single character" - ): + with pytest.raises(ValueError, match="Delimiter must be a single character"): ctx.register_csv("csv4", path, delimiter="wrong") with pytest.raises( @@ -134,9 +132,7 @@ def test_register_parquet_partitioned(ctx, tmp_path): ) assert ctx.tables() == {"datapp"} - result = ctx.sql( - "SELECT grp, COUNT(*) AS cnt FROM datapp GROUP BY grp" - ).collect() + result = ctx.sql("SELECT grp, COUNT(*) AS cnt FROM datapp GROUP BY grp").collect() result = pa.Table.from_batches(result) rd = result.to_pydict() @@ -240,9 +236,7 @@ def test_execute(ctx, tmp_path): assert ctx.tables() == {"t"} # count - result = ctx.sql( - "SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL" - ).collect() + result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").collect() expected = pa.array([7], pa.int64()) expected = [pa.RecordBatch.from_arrays([expected], ["cnt"])] @@ -280,9 +274,7 @@ def test_execute(ctx, tmp_path): ).collect() expected_a = pa.array([50.0219, 50.0152], pa.float64()) expected_cast = pa.array([50, 50], pa.int32()) - expected = [ - pa.RecordBatch.from_arrays([expected_a, expected_cast], ["a", "a_int"]) - ] + expected = [pa.RecordBatch.from_arrays([expected_a, expected_cast], ["a", "a_int"])] np.testing.assert_equal(expected[0].column(1), expected[0].column(1)) @@ -302,9 +294,7 @@ def test_cast(ctx, tmp_path): "float", ] - select = ", ".join( - [f"CAST(9 AS {t}) AS A{i}" for i, t in enumerate(valid_types)] - ) + select = ", ".join([f"CAST(9 AS {t}) AS A{i}" for i, t in enumerate(valid_types)]) # can execute, which implies that we can cast ctx.sql(f"SELECT {select} FROM t").collect() @@ -333,14 +323,10 @@ def test_udf( ctx, tmp_path, fn, input_types, output_type, input_values, expected_values ): # write to disk - path = helpers.write_parquet( - tmp_path / "a.parquet", pa.array(input_values) - ) + path = helpers.write_parquet(tmp_path / "a.parquet", pa.array(input_values)) ctx.register_parquet("t", path) - func = udf( - fn, input_types, output_type, name="func", volatility="immutable" - ) + func = udf(fn, input_types, output_type, name="func", volatility="immutable") ctx.register_udf(func) batches = ctx.sql("SELECT func(a) AS tt FROM t").collect() diff --git a/datafusion/tests/test_substrait.py b/datafusion/tests/test_substrait.py index 7c7a2c1f2..62f6413a3 100644 --- a/datafusion/tests/test_substrait.py +++ b/datafusion/tests/test_substrait.py @@ -38,22 +38,14 @@ def test_substrait_serialization(ctx): assert ctx.tables() == {"t"} # For now just make sure the method calls blow up - substrait_plan = ss.substrait.serde.serialize_to_plan( - "SELECT * FROM t", ctx - ) + substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM t", ctx) substrait_bytes = substrait_plan.encode() - assert type(substrait_bytes) is bytes - substrait_bytes = ss.substrait.serde.serialize_bytes( - "SELECT * FROM t", ctx - ) + assert isinstance(substrait_bytes, bytes) + substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM t", ctx) substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes) - logical_plan = ss.substrait.consumer.from_substrait_plan( - ctx, substrait_plan - ) + logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan) # demonstrate how to create a DataFrame from a deserialized logical plan df = ctx.create_dataframe_from_logical_plan(logical_plan) - substrait_plan = ss.substrait.producer.to_substrait_plan( - df.logical_plan(), ctx - ) + substrait_plan = ss.substrait.producer.to_substrait_plan(df.logical_plan(), ctx) diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py index 30a01116b..d3dd7c5dd 100644 --- a/dev/release/check-rat-report.py +++ b/dev/release/check-rat-report.py @@ -23,9 +23,7 @@ import xml.etree.ElementTree as ET if len(sys.argv) != 3: - sys.stderr.write( - "Usage: %s exclude_globs.lst rat_report.xml\n" % sys.argv[0] - ) + sys.stderr.write("Usage: %s exclude_globs.lst rat_report.xml\n" % sys.argv[0]) sys.exit(1) exclude_globs_filename = sys.argv[1] diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index e97f00304..01d640669 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -27,9 +27,7 @@ def print_pulls(repo_name, title, pulls): print("**{}:**".format(title)) print() for pull, commit in pulls: - url = "https://github.com/{}/pull/{}".format( - repo_name, pull.number - ) + url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) print( "- {} [#{}]({}) ({})".format( pull.title, pull.number, url, commit.author.login @@ -40,9 +38,7 @@ def print_pulls(repo_name, title, pulls): def generate_changelog(repo, repo_name, tag1, tag2): # get a list of commits between two tags - print( - f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr - ) + print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) comparison = repo.compare(tag1, tag2) # get the pull requests for these commits diff --git a/examples/sql-on-polars.py b/examples/sql-on-polars.py index dd7a9e021..ffcb12b70 100644 --- a/examples/sql-on-polars.py +++ b/examples/sql-on-polars.py @@ -20,7 +20,5 @@ ctx = SessionContext() ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") -df = ctx.sql( - "select passenger_count, count(*) from taxi group by passenger_count" -) +df = ctx.sql("select passenger_count, count(*) from taxi group by passenger_count") print(df) diff --git a/examples/sql-using-python-udaf.py b/examples/sql-using-python-udaf.py index 3326c4a1c..7ccf5d3cb 100644 --- a/examples/sql-using-python-udaf.py +++ b/examples/sql-using-python-udaf.py @@ -30,15 +30,11 @@ def __init__(self): def update(self, values: pa.Array) -> None: # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pa.scalar( - self._sum.as_py() + pa.compute.sum(values).as_py() - ) + self._sum = pa.scalar(self._sum.as_py() + pa.compute.sum(values).as_py()) def merge(self, states: pa.Array) -> None: # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pa.scalar( - self._sum.as_py() + pa.compute.sum(states).as_py() - ) + self._sum = pa.scalar(self._sum.as_py() + pa.compute.sum(states).as_py()) def state(self) -> pa.Array: return pa.array([self._sum.as_py()]) diff --git a/examples/substrait.py b/examples/substrait.py index c579751d2..23cd74649 100644 --- a/examples/substrait.py +++ b/examples/substrait.py @@ -23,9 +23,7 @@ ctx = SessionContext() # Register table with context -ctx.register_csv( - "aggregate_test_data", "./testing/data/csv/aggregate_test_100.csv" -) +ctx.register_csv("aggregate_test_data", "./testing/data/csv/aggregate_test_100.csv") substrait_plan = ss.substrait.serde.serialize_to_plan( "SELECT * FROM aggregate_test_data", ctx @@ -49,9 +47,7 @@ substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes) # type(df_logical_plan) -> -df_logical_plan = ss.substrait.consumer.from_substrait_plan( - ctx, substrait_plan -) +df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan) # Back to Substrait Plan just for demonstration purposes # type(substrait_plan) -> diff --git a/requirements-310.txt b/requirements-310.txt index 517c31905..915f74d54 100644 --- a/requirements-310.txt +++ b/requirements-310.txt @@ -2,32 +2,16 @@ # This file is autogenerated by pip-compile with Python 3.10 # by the following command: # -# pip-compile --generate-hashes --resolver=backtracking requirements.in +# pip-compile --generate-hashes --output-file=requirements-310.txt # attrs==21.2.0 \ --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb # via pytest -black==21.9b0 \ - --hash=sha256:380f1b5da05e5a1429225676655dddb96f5ae8c75bdf91e53d798871b902a115 \ - --hash=sha256:7de4cfc7eb6b710de325712d40125689101d21d25283eed7e9998722cf10eb91 - # via -r requirements.in -click==8.0.3 \ - --hash=sha256:353f466495adaeb40b6b5f592f9f91cb22372351c84caeb068132442a4518ef3 \ - --hash=sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b - # via black -flake8==4.0.1 \ - --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ - --hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d - # via -r requirements.in iniconfig==1.1.1 \ --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 # via pytest -isort==5.9.3 \ - --hash=sha256:9c2ea1e62d871267b78307fe511c0838ba0da28698c5732d54e2790bf3ba9899 \ - --hash=sha256:e17d6e2b81095c9db0a03a8025a957f334d6ea30b26f9ec70805411e5c7c81f2 - # via -r requirements.in maturin==0.15.1 \ --hash=sha256:0e89a87549d671056f9358832c8aa1bb522abcf13eeca83327a58b091d4f5a98 \ --hash=sha256:229d7eb6e14455a3c69a10a4546f082c7bd5490b8ec7f50d5d10edcea600dc64 \ @@ -43,10 +27,6 @@ maturin==0.15.1 \ --hash=sha256:dabb8ff46461c6fb1d68e8972a172cf1dede3c9825a41e4a6caecc95c26ca3b4 \ --hash=sha256:f604b65fd9f0b94856e88cf8b345e21a27276297e6df4ad9305937887feda13b # via -r requirements.in -mccabe==0.6.1 \ - --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ - --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f - # via flake8 mypy==0.910 \ --hash=sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9 \ --hash=sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a \ @@ -75,9 +55,7 @@ mypy==0.910 \ mypy-extensions==0.4.3 \ --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 - # via - # black - # mypy + # via mypy numpy==1.21.3 \ --hash=sha256:043e83bfc274649c82a6f09836943e4a4aebe5e33656271c7dbf9621dd58b8ec \ --hash=sha256:160ccc1bed3a8371bf0d760971f09bfe80a3e18646620e9ded0ad159d9749baa \ @@ -119,14 +97,6 @@ packaging==21.0 \ --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 # via pytest -pathspec==0.9.0 \ - --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ - --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 - # via black -platformdirs==2.4.0 \ - --hash=sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2 \ - --hash=sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d - # via black pluggy==1.0.0 \ --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 @@ -135,52 +105,44 @@ py==1.10.0 \ --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \ --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a # via pytest -pyarrow==6.0.0 \ - --hash=sha256:004185e0babc6f3c3fba6ba4f106e406a0113d0f82bb9ad9a8571a1978c45d04 \ - --hash=sha256:0204e80777ab8f4e9abd3a765a8ec07ed1e3c4630bacda50d2ce212ef0f3826f \ - --hash=sha256:072c1a0fca4509eefd7d018b78542fb7e5c63aaf5698f1c0a6e45628ae17ba44 \ - --hash=sha256:15dc0d673d3f865ca63c877bd7a2eced70b0a08969fb733a28247134b8a1f18b \ - --hash=sha256:1c38263ea438a1666b13372e7565450cfeec32dbcd1c2595749476a58465eaec \ - --hash=sha256:281ce5fa03621d786a9beb514abb09846db7f0221b50eabf543caa24037eaacd \ - --hash=sha256:2d2c681659396c745e4f1988d5dd41dcc3ad557bb8d4a8c2e44030edafc08a91 \ - --hash=sha256:376c4b5f248ae63df21fe15c194e9013753164be2d38f4b3fb8bde63ac5a1958 \ - --hash=sha256:465f87fa0be0b2928b2beeba22b5813a0203fb05d90fd8563eea48e08ecc030e \ - --hash=sha256:477c746ef42c039348a288584800e299456c80c5691401bb9b19aa9c02a427b7 \ - --hash=sha256:5144bd9db2920c7cb566c96462d62443cc239104f94771d110f74393f2fb42a2 \ - --hash=sha256:5408fa8d623e66a0445f3fb0e4027fd219bf99bfb57422d543d7b7876e2c5b55 \ - --hash=sha256:5be62679201c441356d3f2a739895dcc8d4d299f2a6eabcd2163bfb6a898abba \ - --hash=sha256:5c666bc6a1cebf01206e2dc1ab05f25f39f35d3a499e0ef5cd635225e07306ca \ - --hash=sha256:6163d82cca7541774b00503c295fe86a1722820eddb958b57f091bb6f5b0a6db \ - --hash=sha256:6a1d9a2f4ee812ed0bd4182cabef99ea914ac297274f0de086f2488093d284ef \ - --hash=sha256:7a683f71b848eb6310b4ec48c0def55dac839e9994c1ac874c9b2d3d5625def1 \ - --hash=sha256:82fe80309e01acf29e3943a1f6d3c98ec109fe1d356bc1ac37d639bcaadcf684 \ - --hash=sha256:8c23f8cdecd3d9e49f9b0f9a651ae5549d1d32fd4901fb1bdc2d327edfba844f \ - --hash=sha256:8d41dfb09ba9236cca6245f33088eb42f3c54023da281139241e0f9f3b4b754e \ - --hash=sha256:a19e58dfb04e451cd8b7bdec3ac8848373b95dfc53492c9a69789aa9074a3c1b \ - --hash=sha256:a50d2f77b86af38ceabf45617208b9105d20e7a5eebc584e7c8c0acededd82ce \ - --hash=sha256:a5bed4f948c032c40597302e9bdfa65f62295240306976ecbe43a54924c6f94f \ - --hash=sha256:ac941a147d14993987cc8b605b721735a34b3e54d167302501fb4db1ad7382c7 \ - --hash=sha256:b86d175262db1eb46afdceb36d459409eb6f8e532d3dec162f8bf572c7f57623 \ - --hash=sha256:bf3400780c4d3c9cb43b1e8a1aaf2e1b7199a0572d0a645529d2784e4d0d8497 \ - --hash=sha256:c7a6e7e0bf8779e9c3428ced85507541f3da9a0675e2f4781d4eb2c7042cbf81 \ - --hash=sha256:cc1d4a70efd583befe92d4ea6f74ed2e0aa31ccdde767cd5cae8e77c65a1c2d4 \ - --hash=sha256:d046dc78a9337baa6415be915c5a16222505233e238a1017f368243c89817eea \ - --hash=sha256:da7860688c33ca88ac05f1a487d32d96d9caa091412496c35f3d1d832145675a \ - --hash=sha256:ddf2e6e3b321adaaf716f2d5af8e92d205a9671e0cb7c0779710a567fd1dd580 \ - --hash=sha256:e81508239a71943759cee272ce625ae208092dd36ef2c6713fccee30bbcf52bb \ - --hash=sha256:ea64a48a85c631eb2a0ea13ccdec5143c85b5897836b16331ee4289d27a57247 \ - --hash=sha256:ed0be080cf595ea15ff1c9ff4097bbf1fcc4b50847d98c0a3c0412fbc6ede7e9 \ - --hash=sha256:fb701ec4a94b92102606d4e88f0b8eba34f09a5ad8e014eaa4af76f42b7f62ae \ - --hash=sha256:fbda7595f24a639bcef3419ecfac17216efacb09f7b0f1b4c4c97f900d65ca0e +pyarrow==15.0.0 \ + --hash=sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4 \ + --hash=sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970 \ + --hash=sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555 \ + --hash=sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b \ + --hash=sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d \ + --hash=sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2 \ + --hash=sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99 \ + --hash=sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac \ + --hash=sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6 \ + --hash=sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5 \ + --hash=sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f \ + --hash=sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3 \ + --hash=sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb \ + --hash=sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642 \ + --hash=sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2 \ + --hash=sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f \ + --hash=sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929 \ + --hash=sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e \ + --hash=sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83 \ + --hash=sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340 \ + --hash=sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540 \ + --hash=sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed \ + --hash=sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351 \ + --hash=sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177 \ + --hash=sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4 \ + --hash=sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7 \ + --hash=sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593 \ + --hash=sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7 \ + --hash=sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31 \ + --hash=sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b \ + --hash=sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e \ + --hash=sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6 \ + --hash=sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4 \ + --hash=sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40 \ + --hash=sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5 \ + --hash=sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19 # via -r requirements.in -pycodestyle==2.8.0 \ - --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \ - --hash=sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f - # via flake8 -pyflakes==2.4.0 \ - --hash=sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c \ - --hash=sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e - # via flake8 pyparsing==3.0.3 \ --hash=sha256:9e3511118010f112a4b4b435ae50e1eaa610cda191acb9e421d60cf5fde83455 \ --hash=sha256:f8d3fe9fc404576c5164f0f0c4e382c96b85265e023c409c43d48f65da9d60d0 @@ -189,44 +151,25 @@ pytest==6.2.5 \ --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89 \ --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134 # via -r requirements.in -regex==2021.10.23 \ - --hash=sha256:0c186691a7995ef1db61205e00545bf161fb7b59cdb8c1201c89b333141c438a \ - --hash=sha256:0dcc0e71118be8c69252c207630faf13ca5e1b8583d57012aae191e7d6d28b84 \ - --hash=sha256:0f7552429dd39f70057ac5d0e897e5bfe211629652399a21671e53f2a9693a4e \ - --hash=sha256:129472cd06062fb13e7b4670a102951a3e655e9b91634432cfbdb7810af9d710 \ - --hash=sha256:13ec99df95003f56edcd307db44f06fbeb708c4ccdcf940478067dd62353181e \ - --hash=sha256:1f2b59c28afc53973d22e7bc18428721ee8ca6079becf1b36571c42627321c65 \ - --hash=sha256:2b20f544cbbeffe171911f6ce90388ad36fe3fad26b7c7a35d4762817e9ea69c \ - --hash=sha256:2fb698037c35109d3c2e30f2beb499e5ebae6e4bb8ff2e60c50b9a805a716f79 \ - --hash=sha256:34d870f9f27f2161709054d73646fc9aca49480617a65533fc2b4611c518e455 \ - --hash=sha256:391703a2abf8013d95bae39145d26b4e21531ab82e22f26cd3a181ee2644c234 \ - --hash=sha256:450dc27483548214314640c89a0f275dbc557968ed088da40bde7ef8fb52829e \ - --hash=sha256:45b65d6a275a478ac2cbd7fdbf7cc93c1982d613de4574b56fd6972ceadb8395 \ - --hash=sha256:5095a411c8479e715784a0c9236568ae72509450ee2226b649083730f3fadfc6 \ - --hash=sha256:530fc2bbb3dc1ebb17f70f7b234f90a1dd43b1b489ea38cea7be95fb21cdb5c7 \ - --hash=sha256:56f0c81c44638dfd0e2367df1a331b4ddf2e771366c4b9c5d9a473de75e3e1c7 \ - --hash=sha256:5e9c9e0ce92f27cef79e28e877c6b6988c48b16942258f3bc55d39b5f911df4f \ - --hash=sha256:6d7722136c6ed75caf84e1788df36397efdc5dbadab95e59c2bba82d4d808a4c \ - --hash=sha256:74d071dbe4b53c602edd87a7476ab23015a991374ddb228d941929ad7c8c922e \ - --hash=sha256:7b568809dca44cb75c8ebb260844ea98252c8c88396f9d203f5094e50a70355f \ - --hash=sha256:80bb5d2e92b2258188e7dcae5b188c7bf868eafdf800ea6edd0fbfc029984a88 \ - --hash=sha256:8d1cdcda6bd16268316d5db1038965acf948f2a6f43acc2e0b1641ceab443623 \ - --hash=sha256:9f665677e46c5a4d288ece12fdedf4f4204a422bb28ff05f0e6b08b7447796d1 \ - --hash=sha256:a30513828180264294953cecd942202dfda64e85195ae36c265daf4052af0464 \ - --hash=sha256:a7a986c45d1099a5de766a15de7bee3840b1e0e1a344430926af08e5297cf666 \ - --hash=sha256:a940ca7e7189d23da2bfbb38973832813eab6bd83f3bf89a977668c2f813deae \ - --hash=sha256:ab7c5684ff3538b67df3f93d66bd3369b749087871ae3786e70ef39e601345b0 \ - --hash=sha256:be04739a27be55631069b348dda0c81d8ea9822b5da10b8019b789e42d1fe452 \ - --hash=sha256:c0938ddd60cc04e8f1faf7a14a166ac939aac703745bfcd8e8f20322a7373019 \ - --hash=sha256:cb46b542133999580ffb691baf67410306833ee1e4f58ed06b6a7aaf4e046952 \ - --hash=sha256:d134757a37d8640f3c0abb41f5e68b7cf66c644f54ef1cb0573b7ea1c63e1509 \ - --hash=sha256:de557502c3bec8e634246588a94e82f1ee1b9dfcfdc453267c4fb652ff531570 \ - --hash=sha256:ded0c4a3eee56b57fcb2315e40812b173cafe79d2f992d50015f4387445737fa \ - --hash=sha256:e1dae12321b31059a1a72aaa0e6ba30156fe7e633355e445451e4021b8e122b6 \ - --hash=sha256:eb672217f7bd640411cfc69756ce721d00ae600814708d35c930930f18e8029f \ - --hash=sha256:ee684f139c91e69fe09b8e83d18b4d63bf87d9440c1eb2eeb52ee851883b1b29 \ - --hash=sha256:f3f9a91d3cc5e5b0ddf1043c0ae5fa4852f18a1c0050318baf5fc7930ecc1f9c - # via black +ruff==0.3.0 \ + --hash=sha256:0886184ba2618d815067cf43e005388967b67ab9c80df52b32ec1152ab49f53a \ + --hash=sha256:128265876c1d703e5f5e5a4543bd8be47c73a9ba223fd3989d4aa87dd06f312f \ + --hash=sha256:19eacceb4c9406f6c41af806418a26fdb23120dfe53583df76d1401c92b7c14b \ + --hash=sha256:23dbb808e2f1d68eeadd5f655485e235c102ac6f12ad31505804edced2a5ae77 \ + --hash=sha256:2f7dbba46e2827dfcb0f0cc55fba8e96ba7c8700e0a866eb8cef7d1d66c25dcb \ + --hash=sha256:3ef655c51f41d5fa879f98e40c90072b567c666a7114fa2d9fe004dffba00932 \ + --hash=sha256:5da894a29ec018a8293d3d17c797e73b374773943e8369cfc50495573d396933 \ + --hash=sha256:755c22536d7f1889be25f2baf6fedd019d0c51d079e8417d4441159f3bcd30c2 \ + --hash=sha256:7deb528029bacf845bdbb3dbb2927d8ef9b4356a5e731b10eef171e3f0a85944 \ + --hash=sha256:9343690f95710f8cf251bee1013bf43030072b9f8d012fbed6ad702ef70d360a \ + --hash=sha256:a1f3ed501a42f60f4dedb7805fa8d4534e78b4e196f536bac926f805f0743d49 \ + --hash=sha256:b08b356d06a792e49a12074b62222f9d4ea2a11dca9da9f68163b28c71bf1dd4 \ + --hash=sha256:cc30a9053ff2f1ffb505a585797c23434d5f6c838bacfe206c0e6cf38c921a1e \ + --hash=sha256:d0d3d7ef3d4f06433d592e5f7d813314a34601e6c5be8481cccb7fa760aa243e \ + --hash=sha256:dd73fe7f4c28d317855da6a7bc4aa29a1500320818dd8f27df95f70a01b8171f \ + --hash=sha256:e1e0d4381ca88fb2b73ea0766008e703f33f460295de658f5467f6f229658c19 \ + --hash=sha256:e3a4a6d46aef0a84b74fcd201a4401ea9a6cd85614f6a9435f2d33dd8cefbf83 + # via -r requirements.in toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f @@ -237,13 +180,9 @@ toml==0.10.2 \ tomli==1.2.2 \ --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade - # via - # black - # maturin + # via maturin typing-extensions==3.10.0.2 \ --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 - # via - # black - # mypy + # via mypy diff --git a/requirements-311.txt b/requirements-311.txt index c5ae2ef79..876311e6c 100644 --- a/requirements-311.txt +++ b/requirements-311.txt @@ -4,46 +4,10 @@ # # pip-compile --generate-hashes --output-file=requirements-311.txt # -black==24.2.0 \ - --hash=sha256:057c3dc602eaa6fdc451069bd027a1b2635028b575a6c3acfd63193ced20d9c8 \ - --hash=sha256:08654d0797e65f2423f850fc8e16a0ce50925f9337fb4a4a176a7aa4026e63f8 \ - --hash=sha256:163baf4ef40e6897a2a9b83890e59141cc8c2a98f2dda5080dc15c00ee1e62cd \ - --hash=sha256:1e08fb9a15c914b81dd734ddd7fb10513016e5ce7e6704bdd5e1251ceee51ac9 \ - --hash=sha256:4dd76e9468d5536abd40ffbc7a247f83b2324f0c050556d9c371c2b9a9a95e31 \ - --hash=sha256:4f9de21bafcba9683853f6c96c2d515e364aee631b178eaa5145fc1c61a3cc92 \ - --hash=sha256:61a0391772490ddfb8a693c067df1ef5227257e72b0e4108482b8d41b5aee13f \ - --hash=sha256:6981eae48b3b33399c8757036c7f5d48a535b962a7c2310d19361edeef64ce29 \ - --hash=sha256:7e53a8c630f71db01b28cd9602a1ada68c937cbf2c333e6ed041390d6968faf4 \ - --hash=sha256:810d445ae6069ce64030c78ff6127cd9cd178a9ac3361435708b907d8a04c693 \ - --hash=sha256:93601c2deb321b4bad8f95df408e3fb3943d85012dddb6121336b8e24a0d1218 \ - --hash=sha256:992e451b04667116680cb88f63449267c13e1ad134f30087dec8527242e9862a \ - --hash=sha256:9db528bccb9e8e20c08e716b3b09c6bdd64da0dd129b11e160bf082d4642ac23 \ - --hash=sha256:a0057f800de6acc4407fe75bb147b0c2b5cbb7c3ed110d3e5999cd01184d53b0 \ - --hash=sha256:ba15742a13de85e9b8f3239c8f807723991fbfae24bad92d34a2b12e81904982 \ - --hash=sha256:bce4f25c27c3435e4dace4815bcb2008b87e167e3bf4ee47ccdc5ce906eb4894 \ - --hash=sha256:ca610d29415ee1a30a3f30fab7a8f4144e9d34c89a235d81292a1edb2b55f540 \ - --hash=sha256:d533d5e3259720fdbc1b37444491b024003e012c5173f7d06825a77508085430 \ - --hash=sha256:d84f29eb3ee44859052073b7636533ec995bd0f64e2fb43aeceefc70090e752b \ - --hash=sha256:e37c99f89929af50ffaf912454b3e3b47fd64109659026b678c091a4cd450fb2 \ - --hash=sha256:e8a6ae970537e67830776488bca52000eaa37fa63b9988e8c487458d9cd5ace6 \ - --hash=sha256:faf2ee02e6612577ba0181f4347bcbcf591eb122f7841ae5ba233d12c39dcb4d - # via -r requirements.in -click==8.1.7 \ - --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ - --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de - # via black -flake8==7.0.0 \ - --hash=sha256:33f96621059e65eec474169085dc92bf26e7b2d47366b70be2f67ab80dc25132 \ - --hash=sha256:a6dfbb75e03252917f2473ea9653f7cd799c3064e54d4c8140044c5c065f53c3 - # via -r requirements.in iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via pytest -isort==5.13.2 \ - --hash=sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109 \ - --hash=sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6 - # via -r requirements.in maturin==1.4.0 \ --hash=sha256:01473dc30aed8f2cee3572b3e99e3ea75bf09c84b028bf6077f7643a189699c8 \ --hash=sha256:076970a73da7fa3648204a584cd347b899c1ea67f8124b212bccd06728e63ed9 \ @@ -59,10 +23,6 @@ maturin==1.4.0 \ --hash=sha256:ff95a4494d9e57b6e74d4d7f8a9a2ee8ed29bd7f0e61855656ad959a432c0efc \ --hash=sha256:ffe4e967080ceb83c156e73a37d3974b30cad01c376a86dc39a76a0c6bccf9b0 # via -r requirements.in -mccabe==0.7.0 \ - --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ - --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e - # via flake8 mypy==1.8.0 \ --hash=sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6 \ --hash=sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d \ @@ -95,9 +55,7 @@ mypy==1.8.0 \ mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 - # via - # black - # mypy + # via mypy numpy==1.26.4 \ --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ @@ -141,17 +99,7 @@ numpy==1.26.4 \ packaging==23.2 \ --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 - # via - # black - # pytest -pathspec==0.12.1 \ - --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ - --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 - # via black -platformdirs==4.2.0 \ - --hash=sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068 \ - --hash=sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768 - # via black + # via pytest pluggy==1.4.0 \ --hash=sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 \ --hash=sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be @@ -194,18 +142,29 @@ pyarrow==15.0.0 \ --hash=sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5 \ --hash=sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19 # via -r requirements.in -pycodestyle==2.11.1 \ - --hash=sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f \ - --hash=sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67 - # via flake8 -pyflakes==3.2.0 \ - --hash=sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f \ - --hash=sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a - # via flake8 pytest==8.0.2 \ --hash=sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd \ --hash=sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096 # via -r requirements.in +ruff==0.3.0 \ + --hash=sha256:0886184ba2618d815067cf43e005388967b67ab9c80df52b32ec1152ab49f53a \ + --hash=sha256:128265876c1d703e5f5e5a4543bd8be47c73a9ba223fd3989d4aa87dd06f312f \ + --hash=sha256:19eacceb4c9406f6c41af806418a26fdb23120dfe53583df76d1401c92b7c14b \ + --hash=sha256:23dbb808e2f1d68eeadd5f655485e235c102ac6f12ad31505804edced2a5ae77 \ + --hash=sha256:2f7dbba46e2827dfcb0f0cc55fba8e96ba7c8700e0a866eb8cef7d1d66c25dcb \ + --hash=sha256:3ef655c51f41d5fa879f98e40c90072b567c666a7114fa2d9fe004dffba00932 \ + --hash=sha256:5da894a29ec018a8293d3d17c797e73b374773943e8369cfc50495573d396933 \ + --hash=sha256:755c22536d7f1889be25f2baf6fedd019d0c51d079e8417d4441159f3bcd30c2 \ + --hash=sha256:7deb528029bacf845bdbb3dbb2927d8ef9b4356a5e731b10eef171e3f0a85944 \ + --hash=sha256:9343690f95710f8cf251bee1013bf43030072b9f8d012fbed6ad702ef70d360a \ + --hash=sha256:a1f3ed501a42f60f4dedb7805fa8d4534e78b4e196f536bac926f805f0743d49 \ + --hash=sha256:b08b356d06a792e49a12074b62222f9d4ea2a11dca9da9f68163b28c71bf1dd4 \ + --hash=sha256:cc30a9053ff2f1ffb505a585797c23434d5f6c838bacfe206c0e6cf38c921a1e \ + --hash=sha256:d0d3d7ef3d4f06433d592e5f7d813314a34601e6c5be8481cccb7fa760aa243e \ + --hash=sha256:dd73fe7f4c28d317855da6a7bc4aa29a1500320818dd8f27df95f70a01b8171f \ + --hash=sha256:e1e0d4381ca88fb2b73ea0766008e703f33f460295de658f5467f6f229658c19 \ + --hash=sha256:e3a4a6d46aef0a84b74fcd201a4401ea9a6cd85614f6a9435f2d33dd8cefbf83 + # via -r requirements.in toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f diff --git a/requirements.in b/requirements.in index 09872b03b..2365d23e0 100644 --- a/requirements.in +++ b/requirements.in @@ -15,13 +15,11 @@ # specific language governing permissions and limitations # under the License. -black -flake8 -isort maturin>=0.15 mypy numpy pyarrow>=11.0.0 pytest +ruff toml importlib_metadata; python_version < "3.8" From 1ff22bbde774471ba324e0f5d8fafd4384656f17 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 4 Mar 2024 09:55:25 -0700 Subject: [PATCH 119/413] remove SQL om polars/pandas/cudf experiment (#602) --- README.md | 31 +++------ datafusion/context.py | 142 -------------------------------------- datafusion/cudf.py | 97 -------------------------- datafusion/pandas.py | 93 ------------------------- datafusion/polars.py | 100 --------------------------- examples/sql-on-cudf.py | 24 ------- examples/sql-on-pandas.py | 24 ------- examples/sql-on-polars.py | 24 ------- 8 files changed, 10 insertions(+), 525 deletions(-) delete mode 100644 datafusion/context.py delete mode 100644 datafusion/cudf.py delete mode 100644 datafusion/pandas.py delete mode 100644 datafusion/polars.py delete mode 100644 examples/sql-on-cudf.py delete mode 100644 examples/sql-on-pandas.py delete mode 100644 examples/sql-on-polars.py diff --git a/README.md b/README.md index a2e3efd4e..4345b52ed 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,16 @@ This is a Python library that binds to [Apache Arrow](https://arrow.apache.org/) in-memory query engine [DataFusion](https://github.com/apache/arrow-datafusion). -DataFusion's Python bindings can be used as an end-user tool as well as providing a foundation for building new systems. +DataFusion's Python bindings can be used as a foundation for building new data systems in Python. Here are some examples: + +- [Dask SQL](https://github.com/dask-contrib/dask-sql) uses DataFusion's Python bindings for SQL parsing, query + planning, and logical plan optimizations, and then transpiles the logical plan to Dask operations for execution. +- [DataFusion Ballista](https://github.com/apache/arrow-ballista) is a distributed SQL query engine that extends + DataFusion's Python bindings for distributed use cases. + +It is also possible to use these Python bindings directly for DataFrame and SQL operations, but you may find that +[Polars](http://pola.rs/) and [DuckDB](http://www.duckdb.org/) are more suitable for this use case, since they have +more of an end-user focus and are more actively maintained than these Python bindings. ## Features @@ -35,20 +44,6 @@ DataFusion's Python bindings can be used as an end-user tool as well as providin - Serialize and deserialize query plans in Substrait format. - Experimental support for transpiling SQL queries to DataFrame calls with Polars, Pandas, and cuDF. -## Comparison with other projects - -Here is a comparison with similar projects that may help understand when DataFusion might be suitable and unsuitable -for your needs: - -- [DuckDB](http://www.duckdb.org/) is an open source, in-process analytic database. Like DataFusion, it supports - very fast execution, both from its custom file format and directly from Parquet files. Unlike DataFusion, it is - written in C/C++ and it is primarily used directly by users as a serverless database and query system rather than - as a library for building such database systems. - -- [Polars](http://pola.rs/) is one of the fastest DataFrame libraries at the time of writing. Like DataFusion, it - is also written in Rust and uses the Apache Arrow memory model, but unlike DataFusion it does not provide full SQL - support, nor as many extension points. - ## Example Usage The following example demonstrates running a SQL query against a Parquet file using DataFusion, storing the results @@ -143,12 +138,6 @@ See [examples](examples/README.md) for more information. - [Serialize query plans using Substrait](./examples/substrait.py) -### Executing SQL against DataFrame Libraries (Experimental) - -- [Executing SQL on Polars](./examples/sql-on-polars.py) -- [Executing SQL on Pandas](./examples/sql-on-pandas.py) -- [Executing SQL on cuDF](./examples/sql-on-cudf.py) - ## How to install (from pip) ### Pip diff --git a/datafusion/context.py b/datafusion/context.py deleted file mode 100644 index 30602402c..000000000 --- a/datafusion/context.py +++ /dev/null @@ -1,142 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from abc import ABC, abstractmethod -from typing import Any, Dict, List - -from datafusion.common import SqlSchema, SqlTable - - -class BaseSessionContext(ABC): - """ - Abstraction defining all methods, properties, and common functionality - shared amongst implementations using DataFusion as their SQL Parser/Engine - """ - - DEFAULT_CATALOG_NAME = "root" - DEFAULT_SCHEMA_NAME = "datafusion" - - @abstractmethod - def create_schema( - self, - schema_name: str, - **kwargs, - ): - """ - Creates/Registers a logical container that holds database - objects such as tables, views, indexes, and other - related objects. It provides a way to group related database - objects together. A schema can be owned by a database - user and can be used to separate objects in different - logical groups for easy management. - """ - pass - - @abstractmethod - def update_schema( - self, - schema_name: str, - new_schema: SqlSchema, - **kwargs, - ): - """ - Updates an existing schema in the SessionContext - """ - pass - - @abstractmethod - def drop_schema( - self, - schema_name: str, - **kwargs, - ): - """ - Drops the specified Schema, based on name, from the current context - """ - pass - - @abstractmethod - def show_schemas(self, **kwargs) -> Dict[str, SqlSchema]: - """ - Return all schemas in the current SessionContext impl. - """ - pass - - @abstractmethod - def create_table( - self, - schema_name: str, - table_name: str, - input_source: Any, - **kwargs, - ): - """ - Creates/Registers a table in the specied schema instance - """ - pass - - @abstractmethod - def update_table( - self, - schema_name: str, - table_name: str, - new_table: SqlTable, - **kwargs, - ): - """ - Updates an existing table in the SessionContext - """ - pass - - @abstractmethod - def drop_table( - self, - schema_name: str, - table_name: str, - **kwargs, - ): - """ - Drops the specified table, based on name, from the current context - """ - pass - - @abstractmethod - def show_tables(self, **kwargs) -> List[SqlTable]: - """ - Return all tables in the current SessionContext impl. - """ - pass - - @abstractmethod - def register_table( - self, - table_name: str, - path: str, - **kwargs, - ): - pass - - # TODO: Remove abstraction, this functionality can be shared - # between all implementing classes since it just prints the - # logical plan from DataFusion - @abstractmethod - def explain(self, sql): - pass - - @abstractmethod - def sql(self, sql): - pass diff --git a/datafusion/cudf.py b/datafusion/cudf.py deleted file mode 100644 index d8bc8e6d0..000000000 --- a/datafusion/cudf.py +++ /dev/null @@ -1,97 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging -import cudf -from datafusion.context import BaseSessionContext -from datafusion.expr import Projection, TableScan, Column - -from datafusion.common import SqlSchema - -logger = logging.getLogger(__name__) - - -class SessionContext(BaseSessionContext): - def __init__(self, context, logging_level=logging.INFO): - """ - Create a new Session. - """ - # Cudf requires a provided context - self.context = context - - # Set the logging level for this SQL context - logging.basicConfig(level=logging_level) - - # Name of the root catalog - self.catalog_name = self.DEFAULT_CATALOG_NAME - # Name of the root schema - self.schema_name = self.DEFAULT_SCHEMA_NAME - # Add the schema to the context - sch = SqlSchema(self.schema_name) - self.schemas = {} - self.schemas[self.schema_name] = sch - self.context.register_schema(self.schema_name, sch) - - def to_cudf_expr(self, expr): - # get Python wrapper for logical expression - expr = expr.to_variant() - - if isinstance(expr, Column): - return expr.name() - else: - raise Exception("unsupported expression: {}".format(expr)) - - def to_cudf_df(self, plan): - # recurse down first to translate inputs into pandas data frames - inputs = [self.to_cudf_df(x) for x in plan.inputs()] - - # get Python wrapper for logical operator node - node = plan.to_variant() - - if isinstance(node, Projection): - args = [self.to_cudf_expr(expr) for expr in node.projections()] - return inputs[0][args] - elif isinstance(node, TableScan): - return cudf.read_parquet(self.parquet_tables[node.table_name()]) - else: - raise Exception("unsupported logical operator: {}".format(type(node))) - - def create_schema(self, schema_name: str, **kwargs): - logger.debug(f"Creating schema: {schema_name}") - self.schemas[schema_name] = SqlSchema(schema_name) - self.context.register_schema(schema_name, SqlSchema(schema_name)) - - def update_schema(self, schema_name: str, new_schema: SqlSchema, **kwargs): - self.schemas[schema_name] = new_schema - - def drop_schema(self, schema_name, **kwargs): - del self.schemas[schema_name] - - def show_schemas(self, **kwargs): - return self.schemas - - def register_table(self, name, path, **kwargs): - self.parquet_tables[name] = path - self.datafusion_ctx.register_parquet(name, path) - - def explain(self, sql): - super.explain() - - def sql(self, sql): - datafusion_df = self.datafusion_ctx.sql(sql) - plan = datafusion_df.logical_plan() - return self.to_cudf_df(plan) diff --git a/datafusion/pandas.py b/datafusion/pandas.py deleted file mode 100644 index cf1fa6f7b..000000000 --- a/datafusion/pandas.py +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import logging -import pandas as pd -import datafusion -from datafusion.common import SqlSchema -from datafusion.context import BaseSessionContext -from datafusion.expr import Projection, TableScan, Column - -logger = logging.getLogger(__name__) - - -class SessionContext(BaseSessionContext): - def __init__(self, logging_level=logging.INFO): - self.datafusion_ctx = datafusion.SessionContext() - self.parquet_tables = {} - - # Set the logging level for this SQL context - logging.basicConfig(level=logging_level) - - # Name of the root catalog - self.catalog_name = self.DEFAULT_CATALOG_NAME - # Name of the root schema - self.schema_name = self.DEFAULT_SCHEMA_NAME - # Add the schema to the context - sch = SqlSchema(self.schema_name) - self.schemas[self.schema_name] = sch - self.context.register_schema(self.schema_name, sch) - - def to_pandas_expr(self, expr): - # get Python wrapper for logical expression - expr = expr.to_variant() - - if isinstance(expr, Column): - return expr.name() - else: - raise Exception("unsupported expression: {}".format(expr)) - - def to_pandas_df(self, plan): - # recurse down first to translate inputs into pandas data frames - inputs = [self.to_pandas_df(x) for x in plan.inputs()] - - # get Python wrapper for logical operator node - node = plan.to_variant() - - if isinstance(node, Projection): - args = [self.to_pandas_expr(expr) for expr in node.projections()] - return inputs[0][args] - elif isinstance(node, TableScan): - return pd.read_parquet(self.parquet_tables[node.table_name()]) - else: - raise Exception("unsupported logical operator: {}".format(type(node))) - - def create_schema(self, schema_name: str, **kwargs): - logger.debug(f"Creating schema: {schema_name}") - self.schemas[schema_name] = SqlSchema(schema_name) - self.context.register_schema(schema_name, SqlSchema(schema_name)) - - def update_schema(self, schema_name: str, new_schema: SqlSchema, **kwargs): - self.schemas[schema_name] = new_schema - - def drop_schema(self, schema_name, **kwargs): - del self.schemas[schema_name] - - def show_schemas(self, **kwargs): - return self.schemas - - def register_table(self, name, path, **kwargs): - self.parquet_tables[name] = path - self.datafusion_ctx.register_parquet(name, path) - - def explain(self, sql): - super.explain() - - def sql(self, sql): - datafusion_df = self.datafusion_ctx.sql(sql) - plan = datafusion_df.logical_plan() - return self.to_pandas_df(plan) diff --git a/datafusion/polars.py b/datafusion/polars.py deleted file mode 100644 index ac5e26e3e..000000000 --- a/datafusion/polars.py +++ /dev/null @@ -1,100 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -import logging -import polars -import datafusion -from datafusion.context import BaseSessionContext -from datafusion.expr import Projection, TableScan, Aggregate -from datafusion.expr import Column, AggregateFunction - -from datafusion.common import SqlSchema - -logger = logging.getLogger(__name__) - - -class SessionContext(BaseSessionContext): - def __init__(self, logging_level=logging.INFO): - self.datafusion_ctx = datafusion.SessionContext() - self.parquet_tables = {} - - def to_polars_expr(self, expr): - # get Python wrapper for logical expression - expr = expr.to_variant() - - if isinstance(expr, Column): - return polars.col(expr.name()) - else: - raise Exception("unsupported expression: {}".format(expr)) - - def to_polars_df(self, plan): - # recurse down first to translate inputs into Polars data frames - inputs = [self.to_polars_df(x) for x in plan.inputs()] - - # get Python wrapper for logical operator node - node = plan.to_variant() - - if isinstance(node, Projection): - args = [self.to_polars_expr(expr) for expr in node.projections()] - return inputs[0].select(*args) - elif isinstance(node, Aggregate): - groupby_expr = [self.to_polars_expr(expr) for expr in node.group_by_exprs()] - aggs = [] - for expr in node.aggregate_exprs(): - expr = expr.to_variant() - if isinstance(expr, AggregateFunction): - if expr.aggregate_type() == "COUNT": - aggs.append(polars.count().alias("{}".format(expr))) - else: - raise Exception( - "Unsupported aggregate function {}".format( - expr.aggregate_type() - ) - ) - else: - raise Exception("Unsupported aggregate function {}".format(expr)) - df = inputs[0].groupby(groupby_expr).agg(aggs) - return df - elif isinstance(node, TableScan): - return polars.read_parquet(self.parquet_tables[node.table_name()]) - else: - raise Exception("unsupported logical operator: {}".format(type(node))) - - def create_schema(self, schema_name: str, **kwargs): - logger.debug(f"Creating schema: {schema_name}") - self.schemas[schema_name] = SqlSchema(schema_name) - self.context.register_schema(schema_name, SqlSchema(schema_name)) - - def update_schema(self, schema_name: str, new_schema: SqlSchema, **kwargs): - self.schemas[schema_name] = new_schema - - def drop_schema(self, schema_name, **kwargs): - del self.schemas[schema_name] - - def show_schemas(self, **kwargs): - return self.schemas - - def register_table(self, name, path, **kwargs): - self.parquet_tables[name] = path - self.datafusion_ctx.register_parquet(name, path) - - def explain(self, sql): - super.explain() - - def sql(self, sql): - datafusion_df = self.datafusion_ctx.sql(sql) - plan = datafusion_df.logical_plan() - return self.to_polars_df(plan) diff --git a/examples/sql-on-cudf.py b/examples/sql-on-cudf.py deleted file mode 100644 index b64d8f046..000000000 --- a/examples/sql-on-cudf.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datafusion.cudf import SessionContext - - -ctx = SessionContext() -ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") -df = ctx.sql("select passenger_count from taxi") -print(df) diff --git a/examples/sql-on-pandas.py b/examples/sql-on-pandas.py deleted file mode 100644 index e3312a201..000000000 --- a/examples/sql-on-pandas.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datafusion.pandas import SessionContext - - -ctx = SessionContext() -ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") -df = ctx.sql("select passenger_count from taxi") -print(df) diff --git a/examples/sql-on-polars.py b/examples/sql-on-polars.py deleted file mode 100644 index ffcb12b70..000000000 --- a/examples/sql-on-polars.py +++ /dev/null @@ -1,24 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from datafusion.polars import SessionContext - - -ctx = SessionContext() -ctx.register_table("taxi", "yellow_tripdata_2021-01.parquet") -df = ctx.sql("select passenger_count, count(*) from taxi group by passenger_count") -print(df) From 18ac182fb236741061367c5acfe4b60b6d8c4a5f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 11:55:53 -0800 Subject: [PATCH 120/413] build(deps): bump object_store from 0.9.0 to 0.9.1 (#611) Bumps [object_store](https://github.com/apache/arrow-rs) from 0.9.0 to 0.9.1. - [Changelog](https://github.com/apache/arrow-rs/blob/master/CHANGELOG-old.md) - [Commits](https://github.com/apache/arrow-rs/commits) --- updated-dependencies: - dependency-name: object_store dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31e7bad01..c86544fa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1835,9 +1835,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d139f545f64630e2e3688fd9f81c470888ab01edeb72d13b4e86c566f1130000" +checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", "base64", @@ -1847,6 +1847,7 @@ dependencies = [ "humantime", "hyper", "itertools 0.12.0", + "md-5", "parking_lot", "percent-encoding", "quick-xml", diff --git a/Cargo.toml b/Cargo.toml index f8c6bf9f7..6afcde957 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,7 @@ uuid = { version = "1.3", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.9.0", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.9.1", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" syn = "2.0.43" From 6a895c611118089916af674e1db2aaac954a42e5 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Thu, 21 Mar 2024 19:26:55 +0000 Subject: [PATCH 121/413] More missing array funcs (#605) * Add `array_distinct` function * Add `range` function * Add `list_distinct` alias * Add `array_intersect` scalar function * Add `array_union` scalar function * Add `array_except` scalar function * Add `array_resize` scalar function --- datafusion/tests/test_functions.py | 53 ++++++++++++++++++++++++++++++ src/functions.rs | 22 +++++++++++++ 2 files changed, 75 insertions(+) diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index 1a18fd823..fa9a495dd 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -220,6 +220,15 @@ def py_arr_replace(arr, from_, to, n=None): return new_arr + def py_arr_resize(arr, size, value): + arr = np.asarray(arr) + return np.pad( + arr, + [(0, size - arr.shape[0])], + "constant", + constant_values=value, + ) + def py_flatten(arr): result = [] for elem in arr: @@ -259,6 +268,14 @@ def py_flatten(arr): f.array_dims(col), lambda: [[len(r)] for r in data], ], + [ + f.array_distinct(col), + lambda: [list(set(r)) for r in data], + ], + [ + f.list_distinct(col), + lambda: [list(set(r)) for r in data], + ], [ f.list_dims(col), lambda: [[len(r)] for r in data], @@ -415,7 +432,43 @@ def py_flatten(arr): f.list_slice(col, literal(-1), literal(2)), lambda: [arr[-1:2] for arr in data], ], + [ + f.array_intersect(col, literal([3.0, 4.0])), + lambda: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], + ], + [ + f.list_intersect(col, literal([3.0, 4.0])), + lambda: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], + ], + [ + f.array_union(col, literal([12.0, 999.0])), + lambda: [np.union1d(arr, [12.0, 999.0]) for arr in data], + ], + [ + f.list_union(col, literal([12.0, 999.0])), + lambda: [np.union1d(arr, [12.0, 999.0]) for arr in data], + ], + [ + f.array_except(col, literal([3.0])), + lambda: [np.setdiff1d(arr, [3.0]) for arr in data], + ], + [ + f.list_except(col, literal([3.0])), + lambda: [np.setdiff1d(arr, [3.0]) for arr in data], + ], + [ + f.array_resize(col, literal(10), literal(0.0)), + lambda: [py_arr_resize(arr, 10, 0.0) for arr in data], + ], + [ + f.list_resize(col, literal(10), literal(0.0)), + lambda: [py_arr_resize(arr, 10, 0.0) for arr in data], + ], [f.flatten(literal(data)), lambda: [py_flatten(data)]], + [ + f.range(literal(1), literal(5), literal(2)), + lambda: [np.arange(1, 5, 2)], + ], ] for stmt, py_expr in test_items: diff --git a/src/functions.rs b/src/functions.rs index 757fb31d1..666e1ec37 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -391,6 +391,7 @@ scalar_function!(trunc, Trunc); scalar_function!(upper, Upper, "Converts the string to all upper case."); scalar_function!(make_array, MakeArray); scalar_function!(array, MakeArray); +scalar_function!(range, Range); scalar_function!(uuid, Uuid); scalar_function!(r#struct, Struct); // Use raw identifier since struct is a keyword scalar_function!(from_unixtime, FromUnixtime); @@ -405,6 +406,8 @@ scalar_function!(list_push_back, ArrayAppend); scalar_function!(array_concat, ArrayConcat); scalar_function!(array_cat, ArrayConcat); scalar_function!(array_dims, ArrayDims); +scalar_function!(array_distinct, ArrayDistinct); +scalar_function!(list_distinct, ArrayDistinct); scalar_function!(list_dims, ArrayDims); scalar_function!(array_element, ArrayElement); scalar_function!(array_extract, ArrayElement); @@ -444,6 +447,14 @@ scalar_function!(array_replace_all, ArrayReplaceAll); scalar_function!(list_replace_all, ArrayReplaceAll); scalar_function!(array_slice, ArraySlice); scalar_function!(list_slice, ArraySlice); +scalar_function!(array_intersect, ArrayIntersect); +scalar_function!(list_intersect, ArrayIntersect); +scalar_function!(array_union, ArrayUnion); +scalar_function!(list_union, ArrayUnion); +scalar_function!(array_except, ArrayExcept); +scalar_function!(list_except, ArrayExcept); +scalar_function!(array_resize, ArrayResize); +scalar_function!(list_resize, ArrayResize); scalar_function!(flatten, Flatten); aggregate_function!(approx_distinct, ApproxDistinct); @@ -499,6 +510,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(approx_percentile_cont))?; m.add_wrapped(wrap_pyfunction!(approx_percentile_cont_with_weight))?; m.add_wrapped(wrap_pyfunction!(array))?; + m.add_wrapped(wrap_pyfunction!(range))?; m.add_wrapped(wrap_pyfunction!(array_agg))?; m.add_wrapped(wrap_pyfunction!(arrow_typeof))?; m.add_wrapped(wrap_pyfunction!(ascii))?; @@ -644,6 +656,8 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(array_concat))?; m.add_wrapped(wrap_pyfunction!(array_cat))?; m.add_wrapped(wrap_pyfunction!(array_dims))?; + m.add_wrapped(wrap_pyfunction!(array_distinct))?; + m.add_wrapped(wrap_pyfunction!(list_distinct))?; m.add_wrapped(wrap_pyfunction!(list_dims))?; m.add_wrapped(wrap_pyfunction!(array_element))?; m.add_wrapped(wrap_pyfunction!(array_extract))?; @@ -661,6 +675,14 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(array_positions))?; m.add_wrapped(wrap_pyfunction!(list_positions))?; m.add_wrapped(wrap_pyfunction!(array_to_string))?; + m.add_wrapped(wrap_pyfunction!(array_intersect))?; + m.add_wrapped(wrap_pyfunction!(list_intersect))?; + m.add_wrapped(wrap_pyfunction!(array_union))?; + m.add_wrapped(wrap_pyfunction!(list_union))?; + m.add_wrapped(wrap_pyfunction!(array_except))?; + m.add_wrapped(wrap_pyfunction!(list_except))?; + m.add_wrapped(wrap_pyfunction!(array_resize))?; + m.add_wrapped(wrap_pyfunction!(list_resize))?; m.add_wrapped(wrap_pyfunction!(array_join))?; m.add_wrapped(wrap_pyfunction!(list_to_string))?; m.add_wrapped(wrap_pyfunction!(list_join))?; From 5e97701901f36a404b09e678b7fbfb02faa5c2f5 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Thu, 21 Mar 2024 20:27:15 +0100 Subject: [PATCH 122/413] feat: add execute_stream and execute_stream_partitioned (#610) --- datafusion/tests/test_dataframe.py | 30 ++++++++++++++++++++ src/dataframe.rs | 45 ++++++++++++++++++++++++++---- src/record_batch.rs | 9 ++++++ 3 files changed, 79 insertions(+), 5 deletions(-) diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index 9944abeef..c8c74fa2c 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -623,6 +623,36 @@ def test_to_arrow_table(df): assert set(pyarrow_table.column_names) == {"a", "b", "c"} +def test_execute_stream(df): + stream = df.execute_stream() + assert all(batch is not None for batch in stream) + assert not list(stream) # after one iteration the generator must be exhausted + + +@pytest.mark.parametrize("schema", [True, False]) +def test_execute_stream_to_arrow_table(df, schema): + stream = df.execute_stream() + + if schema: + pyarrow_table = pa.Table.from_batches( + (batch.to_pyarrow() for batch in stream), schema=df.schema() + ) + else: + pyarrow_table = pa.Table.from_batches((batch.to_pyarrow() for batch in stream)) + + assert isinstance(pyarrow_table, pa.Table) + assert pyarrow_table.shape == (3, 3) + assert set(pyarrow_table.column_names) == {"a", "b", "c"} + + +def test_execute_stream_partitioned(df): + streams = df.execute_stream_partitioned() + assert all(batch is not None for stream in streams for batch in stream) + assert all( + not list(stream) for stream in streams + ) # after one iteration all generators must be exhausted + + def test_empty_to_arrow_table(df): # Convert empty datafusion dataframe to pyarrow Table pyarrow_table = df.limit(0).to_arrow_table() diff --git a/src/dataframe.rs b/src/dataframe.rs index 1e8790990..92a25188d 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -15,21 +15,27 @@ // specific language governing permissions and limitations // under the License. -use crate::physical_plan::PyExecutionPlan; -use crate::sql::logical::PyLogicalPlan; -use crate::utils::wait_for_future; -use crate::{errors::DataFusionError, expr::PyExpr}; +use std::sync::Arc; + use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; +use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyTuple; -use std::sync::Arc; +use tokio::task::JoinHandle; + +use crate::errors::py_datafusion_err; +use crate::physical_plan::PyExecutionPlan; +use crate::record_batch::PyRecordBatchStream; +use crate::sql::logical::PyLogicalPlan; +use crate::utils::{get_tokio_runtime, wait_for_future}; +use crate::{errors::DataFusionError, expr::PyExpr}; /// A PyDataFrame is a representation of a logical plan and an API to compose statements. /// Use it to build a plan and `.collect()` to execute the plan and collect the result. @@ -399,6 +405,35 @@ impl PyDataFrame { }) } + fn execute_stream(&self, py: Python) -> PyResult { + // create a Tokio runtime to run the async code + let rt = &get_tokio_runtime(py).0; + let df = self.df.as_ref().clone(); + let fut: JoinHandle> = + rt.spawn(async move { df.execute_stream().await }); + let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; + Ok(PyRecordBatchStream::new(stream?)) + } + + fn execute_stream_partitioned(&self, py: Python) -> PyResult> { + // create a Tokio runtime to run the async code + let rt = &get_tokio_runtime(py).0; + let df = self.df.as_ref().clone(); + let fut: JoinHandle>> = + rt.spawn(async move { df.execute_stream_partitioned().await }); + let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; + + match stream { + Ok(batches) => Ok(batches + .into_iter() + .map(|batch_stream| PyRecordBatchStream::new(batch_stream)) + .collect()), + _ => Err(PyValueError::new_err( + "Unable to execute stream partitioned", + )), + } + } + /// Convert to pandas dataframe with pyarrow /// Collect the batches, pass to Arrow Table & then convert to Pandas DataFrame fn to_pandas(&self, py: Python) -> PyResult { diff --git a/src/record_batch.rs b/src/record_batch.rs index aa3a392da..427807f22 100644 --- a/src/record_batch.rs +++ b/src/record_batch.rs @@ -20,6 +20,7 @@ use datafusion::arrow::pyarrow::ToPyArrow; use datafusion::arrow::record_batch::RecordBatch; use datafusion::physical_plan::SendableRecordBatchStream; use futures::StreamExt; +use pyo3::prelude::*; use pyo3::{pyclass, pymethods, PyObject, PyResult, Python}; #[pyclass(name = "RecordBatch", module = "datafusion", subclass)] @@ -61,4 +62,12 @@ impl PyRecordBatchStream { Some(Err(e)) => Err(e.into()), } } + + fn __next__(&mut self, py: Python) -> PyResult> { + self.next(py) + } + + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } } From 7d08ec985ccb928d9ec93ae6d554073af9b36431 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 13:43:43 -0700 Subject: [PATCH 123/413] build(deps): bump uuid from 1.7.0 to 1.8.0 (#615) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.7.0 to 1.8.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/1.7.0...1.8.0) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c86544fa7..e7b8d3292 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3148,9 +3148,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ "getrandom", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6afcde957..8d7e72884 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ datafusion-sql = "36.0.0" datafusion-substrait = { version = "36.0.0", optional = true } prost = "0.12" prost-types = "0.12" -uuid = { version = "1.3", features = ["v4"] } +uuid = { version = "1.8", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" From 6e570e211fff70ef69f9fa5a4a3ba2830d4f3e0c Mon Sep 17 00:00:00 2001 From: Giacomo Rebecchi <98255907+giacomorebecchi@users.noreply.github.com> Date: Sat, 13 Apr 2024 23:37:55 +0200 Subject: [PATCH 124/413] Bind SQLOptions and relative ctx method #567 (#588) --- datafusion/__init__.py | 2 ++ datafusion/tests/test_context.py | 46 +++++++++++++++++++++++--- src/context.rs | 57 +++++++++++++++++++++++++++++++- src/lib.rs | 1 + 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/datafusion/__init__.py b/datafusion/__init__.py index cfb673613..c50bf649d 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -33,6 +33,7 @@ SessionConfig, RuntimeConfig, ScalarUDF, + SQLOptions, ) from .common import ( @@ -96,6 +97,7 @@ "DataFrame", "SessionContext", "SessionConfig", + "SQLOptions", "RuntimeConfig", "Expr", "AggregateUDF", diff --git a/datafusion/tests/test_context.py b/datafusion/tests/test_context.py index d48bdd929..962a7ff41 100644 --- a/datafusion/tests/test_context.py +++ b/datafusion/tests/test_context.py @@ -19,16 +19,17 @@ import pyarrow as pa import pyarrow.dataset as ds +import pytest from datafusion import ( + DataFrame, + RuntimeConfig, + SessionConfig, + SessionContext, + SQLOptions, column, literal, - SessionContext, - SessionConfig, - RuntimeConfig, - DataFrame, ) -import pytest def test_create_context_no_args(): @@ -389,3 +390,38 @@ def test_read_parquet(ctx): def test_read_avro(ctx): csv_df = ctx.read_avro(path="testing/data/avro/alltypes_plain.avro") csv_df.show() + + +def test_create_sql_options(): + SQLOptions() + + +def test_sql_with_options_no_ddl(ctx): + sql = "CREATE TABLE IF NOT EXISTS valuetable AS VALUES(1,'HELLO'),(12,'DATAFUSION')" + ctx.sql(sql) + options = SQLOptions().with_allow_ddl(False) + with pytest.raises(Exception, match="DDL"): + ctx.sql_with_options(sql, options=options) + + +def test_sql_with_options_no_dml(ctx): + table_name = "t" + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], + names=["a", "b"], + ) + dataset = ds.dataset([batch]) + ctx.register_dataset(table_name, dataset) + sql = f'INSERT INTO "{table_name}" VALUES (1, 2), (2, 3);' + ctx.sql(sql) + options = SQLOptions().with_allow_dml(False) + with pytest.raises(Exception, match="DML"): + ctx.sql_with_options(sql, options=options) + + +def test_sql_with_options_no_statements(ctx): + sql = "SET time zone = 1;" + ctx.sql(sql) + options = SQLOptions().with_allow_statements(False) + with pytest.raises(Exception, match="SetVariable"): + ctx.sql_with_options(sql, options=options) diff --git a/src/context.rs b/src/context.rs index f34fbce81..825d3e321 100644 --- a/src/context.rs +++ b/src/context.rs @@ -45,7 +45,9 @@ use datafusion::arrow::record_batch::RecordBatch; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; -use datafusion::execution::context::{SessionConfig, SessionContext, SessionState, TaskContext}; +use datafusion::execution::context::{ + SQLOptions, SessionConfig, SessionContext, SessionState, TaskContext, +}; use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; @@ -210,6 +212,43 @@ impl PyRuntimeConfig { } } +/// `PySQLOptions` allows you to specify options to the sql execution. +#[pyclass(name = "SQLOptions", module = "datafusion", subclass)] +#[derive(Clone)] +pub struct PySQLOptions { + pub options: SQLOptions, +} + +impl From for PySQLOptions { + fn from(options: SQLOptions) -> Self { + Self { options } + } +} + +#[pymethods] +impl PySQLOptions { + #[new] + fn new() -> Self { + let options = SQLOptions::new(); + Self { options } + } + + /// Should DDL data modification commands (e.g. `CREATE TABLE`) be run? Defaults to `true`. + fn with_allow_ddl(&self, allow: bool) -> Self { + Self::from(self.options.with_allow_ddl(allow)) + } + + /// Should DML data modification commands (e.g. `INSERT and COPY`) be run? Defaults to `true` + pub fn with_allow_dml(&self, allow: bool) -> Self { + Self::from(self.options.with_allow_dml(allow)) + } + + /// Should Statements such as (e.g. `SET VARIABLE and `BEGIN TRANSACTION` ...`) be run?. Defaults to `true` + pub fn with_allow_statements(&self, allow: bool) -> Self { + Self::from(self.options.with_allow_statements(allow)) + } +} + /// `PySessionContext` is able to plan and execute DataFusion plans. /// It has a powerful optimizer, a physical planner for local execution, and a /// multi-threaded execution engine to perform the execution. @@ -285,6 +324,22 @@ impl PySessionContext { Ok(PyDataFrame::new(df)) } + pub fn sql_with_options( + &mut self, + query: &str, + options: Option, + py: Python, + ) -> PyResult { + let options = if let Some(options) = options { + options.options + } else { + SQLOptions::new() + }; + let result = self.ctx.sql_with_options(query, options); + let df = wait_for_future(py, result).map_err(DataFusionError::from)?; + Ok(PyDataFrame::new(df)) + } + pub fn create_dataframe( &mut self, partitions: PyArrowType>>, diff --git a/src/lib.rs b/src/lib.rs index 49c325a53..a696ebff4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,6 +85,7 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From 7204a354f8d3f3c3fa45ffc3211e252cc3706d47 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Sat, 13 Apr 2024 23:39:51 +0200 Subject: [PATCH 125/413] bugfix: no panic on empty table (#613) --- datafusion/tests/test_context.py | 31 +++++++++++++++++++++++++++++++ src/context.rs | 15 ++++++++++++--- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/datafusion/tests/test_context.py b/datafusion/tests/test_context.py index 962a7ff41..df7e1813b 100644 --- a/datafusion/tests/test_context.py +++ b/datafusion/tests/test_context.py @@ -139,6 +139,37 @@ def test_from_arrow_table_with_name(ctx): assert tables[0] == "tbl" +def test_from_arrow_table_empty(ctx): + data = {"a": [], "b": []} + schema = pa.schema([("a", pa.int32()), ("b", pa.string())]) + table = pa.Table.from_pydict(data, schema=schema) + + # convert to DataFrame + df = ctx.from_arrow_table(table) + tables = list(ctx.tables()) + + assert df + assert len(tables) == 1 + assert isinstance(df, DataFrame) + assert set(df.schema().names) == {"a", "b"} + assert len(df.collect()) == 0 + + +def test_from_arrow_table_empty_no_schema(ctx): + data = {"a": [], "b": []} + table = pa.Table.from_pydict(data) + + # convert to DataFrame + df = ctx.from_arrow_table(table) + tables = list(ctx.tables()) + + assert df + assert len(tables) == 1 + assert isinstance(df, DataFrame) + assert set(df.schema().names) == {"a", "b"} + assert len(df.collect()) == 0 + + def test_from_pylist(ctx): # create a dataframe from Python list data = [ diff --git a/src/context.rs b/src/context.rs index 825d3e321..d9b58609b 100644 --- a/src/context.rs +++ b/src/context.rs @@ -39,7 +39,7 @@ use crate::store::StorageContexts; use crate::udaf::PyAggregateUDF; use crate::udf::PyScalarUDF; use crate::utils::{get_tokio_runtime, wait_for_future}; -use datafusion::arrow::datatypes::{DataType, Schema}; +use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; @@ -344,9 +344,15 @@ impl PySessionContext { &mut self, partitions: PyArrowType>>, name: Option<&str>, + schema: Option>, py: Python, ) -> PyResult { - let schema = partitions.0[0][0].schema(); + let schema = if let Some(schema) = schema { + SchemaRef::from(schema.0) + } else { + partitions.0[0][0].schema() + }; + let table = MemTable::try_new(schema, partitions.0).map_err(DataFusionError::from)?; // generate a random (unique) name for this table if none is provided @@ -428,12 +434,15 @@ impl PySessionContext { // Instantiate pyarrow Table object & convert to batches let table = data.call_method0(py, "to_batches")?; + let schema = data.getattr(py, "schema")?; + let schema = schema.extract::>(py)?; + // Cast PyObject to RecordBatch type // Because create_dataframe() expects a vector of vectors of record batches // here we need to wrap the vector of record batches in an additional vector let batches = table.extract::>>(py)?; let list_of_batches = PyArrowType::from(vec![batches.0]); - self.create_dataframe(list_of_batches, name, py) + self.create_dataframe(list_of_batches, name, Some(schema), py) }) } From 84415ddaa99c4f09005275247eb8e797635de0dc Mon Sep 17 00:00:00 2001 From: Henri Froese Date: Sun, 14 Apr 2024 21:31:52 +0200 Subject: [PATCH 126/413] Expose `register_listing_table` (#618) * Improve documentation It's currently hard to see how datafusion can be configured in python. Adding a small section on configuration, and linking to examples, should help with that. * Fix typo * Fix clippy warning * Support integer table partition columns (tested in next commit) * Expose `register_listing_table` This lets users nicely use `object_store` with python datafusion for partitioned dataset e.g. in S3. Closes #617 --------- Co-authored-by: Henri Froese --- datafusion/tests/test_sql.py | 58 +++++++++++++++++++++++- docs/source/index.rst | 2 + docs/source/user-guide/configuration.rst | 51 +++++++++++++++++++++ src/context.rs | 56 ++++++++++++++++++++++- src/dataframe.rs | 5 +- 5 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 docs/source/user-guide/configuration.rst diff --git a/datafusion/tests/test_sql.py b/datafusion/tests/test_sql.py index 9d4c8f677..8ec2ffb12 100644 --- a/datafusion/tests/test_sql.py +++ b/datafusion/tests/test_sql.py @@ -21,8 +21,9 @@ import pyarrow as pa import pyarrow.dataset as ds import pytest +from datafusion.object_store import LocalFileSystem -from datafusion import udf +from datafusion import udf, col from . import generic as helpers @@ -374,3 +375,58 @@ def test_simple_select(ctx, tmp_path, arr): result = batches[0].column(0) np.testing.assert_equal(result, arr) + + +@pytest.mark.parametrize("file_sort_order", (None, [[col("int").sort(True, True)]])) +@pytest.mark.parametrize("pass_schema", (True, False)) +def test_register_listing_table(ctx, tmp_path, pass_schema, file_sort_order): + dir_root = tmp_path / "dataset_parquet_partitioned" + dir_root.mkdir(exist_ok=False) + (dir_root / "grp=a/date_id=20201005").mkdir(exist_ok=False, parents=True) + (dir_root / "grp=a/date_id=20211005").mkdir(exist_ok=False, parents=True) + (dir_root / "grp=b/date_id=20201005").mkdir(exist_ok=False, parents=True) + + table = pa.Table.from_arrays( + [ + [1, 2, 3, 4, 5, 6, 7], + ["a", "b", "c", "d", "e", "f", "g"], + [1.1, 2.2, 3.3, 4.4, 5.5, 6.6, 7.7], + ], + names=["int", "str", "float"], + ) + pa.parquet.write_table( + table.slice(0, 3), dir_root / "grp=a/date_id=20201005/file.parquet" + ) + pa.parquet.write_table( + table.slice(3, 2), dir_root / "grp=a/date_id=20211005/file.parquet" + ) + pa.parquet.write_table( + table.slice(5, 10), dir_root / "grp=b/date_id=20201005/file.parquet" + ) + + ctx.register_object_store("file://local", LocalFileSystem(), None) + ctx.register_listing_table( + "my_table", + f"file://{dir_root}/", + table_partition_cols=[("grp", "string"), ("date_id", "int")], + file_extension=".parquet", + schema=table.schema if pass_schema else None, + file_sort_order=file_sort_order, + ) + assert ctx.tables() == {"my_table"} + + result = ctx.sql( + "SELECT grp, COUNT(*) AS count FROM my_table GROUP BY grp" + ).collect() + result = pa.Table.from_batches(result) + + rd = result.to_pydict() + assert dict(zip(rd["grp"], rd["count"])) == {"a": 5, "b": 2} + + result = ctx.sql( + "SELECT grp, COUNT(*) AS count FROM my_table WHERE date_id=20201005 GROUP BY grp" + ).collect() + result = pa.Table.from_batches(result) + + rd = result.to_pydict() + assert dict(zip(rd["grp"], rd["count"])) == {"a": 3, "b": 2} diff --git a/docs/source/index.rst b/docs/source/index.rst index 155b0cf9f..6cec3a157 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -75,6 +75,7 @@ Example Github and Issue Tracker Rust's API Docs Code of conduct + Examples .. _toc.guide: .. toctree:: @@ -84,6 +85,7 @@ Example user-guide/introduction user-guide/basics + user-guide/configuration user-guide/common-operations/index user-guide/io/index user-guide/sql diff --git a/docs/source/user-guide/configuration.rst b/docs/source/user-guide/configuration.rst new file mode 100644 index 000000000..63b0dc3e2 --- /dev/null +++ b/docs/source/user-guide/configuration.rst @@ -0,0 +1,51 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Configuration +======== + +Let's look at how we can configure DataFusion. When creating a :code:`SessionContext`, you can pass in +a :code:`SessionConfig` and :code:`RuntimeConfig` object. These two cover a wide range of options. + +.. code-block:: python + + from datafusion import RuntimeConfig, SessionConfig, SessionContext + + # create a session context with default settings + ctx = SessionContext() + print(ctx) + + # create a session context with explicit runtime and config settings + runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) + config = ( + SessionConfig() + .with_create_default_catalog_and_schema(True) + .with_default_catalog_and_schema("foo", "bar") + .with_target_partitions(8) + .with_information_schema(True) + .with_repartition_joins(False) + .with_repartition_aggregations(False) + .with_repartition_windows(False) + .with_parquet_pruning(False) + .set("datafusion.execution.parquet.pushdown_filters", "true") + ) + ctx = SessionContext(config, runtime) + print(ctx) + + +You can read more about available :code:`SessionConfig` options `here `_, +and about :code:`RuntimeConfig` options `here https://docs.rs/datafusion/latest/datafusion/execution/runtime_env/struct.RuntimeConfig.html`_. diff --git a/src/context.rs b/src/context.rs index d9b58609b..f1ecaa328 100644 --- a/src/context.rs +++ b/src/context.rs @@ -43,6 +43,10 @@ use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; +use datafusion::datasource::file_format::parquet::ParquetFormat; +use datafusion::datasource::listing::{ + ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, +}; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; use datafusion::execution::context::{ @@ -283,7 +287,7 @@ impl PySessionContext { }) } - /// Register a an object store with the given name + /// Register an object store with the given name pub fn register_object_store( &mut self, scheme: &str, @@ -317,6 +321,53 @@ impl PySessionContext { Ok(()) } + #[allow(clippy::too_many_arguments)] + #[pyo3(signature = (name, path, table_partition_cols=vec![], + file_extension=".parquet", + schema=None, + file_sort_order=None))] + pub fn register_listing_table( + &mut self, + name: &str, + path: &str, + table_partition_cols: Vec<(String, String)>, + file_extension: &str, + schema: Option>, + file_sort_order: Option>>, + py: Python, + ) -> PyResult<()> { + let options = ListingOptions::new(Arc::new(ParquetFormat::new())) + .with_file_extension(file_extension) + .with_table_partition_cols(convert_table_partition_cols(table_partition_cols)?) + .with_file_sort_order( + file_sort_order + .unwrap_or_default() + .into_iter() + .map(|e| e.into_iter().map(|f| f.into()).collect()) + .collect(), + ); + let table_path = ListingTableUrl::parse(path)?; + let resolved_schema: SchemaRef = match schema { + Some(s) => Arc::new(s.0), + None => { + let state = self.ctx.state(); + let schema = options.infer_schema(&state, &table_path); + wait_for_future(py, schema).map_err(DataFusionError::from)? + } + }; + let config = ListingTableConfig::new(table_path) + .with_listing_options(options) + .with_schema(resolved_schema); + let table = ListingTable::try_new(config)?; + self.register_table( + name, + &PyTable { + table: Arc::new(table), + }, + )?; + Ok(()) + } + /// Returns a PyDataFrame whose plan corresponds to the SQL statement. pub fn sql(&mut self, query: &str, py: Python) -> PyResult { let result = self.ctx.sql(query); @@ -913,8 +964,9 @@ pub fn convert_table_partition_cols( .into_iter() .map(|(name, ty)| match ty.as_str() { "string" => Ok((name, DataType::Utf8)), + "int" => Ok((name, DataType::Int32)), _ => Err(DataFusionError::Common(format!( - "Unsupported data type '{ty}' for partition column" + "Unsupported data type '{ty}' for partition column. Supported types are 'string' and 'int'" ))), }) .collect::, _>>() diff --git a/src/dataframe.rs b/src/dataframe.rs index 92a25188d..a239a35f9 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -424,10 +424,7 @@ impl PyDataFrame { let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; match stream { - Ok(batches) => Ok(batches - .into_iter() - .map(|batch_stream| PyRecordBatchStream::new(batch_stream)) - .collect()), + Ok(batches) => Ok(batches.into_iter().map(PyRecordBatchStream::new).collect()), _ => Err(PyValueError::new_err( "Unable to execute stream partitioned", )), From 7366f8919d9b679baa5d312c515b29cc055867a2 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 23 Apr 2024 10:52:08 -0400 Subject: [PATCH 127/413] Expose unnest feature (#641) * Expose unnest feature * Update dataframe operation name to match rust implementation --- datafusion/tests/test_dataframe.py | 34 ++++++++++++ src/dataframe.rs | 12 +++++ src/expr.rs | 2 + src/expr/unnest.rs | 85 ++++++++++++++++++++++++++++++ src/sql/logical.rs | 2 + 5 files changed, 135 insertions(+) create mode 100644 src/expr/unnest.rs diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index c8c74fa2c..efb1679b9 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -62,6 +62,20 @@ def struct_df(): return ctx.create_dataframe([[batch]]) +@pytest.fixture +def nested_df(): + ctx = SessionContext() + + # create a RecordBatch and a new DataFrame from it + # Intentionally make each array of different length + batch = pa.RecordBatch.from_arrays( + [pa.array([[1], [2, 3], [4, 5, 6], None]), pa.array([7, 8, 9, 10])], + names=["a", "b"], + ) + + return ctx.create_dataframe([[batch]]) + + @pytest.fixture def aggregate_df(): ctx = SessionContext() @@ -160,6 +174,26 @@ def test_with_column_renamed(df): assert result.schema.field(2).name == "sum" +def test_unnest(nested_df): + nested_df = nested_df.unnest_column("a") + + # execute and collect the first (and only) batch + result = nested_df.collect()[0] + + assert result.column(0) == pa.array([1, 2, 3, 4, 5, 6, None]) + assert result.column(1) == pa.array([7, 8, 8, 9, 9, 9, 10]) + + +def test_unnest_without_nulls(nested_df): + nested_df = nested_df.unnest_column("a", preserve_nulls=False) + + # execute and collect the first (and only) batch + result = nested_df.collect()[0] + + assert result.column(0) == pa.array([1, 2, 3, 4, 5, 6]) + assert result.column(1) == pa.array([7, 8, 8, 9, 9, 9]) + + def test_udf(df): # is_null is a pa function over arrays is_null = udf( diff --git a/src/dataframe.rs b/src/dataframe.rs index a239a35f9..a319b3d73 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -25,6 +25,7 @@ use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; +use datafusion_common::UnnestOptions; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::types::PyTuple; @@ -293,6 +294,17 @@ impl PyDataFrame { Ok(Self::new(new_df)) } + #[pyo3(signature = (column, preserve_nulls=true))] + fn unnest_column(&self, column: &str, preserve_nulls: bool) -> PyResult { + let unnest_options = UnnestOptions { preserve_nulls }; + let df = self + .df + .as_ref() + .clone() + .unnest_column_with_options(column, unnest_options)?; + Ok(Self::new(df)) + } + /// Calculate the intersection of two `DataFrame`s. The two `DataFrame`s must have exactly the same schema fn intersect(&self, py_df: PyDataFrame) -> PyResult { let new_df = self diff --git a/src/expr.rs b/src/expr.rs index c0e7019f0..0958c4a98 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -89,6 +89,7 @@ pub mod subquery; pub mod subquery_alias; pub mod table_scan; pub mod union; +pub mod unnest; pub mod window; /// A PyExpr that can be used on a DataFrame @@ -684,6 +685,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/expr/unnest.rs b/src/expr/unnest.rs new file mode 100644 index 000000000..33fb82feb --- /dev/null +++ b/src/expr/unnest.rs @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_expr::logical_plan::Unnest; +use pyo3::prelude::*; +use std::fmt::{self, Display, Formatter}; + +use crate::common::df_schema::PyDFSchema; +use crate::expr::logical_node::LogicalNode; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass(name = "Unnest", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyUnnest { + unnest_: Unnest, +} + +impl From for PyUnnest { + fn from(unnest_: Unnest) -> PyUnnest { + PyUnnest { unnest_ } + } +} + +impl From for Unnest { + fn from(unnest_: PyUnnest) -> Self { + unnest_.unnest_ + } +} + +impl Display for PyUnnest { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Unnest + Inputs: {:?} + Schema: {:?}", + &self.unnest_.input, &self.unnest_.schema, + ) + } +} + +#[pymethods] +impl PyUnnest { + /// Retrieves the input `LogicalPlan` to this `Unnest` node + fn input(&self) -> PyResult> { + Ok(Self::inputs(self)) + } + + /// Resulting Schema for this `Unnest` node instance + fn schema(&self) -> PyResult { + Ok(self.unnest_.schema.as_ref().clone().into()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("Unnest({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("Unnest".to_string()) + } +} + +impl LogicalNode for PyUnnest { + fn inputs(&self) -> Vec { + vec![PyLogicalPlan::from((*self.unnest_.input).clone())] + } + + fn to_variant(&self, py: Python) -> PyResult { + Ok(self.clone().into_py(py)) + } +} diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 3aa8a699f..62515c3dd 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -33,6 +33,7 @@ use crate::expr::sort::PySort; use crate::expr::subquery::PySubquery; use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; +use crate::expr::unnest::PyUnnest; use crate::expr::window::PyWindow; use datafusion_expr::LogicalPlan; use pyo3::prelude::*; @@ -78,6 +79,7 @@ impl PyLogicalPlan { LogicalPlan::TableScan(plan) => PyTableScan::from(plan.clone()).to_variant(py), LogicalPlan::Subquery(plan) => PySubquery::from(plan.clone()).to_variant(py), LogicalPlan::SubqueryAlias(plan) => PySubqueryAlias::from(plan.clone()).to_variant(py), + LogicalPlan::Unnest(plan) => PyUnnest::from(plan.clone()).to_variant(py), LogicalPlan::Window(plan) => PyWindow::from(plan.clone()).to_variant(py), other => Err(py_unsupported_variant_err(format!( "Cannot convert this plan to a LogicalNode: {:?}", From 13d34a5cad6a92e2ab51f03fefb0daeae2ec4a11 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 23 Apr 2024 09:46:31 -0600 Subject: [PATCH 128/413] update domain names and paths in asf yaml (#643) --- .asf.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index de776871c..e96b43cf0 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -16,13 +16,13 @@ # under the License. notifications: - commits: commits@arrow.apache.org - issues: github@arrow.apache.org - pullrequests: github@arrow.apache.org + commits: commits@datafusion.apache.org + issues: github@datafusion.apache.org + pullrequests: github@datafusion.apache.org jira_options: link label worklog github: - description: "Apache Arrow DataFusion Python Bindings" - homepage: https://arrow.apache.org/datafusion-python + description: "Apache DataFusion Python Bindings" + homepage: https://datafusion.apache.org/python enabled_merge_buttons: squash: true merge: false @@ -32,8 +32,8 @@ github: staging: whoami: asf-staging - subdir: datafusion-python + subdir: python publish: whoami: asf-site - subdir: datafusion-python + subdir: python From 88ce6e6395bb8d5b36bfe3a7e8ebaa721d233e6a Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 26 Apr 2024 12:51:19 -0600 Subject: [PATCH 129/413] use python 3.11 to publish docs (#645) * use python 3.11 to publish docs * specify protoc 3.20.2 rather than 3.x --- .github/workflows/build.yml | 4 ++-- .github/workflows/docs.yaml | 4 ++-- .github/workflows/test.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b553749ad..e03c2cbde 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,7 +92,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: "3.x" + version: "3.20.2" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package @@ -153,7 +153,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: "3.x" + version: "3.20.2" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index d9e7ad4ad..cb80049e1 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -35,12 +35,12 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.11" - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.x' + version: '3.20.2' repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 291120bf8..a486f2778 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -54,7 +54,7 @@ jobs: - name: Install Protoc uses: arduino/setup-protoc@v1 with: - version: '3.x' + version: '3.20.2' repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python From 67d4cfb847e42724319aeec9014889262e6ea58a Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Wed, 1 May 2024 15:54:48 -0500 Subject: [PATCH 130/413] docs: update docs CI to install python-311 requirements (#661) PR #645 switched to using python 3.11 to publish docs. This updates the dependency installation to use the appropriate requirements file. --- .github/workflows/docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index cb80049e1..1193d7810 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -48,7 +48,7 @@ jobs: set -x python3 -m venv venv source venv/bin/activate - pip install -r requirements-310.txt + pip install -r requirements-311.txt pip install -r docs/requirements.txt - name: Build Datafusion run: | From ee93cdd6c23a789d816a1c5a555a1e6ecf40a99c Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Wed, 8 May 2024 17:35:55 -0500 Subject: [PATCH 131/413] Upgrade Datafusion to v37.1.0 (#669) * deps: upgrade datafusion to 37.1.0 * feat: re-implement SessionContext::tables The method was removed upstream but is used in many tests for `datafusion-python`. Ref: https://github.com/apache/datafusion/pull/9627 * feat: upgrade dataframe write_parquet and write_json The options to write_parquet changed. write_json has a new argument that I defaulted to None. We can expose that config later. Ref: https://github.com/apache/datafusion/pull/9382 * feat: impl new ExecutionPlanProperties for DatasetExec Ref: https://github.com/apache/datafusion/pull/9346 * feat: add upstream variant and method params - `WindowFunction` and `AggregateFunction` have `null_treatment` options. - `ScalarValue` and `DataType` have new variants - `SchemaProvider::table` now returns a `Result` * lint: allow(deprecated) for make_scalar_function * feat: migrate functions.rs `datafusion` completed an Epic that ported many of the `BuiltInFunctions` enum to `SclarUDF`. I created new macros to simplify the port, and used these macros to refactor a few existing functions. Ref: https://github.com/apache/datafusion/issues/9285 * fixme: commented out last failing test This is a bug upstream in datafusion FAILED datafusion/tests/test_functions.py::test_array_functions - pyo3_runtime.PanicException: range end index 9 out of range for slice of length 8 * chore: update Cargo.toml package info --- Cargo.lock | 325 +++++++++++++---------- Cargo.toml | 22 +- datafusion/tests/test_functions.py | 8 +- src/catalog.rs | 2 +- src/common/data_type.rs | 20 ++ src/context.rs | 14 +- src/dataframe.rs | 18 +- src/dataset_exec.rs | 49 ++-- src/expr.rs | 5 + src/functions.rs | 397 ++++++++++++++++++----------- src/physical_plan.rs | 2 +- src/udf.rs | 4 +- src/utils.rs | 4 +- 13 files changed, 545 insertions(+), 325 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e7b8d3292..5eb791b46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa285343fba4d829d49985bdc541e3789cf6000ed0e84be7c039438df4a4e78c" +checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" +checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" +checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" +checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" dependencies = [ "bytes", "half", @@ -195,28 +195,30 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" +checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "base64", + "atoi", + "base64 0.22.1", "chrono", "comfy-table", "half", "lexical-core", "num", + "ryu", ] [[package]] name = "arrow-csv" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46af72211f0712612f5b18325530b9ad1bfbdc87290d5fbfd32a7da128983781" +checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" dependencies = [ "arrow-array", "arrow-buffer", @@ -233,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" +checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" dependencies = [ "arrow-buffer", "arrow-schema", @@ -245,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" +checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" dependencies = [ "arrow-array", "arrow-buffer", @@ -260,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8950719280397a47d37ac01492e3506a8a724b3fb81001900b866637a829ee0f" +checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" dependencies = [ "arrow-array", "arrow-buffer", @@ -280,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" +checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" dependencies = [ "arrow-array", "arrow-buffer", @@ -295,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" +checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" dependencies = [ "ahash", "arrow-array", @@ -310,18 +312,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" +checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" dependencies = [ "bitflags 2.4.2", ] [[package]] name = "arrow-select" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" +checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" dependencies = [ "ahash", "arrow-array", @@ -333,15 +335,16 @@ dependencies = [ [[package]] name = "arrow-string" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f3b37f2aeece31a2636d1b037dabb69ef590e03bdc7eb68519b51ec86932a7" +checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", + "memchr", "num", "regex", "regex-syntax", @@ -373,7 +376,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -384,7 +387,16 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -423,6 +435,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -715,9 +733,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b360b692bf6c6d6e6b6dbaf41a3be0020daeceac0f406aed54c75331e50dbb" +checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11" dependencies = [ "ahash", "apache-avro", @@ -732,6 +750,7 @@ dependencies = [ "chrono", "dashmap", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-functions", @@ -767,9 +786,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37f343ccc298f440e25aa38ff82678291a7acc24061c7370ba6c0ff5cc811412" +checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3" dependencies = [ "ahash", "apache-avro", @@ -779,6 +798,7 @@ dependencies = [ "arrow-schema", "chrono", "half", + "instant", "libc", "num_cpus", "object_store", @@ -787,11 +807,20 @@ dependencies = [ "sqlparser", ] +[[package]] +name = "datafusion-common-runtime" +version = "37.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e4a44d8ef1b1e85d32234e6012364c411c3787859bb3bba893b0332cb03dfd" +dependencies = [ + "tokio", +] + [[package]] name = "datafusion-execution" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9c93043081487e335399a21ebf8295626367a647ac5cb87d41d18afad7d0f7" +checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95" dependencies = [ "arrow", "chrono", @@ -810,13 +839,14 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e204d89909e678846b6a95f156aafc1ee5b36cb6c9e37ec2e1449b078a38c818" +checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628" dependencies = [ "ahash", "arrow", "arrow-array", + "chrono", "datafusion-common", "paste", "sqlparser", @@ -826,38 +856,54 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98f1c73f7801b2b8ba2297b3ad78ffcf6c1fc6b8171f502987eb9ad5cb244ee7" +checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959" dependencies = [ "arrow", - "base64", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr", "hex", + "itertools 0.12.0", "log", + "md-5", + "regex", + "sha2", + "unicode-segmentation", + "uuid", ] [[package]] name = "datafusion-functions-array" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d16a0ddf2c991526f6ffe2f47a72c6da0b7354d6c32411dd20631fe2e38937" +checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55" dependencies = [ "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions", + "itertools 0.12.0", "log", "paste", ] [[package]] name = "datafusion-optimizer" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ae27e07bf1f04d327be5c2a293470879801ab5535204dc3b16b062fda195496" +checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a" dependencies = [ "arrow", "async-trait", @@ -873,9 +919,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde620cd9ef76a3bca9c754fb68854bd2349c49f55baf97e08001f9e967f6d6b" +checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96" dependencies = [ "ahash", "arrow", @@ -884,7 +930,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-string", - "base64", + "base64 0.22.1", "blake2", "blake3", "chrono", @@ -904,14 +950,13 @@ dependencies = [ "regex", "sha2", "unicode-segmentation", - "uuid", ] [[package]] name = "datafusion-physical-plan" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a4c75fba9ea99d64b2246cbd2fcae2e6fc973e6616b1015237a616036506dd4" +checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9" dependencies = [ "ahash", "arrow", @@ -921,6 +966,7 @@ dependencies = [ "async-trait", "chrono", "datafusion-common", + "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", @@ -935,12 +981,11 @@ dependencies = [ "pin-project-lite", "rand", "tokio", - "uuid", ] [[package]] name = "datafusion-python" -version = "36.0.0" +version = "37.1.0" dependencies = [ "async-trait", "datafusion", @@ -960,7 +1005,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.48", + "syn 2.0.60", "tokio", "url", "uuid", @@ -968,23 +1013,25 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21474a95c3a62d113599d21b439fa15091b538bac06bd20be0bb2e7d22903c09" +checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356" dependencies = [ "arrow", + "arrow-array", "arrow-schema", "datafusion-common", "datafusion-expr", "log", "sqlparser", + "strum 0.26.1", ] [[package]] name = "datafusion-substrait" -version = "36.0.0" +version = "37.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab89c01ef66a59ec92d2360db63893224b4f7e085e2ee6351e0bb77f88931f0" +checksum = "cd3b496697ac22a857c7d497b9d6b40edec19ed2e3e86e2b77051541fefb4c6d" dependencies = [ "async-recursion", "chrono", @@ -1153,7 +1200,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -1287,6 +1334,12 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.4" @@ -1421,9 +1474,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -1435,6 +1488,18 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "integer-encoding" version = "3.0.4" @@ -1840,7 +1905,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" dependencies = [ "async-trait", - "base64", + "base64 0.21.7", "bytes", "chrono", "futures", @@ -1910,9 +1975,9 @@ dependencies = [ [[package]] name = "parquet" -version = "50.0.0" +version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" +checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" dependencies = [ "ahash", "arrow-array", @@ -1922,7 +1987,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.22.1", "brotli", "bytes", "chrono", @@ -2043,14 +2108,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] @@ -2072,7 +2137,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" dependencies = [ "bytes", - "heck", + "heck 0.4.1", "itertools 0.11.0", "log", "multimap", @@ -2095,7 +2160,7 @@ dependencies = [ "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2162,7 +2227,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2171,10 +2236,10 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2278,11 +2343,11 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "regress" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ed9969cad8051328011596bf549629f1b800cf1731e7964b1eef8dfc480d2c2" +checksum = "4f5f39ba4513916c1b2657b72af6ec671f091cd637992f58d0ede5cae4e5dea0" dependencies = [ - "hashbrown 0.13.2", + "hashbrown 0.14.3", "memchr", ] @@ -2292,7 +2357,7 @@ version = "0.11.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -2406,7 +2471,7 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] [[package]] @@ -2415,7 +2480,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" dependencies = [ - "base64", + "base64 0.21.7", "rustls-pki-types", ] @@ -2530,9 +2595,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "seq-macro" @@ -2542,22 +2607,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.199" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2573,9 +2638,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ "itoa", "ryu", @@ -2591,7 +2656,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2608,9 +2673,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.30" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ "indexmap", "itoa", @@ -2667,7 +2732,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "syn 1.0.109", @@ -2697,9 +2762,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.43.1" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f95c4bae5aba7cd30bd506f7140026ade63cff5afd778af8854026f9606bf5d4" +checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" dependencies = [ "log", "sqlparser_derive", @@ -2713,7 +2778,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2743,11 +2808,11 @@ version = "0.25.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2756,21 +2821,21 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] name = "substrait" -version = "0.24.2" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8ffb7a3e7505bb835513e77ebfe67d359e57d684a5972323e3bdefbecc1f25" +checksum = "df9531ae6784dee4c018ebdb0226872b63cc28765bfa65c1e53b6c58584232af" dependencies = [ "git2", - "heck", + "heck 0.5.0", "prettyplease", "prost", "prost-build", @@ -2781,7 +2846,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.48", + "syn 2.0.60", "typify", "walkdir", ] @@ -2805,9 +2870,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -2856,22 +2921,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2934,7 +2999,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -2986,7 +3051,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -3031,7 +3096,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] @@ -3042,9 +3107,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" -version = "0.0.15" +version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63ed4d717aa95e598e2f9183376b060e95669ef8f444701ea6afb990fde1cf69" +checksum = "5c61e9db210bbff218e6535c664b37ec47da449169b98e7866d0580d0db75529" dependencies = [ "typify-impl", "typify-macro", @@ -3052,27 +3117,27 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.15" +version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89057244dfade7c58af9e62beccbcbeb7a7e7701697a33b06dbe0b7331fb79cf" +checksum = "95e32f38493804f88e2dc7a5412eccd872ea5452b4db9b0a77de4df180f2a87e" dependencies = [ - "heck", + "heck 0.4.1", "log", "proc-macro2", "quote", "regress", "schemars", "serde_json", - "syn 2.0.48", + "syn 2.0.60", "thiserror", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.0.15" +version = "0.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ddade397f5957d2cd7fb27f905a9a569db20e8e1e3ea589edce40be07b92825" +checksum = "cc09508b72f63d521d68e42c7f172c7416d67986df44b3c7d1f7f9963948ed32" dependencies = [ "proc-macro2", "quote", @@ -3080,7 +3145,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.48", + "syn 2.0.60", "typify-impl", ] @@ -3125,9 +3190,9 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "untrusted" @@ -3170,9 +3235,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -3214,7 +3279,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", "wasm-bindgen-shared", ] @@ -3248,7 +3313,7 @@ checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3502,7 +3567,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.60", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8d7e72884..f04b23d3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,10 +17,10 @@ [package] name = "datafusion-python" -version = "36.0.0" -homepage = "https://github.com/apache/arrow-datafusion-python" -repository = "https://github.com/apache/arrow-datafusion-python" -authors = ["Apache Arrow "] +version = "37.1.0" +homepage = "https://datafusion.apache.org/python" +repository = "https://github.com/apache/datafusion-python" +authors = ["Apache DataFusion "] description = "Apache Arrow DataFusion DataFrame and SQL Query Engine" readme = "README.md" license = "Apache-2.0" @@ -37,13 +37,13 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "36.0.0", features = ["pyarrow", "avro"] } -datafusion-common = { version = "36.0.0", features = ["pyarrow"] } -datafusion-expr = "36.0.0" -datafusion-functions-array = "36.0.0" -datafusion-optimizer = "36.0.0" -datafusion-sql = "36.0.0" -datafusion-substrait = { version = "36.0.0", optional = true } +datafusion = { version = "37.1.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-common = { version = "37.1.0", features = ["pyarrow"] } +datafusion-expr = "37.1.0" +datafusion-functions-array = "37.1.0" +datafusion-optimizer = "37.1.0" +datafusion-sql = "37.1.0" +datafusion-substrait = { version = "37.1.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.8", features = ["v4"] } diff --git a/datafusion/tests/test_functions.py b/datafusion/tests/test_functions.py index fa9a495dd..d834f587f 100644 --- a/datafusion/tests/test_functions.py +++ b/datafusion/tests/test_functions.py @@ -428,10 +428,10 @@ def py_flatten(arr): f.array_slice(col, literal(2), literal(4)), lambda: [arr[1:4] for arr in data], ], - [ - f.list_slice(col, literal(-1), literal(2)), - lambda: [arr[-1:2] for arr in data], - ], + # [ + # f.list_slice(col, literal(-1), literal(2)), + # lambda: [arr[-1:2] for arr in data], + # ], [ f.array_intersect(col, literal([3.0, 4.0])), lambda: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], diff --git a/src/catalog.rs b/src/catalog.rs index ba7e22255..49fe14046 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -97,7 +97,7 @@ impl PyDatabase { } fn table(&self, name: &str, py: Python) -> PyResult { - if let Some(table) = wait_for_future(py, self.database.table(name)) { + if let Some(table) = wait_for_future(py, self.database.table(name))? { Ok(PyTable::new(table)) } else { Err(DataFusionError::Common(format!("Table not found: {name}")).into()) diff --git a/src/common/data_type.rs b/src/common/data_type.rs index d3203fdcd..49959c457 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -226,6 +226,19 @@ impl DataTypeMap { DataType::RunEndEncoded(_, _) => Err(py_datafusion_err( DataFusionError::NotImplemented(format!("{:?}", arrow_type)), )), + DataType::BinaryView => Err(py_datafusion_err(DataFusionError::NotImplemented( + format!("{:?}", arrow_type), + ))), + DataType::Utf8View => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( + "{:?}", + arrow_type + )))), + DataType::ListView(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( + format!("{:?}", arrow_type), + ))), + DataType::LargeListView(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( + format!("{:?}", arrow_type), + ))), } } @@ -309,6 +322,9 @@ impl DataTypeMap { ScalarValue::DurationMillisecond(_) => Ok(DataType::Duration(TimeUnit::Millisecond)), ScalarValue::DurationMicrosecond(_) => Ok(DataType::Duration(TimeUnit::Microsecond)), ScalarValue::DurationNanosecond(_) => Ok(DataType::Duration(TimeUnit::Nanosecond)), + ScalarValue::Union(_, _, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( + "ScalarValue::LargeList".to_string(), + ))), } } } @@ -598,6 +614,10 @@ impl DataTypeMap { DataType::Decimal256(_, _) => "Decimal256", DataType::Map(_, _) => "Map", DataType::RunEndEncoded(_, _) => "RunEndEncoded", + DataType::BinaryView => "BinaryView", + DataType::Utf8View => "Utf8View", + DataType::ListView(_) => "ListView", + DataType::LargeListView(_) => "LargeListView", }) } } diff --git a/src/context.rs b/src/context.rs index f1ecaa328..9462d0b86 100644 --- a/src/context.rs +++ b/src/context.rs @@ -742,8 +742,18 @@ impl PySessionContext { } pub fn tables(&self) -> HashSet { - #[allow(deprecated)] - self.ctx.tables().unwrap() + self.ctx + .catalog_names() + .into_iter() + .filter_map(|name| self.ctx.catalog(&name)) + .flat_map(move |catalog| { + catalog + .schema_names() + .into_iter() + .filter_map(move |name| catalog.schema(&name)) + }) + .flat_map(|schema| schema.table_names()) + .collect() } pub fn table(&self, name: &str, py: Python) -> PyResult { diff --git a/src/dataframe.rs b/src/dataframe.rs index a319b3d73..f1efc0c7a 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -20,10 +20,10 @@ use std::sync::Arc; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; +use datafusion::config::TableParquetOptions; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; -use datafusion::parquet::file::properties::WriterProperties; use datafusion::prelude::*; use datafusion_common::UnnestOptions; use pyo3::exceptions::{PyTypeError, PyValueError}; @@ -350,7 +350,7 @@ impl PyDataFrame { cl.ok_or(PyValueError::new_err("compression_level is not defined")) } - let compression_type = match compression.to_lowercase().as_str() { + let _validated = match compression.to_lowercase().as_str() { "snappy" => Compression::SNAPPY, "gzip" => Compression::GZIP( GzipLevel::try_new(compression_level.unwrap_or(6)) @@ -375,16 +375,20 @@ impl PyDataFrame { } }; - let writer_properties = WriterProperties::builder() - .set_compression(compression_type) - .build(); + let mut compression_string = compression.to_string(); + if let Some(level) = compression_level { + compression_string.push_str(&format!("({level})")); + } + + let mut options = TableParquetOptions::default(); + options.global.compression = Some(compression_string); wait_for_future( py, self.df.as_ref().clone().write_parquet( path, DataFrameWriteOptions::new(), - Option::from(writer_properties), + Option::from(options), ), )?; Ok(()) @@ -397,7 +401,7 @@ impl PyDataFrame { self.df .as_ref() .clone() - .write_json(path, DataFrameWriteOptions::new()), + .write_json(path, DataFrameWriteOptions::new(), None), )?; Ok(()) } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 89f73a93d..4c9e2f3fe 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -32,11 +32,11 @@ use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::error::{DataFusionError as InnerDataFusionError, Result as DFResult}; use datafusion::execution::context::TaskContext; -use datafusion::physical_expr::PhysicalSortExpr; +use datafusion::physical_expr::{EquivalenceProperties, PhysicalSortExpr}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream, - Statistics, + DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, + Partitioning, SendableRecordBatchStream, Statistics, }; use datafusion_expr::utils::conjunction; use datafusion_expr::Expr; @@ -73,6 +73,7 @@ pub(crate) struct DatasetExec { columns: Option>, filter_expr: Option, projected_statistics: Statistics, + plan_properties: datafusion::physical_plan::PlanProperties, } impl DatasetExec { @@ -134,6 +135,12 @@ impl DatasetExec { .map_err(PyErr::from)?; let projected_statistics = Statistics::new_unknown(&schema); + let plan_properties = datafusion::physical_plan::PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + Partitioning::UnknownPartitioning(fragments.len()), + ExecutionMode::Bounded, + ); + Ok(DatasetExec { dataset: dataset.into(), schema, @@ -141,6 +148,7 @@ impl DatasetExec { columns, filter_expr, projected_statistics, + plan_properties, }) } } @@ -156,18 +164,6 @@ impl ExecutionPlan for DatasetExec { self.schema.clone() } - /// Get the output partitioning of this plan - fn output_partitioning(&self) -> Partitioning { - Python::with_gil(|py| { - let fragments = self.fragments.as_ref(py); - Partitioning::UnknownPartitioning(fragments.len()) - }) - } - - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { - None - } - fn children(&self) -> Vec> { // this is a leaf node and has no children vec![] @@ -240,6 +236,29 @@ impl ExecutionPlan for DatasetExec { fn statistics(&self) -> DFResult { Ok(self.projected_statistics.clone()) } + + fn properties(&self) -> &datafusion::physical_plan::PlanProperties { + &self.plan_properties + } +} + +impl ExecutionPlanProperties for DatasetExec { + /// Get the output partitioning of this plan + fn output_partitioning(&self) -> &Partitioning { + self.plan_properties.output_partitioning() + } + + fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { + None + } + + fn execution_mode(&self) -> datafusion::physical_plan::ExecutionMode { + self.plan_properties.execution_mode + } + + fn equivalence_properties(&self) -> &datafusion::physical_expr::EquivalenceProperties { + &self.plan_properties.eq_properties + } } impl DisplayAs for DatasetExec { diff --git a/src/expr.rs b/src/expr.rs index 0958c4a98..3be0d025c 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -382,6 +382,11 @@ impl PyExpr { "ScalarValue::LargeList".to_string(), ), )), + ScalarValue::Union(_, _, _) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::Union".to_string(), + ), + )), }, _ => Err(py_type_err(format!( "Non Expr::Literal encountered in types: {:?}", diff --git a/src/functions.rs b/src/functions.rs index 666e1ec37..7f6b1a877 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -24,7 +24,7 @@ use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; use datafusion::execution::FunctionRegistry; use datafusion::functions; -use datafusion_common::{Column, TableReference}; +use datafusion_common::{Column, ScalarValue, TableReference}; use datafusion_expr::expr::Alias; use datafusion_expr::{ aggregate_function, @@ -36,56 +36,85 @@ use datafusion_expr::{ }; #[pyfunction] -pub fn isnan(expr: PyExpr) -> PyExpr { - functions::expr_fn::isnan(expr.into()).into() +fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { + datafusion_expr::in_list( + expr.expr, + value.into_iter().map(|x| x.expr).collect::>(), + negated, + ) + .into() } #[pyfunction] -pub fn nullif(expr1: PyExpr, expr2: PyExpr) -> PyExpr { - functions::expr_fn::nullif(expr1.into(), expr2.into()).into() +#[pyo3(signature = (*exprs))] +fn make_array(exprs: Vec) -> PyExpr { + datafusion_functions_array::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) + .into() } #[pyfunction] -pub fn encode(input: PyExpr, encoding: PyExpr) -> PyExpr { - functions::expr_fn::encode(input.into(), encoding.into()).into() +#[pyo3(signature = (*exprs))] +fn array(exprs: Vec) -> PyExpr { + // alias for make_array + make_array(exprs) } #[pyfunction] -pub fn decode(input: PyExpr, encoding: PyExpr) -> PyExpr { - functions::expr_fn::decode(input.into(), encoding.into()).into() +#[pyo3(signature = (*exprs))] +fn array_concat(exprs: Vec) -> PyExpr { + let exprs = exprs.into_iter().map(|x| x.into()).collect(); + datafusion_functions_array::expr_fn::array_concat(exprs).into() } #[pyfunction] -pub fn array_to_string(expr: PyExpr, delim: PyExpr) -> PyExpr { - datafusion_functions_array::expr_fn::array_to_string(expr.into(), delim.into()).into() +#[pyo3(signature = (*exprs))] +fn array_cat(exprs: Vec) -> PyExpr { + array_concat(exprs) } #[pyfunction] -pub fn array_join(expr: PyExpr, delim: PyExpr) -> PyExpr { - // alias for array_to_string - array_to_string(expr, delim) +#[pyo3(signature = (array, element, index = 1))] +fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { + let index = ScalarValue::Int64(index); + let index = Expr::Literal(index); + datafusion_functions_array::expr_fn::array_position(array.into(), element.into(), index).into() } #[pyfunction] -pub fn list_to_string(expr: PyExpr, delim: PyExpr) -> PyExpr { - // alias for array_to_string - array_to_string(expr, delim) +#[pyo3(signature = (array, element, index = 1))] +fn array_indexof(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { + // alias of array_position + array_position(array, element, index) } #[pyfunction] -pub fn list_join(expr: PyExpr, delim: PyExpr) -> PyExpr { - // alias for array_to_string - array_to_string(expr, delim) +#[pyo3(signature = (array, element, index = 1))] +fn list_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { + // alias of array_position + array_position(array, element, index) } #[pyfunction] -fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { - datafusion_expr::in_list( - expr.expr, - value.into_iter().map(|x| x.expr).collect::>(), - negated, - ) - .into() +#[pyo3(signature = (array, element, index = 1))] +fn list_indexof(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { + // alias of array_position + array_position(array, element, index) +} + +#[pyfunction] +#[pyo3(signature = (array, begin, end, stride = 1))] +fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { + let stride = ScalarValue::Int64(stride); + let stride = Expr::Literal(stride); + datafusion_functions_array::expr_fn::array_slice(array.into(), begin.into(), end.into(), stride) + .into() +} + +#[pyfunction] +#[pyo3(signature = (array, begin, end, stride = 1))] +fn list_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { + // alias of array_slice + array_slice(array, begin, end, stride) } /// Computes a binary hash of the given data. type is the algorithm to use. @@ -95,7 +124,7 @@ fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { #[pyo3(signature = (value, method))] fn digest(value: PyExpr, method: PyExpr) -> PyExpr { PyExpr { - expr: datafusion_expr::digest(value.expr, method.expr), + expr: functions::expr_fn::digest(value.expr, method.expr), } } @@ -162,6 +191,7 @@ fn count_star() -> PyResult { distinct: false, filter: None, order_by: None, + null_treatment: None, }), }) } @@ -214,6 +244,7 @@ fn window( .map(|x| x.expr) .collect::>(), window_frame, + null_treatment: None, }), }) } @@ -256,46 +287,124 @@ macro_rules! aggregate_function { distinct, filter: None, order_by: None, + null_treatment: None, }); expr.into() } }; } -scalar_function!(abs, Abs); -scalar_function!(acos, Acos); +/// Generates a [pyo3] wrapper for [datafusion::functions::expr_fn] +/// +/// These functions have explicit named arguments. +macro_rules! expr_fn { + ($NAME: ident) => { + expr_fn!($NAME, $NAME, , stringify!($NAME)); + }; + ($NAME:ident, $($arg:ident)*) => { + expr_fn!($NAME, $NAME, $($arg)*, stringify!($FUNC)); + }; + ($NAME:ident, $FUNC:ident, $($arg:ident)*) => { + expr_fn!($NAME, $FUNC, $($arg)*, stringify!($FUNC)); + }; + ($NAME: ident, $DOC: expr) => { + expr_fn!($NAME, $NAME, ,$DOC); + }; + ($NAME: ident, $($arg:ident)*, $DOC: expr) => { + expr_fn!($NAME, $NAME, $($arg)* ,$DOC); + }; + ($NAME: ident, $FUNC: ident, $($arg:ident)*, $DOC: expr) => { + #[doc = $DOC] + #[pyfunction] + fn $NAME($($arg: PyExpr),*) -> PyExpr { + functions::expr_fn::$FUNC($($arg.into()),*).into() + } + }; +} + +/// Generates a [pyo3] wrapper for [datafusion::functions::expr_fn] +/// +/// These functions take a single `Vec` argument using `pyo3(signature = (*args))`. +macro_rules! expr_fn_vec { + ($NAME: ident) => { + expr_fn_vec!($NAME, $NAME, stringify!($NAME)); + }; + ($NAME: ident, $DOC: expr) => { + expr_fn_vec!($NAME, $NAME, $DOC); + }; + ($NAME: ident, $FUNC: ident, $DOC: expr) => { + #[doc = $DOC] + #[pyfunction] + #[pyo3(signature = (*args))] + fn $NAME(args: Vec) -> PyExpr { + let args = args.into_iter().map(|e| e.into()).collect::>(); + functions::expr_fn::$FUNC(args).into() + } + }; +} + +/// Generates a [pyo3] wrapper for [datafusion_functions_array::expr_fn] +/// +/// These functions have explicit named arguments. +macro_rules! array_fn { + ($NAME: ident) => { + array_fn!($NAME, $NAME, , stringify!($NAME)); + }; + ($NAME:ident, $($arg:ident)*) => { + array_fn!($NAME, $NAME, $($arg)*, stringify!($FUNC)); + }; + ($NAME: ident, $FUNC:ident, $($arg:ident)*) => { + array_fn!($NAME, $FUNC, $($arg)*, stringify!($FUNC)); + }; + ($NAME: ident, $DOC: expr) => { + array_fn!($NAME, $NAME, , $DOC); + }; + ($NAME: ident, $FUNC:ident, $($arg:ident)*, $DOC:expr) => { + #[doc = $DOC] + #[pyfunction] + fn $NAME($($arg: PyExpr),*) -> PyExpr { + datafusion_functions_array::expr_fn::$FUNC($($arg.into()),*).into() + } + }; +} + +expr_fn!(abs, num); +expr_fn!(acos, num); scalar_function!(acosh, Acosh); -scalar_function!(ascii, Ascii, "Returns the numeric code of the first character of the argument. In UTF8 encoding, returns the Unicode code point of the character. In other multibyte encodings, the argument must be an ASCII character."); -scalar_function!(asin, Asin); +expr_fn!(ascii, arg1, "Returns the numeric code of the first character of the argument. In UTF8 encoding, returns the Unicode code point of the character. In other multibyte encodings, the argument must be an ASCII character."); +expr_fn!(asin, num); scalar_function!(asinh, Asinh); scalar_function!(atan, Atan); scalar_function!(atanh, Atanh); scalar_function!(atan2, Atan2); -scalar_function!( +expr_fn!( bit_length, - BitLength, + arg, "Returns number of bits in the string (8 times the octet_length)." ); -scalar_function!(btrim, Btrim, "Removes the longest string containing only characters in characters (a space by default) from the start and end of string."); +expr_fn_vec!(btrim, "Removes the longest string containing only characters in characters (a space by default) from the start and end of string."); scalar_function!(cbrt, Cbrt); scalar_function!(ceil, Ceil); -scalar_function!( +expr_fn!( character_length, - CharacterLength, + string, "Returns number of characters in the string." ); -scalar_function!(length, CharacterLength); -scalar_function!(char_length, CharacterLength); -scalar_function!(chr, Chr, "Returns the character with the given code."); +expr_fn!(length, string); +expr_fn!(char_length, string); +expr_fn!(chr, arg, "Returns the character with the given code."); scalar_function!(coalesce, Coalesce); scalar_function!(cos, Cos); scalar_function!(cosh, Cosh); scalar_function!(degrees, Degrees); +expr_fn!(decode, input encoding); +expr_fn!(encode, input encoding); scalar_function!(exp, Exp); scalar_function!(factorial, Factorial); scalar_function!(floor, Floor); scalar_function!(gcd, Gcd); scalar_function!(initcap, InitCap, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); +expr_fn!(isnan, num); scalar_function!(iszero, Iszero); scalar_function!(lcm, Lcm); scalar_function!(left, Left, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); @@ -303,12 +412,12 @@ scalar_function!(ln, Ln); scalar_function!(log, Log); scalar_function!(log10, Log10); scalar_function!(log2, Log2); -scalar_function!(lower, Lower, "Converts the string to all lower case"); +expr_fn!(lower, arg1, "Converts the string to all lower case"); scalar_function!(lpad, Lpad, "Extends the string to length length by prepending the characters fill (a space by default). If the string is already longer than length then it is truncated (on the right)."); -scalar_function!(ltrim, Ltrim, "Removes the longest string containing only characters in characters (a space by default) from the start of string."); -scalar_function!( +expr_fn_vec!(ltrim, "Removes the longest string containing only characters in characters (a space by default) from the start of string."); +expr_fn!( md5, - MD5, + input_arg, "Computes the MD5 hash of the argument, with the result written in hexadecimal." ); scalar_function!( @@ -316,25 +425,22 @@ scalar_function!( Nanvl, "Returns x if x is not NaN otherwise returns y." ); -scalar_function!(octet_length, OctetLength, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); +expr_fn!(nullif, arg_1 arg_2); +expr_fn_vec!(octet_length, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); scalar_function!(pi, Pi); scalar_function!(power, Power); scalar_function!(pow, Power); scalar_function!(radians, Radians); -scalar_function!(regexp_match, RegexpMatch); -scalar_function!( +expr_fn!(regexp_match, input_arg1 input_arg2); +expr_fn!( regexp_replace, - RegexpReplace, - "Replaces substring(s) matching a POSIX regular expression" + arg1 arg2 arg3 arg4, + "Replaces substring(s) matching a POSIX regular expression." ); -scalar_function!( - repeat, - Repeat, - "Repeats string the specified number of times." -); -scalar_function!( +expr_fn!(repeat, string n, "Repeats string the specified number of times."); +expr_fn!( replace, - Replace, + string from to, "Replaces all occurrences in string of substring from with substring to." ); scalar_function!( @@ -345,117 +451,108 @@ scalar_function!( scalar_function!(right, Right, "Returns last n characters in the string, or when n is negative, returns all but first |n| characters."); scalar_function!(round, Round); scalar_function!(rpad, Rpad, "Extends the string to length length by appending the characters fill (a space by default). If the string is already longer than length then it is truncated."); -scalar_function!(rtrim, Rtrim, "Removes the longest string containing only characters in characters (a space by default) from the end of string."); -scalar_function!(sha224, SHA224); -scalar_function!(sha256, SHA256); -scalar_function!(sha384, SHA384); -scalar_function!(sha512, SHA512); +expr_fn_vec!(rtrim, "Removes the longest string containing only characters in characters (a space by default) from the end of string."); +expr_fn!(sha224, input_arg1); +expr_fn!(sha256, input_arg1); +expr_fn!(sha384, input_arg1); +expr_fn!(sha512, input_arg1); scalar_function!(signum, Signum); scalar_function!(sin, Sin); scalar_function!(sinh, Sinh); -scalar_function!( +expr_fn!( split_part, - SplitPart, + string delimiter index, "Splits string at occurrences of delimiter and returns the n'th field (counting from one)." ); scalar_function!(sqrt, Sqrt); -scalar_function!( - starts_with, - StartsWith, - "Returns true if string starts with prefix." -); +expr_fn!(starts_with, arg1 arg2, "Returns true if string starts with prefix."); scalar_function!(strpos, Strpos, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); scalar_function!(substr, Substr); -scalar_function!(tan, Tan); -scalar_function!(tanh, Tanh); -scalar_function!( +expr_fn!(tan, num); +expr_fn!(tanh, num); +expr_fn!( to_hex, - ToHex, + arg1, "Converts the number to its equivalent hexadecimal representation." ); -scalar_function!(now, Now); -scalar_function!(to_timestamp, ToTimestamp); -scalar_function!(to_timestamp_millis, ToTimestampMillis); -scalar_function!(to_timestamp_micros, ToTimestampMicros); -scalar_function!(to_timestamp_seconds, ToTimestampSeconds); -scalar_function!(current_date, CurrentDate); -scalar_function!(current_time, CurrentTime); -scalar_function!(datepart, DatePart); -scalar_function!(date_part, DatePart); -scalar_function!(date_trunc, DateTrunc); -scalar_function!(datetrunc, DateTrunc); -scalar_function!(date_bin, DateBin); +expr_fn!(now); +expr_fn_vec!(to_timestamp); +expr_fn_vec!(to_timestamp_millis); +expr_fn_vec!(to_timestamp_micros); +expr_fn_vec!(to_timestamp_seconds); +expr_fn!(current_date); +expr_fn!(current_time); +expr_fn!(date_part, part date); +expr_fn!(datepart, date_part, part date); +expr_fn!(date_trunc, part date); +expr_fn!(datetrunc, date_trunc, part date); +expr_fn!(date_bin, stride source origin); + scalar_function!(translate, Translate, "Replaces each character in string that matches a character in the from set with the corresponding character in the to set. If from is longer than to, occurrences of the extra characters in from are deleted."); -scalar_function!(trim, Trim, "Removes the longest string containing only characters in characters (a space by default) from the start, end, or both ends (BOTH is the default) of string."); +expr_fn_vec!(trim, "Removes the longest string containing only characters in characters (a space by default) from the start, end, or both ends (BOTH is the default) of string."); scalar_function!(trunc, Trunc); -scalar_function!(upper, Upper, "Converts the string to all upper case."); -scalar_function!(make_array, MakeArray); -scalar_function!(array, MakeArray); -scalar_function!(range, Range); -scalar_function!(uuid, Uuid); -scalar_function!(r#struct, Struct); // Use raw identifier since struct is a keyword -scalar_function!(from_unixtime, FromUnixtime); -scalar_function!(arrow_typeof, ArrowTypeof); +expr_fn!(upper, arg1, "Converts the string to all upper case."); +expr_fn!(uuid); +expr_fn!(r#struct, args); // Use raw identifier since struct is a keyword +expr_fn!(from_unixtime, unixtime); +expr_fn!(arrow_typeof, arg_1); scalar_function!(random, Random); // Array Functions -scalar_function!(array_append, ArrayAppend); -scalar_function!(array_push_back, ArrayAppend); -scalar_function!(list_append, ArrayAppend); -scalar_function!(list_push_back, ArrayAppend); -scalar_function!(array_concat, ArrayConcat); -scalar_function!(array_cat, ArrayConcat); -scalar_function!(array_dims, ArrayDims); -scalar_function!(array_distinct, ArrayDistinct); -scalar_function!(list_distinct, ArrayDistinct); -scalar_function!(list_dims, ArrayDims); -scalar_function!(array_element, ArrayElement); -scalar_function!(array_extract, ArrayElement); -scalar_function!(list_element, ArrayElement); -scalar_function!(list_extract, ArrayElement); -scalar_function!(array_length, ArrayLength); -scalar_function!(list_length, ArrayLength); -scalar_function!(array_has, ArrayHas); -scalar_function!(array_has_all, ArrayHasAll); -scalar_function!(array_has_any, ArrayHasAny); -scalar_function!(array_position, ArrayPosition); -scalar_function!(array_indexof, ArrayPosition); -scalar_function!(list_position, ArrayPosition); -scalar_function!(list_indexof, ArrayPosition); -scalar_function!(array_positions, ArrayPositions); -scalar_function!(list_positions, ArrayPositions); -scalar_function!(array_ndims, ArrayNdims); -scalar_function!(list_ndims, ArrayNdims); -scalar_function!(array_prepend, ArrayPrepend); -scalar_function!(array_push_front, ArrayPrepend); -scalar_function!(list_prepend, ArrayPrepend); -scalar_function!(list_push_front, ArrayPrepend); -scalar_function!(array_pop_back, ArrayPopBack); -scalar_function!(array_pop_front, ArrayPopFront); -scalar_function!(array_remove, ArrayRemove); -scalar_function!(list_remove, ArrayRemove); -scalar_function!(array_remove_n, ArrayRemoveN); -scalar_function!(list_remove_n, ArrayRemoveN); -scalar_function!(array_remove_all, ArrayRemoveAll); -scalar_function!(list_remove_all, ArrayRemoveAll); -scalar_function!(array_repeat, ArrayRepeat); -scalar_function!(array_replace, ArrayReplace); -scalar_function!(list_replace, ArrayReplace); -scalar_function!(array_replace_n, ArrayReplaceN); -scalar_function!(list_replace_n, ArrayReplaceN); -scalar_function!(array_replace_all, ArrayReplaceAll); -scalar_function!(list_replace_all, ArrayReplaceAll); -scalar_function!(array_slice, ArraySlice); -scalar_function!(list_slice, ArraySlice); -scalar_function!(array_intersect, ArrayIntersect); -scalar_function!(list_intersect, ArrayIntersect); -scalar_function!(array_union, ArrayUnion); -scalar_function!(list_union, ArrayUnion); -scalar_function!(array_except, ArrayExcept); -scalar_function!(list_except, ArrayExcept); -scalar_function!(array_resize, ArrayResize); -scalar_function!(list_resize, ArrayResize); -scalar_function!(flatten, Flatten); +array_fn!(array_append, array element); +array_fn!(array_push_back, array_append, array element); +array_fn!(array_to_string, array delimiter); +array_fn!(array_join, array_to_string, array delimiter); +array_fn!(list_to_string, array_to_string, array delimiter); +array_fn!(list_join, array_to_string, array delimiter); +array_fn!(list_append, array_append, array element); +array_fn!(list_push_back, array_append, array element); +array_fn!(array_dims, array); +array_fn!(array_distinct, array); +array_fn!(list_distinct, array_distinct, array); +array_fn!(list_dims, array_dims, array); +array_fn!(array_element, array element); +array_fn!(array_extract, array_element, array element); +array_fn!(list_element, array_element, array element); +array_fn!(list_extract, array_element, array element); +array_fn!(array_length, array); +array_fn!(list_length, array_length, array); +array_fn!(array_has, first_array second_array); +array_fn!(array_has_all, first_array second_array); +array_fn!(array_has_any, first_array second_array); +array_fn!(array_positions, array_positions, array element); +array_fn!(list_positions, array_positions, array element); +array_fn!(array_ndims, array); +array_fn!(list_ndims, array_ndims, array); +array_fn!(array_prepend, element array); +array_fn!(array_push_front, array_prepend, element array); +array_fn!(list_prepend, array_prepend, element array); +array_fn!(list_push_front, array_prepend, element array); +array_fn!(array_pop_back, array); +array_fn!(array_pop_front, array); +array_fn!(array_remove, array element); +array_fn!(list_remove, array_remove, array element); +array_fn!(array_remove_n, array element max); +array_fn!(list_remove_n, array_remove_n, array element max); +array_fn!(array_remove_all, array element); +array_fn!(list_remove_all, array_remove_all, array element); +array_fn!(array_repeat, element count); +array_fn!(array_replace, array from to); +array_fn!(list_replace, array_replace, array from to); +array_fn!(array_replace_n, array from to max); +array_fn!(list_replace_n, array_replace_n, array from to max); +array_fn!(array_replace_all, array from to); +array_fn!(list_replace_all, array_replace_all, array from to); +array_fn!(array_intersect, first_array second_array); +array_fn!(list_intersect, array_intersect, first_array second_array); +array_fn!(array_union, array1 array2); +array_fn!(list_union, array_union, array1 array2); +array_fn!(array_except, first_array second_array); +array_fn!(list_except, array_except, first_array second_array); +array_fn!(array_resize, array size value); +array_fn!(list_resize, array_resize, array size value); +array_fn!(flatten, array); +array_fn!(range, start stop step); aggregate_function!(approx_distinct, ApproxDistinct); aggregate_function!(approx_median, ApproxMedian); diff --git a/src/physical_plan.rs b/src/physical_plan.rs index ab783221b..51032f2d4 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion::physical_plan::{displayable, ExecutionPlan}; +use datafusion::physical_plan::{displayable, ExecutionPlan, ExecutionPlanProperties}; use std::sync::Arc; use pyo3::prelude::*; diff --git a/src/udf.rs b/src/udf.rs index af17b57e8..69519f499 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -23,7 +23,6 @@ use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; -use datafusion::physical_plan::functions::make_scalar_function; use datafusion::physical_plan::udf::ScalarUDF; use datafusion_expr::create_udf; use datafusion_expr::function::ScalarFunctionImplementation; @@ -35,7 +34,8 @@ use crate::utils::parse_volatility; /// that expects pyarrow arrays. This is more efficient as it performs /// a zero-copy of the contents. fn to_rust_function(func: PyObject) -> ScalarFunctionImplementation { - make_scalar_function( + #[allow(deprecated)] + datafusion::physical_plan::functions::make_scalar_function( move |args: &[ArrayRef]| -> Result { Python::with_gil(|py| { // 1. cast args to Pyarrow arrays diff --git a/src/utils.rs b/src/utils.rs index c5965bd2f..62cf07d9e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -37,9 +37,9 @@ pub(crate) fn get_tokio_runtime(py: Python) -> PyRef { } /// Utility to collect rust futures with GIL released -pub fn wait_for_future(py: Python, f: F) -> F::Output +pub fn wait_for_future(py: Python, f: F) -> F::Output where - F: Send, + F: Future + Send, F::Output: Send, { let runtime: &Runtime = &get_tokio_runtime(py).0; From 7f276516a644cde908cf74795251746d482d1829 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 8 May 2024 17:07:20 -0600 Subject: [PATCH 132/413] Generate changelog (#673) --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ Cargo.toml | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a05429f4..863b142dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,33 @@ # DataFusion Python Changelog +## [37.1.0](https://github.com/apache/arrow-datafusion-python/tree/37.1.0) (2024-05-08) + +**Implemented enhancements:** + +- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/arrow-datafusion-python/pull/610) (mesejo) + +**Documentation updates:** + +- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/arrow-datafusion-python/pull/661) (Michael-J-Ward) + +**Merged pull requests:** + +- Switch to Ruff for Python linting [#529](https://github.com/apache/arrow-datafusion-python/pull/529) (andygrove) +- Remove sql-on-pandas/polars/cudf examples [#602](https://github.com/apache/arrow-datafusion-python/pull/602) (andygrove) +- build(deps): bump object_store from 0.9.0 to 0.9.1 [#611](https://github.com/apache/arrow-datafusion-python/pull/611) (dependabot[bot]) +- More missing array funcs [#605](https://github.com/apache/arrow-datafusion-python/pull/605) (judahrand) +- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/arrow-datafusion-python/pull/610) (mesejo) +- build(deps): bump uuid from 1.7.0 to 1.8.0 [#615](https://github.com/apache/arrow-datafusion-python/pull/615) (dependabot[bot]) +- Bind SQLOptions and relative ctx method #567 [#588](https://github.com/apache/arrow-datafusion-python/pull/588) (giacomorebecchi) +- bugfix: no panic on empty table [#613](https://github.com/apache/arrow-datafusion-python/pull/613) (mesejo) +- Expose `register_listing_table` [#618](https://github.com/apache/arrow-datafusion-python/pull/618) (henrifroese) +- Expose unnest feature [#641](https://github.com/apache/arrow-datafusion-python/pull/641) (timsaucer) +- Update domain names and paths in asf yaml [#643](https://github.com/apache/arrow-datafusion-python/pull/643) (andygrove) +- use python 3.11 to publish docs [#645](https://github.com/apache/arrow-datafusion-python/pull/645) (andygrove) +- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/arrow-datafusion-python/pull/661) (Michael-J-Ward) +- Upgrade Datafusion to v37.1.0 [#669](https://github.com/apache/arrow-datafusion-python/pull/669) (Michael-J-Ward) + ## [36.0.0](https://github.com/apache/arrow-datafusion-python/tree/36.0.0) (2024-03-02) **Implemented enhancements:** diff --git a/Cargo.toml b/Cargo.toml index f04b23d3e..9da36d710 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ version = "37.1.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] -description = "Apache Arrow DataFusion DataFrame and SQL Query Engine" +description = "Apache DataFusion DataFrame and SQL Query Engine" readme = "README.md" license = "Apache-2.0" edition = "2021" From 7fd0c96b6f59c750e7dd59f92beed7d57d371f6a Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 9 May 2024 12:01:06 -0400 Subject: [PATCH 133/413] Add document about basics of working with expressions (#668) --- .../common-operations/expressions.rst | 94 +++++++++++++++++++ .../user-guide/common-operations/index.rst | 1 + 2 files changed, 95 insertions(+) create mode 100644 docs/source/user-guide/common-operations/expressions.rst diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst new file mode 100644 index 000000000..ebb514f14 --- /dev/null +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -0,0 +1,94 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Expressions +=========== + +In DataFusion an expression is an abstraction that represents a computation. +Expressions are used as the primary inputs and ouputs for most functions within +DataFusion. As such, expressions can be combined to create expression trees, a +concept shared across most compilers and databases. + +Column +------ + +The first expression most new users will interact with is the Column, which is created by calling :func:`col`. +This expression represents a column within a DataFrame. The function :func:`col` takes as in input a string +and returns an expression as it's output. + +Literal +------- + +Literal expressions represent a single value. These are helpful in a wide range of operations where +a specific, known value is of interest. You can create a literal expression using the function :func:`lit`. +The type of the object passed to the :func:`lit` function will be used to convert it to a known data type. + +In the following example we create expressions for the column named `color` and the literal scalar string `red`. +The resultant variable `red_units` is itself also an expression. + +.. ipython:: python + + red_units = col("color") == lit("red") + +Boolean +------- + +When combining expressions that evaluate to a boolean value, you can combine these expressions using boolean operators. +It is important to note that in order to combine these expressions, you *must* use bitwise operators. See the following +examples for the and, or, and not operations. + + +.. ipython:: python + + red_or_green_units = (col("color") == lit("red")) | (col("color") == lit("green")) + heavy_red_units = (col("color") == lit("red")) & (col("weight") > lit(42)) + not_red_units = ~(col("color") == lit("red")) + +Functions +--------- + +As mentioned before, most functions in DataFusion return an expression at their output. This allows us to create +a wide variety of expressions built up from other expressions. For example, :func:`.alias` is a function that takes +as it input a single expression and returns an expression in which the name of the expression has changed. + +The following example shows a series of expressions that are built up from functions operating on expressions. + +.. ipython:: python + + from datafusion import SessionContext + from datafusion import column, lit + from datafusion import functions as f + import random + + ctx = SessionContext() + df = ctx.from_pydict( + { + "name": ["Albert", "Becca", "Carlos", "Dante"], + "age": [42, 67, 27, 71], + "years_in_position": [13, 21, 10, 54], + }, + name="employees" + ) + + age_col = col("age") + renamed_age = age_col.alias("age_in_years") + start_age = age_col - col("years_in_position") + started_young = start_age < lit(18) + can_retire = age_col > lit(65) + long_timer = started_young & can_retire + + df.filter(long_timer).select(col("name"), renamed_age, col("years_in_position")) diff --git a/docs/source/user-guide/common-operations/index.rst b/docs/source/user-guide/common-operations/index.rst index 950afb93e..b15b04c62 100644 --- a/docs/source/user-guide/common-operations/index.rst +++ b/docs/source/user-guide/common-operations/index.rst @@ -23,6 +23,7 @@ Common Operations basic-info select-and-filter + expressions joins functions aggregations From c6bdc9fecc4e8e28b055af28887f8cd7c19378ce Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 9 May 2024 12:17:39 -0600 Subject: [PATCH 134/413] chore: Update Python release process now that DataFusion is TLP (#674) --- dev/release/README.md | 59 +++---------- dev/release/create-tarball.sh | 34 ++++---- dev/release/generate-changelog.py | 2 +- dev/release/release-tarball.sh | 12 +-- .../update_change_log-datafusion-python.sh | 33 ------- dev/release/update_change_log.sh | 87 ------------------- dev/release/verify-release-candidate.sh | 28 +++--- 7 files changed, 50 insertions(+), 205 deletions(-) delete mode 100755 dev/release/update_change_log-datafusion-python.sh delete mode 100755 dev/release/update_change_log.sh diff --git a/dev/release/README.md b/dev/release/README.md index 6bd2c1eb2..c4372c832 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -63,7 +63,7 @@ We maintain a `CHANGELOG.md` so our users know what has been changed between rel The changelog is generated using a Python script: ```bash -$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/arrow-datafusion-python 24.0.0 HEAD > dev/changelog/25.0.0.md +$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/datafusion-python 24.0.0 HEAD > dev/changelog/25.0.0.md ``` This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for @@ -83,9 +83,9 @@ This process is not fully automated, so there are some additional manual steps: - Add the following content (copy from the previous version's changelog and update as appropriate: ``` -## [24.0.0](https://github.com/apache/arrow-datafusion-python/tree/24.0.0) (2023-05-06) +## [24.0.0](https://github.com/apache/datafusion-python/tree/24.0.0) (2023-05-06) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/23.0.0...24.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/23.0.0...24.0.0) ``` ### Preparing a Release Candidate @@ -103,42 +103,7 @@ git push apache 0.8.0-rc1 ./dev/release/create-tarball.sh 0.8.0 1 ``` -This will also create the email template to send to the mailing list. Here is an example: - -``` -To: dev@arrow.apache.org -Subject: [VOTE][RUST][DataFusion] Release DataFusion Python Bindings 0.7.0 RC2 -Hi, - -I would like to propose a release of Apache Arrow DataFusion Python Bindings, -version 0.7.0. - -This release candidate is based on commit: bd1b78b6d444b7ab172c6aec23fa58c842a592d7 [1] -The proposed release tarball and signatures are hosted at [2]. -The changelog is located at [3]. -The Python wheels are located at [4]. - -Please download, verify checksums and signatures, run the unit tests, and vote -on the release. The vote will be open for at least 72 hours. - -Only votes from PMC members are binding, but all members of the community are -encouraged to test the release and vote with "(non-binding)". - -The standard verification procedure is documented at https://github.com/apache/arrow-datafusion-python/blob/main/dev/release/README.md#verifying-release-candidates. - -[ ] +1 Release this as Apache Arrow DataFusion Python 0.7.0 -[ ] +0 -[ ] -1 Do not release this as Apache Arrow DataFusion Python 0.7.0 because... - -Here is my vote: - -+1 - -[1]: https://github.com/apache/arrow-datafusion-python/tree/bd1b78b6d444b7ab172c6aec23fa58c842a592d7 -[2]: https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-0.7.0-rc2 -[3]: https://github.com/apache/arrow-datafusion-python/blob/bd1b78b6d444b7ab172c6aec23fa58c842a592d7/CHANGELOG.md -[4]: https://test.pypi.org/project/datafusion/0.7.0/ -``` +This will also create the email template to send to the mailing list. Create a draft email using this content, but do not send until after completing the next step. @@ -151,7 +116,7 @@ This section assumes some familiarity with publishing Python packages to PyPi. F Pushing an `rc` tag to the release branch will cause a GitHub Workflow to run that will build the Python wheels. -Go to https://github.com/apache/arrow-datafusion-python/actions and look for an action named "Python Release Build" +Go to https://github.com/apache/datafusion-python/actions and look for an action named "Python Release Build" that has run against the pushed tag. Click on the action and scroll down to the bottom of the page titled "Artifacts". Download `dist.zip`. It should @@ -266,10 +231,10 @@ git push apache 0.8.0 ### Add the release to Apache Reporter -Add the release to https://reporter.apache.org/addrelease.html?arrow with a version name prefixed with `RS-DATAFUSION-PYTHON`, -for example `RS-DATAFUSION-PYTHON-31.0.0`. +Add the release to https://reporter.apache.org/addrelease.html?datafusion with a version name prefixed with `DATAFUSION-PYTHON`, +for example `DATAFUSION-PYTHON-31.0.0`. -The release information is used to generate a template for a board report (see example +The release information is used to generate a template for a board report (see example from Apache Arrow [here](https://github.com/apache/arrow/pull/14357)). ### Delete old RCs and Releases @@ -284,13 +249,13 @@ Release candidates should be deleted once the release is published. Get a list of DataFusion release candidates: ```bash -svn ls https://dist.apache.org/repos/dist/dev/arrow | grep datafusion-python +svn ls https://dist.apache.org/repos/dist/dev/datafusion | grep datafusion-python ``` Delete a release candidate: ```bash -svn delete -m "delete old DataFusion RC" https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-7.1.0-rc1/ +svn delete -m "delete old DataFusion RC" https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-python-7.1.0-rc1/ ``` #### Deleting old releases from `release` svn @@ -300,11 +265,11 @@ Only the latest release should be available. Delete old releases after publishin Get a list of DataFusion releases: ```bash -svn ls https://dist.apache.org/repos/dist/release/arrow | grep datafusion-python +svn ls https://dist.apache.org/repos/dist/release/datafusion | grep datafusion-python ``` Delete a release: ```bash -svn delete -m "delete old DataFusion release" https://dist.apache.org/repos/dist/release/arrow/arrow-datafusion-python-7.0.0 +svn delete -m "delete old DataFusion release" https://dist.apache.org/repos/dist/release/datafusion/datafusion-python-7.0.0 ``` diff --git a/dev/release/create-tarball.sh b/dev/release/create-tarball.sh index c05da5b75..d6ca76561 100755 --- a/dev/release/create-tarball.sh +++ b/dev/release/create-tarball.sh @@ -21,9 +21,9 @@ # Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/create-tarball.sh # This script creates a signed tarball in -# dev/dist/apache-arrow-datafusion-python--.tar.gz and uploads it to +# dev/dist/apache-datafusion-python--.tar.gz and uploads it to # the "dev" area of the dist.apache.arrow repository and prepares an -# email for sending to the dev@arrow.apache.org list for a formal +# email for sending to the dev@datafusion.apache.org list for a formal # vote. # # See release/README.md for full release instructions @@ -65,25 +65,25 @@ tag="${version}-rc${rc}" echo "Attempting to create ${tarball} from tag ${tag}" release_hash=$(cd "${SOURCE_TOP_DIR}" && git rev-list --max-count=1 ${tag}) -release=apache-arrow-datafusion-python-${version} +release=apache-datafusion-python-${version} distdir=${SOURCE_TOP_DIR}/dev/dist/${release}-rc${rc} tarname=${release}.tar.gz tarball=${distdir}/${tarname} -url="https://dist.apache.org/repos/dist/dev/arrow/${release}-rc${rc}" +url="https://dist.apache.org/repos/dist/dev/datafusion/${release}-rc${rc}" if [ -z "$release_hash" ]; then echo "Cannot continue: unknown git tag: ${tag}" fi -echo "Draft email for dev@arrow.apache.org mailing list" +echo "Draft email for dev@datafusion.apache.org mailing list" echo "" echo "---------------------------------------------------------" cat < ${tarball}.sha256 (cd ${distdir} && shasum -a 512 ${tarname}) > ${tarball}.sha512 -echo "Uploading to apache dist/dev to ${url}" -svn co --depth=empty https://dist.apache.org/repos/dist/dev/arrow ${SOURCE_TOP_DIR}/dev/dist +echo "Uploading to datafusion dist/dev to ${url}" +svn co --depth=empty https://dist.apache.org/repos/dist/dev/datafusion ${SOURCE_TOP_DIR}/dev/dist svn add ${distdir} -svn ci -m "Apache Arrow DataFusion Python ${version} ${rc}" ${distdir} +svn ci -m "Apache DataFusion Python ${version} ${rc}" ${distdir} diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 01d640669..af097ce98 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -102,7 +102,7 @@ def cli(args=None): parser = argparse.ArgumentParser() parser.add_argument( - "project", help="The project name e.g. apache/arrow-datafusion-python" + "project", help="The project name e.g. apache/datafusion-python" ) parser.add_argument("tag1", help="The previous release tag") parser.add_argument("tag2", help="The current release tag") diff --git a/dev/release/release-tarball.sh b/dev/release/release-tarball.sh index f5e8eb1bf..8c305a676 100755 --- a/dev/release/release-tarball.sh +++ b/dev/release/release-tarball.sh @@ -43,7 +43,7 @@ fi version=$1 rc=$2 -tmp_dir=tmp-apache-arrow-datafusion-python-dist +tmp_dir=tmp-apache-datafusion-python-dist echo "Recreate temporary directory: ${tmp_dir}" rm -rf ${tmp_dir} @@ -52,23 +52,23 @@ mkdir -p ${tmp_dir} echo "Clone dev dist repository" svn \ co \ - https://dist.apache.org/repos/dist/dev/arrow/apache-arrow-datafusion-python-${version}-rc${rc} \ + https://dist.apache.org/repos/dist/dev/datafusion/apache-datafusion-python-${version}-rc${rc} \ ${tmp_dir}/dev echo "Clone release dist repository" -svn co https://dist.apache.org/repos/dist/release/arrow ${tmp_dir}/release +svn co https://dist.apache.org/repos/dist/release/datafusion ${tmp_dir}/release echo "Copy ${version}-rc${rc} to release working copy" -release_version=arrow-datafusion-python-${version} +release_version=datafusion-python-${version} mkdir -p ${tmp_dir}/release/${release_version} cp -r ${tmp_dir}/dev/* ${tmp_dir}/release/${release_version}/ svn add ${tmp_dir}/release/${release_version} echo "Commit release" -svn ci -m "Apache Arrow DataFusion Python ${version}" ${tmp_dir}/release +svn ci -m "Apache DataFusion Python ${version}" ${tmp_dir}/release echo "Clean up" rm -rf ${tmp_dir} echo "Success! The release is available here:" -echo " https://dist.apache.org/repos/dist/release/arrow/${release_version}" +echo " https://dist.apache.org/repos/dist/release/datafusion/${release_version}" diff --git a/dev/release/update_change_log-datafusion-python.sh b/dev/release/update_change_log-datafusion-python.sh deleted file mode 100755 index a11447f0b..000000000 --- a/dev/release/update_change_log-datafusion-python.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Usage: -# CHANGELOG_GITHUB_TOKEN= ./update_change_log-datafusion.sh main 8.0.0 7.1.0 -# CHANGELOG_GITHUB_TOKEN= ./update_change_log-datafusion.sh maint-7.x 7.1.0 7.0.0 - -RELEASE_BRANCH=$1 -RELEASE_TAG=$2 -BASE_TAG=$3 - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -${SOURCE_DIR}/update_change_log.sh \ - "${BASE_TAG}" \ - --future-release "${RELEASE_TAG}" \ - --release-branch "${RELEASE_BRANCH}" diff --git a/dev/release/update_change_log.sh b/dev/release/update_change_log.sh deleted file mode 100755 index a0b398131..000000000 --- a/dev/release/update_change_log.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -# Adapted from https://github.com/apache/arrow-rs/tree/master/dev/release/update_change_log.sh - -# invokes the changelog generator from -# https://github.com/github-changelog-generator/github-changelog-generator -# -# With the config located in -# arrow-datafusion/.github_changelog_generator -# -# Usage: -# CHANGELOG_GITHUB_TOKEN= ./update_change_log.sh - -set -e - -SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SOURCE_TOP_DIR="$(cd "${SOURCE_DIR}/../../" && pwd)" - -if [[ "$#" -lt 1 ]]; then - echo "USAGE: $0 SINCE_TAG EXTRA_ARGS..." - exit 1 -fi - -SINCE_TAG=$1 -shift 1 - -OUTPUT_PATH="CHANGELOG.md" - -pushd ${SOURCE_TOP_DIR} - -# reset content in changelog -git checkout "${SINCE_TAG}" "${OUTPUT_PATH}" -# remove license header so github-changelog-generator has a clean base to append -sed -i.bak '1,18d' "${OUTPUT_PATH}" - -docker run -it --rm \ - --cpus "0.1" \ - -e CHANGELOG_GITHUB_TOKEN=$CHANGELOG_GITHUB_TOKEN \ - -v "$(pwd)":/usr/local/src/your-app \ - githubchangeloggenerator/github-changelog-generator \ - --user apache \ - --project arrow-datafusion-python \ - --since-tag "${SINCE_TAG}" \ - --base "${OUTPUT_PATH}" \ - --output "${OUTPUT_PATH}" \ - "$@" - -sed -i.bak "s/\\\n/\n\n/" "${OUTPUT_PATH}" - -echo ' -' | cat - "${OUTPUT_PATH}" > "${OUTPUT_PATH}".tmp -mv "${OUTPUT_PATH}".tmp "${OUTPUT_PATH}" diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index be86f69e0..14c0baee8 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -32,8 +32,8 @@ set -x set -o pipefail SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")" && pwd)" -ARROW_DIR="$(dirname $(dirname ${SOURCE_DIR}))" -ARROW_DIST_URL='https://dist.apache.org/repos/dist/dev/arrow' +DATAFUSION_PYTHON_DIR="$(dirname $(dirname ${SOURCE_DIR}))" +DATAFUSION_PYTHON_DIST_URL='https://dist.apache.org/repos/dist/dev/datafusion' download_dist_file() { curl \ @@ -41,11 +41,11 @@ download_dist_file() { --show-error \ --fail \ --location \ - --remote-name $ARROW_DIST_URL/$1 + --remote-name $DATAFUSION_PYTHON_DIST_URL/$1 } download_rc_file() { - download_dist_file apache-arrow-datafusion-python-${VERSION}-rc${RC_NUMBER}/$1 + download_dist_file apache-datafusion-python-${VERSION}-rc${RC_NUMBER}/$1 } import_gpg_keys() { @@ -89,19 +89,19 @@ verify_dir_artifact_signatures() { setup_tempdir() { cleanup() { if [ "${TEST_SUCCESS}" = "yes" ]; then - rm -fr "${ARROW_TMPDIR}" + rm -fr "${DATAFUSION_PYTHON_TMPDIR}" else - echo "Failed to verify release candidate. See ${ARROW_TMPDIR} for details." + echo "Failed to verify release candidate. See ${DATAFUSION_PYTHON_TMPDIR} for details." fi } - if [ -z "${ARROW_TMPDIR}" ]; then - # clean up automatically if ARROW_TMPDIR is not defined - ARROW_TMPDIR=$(mktemp -d -t "$1.XXXXX") + if [ -z "${DATAFUSION_PYTHON_TMPDIR}" ]; then + # clean up automatically if DATAFUSION_PYTHON_TMPDIR is not defined + DATAFUSION_PYTHON_TMPDIR=$(mktemp -d -t "$1.XXXXX") trap cleanup EXIT else # don't clean up automatically - mkdir -p "${ARROW_TMPDIR}" + mkdir -p "${DATAFUSION_PYTHON_TMPDIR}" fi } @@ -142,11 +142,11 @@ test_source_distribution() { TEST_SUCCESS=no -setup_tempdir "arrow-${VERSION}" -echo "Working in sandbox ${ARROW_TMPDIR}" -cd ${ARROW_TMPDIR} +setup_tempdir "datafusion-python-${VERSION}" +echo "Working in sandbox ${DATAFUSION_PYTHON_TMPDIR}" +cd ${DATAFUSION_PYTHON_TMPDIR} -dist_name="apache-arrow-datafusion-python-${VERSION}" +dist_name="apache-datafusion-python-${VERSION}" import_gpg_keys fetch_archive ${dist_name} tar xf ${dist_name}.tar.gz From 7d4a40cc73c37bcf41ccd3c6480627b03c9f93cc Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sat, 11 May 2024 13:32:01 -0500 Subject: [PATCH 135/413] Fix Docs (#676) --- .github/workflows/docs.yaml | 6 ++++++ docs/source/api/object_store.rst | 2 +- docs/source/user-guide/common-operations/functions.rst | 3 +-- docs/source/user-guide/configuration.rst | 4 ++-- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 1193d7810..855164852 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -4,6 +4,9 @@ on: - main tags-ignore: - "**-rc**" + pull_request: + branches: + - main name: Deploy DataFusion Python site @@ -13,6 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set target branch + if: github.event_name == 'push' && github.ref == 'refs/heads/main' id: target-branch run: | set -x @@ -27,6 +31,7 @@ jobs: - name: Checkout docs sources uses: actions/checkout@v3 - name: Checkout docs target branch + if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: actions/checkout@v3 with: fetch-depth: 0 @@ -64,6 +69,7 @@ jobs: make html - name: Copy & push the generated HTML + if: github.event_name == 'push' && github.ref == 'refs/heads/main' run: | set -x cd docs-target diff --git a/docs/source/api/object_store.rst b/docs/source/api/object_store.rst index eeb6c4326..8d78f0724 100644 --- a/docs/source/api/object_store.rst +++ b/docs/source/api/object_store.rst @@ -19,7 +19,7 @@ .. currentmodule:: datafusion.object_store ObjectStore -========= +=========== .. autosummary:: :toctree: ../generated/ diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index 7e5c592d8..50b493098 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -92,13 +92,12 @@ DataFusion offers a range of helpful options. f.left(col('"Name"'), literal(4)).alias("code") ) -This also includes the functions for regular expressions :func:`.regexp_replace` and :func:`.regexp_match` +This also includes the functions for regular expressions like :func:`.regexp_match` .. ipython:: python df.select( f.regexp_match(col('"Name"'), literal("Char")).alias("dragons"), - f.regexp_replace(col('"Name"'), literal("saur"), literal("fleur")).alias("flowers") ) diff --git a/docs/source/user-guide/configuration.rst b/docs/source/user-guide/configuration.rst index 63b0dc3e2..0c1a4818a 100644 --- a/docs/source/user-guide/configuration.rst +++ b/docs/source/user-guide/configuration.rst @@ -16,7 +16,7 @@ .. under the License. Configuration -======== +============= Let's look at how we can configure DataFusion. When creating a :code:`SessionContext`, you can pass in a :code:`SessionConfig` and :code:`RuntimeConfig` object. These two cover a wide range of options. @@ -48,4 +48,4 @@ a :code:`SessionConfig` and :code:`RuntimeConfig` object. These two cover a wide You can read more about available :code:`SessionConfig` options `here `_, -and about :code:`RuntimeConfig` options `here https://docs.rs/datafusion/latest/datafusion/execution/runtime_env/struct.RuntimeConfig.html`_. +and about :code:`RuntimeConfig` options `here `_. From d71c436ae2006843dc720bfdfcb8b3aeb434815e Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 13 May 2024 10:00:18 -0400 Subject: [PATCH 136/413] Add examples from TPC-H (#666) * Update location of docker image * Initial commit for queries 1-3 * Commit queries 4-7 of TPC-H in examples * Add required license text * Add additional text around why to use a case statement in the example * add market share example * Add example for product type profit measure * Inital commit returned item report * Linting * Initial commit of q11 example * Initial commit of q12 from tpc-h * Initial commit for customer distribution example * Initial commit of promotion effect example * Initial commit of q15 in tph-c, top supplier * Initial commit of q16 in tph-c, part supplier relationship * Initial commit of q17 in tph-c, small quatity order * Initial commit of q18 in tph-c, large volume customer * Initial commit of q19 in tph-c, discounted revenue * Initial commit of q20 in tph-c, potential part promotion * Initial commit of q21 in tph-c, supplier who kept order waiting * Initial commit of q22 in tph-c, global sales opportunity * Adding readme information and marking text as copyrighted * Minimum part cost must be identified per part not across all parts that match the filters * Change ordering of output rows to match spec * Set parameter to match spec * Set parameter to match spec * setting values to match spec * Linting * Expand on readme to link to examples within tpch folder * Minor typo --- benchmarks/tpch/tpch-gen.sh | 6 +- examples/README.md | 64 +++++++ examples/tpch/.gitignore | 2 + examples/tpch/README.md | 57 ++++++ examples/tpch/convert_data_to_parquet.py | 142 ++++++++++++++ examples/tpch/q01_pricing_summary_report.py | 90 +++++++++ examples/tpch/q02_minimum_cost_supplier.py | 139 ++++++++++++++ examples/tpch/q03_shipping_priority.py | 86 +++++++++ examples/tpch/q04_order_priority_checking.py | 80 ++++++++ examples/tpch/q05_local_supplier_volume.py | 102 ++++++++++ .../tpch/q06_forecasting_revenue_change.py | 87 +++++++++ examples/tpch/q07_volume_shipping.py | 123 ++++++++++++ examples/tpch/q08_market_share.py | 175 ++++++++++++++++++ .../tpch/q09_product_type_profit_measure.py | 93 ++++++++++ examples/tpch/q10_returned_item_reporting.py | 108 +++++++++++ .../q11_important_stock_identification.py | 82 ++++++++ examples/tpch/q12_ship_mode_order_priority.py | 112 +++++++++++ examples/tpch/q13_customer_distribution.py | 64 +++++++ examples/tpch/q14_promotion_effect.py | 81 ++++++++ examples/tpch/q15_top_supplier.py | 87 +++++++++ .../tpch/q16_part_supplier_relationship.py | 85 +++++++++ examples/tpch/q17_small_quantity_order.py | 69 +++++++ examples/tpch/q18_large_volume_customer.py | 65 +++++++ examples/tpch/q19_discounted_revenue.py | 137 ++++++++++++++ examples/tpch/q20_potential_part_promotion.py | 97 ++++++++++ .../tpch/q21_suppliers_kept_orders_waiting.py | 114 ++++++++++++ examples/tpch/q22_global_sales_opportunity.py | 76 ++++++++ 27 files changed, 2420 insertions(+), 3 deletions(-) create mode 100644 examples/tpch/.gitignore create mode 100644 examples/tpch/README.md create mode 100644 examples/tpch/convert_data_to_parquet.py create mode 100644 examples/tpch/q01_pricing_summary_report.py create mode 100644 examples/tpch/q02_minimum_cost_supplier.py create mode 100644 examples/tpch/q03_shipping_priority.py create mode 100644 examples/tpch/q04_order_priority_checking.py create mode 100644 examples/tpch/q05_local_supplier_volume.py create mode 100644 examples/tpch/q06_forecasting_revenue_change.py create mode 100644 examples/tpch/q07_volume_shipping.py create mode 100644 examples/tpch/q08_market_share.py create mode 100644 examples/tpch/q09_product_type_profit_measure.py create mode 100644 examples/tpch/q10_returned_item_reporting.py create mode 100644 examples/tpch/q11_important_stock_identification.py create mode 100644 examples/tpch/q12_ship_mode_order_priority.py create mode 100644 examples/tpch/q13_customer_distribution.py create mode 100644 examples/tpch/q14_promotion_effect.py create mode 100644 examples/tpch/q15_top_supplier.py create mode 100644 examples/tpch/q16_part_supplier_relationship.py create mode 100644 examples/tpch/q17_small_quantity_order.py create mode 100644 examples/tpch/q18_large_volume_customer.py create mode 100644 examples/tpch/q19_discounted_revenue.py create mode 100644 examples/tpch/q20_potential_part_promotion.py create mode 100644 examples/tpch/q21_suppliers_kept_orders_waiting.py create mode 100644 examples/tpch/q22_global_sales_opportunity.py diff --git a/benchmarks/tpch/tpch-gen.sh b/benchmarks/tpch/tpch-gen.sh index e27472a3d..15cab12a5 100755 --- a/benchmarks/tpch/tpch-gen.sh +++ b/benchmarks/tpch/tpch-gen.sh @@ -29,7 +29,7 @@ FILE=./data/supplier.tbl if test -f "$FILE"; then echo "$FILE exists." else - docker run -v `pwd`/data:/data -it --rm ghcr.io/databloom-ai/tpch-docker:main -vf -s $1 + docker run -v `pwd`/data:/data -it --rm ghcr.io/scalytics/tpch-docker:main -vf -s $1 # workaround for https://github.com/apache/arrow-datafusion/issues/6147 mv data/customer.tbl data/customer.csv @@ -49,5 +49,5 @@ FILE=./data/answers/q1.out if test -f "$FILE"; then echo "$FILE exists." else - docker run -v `pwd`/data:/data -it --entrypoint /bin/bash --rm ghcr.io/databloom-ai/tpch-docker:main -c "cp /opt/tpch/2.18.0_rc2/dbgen/answers/* /data/answers/" -fi \ No newline at end of file + docker run -v `pwd`/data:/data -it --entrypoint /bin/bash --rm ghcr.io/scalytics/tpch-docker:main -c "cp /opt/tpch/2.18.0_rc2/dbgen/answers/* /data/answers/" +fi diff --git a/examples/README.md b/examples/README.md index 82405955b..0ef194afe 100644 --- a/examples/README.md +++ b/examples/README.md @@ -52,3 +52,67 @@ Here is a direct link to the file used in the examples: - [Executing SQL on Polars](./sql-on-polars.py) - [Executing SQL on Pandas](./sql-on-pandas.py) - [Executing SQL on cuDF](./sql-on-cudf.py) + +## TPC-H Examples + +Within the subdirectory `tpch` there are 22 examples that reproduce queries in +the TPC-H specification. These include realistic data that can be generated at +arbitrary scale and allow the user to see use cases for a variety of data frame +operations. + +In the list below we describe which new operations can be found in the examples. +The queries are designed to be of increasing complexity, so it is recommended to +review them in order. For brevity, the following list does not include operations +found in previous examples. + +- [Convert CSV to Parquet](./tpch/convert_data_to_parquet.py) + - Read from a CSV files where the delimiter is something other than a comma + - Specify schema during CVS reading + - Write to a parquet file +- [Pricing Summary Report](./tpch/q01_pricing_summary_report.py) + - Aggregation computing the maximum value, average, sum, and number of entries + - Filter data by date and interval + - Sorting +- [Minimum Cost Supplier](./tpch/q02_minimum_cost_supplier.py) + - Window operation to find minimum + - Sorting in descending order +- [Shipping Priority](./tpch/q03_shipping_priority.py) +- [Order Priority Checking](./tpch/q04_order_priority_checking.py) + - Aggregating multiple times in one data frame +- [Local Supplier Volume](./tpch/q05_local_supplier_volume.py) +- [Forecasting Revenue Change](./tpch/q06_forecasting_revenue_change.py) + - Using collect and extracting values as a python object +- [Volume Shipping](./tpch/q07_volume_shipping.py) + - Finding multiple distinct and mutually exclusive values within one dataframe + - Using `case` and `when` statements +- [Market Share](./tpch/q08_market_share.py) + - The operations in this query are similar to those in the prior examples, but + it is a more complex example of using filters, joins, and aggregates + - Using left outer joins +- [Product Type Profit Measure](./tpch/q09_product_type_profit_measure.py) + - Extract year from a date +- [Returned Item Reporting](./tpch/q10_returned_item_reporting.py) +- [Important Stock Identification](./tpch/q11_important_stock_identification.py) +- [Shipping Modes and Order](./tpch/q12_ship_mode_order_priority.py) + - Finding non-null values using a boolean operation in a filter + - Case statement with default value +- [Customer Distribution](./tpch/q13_customer_distribution.py) +- [Promotion Effect](./tpch/q14_promotion_effect.py) +- [Top Supplier](./tpch/q15_top_supplier.py) +- [Parts/Supplier Relationship](./tpch/q16_part_supplier_relationship.py) + - Using anti joins + - Using regular expressions (regex) + - Creating arrays of literal values + - Determine if an element exists within an array +- [Small-Quantity-Order Revenue](./tpch/q17_small_quantity_order.py) +- [Large Volume Customer](./tpch/q18_large_volume_customer.py) +- [Discounted Revenue](./tpch/q19_discounted_revenue.py) + - Creating a user defined function (UDF) + - Convert pyarrow Array to python values + - Filtering based on a UDF +- [Potential Part Promotion](./tpch/q20_potential_part_promotion.py) + - Extracting part of a string using substr +- [Suppliers Who Kept Orders Waiting](./tpch/q21_suppliers_kept_orders_waiting.py) + - Using array aggregation + - Determining the size of array elements +- [Global Sales Opportunity](./tpch/q22_global_sales_opportunity.py) diff --git a/examples/tpch/.gitignore b/examples/tpch/.gitignore new file mode 100644 index 000000000..9e67bd47d --- /dev/null +++ b/examples/tpch/.gitignore @@ -0,0 +1,2 @@ +data + diff --git a/examples/tpch/README.md b/examples/tpch/README.md new file mode 100644 index 000000000..7c52c8230 --- /dev/null +++ b/examples/tpch/README.md @@ -0,0 +1,57 @@ + + +# DataFusion Python Examples for TPC-H + +These examples reproduce the problems listed in the Transaction Process Council +TPC-H benchmark. The purpose of these examples is to demonstrate how to use +different aspects of Data Fusion and not necessarily geared towards creating the +most performant queries possible. Within each example is a description of the +problem. For users who are familiar with SQL style commands, you can compare the +approaches in these examples with those listed in the specification. + +- https://www.tpc.org/tpch/ + +The examples provided are based on version 2.18.0 of the TPC-H specification. + +## Data Setup + +To run these examples, you must first generate a dataset. The `dbgen` tool +provided by TPC can create datasets of arbitrary scale. For testing it is +typically sufficient to create a 1 gigabyte dataset. For convenience, this +repository has a script which uses docker to create this dataset. From the +`benchmarks/tpch` directory execute the following script. + +```bash +./tpch-gen.sh 1 +``` + +The examples provided use parquet files for the tables generated by `dbgen`. +A python script is provided to convert the text files from `dbgen` into parquet +files expected by the examples. From the `examples/tpch` directory you can +execute the following command to create the necessary parquet files. + +```bash +python convert_data_to_parquet.py +``` + +## Description of Examples + +For easier access, a description of the techniques demonstrated in each file +is in the README.md file in the `examples` directory. diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py new file mode 100644 index 000000000..178b7fb39 --- /dev/null +++ b/examples/tpch/convert_data_to_parquet.py @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +This is a utility function that will consumer the data generated by dbgen from TPC-H and convert +it into a parquet file with the column names as expected by the TPC-H specification. It assumes +the data generated resides in a path ../../benchmarks/tpch/data relative to the current file, +as will be generated by the script provided in this repository. +""" + +import os +import pyarrow +import datafusion + +ctx = datafusion.SessionContext() + +all_schemas = {} + +all_schemas["customer"] = [ + ("C_CUSTKEY", pyarrow.int32()), + ("C_NAME", pyarrow.string()), + ("C_ADDRESS", pyarrow.string()), + ("C_NATIONKEY", pyarrow.int32()), + ("C_PHONE", pyarrow.string()), + ("C_ACCTBAL", pyarrow.float32()), + ("C_MKTSEGMENT", pyarrow.string()), + ("C_COMMENT", pyarrow.string()), +] + +all_schemas["lineitem"] = [ + ("L_ORDERKEY", pyarrow.int32()), + ("L_PARTKEY", pyarrow.int32()), + ("L_SUPPKEY", pyarrow.int32()), + ("L_LINENUMBER", pyarrow.int32()), + ("L_QUANTITY", pyarrow.float32()), + ("L_EXTENDEDPRICE", pyarrow.float32()), + ("L_DISCOUNT", pyarrow.float32()), + ("L_TAX", pyarrow.float32()), + ("L_RETURNFLAG", pyarrow.string()), + ("L_LINESTATUS", pyarrow.string()), + ("L_SHIPDATE", pyarrow.date32()), + ("L_COMMITDATE", pyarrow.date32()), + ("L_RECEIPTDATE", pyarrow.date32()), + ("L_SHIPINSTRUCT", pyarrow.string()), + ("L_SHIPMODE", pyarrow.string()), + ("L_COMMENT", pyarrow.string()), +] + +all_schemas["nation"] = [ + ("N_NATIONKEY", pyarrow.int32()), + ("N_NAME", pyarrow.string()), + ("N_REGIONKEY", pyarrow.int32()), + ("N_COMMENT", pyarrow.string()), +] + +all_schemas["orders"] = [ + ("O_ORDERKEY", pyarrow.int32()), + ("O_CUSTKEY", pyarrow.int32()), + ("O_ORDERSTATUS", pyarrow.string()), + ("O_TOTALPRICE", pyarrow.float32()), + ("O_ORDERDATE", pyarrow.date32()), + ("O_ORDERPRIORITY", pyarrow.string()), + ("O_CLERK", pyarrow.string()), + ("O_SHIPPRIORITY", pyarrow.int32()), + ("O_COMMENT", pyarrow.string()), +] + +all_schemas["part"] = [ + ("P_PARTKEY", pyarrow.int32()), + ("P_NAME", pyarrow.string()), + ("P_MFGR", pyarrow.string()), + ("P_BRAND", pyarrow.string()), + ("P_TYPE", pyarrow.string()), + ("P_SIZE", pyarrow.int32()), + ("P_CONTAINER", pyarrow.string()), + ("P_RETAILPRICE", pyarrow.float32()), + ("P_COMMENT", pyarrow.string()), +] + +all_schemas["partsupp"] = [ + ("PS_PARTKEY", pyarrow.int32()), + ("PS_SUPPKEY", pyarrow.int32()), + ("PS_AVAILQTY", pyarrow.int32()), + ("PS_SUPPLYCOST", pyarrow.float32()), + ("PS_COMMENT", pyarrow.string()), +] + +all_schemas["region"] = [ + ("r_REGIONKEY", pyarrow.int32()), + ("r_NAME", pyarrow.string()), + ("r_COMMENT", pyarrow.string()), +] + +all_schemas["supplier"] = [ + ("S_SUPPKEY", pyarrow.int32()), + ("S_NAME", pyarrow.string()), + ("S_ADDRESS", pyarrow.string()), + ("S_NATIONKEY", pyarrow.int32()), + ("S_PHONE", pyarrow.string()), + ("S_ACCTBAL", pyarrow.float32()), + ("S_COMMENT", pyarrow.string()), +] + +curr_dir = os.path.dirname(os.path.abspath(__file__)) +for filename, curr_schema in all_schemas.items(): + + # For convenience, go ahead and convert the schema column names to lowercase + curr_schema = [(s[0].lower(), s[1]) for s in curr_schema] + + # Pre-collect the output columns so we can ignore the null field we add + # in to handle the trailing | in the file + output_cols = [r[0] for r in curr_schema] + + # Trailing | requires extra field for in processing + curr_schema.append(("some_null", pyarrow.null())) + + schema = pyarrow.schema(curr_schema) + + source_file = os.path.abspath( + os.path.join(curr_dir, f"../../benchmarks/tpch/data/{filename}.csv") + ) + dest_file = os.path.abspath(os.path.join(curr_dir, f"./data/{filename}.parquet")) + + df = ctx.read_csv(source_file, schema=schema, has_header=False, delimiter="|") + + df = df.select_columns(*output_cols) + + df.write_parquet(dest_file, compression="snappy") diff --git a/examples/tpch/q01_pricing_summary_report.py b/examples/tpch/q01_pricing_summary_report.py new file mode 100644 index 000000000..1aafccab0 --- /dev/null +++ b/examples/tpch/q01_pricing_summary_report.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 1: + +The Pricing Summary Report Query provides a summary pricing report for all lineitems shipped as of +a given date. The date is within 60 - 120 days of the greatest ship date contained in the database. +The query lists totals for extended price, discounted extended price, discounted extended price +plus tax, average quantity, average extended price, and average discount. These aggregates are +grouped by RETURNFLAG and LINESTATUS, and listed in ascending order of RETURNFLAG and LINESTATUS. +A count of the number of lineitems in each group is included. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +ctx = SessionContext() + +df = ctx.read_parquet("data/lineitem.parquet") + +# It may be that the date can be hard coded, based on examples shown. +# This approach will work with any date range in the provided data set. + +greatest_ship_date = df.aggregate( + [], [F.max(col("l_shipdate")).alias("shipdate")] +).collect()[0]["shipdate"][0] + +# From the given problem, this is how close to the last date in the database we +# want to report results for. It should be between 60-120 days before the end. +DAYS_BEFORE_FINAL = 68 + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval = pa.scalar((0, 0, DAYS_BEFORE_FINAL), type=pa.month_day_nano_interval()) + +print("Final date in database:", greatest_ship_date) + +# Filter data to the dates of interest +df = df.filter(col("l_shipdate") <= lit(greatest_ship_date) - lit(interval)) + +# Aggregate the results + +df = df.aggregate( + [col("l_returnflag"), col("l_linestatus")], + [ + F.sum(col("l_quantity")).alias("sum_qty"), + F.sum(col("l_extendedprice")).alias("sum_base_price"), + F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias( + "sum_disc_price" + ), + F.sum( + col("l_extendedprice") + * (lit(1.0) - col("l_discount")) + * (lit(1.0) + col("l_tax")) + ).alias("sum_charge"), + F.avg(col("l_quantity")).alias("avg_qty"), + F.avg(col("l_extendedprice")).alias("avg_price"), + F.avg(col("l_discount")).alias("avg_disc"), + F.count(col("l_returnflag")).alias( + "count_order" + ), # Counting any column should return same result + ], +) + +# Sort per the expected result + +df = df.sort(col("l_returnflag").sort(), col("l_linestatus").sort()) + +# Note: There appears to be a discrepancy between what is returned here and what is in the generated +# answers file for the case of return flag N and line status O, but I did not investigate further. + +df.show() diff --git a/examples/tpch/q02_minimum_cost_supplier.py b/examples/tpch/q02_minimum_cost_supplier.py new file mode 100644 index 000000000..262e2cf46 --- /dev/null +++ b/examples/tpch/q02_minimum_cost_supplier.py @@ -0,0 +1,139 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 2: + +The Minimum Cost Supplier Query finds, in a given region, for each part of a certain type and size, +the supplier who can supply it at minimum cost. If several suppliers in that region offer the +desired part type and size at the same (minimum) cost, the query lists the parts from suppliers with +the 100 highest account balances. For each supplier, the query lists the supplier's account balance, +name and nation; the part's number and manufacturer; the supplier's address, phone number and +comment information. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +import datafusion +from datafusion import SessionContext, col, lit, functions as F + +# This is the part we're looking for +SIZE_OF_INTEREST = 15 +TYPE_OF_INTEREST = "BRASS" +REGION_OF_INTEREST = "EUROPE" + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns( + "p_partkey", "p_mfgr", "p_type", "p_size" +) +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_acctbal", + "s_name", + "s_address", + "s_phone", + "s_comment", + "s_nationkey", + "s_suppkey", +) +df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( + "ps_partkey", "ps_suppkey", "ps_supplycost" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_regionkey", "n_name" +) +df_region = ctx.read_parquet("data/region.parquet").select_columns( + "r_regionkey", "r_name" +) + +# Filter down parts. Part names contain the type of interest, so we can use strpos to find where +# in the p_type column the word is. `strpos` will return 0 if not found, otherwise the position +# in the string where it is located. + +df_part = df_part.filter( + F.strpos(col("p_type"), lit(TYPE_OF_INTEREST)) > lit(0) +).filter(col("p_size") == lit(SIZE_OF_INTEREST)) + +# Filter regions down to the one of interest + +df_region = df_region.filter(col("r_name") == lit(REGION_OF_INTEREST)) + +# Now that we have the region, find suppliers in that region. Suppliers are tied to their nation +# and nations are tied to the region. + +df_nation = df_nation.join(df_region, (["n_regionkey"], ["r_regionkey"]), how="inner") +df_supplier = df_supplier.join( + df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner" +) + +# Now that we know who the potential suppliers are for the part, we can limit out part +# supplies table down. We can further join down to the specific parts we've identified +# as matching the request + +df = df_partsupp.join(df_supplier, (["ps_suppkey"], ["s_suppkey"]), how="inner") + +# Locate the minimum cost across all suppliers. There are multiple ways you could do this, +# but one way is to create a window function across all suppliers, find the minimum, and +# create a column of that value. We can then filter down any rows for which the cost and +# minimum do not match. + +# The default window frame as of 5/6/2024 is from unbounded preceeding to the current row. +# We want to evaluate the entire data frame, so we specify this. +window_frame = datafusion.WindowFrame("rows", None, None) +df = df.with_column( + "min_cost", + F.window( + "min", + [col("ps_supplycost")], + partition_by=[col("ps_partkey")], + window_frame=window_frame, + ), +) + +df = df.filter(col("min_cost") == col("ps_supplycost")) + +df = df.join(df_part, (["ps_partkey"], ["p_partkey"]), how="inner") + +# From the problem statement, these are the values we wish to output + +df = df.select_columns( + "s_acctbal", + "s_name", + "n_name", + "p_partkey", + "p_mfgr", + "s_address", + "s_phone", + "s_comment", +) + +# Sort and display 100 entries +df = df.sort( + col("s_acctbal").sort(ascending=False), + col("n_name").sort(), + col("s_name").sort(), + col("p_partkey").sort(), +) + +df = df.limit(100) + +# Show results + +df.show() diff --git a/examples/tpch/q03_shipping_priority.py b/examples/tpch/q03_shipping_priority.py new file mode 100644 index 000000000..78993e9e4 --- /dev/null +++ b/examples/tpch/q03_shipping_priority.py @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 3: + +The Shipping Priority Query retrieves the shipping priority and potential revenue, defined as the +sum of l_extendedprice * (1-l_discount), of the orders having the largest revenue among those that +had not been shipped as of a given date. Orders are listed in decreasing order of revenue. If more +than 10 unshipped orders exist, only the 10 orders with the largest revenue are listed. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, col, lit, functions as F + +SEGMENT_OF_INTEREST = "BUILDING" +DATE_OF_INTEREST = "1995-03-15" + +# Load the dataframes we need + +ctx = SessionContext() + +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_mktsegment", "c_custkey" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderdate", "o_shippriority", "o_custkey", "o_orderkey" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_orderkey", "l_extendedprice", "l_discount", "l_shipdate" +) + +# Limit dataframes to the rows of interest + +df_customer = df_customer.filter(col("c_mktsegment") == lit(SEGMENT_OF_INTEREST)) +df_orders = df_orders.filter(col("o_orderdate") < lit(DATE_OF_INTEREST)) +df_lineitem = df_lineitem.filter(col("l_shipdate") > lit(DATE_OF_INTEREST)) + +# Join all 3 dataframes + +df = df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="inner").join( + df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner" +) + +# Compute the revenue + +df = df.aggregate( + [col("l_orderkey")], + [ + F.first_value(col("o_orderdate")).alias("o_orderdate"), + F.first_value(col("o_shippriority")).alias("o_shippriority"), + F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias("revenue"), + ], +) + +# Sort by priority + +df = df.sort(col("revenue").sort(ascending=False), col("o_orderdate").sort()) + +# Only return 100 results + +df = df.limit(100) + +# Change the order that the columns are reported in just to match the spec + +df = df.select_columns("l_orderkey", "revenue", "o_orderdate", "o_shippriority") + +# Show result + +df.show() diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py new file mode 100644 index 000000000..b691d5b19 --- /dev/null +++ b/examples/tpch/q04_order_priority_checking.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 4: + +The Order Priority Checking Query counts the number of orders ordered in a given quarter of a given +year in which at least one lineitem was received by the customer later than its committed date. The +query lists the count of such orders for each order priority sorted in ascending priority order. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +# Ideally we could put 3 months into the interval. See note below. +INTERVAL_DAYS = 92 +DATE_OF_INTEREST = "1993-07-01" + +# Load the dataframes we need + +ctx = SessionContext() + +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderdate", "o_orderpriority", "o_orderkey" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_orderkey", "l_commitdate", "l_receiptdate" +) + +# Create a date object from the string +date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval()) + +# Limit results to cases where commitment date before receipt date +# Aggregate the results so we only get one row to join with the order table. +# Alterately, and likely more idomatic is instead of `.aggregate` you could +# do `.select_columns("l_orderkey").distinct()`. The goal here is to show +# mulitple examples of how to use Data Fusion. +df_lineitem = df_lineitem.filter(col("l_commitdate") < col("l_receiptdate")).aggregate( + [col("l_orderkey")], [] +) + +# Limit orders to date range of interest +df_orders = df_orders.filter(col("o_orderdate") >= lit(date)).filter( + col("o_orderdate") < lit(date) + lit(interval) +) + +# Perform the join to find only orders for which there are lineitems outside of expected range +df = df_orders.join(df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner") + +# Based on priority, find the number of entries +df = df.aggregate( + [col("o_orderpriority")], [F.count(col("o_orderpriority")).alias("order_count")] +) + +# Sort the results +df = df.sort(col("o_orderpriority").sort()) + +df.show() diff --git a/examples/tpch/q05_local_supplier_volume.py b/examples/tpch/q05_local_supplier_volume.py new file mode 100644 index 000000000..7cb6e6324 --- /dev/null +++ b/examples/tpch/q05_local_supplier_volume.py @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 5: + +The Local Supplier Volume Query lists for each nation in a region the revenue volume that resulted +from lineitem transactions in which the customer ordering parts and the supplier filling them were +both within that nation. The query is run in order to determine whether to institute local +distribution centers in a given region. The query considers only parts ordered in a given year. The +query displays the nations and revenue volume in descending order by revenue. Revenue volume for all +qualifying lineitems in a particular nation is defined as sum(l_extendedprice * (1 - l_discount)). + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + + +DATE_OF_INTEREST = "1994-01-01" +INTERVAL_DAYS = 365 +REGION_OF_INTEREST = "ASIA" + +date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval()) + +# Load the dataframes we need + +ctx = SessionContext() + +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_custkey", "c_nationkey" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_custkey", "o_orderkey", "o_orderdate" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_orderkey", "l_suppkey", "l_extendedprice", "l_discount" +) +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_nationkey" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_regionkey", "n_name" +) +df_region = ctx.read_parquet("data/region.parquet").select_columns( + "r_regionkey", "r_name" +) + +# Restrict dataframes to cases of interest +df_orders = df_orders.filter(col("o_orderdate") >= lit(date)).filter( + col("o_orderdate") < lit(date) + lit(interval) +) + +df_region = df_region.filter(col("r_name") == lit(REGION_OF_INTEREST)) + +# Join all the dataframes + +df = ( + df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="inner") + .join(df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner") + .join( + df_supplier, + (["l_suppkey", "c_nationkey"], ["s_suppkey", "s_nationkey"]), + how="inner", + ) + .join(df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner") + .join(df_region, (["n_regionkey"], ["r_regionkey"]), how="inner") +) + +# Compute the final result + +df = df.aggregate( + [col("n_name")], + [F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias("revenue")], +) + +# Sort in descending order + +df = df.sort(col("revenue").sort(ascending=False)) + +df.show() diff --git a/examples/tpch/q06_forecasting_revenue_change.py b/examples/tpch/q06_forecasting_revenue_change.py new file mode 100644 index 000000000..5fbb91778 --- /dev/null +++ b/examples/tpch/q06_forecasting_revenue_change.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 6: + +The Forecasting Revenue Change Query considers all the lineitems shipped in a given year with +discounts between DISCOUNT-0.01 and DISCOUNT+0.01. The query lists the amount by which the total +revenue would have increased if these discounts had been eliminated for lineitems with l_quantity +less than quantity. Note that the potential revenue increase is equal to the sum of +[l_extendedprice * l_discount] for all lineitems with discounts and quantities in the qualifying +range. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +# Variables from the example query + +DATE_OF_INTEREST = "1994-01-01" +DISCOUT = 0.06 +DELTA = 0.01 +QUANTITY = 24 + +INTERVAL_DAYS = 365 + +date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval()) + +# Load the dataframes we need + +ctx = SessionContext() + +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_shipdate", "l_quantity", "l_extendedprice", "l_discount" +) + +# Filter down to lineitems of interest + +df = ( + df_lineitem.filter(col("l_shipdate") >= lit(date)) + .filter(col("l_shipdate") < lit(date) + lit(interval)) + .filter(col("l_discount") >= lit(DISCOUT) - lit(DELTA)) + .filter(col("l_discount") <= lit(DISCOUT) + lit(DELTA)) + .filter(col("l_quantity") < lit(QUANTITY)) +) + +# Add up all the "lost" revenue + +df = df.aggregate( + [], [F.sum(col("l_extendedprice") * col("l_discount")).alias("revenue")] +) + +# Show the single result. We could do a `show()` but since we want to demonstrate features of how +# to use Data Fusion, instead collect the result as a python object and print it out. + +# collect() should give a list of record batches. This is a small query, so we should get a +# single batch back, hence the index [0]. Within each record batch we only care about the +# single column result `revenue`. Since we have only one row returned because we aggregated +# over the entire dataframe, we can index it at 0. Then convert the DoubleScalar into a +# simple python object. + +revenue = df.collect()[0]["revenue"][0].as_py() + +# Note: the output value from this query may be dependant on the size of the database generated +print(f"Potential lost revenue: {revenue:.2f}") diff --git a/examples/tpch/q07_volume_shipping.py b/examples/tpch/q07_volume_shipping.py new file mode 100644 index 000000000..3c87f9375 --- /dev/null +++ b/examples/tpch/q07_volume_shipping.py @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 7: + +The Volume Shipping Query finds, for two given nations, the gross discounted revenues derived from +lineitems in which parts were shipped from a supplier in either nation to a customer in the other +nation during 1995 and 1996. The query lists the supplier nation, the customer nation, the year, +and the revenue from shipments that took place in that year. The query orders the answer by +Supplier nation, Customer nation, and year (all ascending). + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +# Variables of interest to query over + +nation_1 = lit("FRANCE") +nation_2 = lit("GERMANY") + +START_DATE = "1995-01-01" +END_DATE = "1996-12-31" + +start_date = lit(datetime.strptime(START_DATE, "%Y-%m-%d").date()) +end_date = lit(datetime.strptime(END_DATE, "%Y-%m-%d").date()) + + +# Load the dataframes we need + +ctx = SessionContext() + +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_nationkey" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_shipdate", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_custkey" +) +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_custkey", "c_nationkey" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name" +) + + +# Filter to time of interest +df_lineitem = df_lineitem.filter(col("l_shipdate") >= start_date).filter( + col("l_shipdate") <= end_date +) + + +# A simpler way to do the following operation is to use a filter, but we also want to demonstrate +# how to use case statements. Here we are assigning `n_name` to be itself when it is either of +# the two nations of interest. Since there is no `otherwise()` statement, any values that do +# not match these will result in a null value and then get filtered out. +# +# To do the same using a simle filter would be: +# df_nation = df_nation.filter((F.col("n_name") == nation_1) | (F.col("n_name") == nation_2)) +df_nation = df_nation.with_column( + "n_name", + F.case(col("n_name")) + .when(nation_1, col("n_name")) + .when(nation_2, col("n_name")) + .end(), +).filter(~col("n_name").is_null()) + + +# Limit suppliers to either nation +df_supplier = df_supplier.join( + df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner" +).select(col("s_suppkey"), col("n_name").alias("supp_nation")) + +# Limit customers to either nation +df_customer = df_customer.join( + df_nation, (["c_nationkey"], ["n_nationkey"]), how="inner" +).select(col("c_custkey"), col("n_name").alias("cust_nation")) + +# Join up all the data frames from line items, and make sure the supplier and customer are in +# different nations. +df = ( + df_lineitem.join(df_orders, (["l_orderkey"], ["o_orderkey"]), how="inner") + .join(df_customer, (["o_custkey"], ["c_custkey"]), how="inner") + .join(df_supplier, (["l_suppkey"], ["s_suppkey"]), how="inner") + .filter(col("cust_nation") != col("supp_nation")) +) + +# Extract out two values for every line item +df = df.with_column( + "l_year", F.datepart(lit("year"), col("l_shipdate")).cast(pa.int32()) +).with_column("volume", col("l_extendedprice") * (lit(1.0) - col("l_discount"))) + +# Aggregate the results +df = df.aggregate( + [col("supp_nation"), col("cust_nation"), col("l_year")], + [F.sum(col("volume")).alias("revenue")], +) + +# Sort based on problem statement requirements +df = df.sort(col("supp_nation").sort(), col("cust_nation").sort(), col("l_year").sort()) + +df.show() diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py new file mode 100644 index 000000000..a415156ec --- /dev/null +++ b/examples/tpch/q08_market_share.py @@ -0,0 +1,175 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 8: + +The market share for a given nation within a given region is defined as the fraction of the +revenue, the sum of [l_extendedprice * (1-l_discount)], from the products of a specified type in +that region that was supplied by suppliers from the given nation. The query determines this for the +years 1995 and 1996 presented in this order. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +supplier_nation = lit("BRAZIL") +customer_region = lit("AMERICA") +part_of_interest = lit("ECONOMY ANODIZED STEEL") + +START_DATE = "1995-01-01" +END_DATE = "1996-12-31" + +start_date = lit(datetime.strptime(START_DATE, "%Y-%m-%d").date()) +end_date = lit(datetime.strptime(END_DATE, "%Y-%m-%d").date()) + + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_type") +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_nationkey" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_partkey", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_custkey", "o_orderdate" +) +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_custkey", "c_nationkey" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name", "n_regionkey" +) +df_region = ctx.read_parquet("data/region.parquet").select_columns( + "r_regionkey", "r_name" +) + +# Limit possible parts to the one specified +df_part = df_part.filter(col("p_type") == part_of_interest) + +# Limit orders to those in the specified range + +df_orders = df_orders.filter(col("o_orderdate") >= start_date).filter( + col("o_orderdate") <= end_date +) + +# Part 1: Find customers in the region + +# We want customers in region specified by region_of_interest. This will be used to compute +# the total sales of the part of interest. We want to know of those sales what fraction +# was supplied by the nation of interest. There is no guarantee that the nation of +# interest is within the region of interest. + +# First we find all the sales that make up the basis. + +df_regional_customers = df_region.filter(col("r_name") == customer_region) + +# After this join we have all of the possible sales nations +df_regional_customers = df_regional_customers.join( + df_nation, (["r_regionkey"], ["n_regionkey"]), how="inner" +) + +# Now find the possible customers +df_regional_customers = df_regional_customers.join( + df_customer, (["n_nationkey"], ["c_nationkey"]), how="inner" +) + +# Next find orders for these customers +df_regional_customers = df_regional_customers.join( + df_orders, (["c_custkey"], ["o_custkey"]), how="inner" +) + +# Find all line items from these orders +df_regional_customers = df_regional_customers.join( + df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner" +) + +# Limit to the part of interest +df_regional_customers = df_regional_customers.join( + df_part, (["l_partkey"], ["p_partkey"]), how="inner" +) + +# Compute the volume for each line item +df_regional_customers = df_regional_customers.with_column( + "volume", col("l_extendedprice") * (lit(1.0) - col("l_discount")) +) + +# Part 2: Find suppliers from the nation + +# Now that we have all of the sales of that part in the specified region, we need +# to determine which of those came from suppliers in the nation we are interested in. + +df_national_suppliers = df_nation.filter(col("n_name") == supplier_nation) + +# Determine the suppliers by the limited nation key we have in our single row df above +df_national_suppliers = df_national_suppliers.join( + df_supplier, (["n_nationkey"], ["s_nationkey"]), how="inner" +) + +# When we join to the customer dataframe, we don't want to confuse other columns, so only +# select the supplier key that we need +df_national_suppliers = df_national_suppliers.select_columns("s_suppkey") + + +# Part 3: Combine suppliers and customers and compute the market share + +# Now we can do a left outer join on the suppkey. Those line items from other suppliers +# will get a null value. We can check for the existence of this null to compute a volume +# column only from suppliers in the nation we are evaluating. + +df = df_regional_customers.join( + df_national_suppliers, (["l_suppkey"], ["s_suppkey"]), how="left" +) + +# Use a case statement to compute the volume sold by suppliers in the nation of interest +df = df.with_column( + "national_volume", + F.case(col("s_suppkey").is_null()) + .when(lit(False), col("volume")) + .otherwise(lit(0.0)), +) + +df = df.with_column( + "o_year", F.datepart(lit("year"), col("o_orderdate")).cast(pa.int32()) +) + + +# Lastly, sum up the results + +df = df.aggregate( + [col("o_year")], + [ + F.sum(col("volume")).alias("volume"), + F.sum(col("national_volume")).alias("national_volume"), + ], +) + +df = df.select( + col("o_year"), (F.col("national_volume") / F.col("volume")).alias("mkt_share") +) + +df = df.sort(col("o_year").sort()) + +df.show() diff --git a/examples/tpch/q09_product_type_profit_measure.py b/examples/tpch/q09_product_type_profit_measure.py new file mode 100644 index 000000000..4fdfc1cba --- /dev/null +++ b/examples/tpch/q09_product_type_profit_measure.py @@ -0,0 +1,93 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 9: + +The Product Type Profit Measure Query finds, for each nation and each year, the profit for all parts +ordered in that year that contain a specified substring in their names and that were filled by a +supplier in that nation. The profit is defined as the sum of +[(l_extendedprice*(1-l_discount)) - (ps_supplycost * l_quantity)] for all lineitems describing +parts in the specified line. The query lists the nations in ascending alphabetical order and, for +each nation, the year and profit in descending order by year (most recent first). + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +part_color = lit("green") + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_name") +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_nationkey" +) +df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( + "ps_suppkey", "ps_partkey", "ps_supplycost" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_partkey", + "l_extendedprice", + "l_discount", + "l_suppkey", + "l_orderkey", + "l_quantity", +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_custkey", "o_orderdate" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name", "n_regionkey" +) + +# Limit possible parts to the color specified +df = df_part.filter(F.strpos(col("p_name"), part_color) > lit(0)) + +# We have a series of joins that get us to limit down to the line items we need +df = df.join(df_lineitem, (["p_partkey"], ["l_partkey"]), how="inner") +df = df.join(df_supplier, (["l_suppkey"], ["s_suppkey"]), how="inner") +df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), how="inner") +df = df.join( + df_partsupp, (["l_suppkey", "l_partkey"], ["ps_suppkey", "ps_partkey"]), how="inner" +) +df = df.join(df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner") + +# Compute the intermediate values and limit down to the expressions we need +df = df.select( + col("n_name").alias("nation"), + F.datepart(lit("year"), col("o_orderdate")).cast(pa.int32()).alias("o_year"), + ( + col("l_extendedprice") * (lit(1.0) - col("l_discount")) + - (col("ps_supplycost") * col("l_quantity")) + ).alias("amount"), +) + +# Sum up the values by nation and year +df = df.aggregate( + [col("nation"), col("o_year")], [F.sum(col("amount")).alias("profit")] +) + +# Sort according to the problem specification +df = df.sort(col("nation").sort(), col("o_year").sort(ascending=False)) + +df.show() diff --git a/examples/tpch/q10_returned_item_reporting.py b/examples/tpch/q10_returned_item_reporting.py new file mode 100644 index 000000000..1879027c1 --- /dev/null +++ b/examples/tpch/q10_returned_item_reporting.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 10: + +The Returned Item Reporting Query finds the top 20 customers, in terms of their effect on lost +revenue for a given quarter, who have returned parts. The query considers only parts that were +ordered in the specified quarter. The query lists the customer's name, address, nation, phone +number, account balance, comment information and revenue lost. The customers are listed in +descending order of lost revenue. Revenue lost is defined as +sum(l_extendedprice*(1-l_discount)) for all qualifying lineitems. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +DATE_START_OF_QUARTER = "1993-10-01" + +date_start_of_quarter = lit(datetime.strptime(DATE_START_OF_QUARTER, "%Y-%m-%d").date()) + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval_one_quarter = lit(pa.scalar((0, 0, 120), type=pa.month_day_nano_interval())) + +# Load the dataframes we need + +ctx = SessionContext() + +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_custkey", + "c_nationkey", + "c_name", + "c_acctbal", + "c_address", + "c_phone", + "c_comment", +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_extendedprice", "l_discount", "l_orderkey", "l_returnflag" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_custkey", "o_orderdate" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name", "n_regionkey" +) + +# limit to returns +df_lineitem = df_lineitem.filter(col("l_returnflag") == lit("R")) + + +# Rather than aggregate by all of the customer fields as you might do looking at the specification, +# we can aggregate by o_custkey and then join in the customer data at the end. + +df = df_orders.filter(col("o_orderdate") >= date_start_of_quarter).filter( + col("o_orderdate") < date_start_of_quarter + interval_one_quarter +) + +df = df.join(df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner") + +# Compute the revenue +df = df.aggregate( + [col("o_custkey")], + [F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias("revenue")], +) + +# Now join in the customer data +df = df.join(df_customer, (["o_custkey"], ["c_custkey"]), how="inner") +df = df.join(df_nation, (["c_nationkey"], ["n_nationkey"]), how="inner") + +# These are the columns the problem statement requires +df = df.select_columns( + "c_custkey", + "c_name", + "revenue", + "c_acctbal", + "n_name", + "c_address", + "c_phone", + "c_comment", +) + +# Sort the results in descending order +df = df.sort(col("revenue").sort(ascending=False)) + +# Only return the top 20 results +df = df.limit(20) + +df.show() diff --git a/examples/tpch/q11_important_stock_identification.py b/examples/tpch/q11_important_stock_identification.py new file mode 100644 index 000000000..78fe26dbf --- /dev/null +++ b/examples/tpch/q11_important_stock_identification.py @@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 11: + +The Important Stock Identification Query finds, from scanning the available stock of suppliers +in a given nation, all the parts that represent a significant percentage of the total value of +all available parts. The query displays the part number and the value of those parts in +descending order of value. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, WindowFrame, col, lit, functions as F + +NATION = "GERMANY" +FRACTION = 0.0001 + +# Load the dataframes we need + +ctx = SessionContext() + +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_nationkey" +) +df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( + "ps_supplycost", "ps_availqty", "ps_suppkey", "ps_partkey" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name" +) + +# limit to returns +df_nation = df_nation.filter(col("n_name") == lit(NATION)) + +# Find part supplies of within this target nation + +df = df_nation.join(df_supplier, (["n_nationkey"], ["s_nationkey"]), how="inner") + +df = df.join(df_partsupp, (["s_suppkey"], ["ps_suppkey"]), how="inner") + + +# Compute the value of individual parts +df = df.with_column("value", col("ps_supplycost") * col("ps_availqty")) + +# Compute total value of specific parts +df = df.aggregate([col("ps_partkey")], [F.sum(col("value")).alias("value")]) + +# By default window functions go from unbounded preceeding to current row, but we want +# to compute this sum across all rows +window_frame = WindowFrame("rows", None, None) + +df = df.with_column( + "total_value", F.window("sum", [col("value")], window_frame=window_frame) +) + +# Limit to the parts for which there is a significant value based on the fraction of the total +df = df.filter(col("value") / col("total_value") > lit(FRACTION)) + +# We only need to report on these two columns +df = df.select_columns("ps_partkey", "value") + +# Sort in descending order of value +df = df.sort(col("value").sort(ascending=False)) + +df.show() diff --git a/examples/tpch/q12_ship_mode_order_priority.py b/examples/tpch/q12_ship_mode_order_priority.py new file mode 100644 index 000000000..e76efa54e --- /dev/null +++ b/examples/tpch/q12_ship_mode_order_priority.py @@ -0,0 +1,112 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 12: + +The Shipping Modes and Order Priority Query counts, by ship mode, for lineitems actually received +by customers in a given year, the number of lineitems belonging to orders for which the +l_receiptdate exceeds the l_commitdate for two different specified ship modes. Only lineitems that +were actually shipped before the l_commitdate are considered. The late lineitems are partitioned +into two groups, those with priority URGENT or HIGH, and those with a priority other than URGENT or +HIGH. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +SHIP_MODE_1 = "MAIL" +SHIP_MODE_2 = "SHIP" +DATE_OF_INTEREST = "1994-01-01" + +# Load the dataframes we need + +ctx = SessionContext() + +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_orderpriority" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_orderkey", "l_shipmode", "l_commitdate", "l_shipdate", "l_receiptdate" +) + +date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval = pa.scalar((0, 0, 365), type=pa.month_day_nano_interval()) + + +df = df_lineitem.filter(col("l_receiptdate") >= lit(date)).filter( + col("l_receiptdate") < lit(date) + lit(interval) +) + +# Note: It is not recommended to use array_has because it treats the second argument as an argument +# so if you pass it col("l_shipmode") it will pass the entire array to process which is very slow. +# Instead check the position of the entry is not null. +df = df.filter( + ~F.array_position( + F.make_array(lit(SHIP_MODE_1), lit(SHIP_MODE_2)), col("l_shipmode") + ).is_null() +) + +# Since we have only two values, it's much easier to do this as a filter where the l_shipmode +# matches either of the two values, but we want to show doing some array operations in this +# example. If you want to see this done with filters, comment out the above line and uncomment +# this one. +# df = df.filter((col("l_shipmode") == lit(SHIP_MODE_1)) | (col("l_shipmode") == lit(SHIP_MODE_2))) + + +# We need order priority, so join order df to line item +df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), how="inner") + +# Restrict to line items we care about based on the problem statement. +df = df.filter(col("l_commitdate") < col("l_receiptdate")) + +df = df.filter(col("l_shipdate") < col("l_commitdate")) + +df = df.with_column( + "high_line_value", + F.case(col("o_orderpriority")) + .when(lit("1-URGENT"), lit(1)) + .when(lit("2-HIGH"), lit(1)) + .otherwise(lit(0)), +) + +# Aggregate the results +df = df.aggregate( + [col("l_shipmode")], + [ + F.sum(col("high_line_value")).alias("high_line_count"), + F.count(col("high_line_value")).alias("all_lines_count"), + ], +) + +# Compute the final output +df = df.select( + col("l_shipmode"), + col("high_line_count"), + (col("all_lines_count") - col("high_line_count")).alias("low_line_count"), +) + +df = df.sort(col("l_shipmode").sort()) + +df.show() diff --git a/examples/tpch/q13_customer_distribution.py b/examples/tpch/q13_customer_distribution.py new file mode 100644 index 000000000..1eb9ca303 --- /dev/null +++ b/examples/tpch/q13_customer_distribution.py @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 13: + +This query determines the distribution of customers by the number of orders they have made, +including customers who have no record of orders, past or present. It counts and reports how many +customers have no orders, how many have 1, 2, 3, etc. A check is made to ensure that the orders +counted do not fall into one of several special categories of orders. Special categories are +identified in the order comment column by looking for a particular pattern. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, col, lit, functions as F + +WORD_1 = "special" +WORD_2 = "requests" + +# Load the dataframes we need + +ctx = SessionContext() + +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_custkey", "o_comment" +) +df_customer = ctx.read_parquet("data/customer.parquet").select_columns("c_custkey") + +# Use a regex to remove special cases +df_orders = df_orders.filter( + F.regexp_match(col("o_comment"), lit(f"{WORD_1}.?*{WORD_2}")).is_null() +) + +# Since we may have customers with no orders we must do a left join +df = df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="left") + +# Find the number of orders for each customer +df = df.aggregate([col("c_custkey")], [F.count(col("c_custkey")).alias("c_count")]) + +# Ultimately we want to know the number of customers that have that customer count +df = df.aggregate([col("c_count")], [F.count(col("c_count")).alias("custdist")]) + +# We want to order the results by the highest number of customers per count +df = df.sort( + col("custdist").sort(ascending=False), col("c_count").sort(ascending=False) +) + +df.show() diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py new file mode 100644 index 000000000..9ec38366b --- /dev/null +++ b/examples/tpch/q14_promotion_effect.py @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 14: + +The Promotion Effect Query determines what percentage of the revenue in a given year and month was +derived from promotional parts. The query considers only parts actually shipped in that month and +gives the percentage. Revenue is defined as (l_extendedprice * (1-l_discount)). + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +DATE = "1995-09-01" + +date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date()) +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval_one_month = lit(pa.scalar((0, 0, 30), type=pa.month_day_nano_interval())) + +# Load the dataframes we need + +ctx = SessionContext() + +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_partkey", "l_shipdate", "l_extendedprice", "l_discount" +) +df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_type") + + +# Check part type begins with PROMO +df_part = df_part.filter( + F.substr(col("p_type"), lit(0), lit(6)) == lit("PROMO") +).with_column("promo_factor", lit(1.0)) + +df_lineitem = df_lineitem.filter(col("l_shipdate") >= date_of_interest).filter( + col("l_shipdate") < date_of_interest + interval_one_month +) + +# Left join so we can sum up the promo parts different from other parts +df = df_lineitem.join(df_part, (["l_partkey"], ["p_partkey"]), "left") + +# Make a factor of 1.0 if it is a promotion, 0.0 otherwise +df = df.with_column("promo_factor", F.coalesce(col("promo_factor"), lit(0.0))) +df = df.with_column("revenue", col("l_extendedprice") * (lit(1.0) - col("l_discount"))) + + +# Sum up the promo and total revenue +df = df.aggregate( + [], + [ + F.sum(col("promo_factor") * col("revenue")).alias("promo_revenue"), + F.sum(col("revenue")).alias("total_revenue"), + ], +) + +# Return the percentage of revenue from promotions +df = df.select( + (lit(100.0) * col("promo_revenue") / col("total_revenue")).alias("promo_revenue") +) + +df.show() diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py new file mode 100644 index 000000000..7113e04fe --- /dev/null +++ b/examples/tpch/q15_top_supplier.py @@ -0,0 +1,87 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 15: + +The Top Supplier Query finds the supplier who contributed the most to the overall revenue for parts +shipped during a given quarter of a given year. In case of a tie, the query lists all suppliers +whose contribution was equal to the maximum, presented in supplier number order. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, WindowFrame, col, lit, functions as F + +DATE = "1996-01-01" + +date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date()) +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval_3_months = lit(pa.scalar((0, 0, 90), type=pa.month_day_nano_interval())) + +# Load the dataframes we need + +ctx = SessionContext() + +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_suppkey", "l_shipdate", "l_extendedprice", "l_discount" +) +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", + "s_name", + "s_address", + "s_phone", +) + +# Limit line items to the quarter of interest +df_lineitem = df_lineitem.filter(col("l_shipdate") >= date_of_interest).filter( + col("l_shipdate") < date_of_interest + interval_3_months +) + +df = df_lineitem.aggregate( + [col("l_suppkey")], + [ + F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias( + "total_revenue" + ) + ], +) + +# Use a window function to find the maximum revenue across the entire dataframe +window_frame = WindowFrame("rows", None, None) +df = df.with_column( + "max_revenue", F.window("max", [col("total_revenue")], window_frame=window_frame) +) + +# Find all suppliers whose total revenue is the same as the maximum +df = df.filter(col("total_revenue") == col("max_revenue")) + +# Now that we know the supplier(s) with maximum revenue, get the rest of their information +# from the supplier table +df = df.join(df_supplier, (["l_suppkey"], ["s_suppkey"]), "inner") + +# Return only the colums requested +df = df.select_columns("s_suppkey", "s_name", "s_address", "s_phone", "total_revenue") + +# If we have more than one, sort by supplier number (suppkey) +df = df.sort(col("s_suppkey").sort()) + +df.show() diff --git a/examples/tpch/q16_part_supplier_relationship.py b/examples/tpch/q16_part_supplier_relationship.py new file mode 100644 index 000000000..5f941d5ad --- /dev/null +++ b/examples/tpch/q16_part_supplier_relationship.py @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 16: + +The Parts/Supplier Relationship Query counts the number of suppliers who can supply parts that +satisfy a particular customer's requirements. The customer is interested in parts of eight +different sizes as long as they are not of a given type, not of a given brand, and not from a +supplier who has had complaints registered at the Better Business Bureau. Results must be presented +in descending count and ascending brand, type, and size. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +BRAND = "Brand#45" +TYPE_TO_IGNORE = "MEDIUM POLISHED" +SIZES_OF_INTEREST = [49, 14, 23, 45, 19, 3, 36, 9] + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns( + "p_partkey", "p_brand", "p_type", "p_size" +) +df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( + "ps_suppkey", "ps_partkey" +) +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_comment" +) + +df_unwanted_suppliers = df_supplier.filter( + ~F.regexp_match(col("s_comment"), lit("Customer.?*Complaints")).is_null() +) + +# Remove unwanted suppliers +df_partsupp = df_partsupp.join( + df_unwanted_suppliers, (["ps_suppkey"], ["s_suppkey"]), "anti" +) + +# Select the parts we are interested in +df_part = df_part.filter(col("p_brand") == lit(BRAND)) +df_part = df_part.filter( + F.substr(col("p_type"), lit(0), lit(len(TYPE_TO_IGNORE) + 1)) != lit(TYPE_TO_IGNORE) +) + +# Python conversion of integer to literal casts it to int64 but the data for +# part size is stored as an int32, so perform a cast. Then check to find if the part +# size is within the array of possible sizes by checking the position of it is not +# null. +p_sizes = F.make_array(*[lit(s).cast(pa.int32()) for s in SIZES_OF_INTEREST]) +df_part = df_part.filter(~F.array_position(p_sizes, col("p_size")).is_null()) + +df = df_part.join(df_partsupp, (["p_partkey"], ["ps_partkey"]), "inner") + +df = df.select_columns("p_brand", "p_type", "p_size", "ps_suppkey").distinct() + +df = df.aggregate( + [col("p_brand"), col("p_type"), col("p_size")], + [F.count(col("ps_suppkey")).alias("supplier_cnt")], +) + +df = df.sort(col("supplier_cnt").sort(ascending=False)) + +df.show() diff --git a/examples/tpch/q17_small_quantity_order.py b/examples/tpch/q17_small_quantity_order.py new file mode 100644 index 000000000..aae238b2f --- /dev/null +++ b/examples/tpch/q17_small_quantity_order.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 17: + +The Small-Quantity-Order Revenue Query considers parts of a given brand and with a given container +type and determines the average lineitem quantity of such parts ordered for all orders (past and +pending) in the 7-year database. What would be the average yearly gross (undiscounted) loss in +revenue if orders for these parts with a quantity of less than 20% of this average were no longer +taken? + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, WindowFrame, col, lit, functions as F + +BRAND = "Brand#23" +CONTAINER = "MED BOX" + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns( + "p_partkey", "p_brand", "p_container" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_partkey", "l_quantity", "l_extendedprice" +) + +# Limit to the problem statement's brand and container types +df = df_part.filter(col("p_brand") == lit(BRAND)).filter( + col("p_container") == lit(CONTAINER) +) + +# Combine data +df = df.join(df_lineitem, (["p_partkey"], ["l_partkey"]), "inner") + +# Find the average quantity +window_frame = WindowFrame("rows", None, None) +df = df.with_column( + "avg_quantity", F.window("avg", [col("l_quantity")], window_frame=window_frame) +) + +df = df.filter(col("l_quantity") < lit(0.2) * col("avg_quantity")) + +# Compute the total +df = df.aggregate([], [F.sum(col("l_extendedprice")).alias("total")]) + +# Divide by number of years in the problem statement to get average +df = df.select((col("total") / lit(7.0)).alias("avg_yearly")) + +df.show() diff --git a/examples/tpch/q18_large_volume_customer.py b/examples/tpch/q18_large_volume_customer.py new file mode 100644 index 000000000..96ca08ff7 --- /dev/null +++ b/examples/tpch/q18_large_volume_customer.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 18: + +The Large Volume Customer Query finds a list of the top 100 customers who have ever placed large +quantity orders. The query lists the customer name, customer key, the order key, date and total +price and the quantity for the order. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, col, lit, functions as F + +QUANTITY = 300 + +# Load the dataframes we need + +ctx = SessionContext() + +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_custkey", "c_name" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_custkey", "o_orderdate", "o_totalprice" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_orderkey", "l_quantity", "l_extendedprice" +) + +df = df_lineitem.aggregate( + [col("l_orderkey")], [F.sum(col("l_quantity")).alias("total_quantity")] +) + +# Limit to orders in which the total quantity is above a threshold +df = df.filter(col("total_quantity") > lit(QUANTITY)) + +# We've identified the orders of interest, now join the additional data +# we are required to report on +df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), "inner") +df = df.join(df_customer, (["o_custkey"], ["c_custkey"]), "inner") + +df = df.select_columns( + "c_name", "c_custkey", "o_orderkey", "o_orderdate", "o_totalprice", "total_quantity" +) + +df = df.sort(col("o_totalprice").sort(ascending=False), col("o_orderdate").sort()) + +df.show() diff --git a/examples/tpch/q19_discounted_revenue.py b/examples/tpch/q19_discounted_revenue.py new file mode 100644 index 000000000..20ad48a77 --- /dev/null +++ b/examples/tpch/q19_discounted_revenue.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 19: + +The Discounted Revenue query finds the gross discounted revenue for all orders for three different +types of parts that were shipped by air and delivered in person. Parts are selected based on the +combination of specific brands, a list of containers, and a range of sizes. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +import pyarrow as pa +from datafusion import SessionContext, col, lit, udf, functions as F + +items_of_interest = { + "Brand#12": { + "min_quantity": 1, + "containers": ["SM CASE", "SM BOX", "SM PACK", "SM PKG"], + "max_size": 5, + }, + "Brand#23": { + "min_quantity": 10, + "containers": ["MED BAG", "MED BOX", "MED PKG", "MED PACK"], + "max_size": 10, + }, + "Brand#34": { + "min_quantity": 20, + "containers": ["LG CASE", "LG BOX", "LG PACK", "LG PKG"], + "max_size": 15, + }, +} + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns( + "p_partkey", "p_brand", "p_container", "p_size" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_partkey", + "l_quantity", + "l_shipmode", + "l_shipinstruct", + "l_extendedprice", + "l_discount", +) + +# These limitations apply to all line items, so go ahead and do them first + +df = df_lineitem.filter(col("l_shipinstruct") == lit("DELIVER IN PERSON")) + +# Small note: The data generated uses "REG AIR" but the spec says "AIR REG" +df = df.filter( + (col("l_shipmode") == lit("AIR")) | (col("l_shipmode") == lit("REG AIR")) +) + +df = df.join(df_part, (["l_partkey"], ["p_partkey"]), "inner") + + +# Create the user defined function (UDF) definition that does the work +def is_of_interest( + brand_arr: pa.Array, + container_arr: pa.Array, + quantity_arr: pa.Array, + size_arr: pa.Array, +) -> pa.Array: + """ + The purpose of this function is to demonstrate how a UDF works, taking as input a pyarrow Array + and generating a resultant Array. The length of the inputs should match and there should be the + same number of rows in the output. + """ + result = [] + for idx, brand in enumerate(brand_arr): + brand = brand.as_py() + if brand in items_of_interest: + values_of_interest = items_of_interest[brand] + + container_matches = ( + container_arr[idx].as_py() in values_of_interest["containers"] + ) + + quantity = quantity_arr[idx].as_py() + quantity_matches = ( + values_of_interest["min_quantity"] + <= quantity + <= values_of_interest["min_quantity"] + 10 + ) + + size = size_arr[idx].as_py() + size_matches = 1 <= size <= values_of_interest["max_size"] + + result.append(container_matches and quantity_matches and size_matches) + else: + result.append(False) + + return pa.array(result) + + +# Turn the above function into a UDF that DataFusion can understand +is_of_interest_udf = udf( + is_of_interest, + [pa.utf8(), pa.utf8(), pa.float32(), pa.int32()], + pa.bool_(), + "stable", +) + +# Filter results using the above UDF +df = df.filter( + is_of_interest_udf( + col("p_brand"), col("p_container"), col("l_quantity"), col("p_size") + ) +) + +df = df.aggregate( + [], + [F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias("revenue")], +) + +df.show() diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py new file mode 100644 index 000000000..09686db05 --- /dev/null +++ b/examples/tpch/q20_potential_part_promotion.py @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 20: + +The Potential Part Promotion query identifies suppliers who have an excess of a given part +available; an excess is defined to be more than 50% of the parts like the given part that the +supplier shipped in a given year for a given nation. Only parts whose names share a certain naming +convention are considered. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datetime import datetime +import pyarrow as pa +from datafusion import SessionContext, col, lit, functions as F + +COLOR_OF_INTEREST = "forest" +DATE_OF_INTEREST = "1994-01-01" +NATION_OF_INTEREST = "CANADA" + +# Load the dataframes we need + +ctx = SessionContext() + +df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_name") +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_shipdate", "l_partkey", "l_suppkey", "l_quantity" +) +df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( + "ps_partkey", "ps_suppkey", "ps_availqty" +) +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_address", "s_name", "s_nationkey" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name" +) + +date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() + +# Note: this is a hack on setting the values. It should be set differently once +# https://github.com/apache/datafusion-python/issues/665 is resolved. +interval = pa.scalar((0, 0, 365), type=pa.month_day_nano_interval()) + +# Filter down dataframes +df_nation = df_nation.filter(col("n_name") == lit(NATION_OF_INTEREST)) +df_part = df_part.filter( + F.substr(col("p_name"), lit(0), lit(len(COLOR_OF_INTEREST) + 1)) + == lit(COLOR_OF_INTEREST) +) + +df = df_lineitem.filter(col("l_shipdate") >= lit(date)).filter( + col("l_shipdate") < lit(date) + lit(interval) +) + +# This will filter down the line items to the parts of interest +df = df.join(df_part, (["l_partkey"], ["p_partkey"]), "inner") + +# Compute the total sold and limit ourselves to indivdual supplier/part combinations +df = df.aggregate( + [col("l_partkey"), col("l_suppkey")], [F.sum(col("l_quantity")).alias("total_sold")] +) + +df = df.join( + df_partsupp, (["l_partkey", "l_suppkey"], ["ps_partkey", "ps_suppkey"]), "inner" +) + +# Find cases of excess quantity +df.filter(col("ps_availqty") > lit(0.5) * col("total_sold")) + +# We could do these joins earlier, but now limit to the nation of interest suppliers +df = df.join(df_supplier, (["ps_suppkey"], ["s_suppkey"]), "inner") +df = df.join(df_nation, (["s_nationkey"], ["n_nationkey"]), "inner") + +# Restrict to the requested data per the problem statement +df = df.select_columns("s_name", "s_address") + +df = df.sort(col("s_name").sort()) + +df.show() diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py new file mode 100644 index 000000000..2f58d6e79 --- /dev/null +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 21: + +The Suppliers Who Kept Orders Waiting query identifies suppliers, for a given nation, whose product +was part of a multi-supplier order (with current status of 'F') where they were the only supplier +who failed to meet the committed delivery date. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, col, lit, functions as F + +NATION_OF_INTEREST = "SAUDI ARABIA" + +# Load the dataframes we need + +ctx = SessionContext() + +df_orders = ctx.read_parquet("data/orders.parquet").select_columns( + "o_orderkey", "o_orderstatus" +) +df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( + "l_orderkey", "l_receiptdate", "l_commitdate", "l_suppkey" +) +df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( + "s_suppkey", "s_name", "s_nationkey" +) +df_nation = ctx.read_parquet("data/nation.parquet").select_columns( + "n_nationkey", "n_name" +) + +# Limit to suppliers in the nation of interest +df_suppliers_of_interest = df_nation.filter(col("n_name") == lit(NATION_OF_INTEREST)) + +df_suppliers_of_interest = df_suppliers_of_interest.join( + df_supplier, (["n_nationkey"], ["s_nationkey"]), "inner" +) + +# Find the failed orders and all their line items +df = df_orders.filter(col("o_orderstatus") == lit("F")) + +df = df_lineitem.join(df, (["l_orderkey"], ["o_orderkey"]), "inner") + +# Identify the line items for which the order is failed due to. +df = df.with_column( + "failed_supp", + F.case(col("l_receiptdate") > col("l_commitdate")) + .when(lit(True), col("l_suppkey")) + .end(), +) + +# There are other ways we could do this but the purpose of this example is to work with rows where +# an element is an array of values. In this case, we will create two columns of arrays. One will be +# an array of all of the suppliers who made up this order. That way we can filter the dataframe for +# only orders where this array is larger than one for multiple supplier orders. The second column +# is all of the suppliers who failed to make their commitment. We can filter the second column for +# arrays with size one. That combination will give us orders that had multiple suppliers where only +# one failed. Use distinct=True in the blow aggregation so we don't get multipe line items from the +# same supplier reported in either array. +df = df.aggregate( + [col("o_orderkey")], + [ + F.array_agg(col("l_suppkey"), distinct=True).alias("all_suppliers"), + F.array_agg(col("failed_supp"), distinct=True).alias("failed_suppliers"), + ], +) + +# Remove the null entries that will get returned by array_agg so we can test to see where we only +# have a single failed supplier in a multiple supplier order +df = df.with_column( + "failed_suppliers", F.array_remove(col("failed_suppliers"), lit(None)) +) + +# This is the check described above which will identify single failed supplier in a multiple +# supplier order. +df = df.filter(F.array_length(col("failed_suppliers")) == lit(1)).filter( + F.array_length(col("all_suppliers")) > lit(1) +) + +# Since we have an array we know is exactly one element long, we can extract that single value. +df = df.select( + col("o_orderkey"), F.array_element(col("failed_suppliers"), lit(1)).alias("suppkey") +) + +# Join to the supplier of interest list for the nation of interest +df = df.join(df_suppliers_of_interest, (["suppkey"], ["s_suppkey"]), "inner") + +# Count how many orders that supplier is the only failed supplier for +df = df.aggregate([col("s_name")], [F.count(col("o_orderkey")).alias("numwait")]) + +# Return in descending order +df = df.sort(col("numwait").sort(ascending=False)) + +df = df.limit(100) + +df.show() diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py new file mode 100644 index 000000000..d2d0c5a0d --- /dev/null +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +TPC-H Problem Statement Query 22: + +This query counts how many customers within a specific range of country codes have not placed +orders for 7 years but who have a greater than average “positive” account balance. It also reflects +the magnitude of that balance. Country code is defined as the first two characters of c_phone. + +The above problem statement text is copyrighted by the Transaction Processing Performance Council +as part of their TPC Benchmark H Specification revision 2.18.0. +""" + +from datafusion import SessionContext, WindowFrame, col, lit, functions as F + +NATION_CODE = 13 + +# Load the dataframes we need + +ctx = SessionContext() + +df_customer = ctx.read_parquet("data/customer.parquet").select_columns( + "c_phone", "c_acctbal", "c_custkey" +) +df_orders = ctx.read_parquet("data/orders.parquet").select_columns("o_custkey") + +# The nation code is a two digit number, but we need to convert it to a string literal +nation_code = lit(str(NATION_CODE)) + +# Use the substring operation to extract the first two charaters of the phone number +df = df_customer.with_column("cntrycode", F.substr(col("c_phone"), lit(0), lit(3))) + +# Limit our search to customers with some balance and in the country code above +df = df.filter(col("c_acctbal") > lit(0.0)) +df = df.filter(nation_code == col("cntrycode")) + +# Compute the average balance. By default, the window frame is from unbounded preceeding to the +# current row. We want our frame to cover the entire data frame. +window_frame = WindowFrame("rows", None, None) +df = df.with_column( + "avg_balance", F.window("avg", [col("c_acctbal")], window_frame=window_frame) +) + +# Limit results to customers with above average balance +df = df.filter(col("c_acctbal") > col("avg_balance")) + +# Limit results to customers with no orders +df = df.join(df_orders, (["c_custkey"], ["o_custkey"]), "anti") + +# Count up the customers and the balances +df = df.aggregate( + [col("cntrycode")], + [ + F.count(col("c_custkey")).alias("numcust"), + F.sum(col("c_acctbal")).alias("totacctbal"), + ], +) + +df = df.sort(col("cntrycode").sort()) + +df.show() From d41eba47ab5b88a8e94a83f7e22a863710c9d28b Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Mon, 13 May 2024 13:04:03 -0500 Subject: [PATCH 137/413] ci: clean conda cache before building the packages (#689) --- .github/workflows/conda.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index f8474565d..c33ae551f 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -81,6 +81,9 @@ jobs: which python pip list mamba list + # Clean the conda cache + - name: Clean Conda Cache + run: conda clean --all --yes - name: Build conda packages run: | # suffix for nightly package versions From 01a370e69931311db6ce08337aaefb309a668c3c Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 14 May 2024 08:40:58 -0500 Subject: [PATCH 138/413] Upgrade to datafusion 38 (#691) * chore: upgrade datafusion Deps Ref #690 * update concat and concat_ws to use datafusion_functions Moved in https://github.com/apache/datafusion/pull/10089 * feat: upgrade functions.rs Upstream is continuing it's migration to UDFs. Ref https://github.com/apache/datafusion/pull/10098 Ref https://github.com/apache/datafusion/pull/10372 * fix ScalarUDF import * feat: remove deprecated suppors_filter_pushdown and impl supports_filters_pushdown Deprecated function removed in https://github.com/apache/datafusion/pull/9923 * use `unnest_columns_with_options` instead of deprecated `unnest_column_with_option` * remove ScalarFunction wrappers These relied on upstream BuiltinScalarFunction, which are now removed. Ref https://github.com/apache/datafusion/pull/10098 * update dataframe `test_describe` `null_count` was fixed upstream. Ref https://github.com/apache/datafusion/pull/10260 * remove PyDFField and related methods DFField was removed upstream. Ref: https://github.com/apache/datafusion/pull/9595 * bump `datafusion-python` package version to 38.0.0 * re-implement `PyExpr::column_name` The previous implementation relied on `DFField` which was removed upstream. Ref: https://github.com/apache/datafusion/pull/9595 --- Cargo.lock | 145 +++++++++++++-------------- Cargo.toml | 16 +-- datafusion/__init__.py | 3 - datafusion/tests/test_dataframe.py | 6 +- datafusion/tests/test_imports.py | 7 +- src/common.rs | 2 - src/common/df_field.rs | 111 --------------------- src/dataframe.rs | 2 +- src/dataset.rs | 16 ++- src/expr.rs | 26 +++-- src/expr/scalar_function.rs | 65 ------------ src/functions.rs | 152 +++++++++++++++-------------- src/udf.rs | 2 +- 13 files changed, 181 insertions(+), 372 deletions(-) delete mode 100644 src/common/df_field.rs delete mode 100644 src/expr/scalar_function.rs diff --git a/Cargo.lock b/Cargo.lock index 5eb791b46..6b4568b96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -733,9 +733,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85069782056753459dc47e386219aa1fdac5b731f26c28abb8c0ffd4b7c5ab11" +checksum = "05fb4eeeb7109393a0739ac5b8fd892f95ccef691421491c85544f7997366f68" dependencies = [ "ahash", "apache-avro", @@ -754,6 +754,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", @@ -786,9 +787,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "309d9040751f6dc9e33c85dce6abb55a46ef7ea3644577dd014611c379447ef3" +checksum = "741aeac15c82f239f2fc17deccaab19873abbd62987be20023689b15fa72fa09" dependencies = [ "ahash", "apache-avro", @@ -809,18 +810,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e4a44d8ef1b1e85d32234e6012364c411c3787859bb3bba893b0332cb03dfd" +checksum = "6e8ddfb8d8cb51646a30da0122ecfffb81ca16919ae9a3495a9e7468bdcd52b8" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06a3a29ae36bcde07d179cc33b45656a8e7e4d023623e320e48dcf1200eeee95" +checksum = "282122f90b20e8f98ebfa101e4bf20e718fd2684cf81bef4e8c6366571c64404" dependencies = [ "arrow", "chrono", @@ -839,9 +840,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3542aa322029c2121a671ce08000d4b274171070df13f697b14169ccf4f628" +checksum = "5478588f733df0dfd87a62671c7478f590952c95fa2fa5c137e3ff2929491e22" dependencies = [ "ahash", "arrow", @@ -849,6 +850,7 @@ dependencies = [ "chrono", "datafusion-common", "paste", + "serde_json", "sqlparser", "strum 0.26.1", "strum_macros 0.26.1", @@ -856,9 +858,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd221792c666eac174ecc09e606312844772acc12cbec61a420c2fca1ee70959" +checksum = "f4afd261cea6ac9c3ca1192fd5e9f940596d8e9208c5b1333f4961405db53185" dependencies = [ "arrow", "base64 0.22.1", @@ -869,21 +871,39 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", + "hashbrown 0.14.3", "hex", "itertools 0.12.0", "log", "md-5", + "rand", "regex", "sha2", "unicode-segmentation", "uuid", ] +[[package]] +name = "datafusion-functions-aggregate" +version = "38.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b36a6c4838ab94b5bf8f7a96ce6ce059d805c5d1dcaa6ace49e034eb65cd999" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", + "paste", + "sqlparser", +] + [[package]] name = "datafusion-functions-array" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e501801e84d9c6ef54caaebcda1b18a6196a24176c12fb70e969bc0572e03c55" +checksum = "d5fdd200a6233f48d3362e7ccb784f926f759100e44ae2137a5e2dcb986a59c4" dependencies = [ "arrow", "arrow-array", @@ -901,9 +921,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bd7f5087817deb961764e8c973d243b54f8572db414a8f0a8f33a48f991e0a" +checksum = "54f2820938810e8a2d71228fd6f59f33396aebc5f5f687fcbf14de5aab6a7e1a" dependencies = [ "arrow", "async-trait", @@ -912,6 +932,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "hashbrown 0.14.3", + "indexmap", "itertools 0.12.0", "log", "regex-syntax", @@ -919,9 +940,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cabc0d9aaa0f5eb1b472112f16223c9ffd2fb04e58cbf65c0a331ee6e993f96" +checksum = "9adf8eb12716f52ddf01e09eb6c94d3c9b291e062c05c91b839a448bddba2ff8" dependencies = [ "ahash", "arrow", @@ -931,37 +952,45 @@ dependencies = [ "arrow-schema", "arrow-string", "base64 0.22.1", - "blake2", - "blake3", "chrono", "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", "half", "hashbrown 0.14.3", "hex", "indexmap", "itertools 0.12.0", "log", - "md-5", "paste", "petgraph", - "rand", "regex", - "sha2", - "unicode-segmentation", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "38.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5472c3230584c150197b3f2c23f2392b9dc54dbfb62ad41e7e36447cfce4be" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", ] [[package]] name = "datafusion-physical-plan" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c0523e9c8880f2492a88bbd857dde02bed1ed23f3e9211a89d3d7ec3b44af9" +checksum = "18ae750c38389685a8b62e5b899bbbec488950755ad6d218f3662d35b800c4fe" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ord", "arrow-schema", "async-trait", "chrono", @@ -969,7 +998,9 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate", "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.14.3", @@ -985,7 +1016,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "37.1.0" +version = "38.0.0" dependencies = [ "async-trait", "datafusion", @@ -1013,9 +1044,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb54b42227136f6287573f2434b1de249fe1b8e6cd6cc73a634e4a3ec29356" +checksum = "befc67a3cdfbfa76853f43b10ac27337821bb98e519ab6baf431fcc0bcfcafdb" dependencies = [ "arrow", "arrow-array", @@ -1029,9 +1060,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "37.1.0" +version = "38.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3b496697ac22a857c7d497b9d6b40edec19ed2e3e86e2b77051541fefb4c6d" +checksum = "1f62542caa77df003e23a8bc2f1b8a1ffc682fe447c7fcb4905d109e3d7a5b9d" dependencies = [ "async-recursion", "chrono", @@ -1260,19 +1291,6 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" -[[package]] -name = "git2" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b3ba52851e73b46a4c3df1d89343741112003f0f6f13beb0dfac9e457c3fdcd" -dependencies = [ - "bitflags 2.4.2", - "libc", - "libgit2-sys", - "log", - "url", -] - [[package]] name = "glob" version = "0.3.1" @@ -1654,18 +1672,6 @@ dependencies = [ "rle-decode-fast", ] -[[package]] -name = "libgit2-sys" -version = "0.16.2+1.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" -dependencies = [ - "cc", - "libc", - "libz-sys", - "pkg-config", -] - [[package]] name = "libm" version = "0.2.8" @@ -1682,18 +1688,6 @@ dependencies = [ "libc", ] -[[package]] -name = "libz-sys" -version = "1.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295c17e837573c8c821dbaeb3cceb3d745ad082f7572191409e69cbc1b3fd050" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "linux-raw-sys" version = "0.4.13" @@ -2762,9 +2756,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.44.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf9c7ff146298ffda83a200f8d5084f08dcee1edfc135fcc1d646a45d50ffd6" +checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" dependencies = [ "log", "sqlparser_derive", @@ -2830,11 +2824,10 @@ dependencies = [ [[package]] name = "substrait" -version = "0.28.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9531ae6784dee4c018ebdb0226872b63cc28765bfa65c1e53b6c58584232af" +checksum = "f01344023c2614171a9ffd6e387eea14e12f7387c5b6adb33f1563187d65e376" dependencies = [ - "git2", "heck 0.5.0", "prettyplease", "prost", @@ -3221,12 +3214,6 @@ dependencies = [ "serde", ] -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.4" diff --git a/Cargo.toml b/Cargo.toml index 9da36d710..cde3be222 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "37.1.0" +version = "38.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -37,13 +37,13 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "37.1.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-common = { version = "37.1.0", features = ["pyarrow"] } -datafusion-expr = "37.1.0" -datafusion-functions-array = "37.1.0" -datafusion-optimizer = "37.1.0" -datafusion-sql = "37.1.0" -datafusion-substrait = { version = "37.1.0", optional = true } +datafusion = { version = "38.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-common = { version = "38.0.0", features = ["pyarrow"] } +datafusion-expr = "38.0.0" +datafusion-functions-array = "38.0.0" +datafusion-optimizer = "38.0.0" +datafusion-sql = "38.0.0" +datafusion-substrait = { version = "38.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.8", features = ["v4"] } diff --git a/datafusion/__init__.py b/datafusion/__init__.py index c50bf649d..d0b823bbd 100644 --- a/datafusion/__init__.py +++ b/datafusion/__init__.py @@ -37,7 +37,6 @@ ) from .common import ( - DFField, DFSchema, ) @@ -64,8 +63,6 @@ IsNotFalse, IsNotUnknown, Negative, - ScalarFunction, - BuiltinScalarFunction, InList, Exists, Subquery, diff --git a/datafusion/tests/test_dataframe.py b/datafusion/tests/test_dataframe.py index efb1679b9..2f6a818ea 100644 --- a/datafusion/tests/test_dataframe.py +++ b/datafusion/tests/test_dataframe.py @@ -730,9 +730,9 @@ def test_describe(df): "max", "median", ], - "a": [3.0, 3.0, 2.0, 1.0, 1.0, 3.0, 2.0], - "b": [3.0, 3.0, 5.0, 1.0, 4.0, 6.0, 5.0], - "c": [3.0, 3.0, 7.0, 1.7320508075688772, 5.0, 8.0, 8.0], + "a": [3.0, 0.0, 2.0, 1.0, 1.0, 3.0, 2.0], + "b": [3.0, 0.0, 5.0, 1.0, 4.0, 6.0, 5.0], + "c": [3.0, 0.0, 7.0, 1.7320508075688772, 5.0, 8.0, 8.0], } diff --git a/datafusion/tests/test_imports.py b/datafusion/tests/test_imports.py index 766ddce89..2a8a3de83 100644 --- a/datafusion/tests/test_imports.py +++ b/datafusion/tests/test_imports.py @@ -27,7 +27,6 @@ ) from datafusion.common import ( - DFField, DFSchema, ) @@ -64,8 +63,6 @@ IsNotFalse, IsNotUnknown, Negative, - ScalarFunction, - BuiltinScalarFunction, InList, Exists, Subquery, @@ -139,8 +136,6 @@ def test_class_module_is_datafusion(): IsNotFalse, IsNotUnknown, Negative, - ScalarFunction, - BuiltinScalarFunction, InList, Exists, Subquery, @@ -165,7 +160,7 @@ def test_class_module_is_datafusion(): assert klass.__module__ == "datafusion.expr" # schema - for klass in [DFField, DFSchema]: + for klass in [DFSchema]: assert klass.__module__ == "datafusion.common" diff --git a/src/common.rs b/src/common.rs index 45523173c..682639aca 100644 --- a/src/common.rs +++ b/src/common.rs @@ -18,7 +18,6 @@ use pyo3::prelude::*; pub mod data_type; -pub mod df_field; pub mod df_schema; pub mod function; pub mod schema; @@ -26,7 +25,6 @@ pub mod schema; /// Initializes the `common` module to match the pattern of `datafusion-common` https://docs.rs/datafusion-common/18.0.0/datafusion_common/index.html pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/common/df_field.rs b/src/common/df_field.rs deleted file mode 100644 index 68c05361f..000000000 --- a/src/common/df_field.rs +++ /dev/null @@ -1,111 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion::arrow::datatypes::DataType; -use datafusion_common::{DFField, OwnedTableReference}; -use pyo3::prelude::*; - -use super::data_type::PyDataType; - -/// PyDFField wraps an arrow-datafusion `DFField` struct type -/// and also supplies convenience methods for interacting -/// with the `DFField` instance in the context of Python -#[pyclass(name = "DFField", module = "datafusion.common", subclass)] -#[derive(Debug, Clone)] -pub struct PyDFField { - pub field: DFField, -} - -impl From for DFField { - fn from(py_field: PyDFField) -> DFField { - py_field.field - } -} - -impl From for PyDFField { - fn from(field: DFField) -> PyDFField { - PyDFField { field } - } -} - -#[pymethods] -impl PyDFField { - #[new] - #[pyo3(signature = (qualifier=None, name="", data_type=DataType::Int64.into(), nullable=false))] - fn new(qualifier: Option, name: &str, data_type: PyDataType, nullable: bool) -> Self { - PyDFField { - field: DFField::new( - qualifier.map(OwnedTableReference::from), - name, - data_type.into(), - nullable, - ), - } - } - - // TODO: Need bindings for Array `Field` first - // #[staticmethod] - // #[pyo3(name = "from")] - // fn py_from(field: Field) -> Self {} - - // TODO: Need bindings for Array `Field` first - // #[staticmethod] - // #[pyo3(name = "from_qualified")] - // fn py_from_qualified(field: Field) -> Self {} - - #[pyo3(name = "name")] - fn py_name(&self) -> PyResult { - Ok(self.field.name().clone()) - } - - #[pyo3(name = "data_type")] - fn py_data_type(&self) -> PyResult { - Ok(self.field.data_type().clone().into()) - } - - #[pyo3(name = "is_nullable")] - fn py_is_nullable(&self) -> PyResult { - Ok(self.field.is_nullable()) - } - - #[pyo3(name = "qualified_name")] - fn py_qualified_name(&self) -> PyResult { - Ok(self.field.qualified_name()) - } - - // TODO: Need bindings for `Column` first - // #[pyo3(name = "qualified_column")] - // fn py_qualified_column(&self) -> PyResult {} - - // TODO: Need bindings for `Column` first - // #[pyo3(name = "unqualified_column")] - // fn py_unqualified_column(&self) -> PyResult {} - - #[pyo3(name = "qualifier")] - fn py_qualifier(&self) -> PyResult> { - Ok(self.field.qualifier().map(|q| format!("{}", q))) - } - - // TODO: Need bindings for Arrow `Field` first - // #[pyo3(name = "field")] - // fn py_field(&self) -> PyResult {} - - #[pyo3(name = "strip_qualifier")] - fn py_strip_qualifier(&self) -> PyResult { - Ok(self.field.clone().strip_qualifier().into()) - } -} diff --git a/src/dataframe.rs b/src/dataframe.rs index f1efc0c7a..8f4514398 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -301,7 +301,7 @@ impl PyDataFrame { .df .as_ref() .clone() - .unnest_column_with_options(column, unnest_options)?; + .unnest_columns_with_options(&[column], unnest_options)?; Ok(Self::new(df)) } diff --git a/src/dataset.rs b/src/dataset.rs index 713610c51..fcbb503c0 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -117,10 +117,16 @@ impl TableProvider for Dataset { /// Tests whether the table provider can make use of a filter expression /// to optimise data retrieval. - fn supports_filter_pushdown(&self, filter: &Expr) -> DFResult { - match PyArrowFilterExpression::try_from(filter) { - Ok(_) => Ok(TableProviderFilterPushDown::Exact), - _ => Ok(TableProviderFilterPushDown::Unsupported), - } + fn supports_filters_pushdown( + &self, + filter: &[&Expr], + ) -> DFResult> { + filter + .iter() + .map(|&f| match PyArrowFilterExpression::try_from(f) { + Ok(_) => Ok(TableProviderFilterPushDown::Exact), + _ => Ok(TableProviderFilterPushDown::Unsupported), + }) + .collect() } } diff --git a/src/expr.rs b/src/expr.rs index 3be0d025c..2f1477457 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,20 +15,20 @@ // specific language governing permissions and limitations // under the License. +use datafusion_expr::utils::exprlist_to_fields; +use datafusion_expr::LogicalPlan; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; +use std::sync::Arc; -use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::scalar::ScalarValue; -use datafusion_common::DFField; use datafusion_expr::{ col, expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, - lit, - utils::exprlist_to_fields, - Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, LogicalPlan, - Operator, TryCast, + lit, Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, Operator, + TryCast, }; use crate::common::data_type::{DataTypeMap, RexType}; @@ -80,7 +80,6 @@ pub mod logical_node; pub mod placeholder; pub mod projection; pub mod repartition; -pub mod scalar_function; pub mod scalar_subquery; pub mod scalar_variable; pub mod signature; @@ -567,14 +566,14 @@ impl PyExpr { impl PyExpr { pub fn _column_name(&self, plan: &LogicalPlan) -> Result { let field = Self::expr_to_field(&self.expr, plan)?; - Ok(field.qualified_column().flat_name()) + Ok(field.name().to_owned()) } - /// Create a [DFField] representing an [Expr], given an input [LogicalPlan] to resolve against + /// Create a [Field] representing an [Expr], given an input [LogicalPlan] to resolve against pub fn expr_to_field( expr: &Expr, input_plan: &LogicalPlan, - ) -> Result { + ) -> Result, DataFusionError> { match expr { Expr::Sort(Sort { expr, .. }) => { // DataFusion does not support create_name for sort expressions (since they never @@ -583,16 +582,15 @@ impl PyExpr { } Expr::Wildcard { .. } => { // Since * could be any of the valid column names just return the first one - Ok(input_plan.schema().field(0).clone()) + Ok(Arc::new(input_plan.schema().field(0).clone())) } _ => { let fields = exprlist_to_fields(&[expr.clone()], input_plan).map_err(PyErr::from)?; - Ok(fields[0].clone()) + Ok(fields[0].1.clone()) } } } - fn _types(expr: &Expr) -> PyResult { match expr { Expr::BinaryExpr(BinaryExpr { @@ -665,8 +663,6 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/expr/scalar_function.rs b/src/expr/scalar_function.rs deleted file mode 100644 index 776ca3297..000000000 --- a/src/expr/scalar_function.rs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use crate::expr::PyExpr; -use datafusion_expr::{BuiltinScalarFunction, Expr}; -use pyo3::prelude::*; - -#[pyclass(name = "ScalarFunction", module = "datafusion.expr", subclass)] -#[derive(Clone)] -pub struct PyScalarFunction { - scalar_function: BuiltinScalarFunction, - args: Vec, -} - -impl PyScalarFunction { - pub fn new(scalar_function: BuiltinScalarFunction, args: Vec) -> Self { - Self { - scalar_function, - args, - } - } -} - -#[pyclass(name = "BuiltinScalarFunction", module = "datafusion.expr", subclass)] -#[derive(Clone)] -pub struct PyBuiltinScalarFunction { - scalar_function: BuiltinScalarFunction, -} - -impl From for PyBuiltinScalarFunction { - fn from(scalar_function: BuiltinScalarFunction) -> PyBuiltinScalarFunction { - PyBuiltinScalarFunction { scalar_function } - } -} - -impl From for BuiltinScalarFunction { - fn from(scalar_function: PyBuiltinScalarFunction) -> Self { - scalar_function.scalar_function - } -} - -#[pymethods] -impl PyScalarFunction { - fn fun(&self) -> PyResult { - Ok(self.scalar_function.into()) - } - - fn args(&self) -> PyResult> { - Ok(self.args.iter().map(|e| e.clone().into()).collect()) - } -} diff --git a/src/functions.rs b/src/functions.rs index 7f6b1a877..4b137d90d 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -24,17 +24,46 @@ use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; use datafusion::execution::FunctionRegistry; use datafusion::functions; +use datafusion::functions_aggregate; use datafusion_common::{Column, ScalarValue, TableReference}; use datafusion_expr::expr::Alias; use datafusion_expr::{ aggregate_function, expr::{ - find_df_window_func, AggregateFunction, AggregateFunctionDefinition, ScalarFunction, Sort, - WindowFunction, + find_df_window_func, AggregateFunction, AggregateFunctionDefinition, Sort, WindowFunction, }, - lit, BuiltinScalarFunction, Expr, WindowFunctionDefinition, + lit, Expr, WindowFunctionDefinition, }; +#[pyfunction] +#[pyo3(signature = (y, x, distinct = false, filter = None, order_by = None))] +pub fn covar_samp( + y: PyExpr, + x: PyExpr, + distinct: bool, + filter: Option, + order_by: Option>, + // null_treatment: Option, +) -> PyExpr { + let filter = filter.map(|x| Box::new(x.expr)); + let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); + functions_aggregate::expr_fn::covar_samp(y.expr, x.expr, distinct, filter, order_by, None) + .into() +} + +#[pyfunction] +#[pyo3(signature = (y, x, distinct = false, filter = None, order_by = None))] +pub fn covar( + y: PyExpr, + x: PyExpr, + distinct: bool, + filter: Option, + order_by: Option>, +) -> PyExpr { + // alias for covar_samp + covar_samp(y, x, distinct, filter, order_by) +} + #[pyfunction] fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { datafusion_expr::in_list( @@ -134,7 +163,7 @@ fn digest(value: PyExpr, method: PyExpr) -> PyExpr { #[pyo3(signature = (*args))] fn concat(args: Vec) -> PyResult { let args = args.into_iter().map(|e| e.expr).collect::>(); - Ok(datafusion_expr::concat(&args).into()) + Ok(functions::string::expr_fn::concat(args).into()) } /// Concatenates all but the first argument, with separators. @@ -144,7 +173,7 @@ fn concat(args: Vec) -> PyResult { #[pyo3(signature = (sep, *args))] fn concat_ws(sep: String, args: Vec) -> PyResult { let args = args.into_iter().map(|e| e.expr).collect::>(); - Ok(datafusion_expr::concat_ws(lit(sep), args).into()) + Ok(functions::string::expr_fn::concat_ws(lit(sep), args).into()) } /// Creates a new Sort Expr @@ -249,27 +278,6 @@ fn window( }) } -macro_rules! scalar_function { - ($NAME: ident, $FUNC: ident) => { - scalar_function!($NAME, $FUNC, stringify!($NAME)); - }; - - ($NAME: ident, $FUNC: ident, $DOC: expr) => { - #[doc = $DOC] - #[pyfunction] - #[pyo3(signature = (*args))] - fn $NAME(args: Vec) -> PyExpr { - let expr = datafusion_expr::Expr::ScalarFunction(ScalarFunction { - func_def: datafusion_expr::ScalarFunctionDefinition::BuiltIn( - BuiltinScalarFunction::$FUNC, - ), - args: args.into_iter().map(|e| e.into()).collect(), - }); - expr.into() - } - }; -} - macro_rules! aggregate_function { ($NAME: ident, $FUNC: ident) => { aggregate_function!($NAME, $FUNC, stringify!($NAME)); @@ -370,21 +378,21 @@ macro_rules! array_fn { expr_fn!(abs, num); expr_fn!(acos, num); -scalar_function!(acosh, Acosh); +expr_fn!(acosh, num); expr_fn!(ascii, arg1, "Returns the numeric code of the first character of the argument. In UTF8 encoding, returns the Unicode code point of the character. In other multibyte encodings, the argument must be an ASCII character."); expr_fn!(asin, num); -scalar_function!(asinh, Asinh); -scalar_function!(atan, Atan); -scalar_function!(atanh, Atanh); -scalar_function!(atan2, Atan2); +expr_fn!(asinh, num); +expr_fn!(atan, num); +expr_fn!(atanh, num); +expr_fn!(atan2, y x); expr_fn!( bit_length, arg, "Returns number of bits in the string (8 times the octet_length)." ); expr_fn_vec!(btrim, "Removes the longest string containing only characters in characters (a space by default) from the start and end of string."); -scalar_function!(cbrt, Cbrt); -scalar_function!(ceil, Ceil); +expr_fn!(cbrt, num); +expr_fn!(ceil, num); expr_fn!( character_length, string, @@ -393,44 +401,44 @@ expr_fn!( expr_fn!(length, string); expr_fn!(char_length, string); expr_fn!(chr, arg, "Returns the character with the given code."); -scalar_function!(coalesce, Coalesce); -scalar_function!(cos, Cos); -scalar_function!(cosh, Cosh); -scalar_function!(degrees, Degrees); +expr_fn_vec!(coalesce); +expr_fn!(cos, num); +expr_fn!(cosh, num); +expr_fn!(degrees, num); expr_fn!(decode, input encoding); expr_fn!(encode, input encoding); -scalar_function!(exp, Exp); -scalar_function!(factorial, Factorial); -scalar_function!(floor, Floor); -scalar_function!(gcd, Gcd); -scalar_function!(initcap, InitCap, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); +expr_fn!(exp, num); +expr_fn!(factorial, num); +expr_fn!(floor, num); +expr_fn!(gcd, x y); +expr_fn!(initcap, string, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); expr_fn!(isnan, num); -scalar_function!(iszero, Iszero); -scalar_function!(lcm, Lcm); -scalar_function!(left, Left, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); -scalar_function!(ln, Ln); -scalar_function!(log, Log); -scalar_function!(log10, Log10); -scalar_function!(log2, Log2); +expr_fn!(iszero, num); +expr_fn!(lcm, x y); +expr_fn!(left, string n, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); +expr_fn!(ln, num); +expr_fn!(log, base num); +expr_fn!(log10, num); +expr_fn!(log2, num); expr_fn!(lower, arg1, "Converts the string to all lower case"); -scalar_function!(lpad, Lpad, "Extends the string to length length by prepending the characters fill (a space by default). If the string is already longer than length then it is truncated (on the right)."); +expr_fn_vec!(lpad, "Extends the string to length length by prepending the characters fill (a space by default). If the string is already longer than length then it is truncated (on the right)."); expr_fn_vec!(ltrim, "Removes the longest string containing only characters in characters (a space by default) from the start of string."); expr_fn!( md5, input_arg, "Computes the MD5 hash of the argument, with the result written in hexadecimal." ); -scalar_function!( +expr_fn!( nanvl, - Nanvl, + x y, "Returns x if x is not NaN otherwise returns y." ); expr_fn!(nullif, arg_1 arg_2); expr_fn_vec!(octet_length, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); -scalar_function!(pi, Pi); -scalar_function!(power, Power); -scalar_function!(pow, Power); -scalar_function!(radians, Radians); +expr_fn!(pi); +expr_fn!(power, base exponent); +expr_fn!(pow, power, base exponent); +expr_fn!(radians, num); expr_fn!(regexp_match, input_arg1 input_arg2); expr_fn!( regexp_replace, @@ -443,31 +451,31 @@ expr_fn!( string from to, "Replaces all occurrences in string of substring from with substring to." ); -scalar_function!( +expr_fn!( reverse, - Reverse, + string, "Reverses the order of the characters in the string." ); -scalar_function!(right, Right, "Returns last n characters in the string, or when n is negative, returns all but first |n| characters."); -scalar_function!(round, Round); -scalar_function!(rpad, Rpad, "Extends the string to length length by appending the characters fill (a space by default). If the string is already longer than length then it is truncated."); +expr_fn!(right, string n, "Returns last n characters in the string, or when n is negative, returns all but first |n| characters."); +expr_fn_vec!(round); +expr_fn_vec!(rpad, "Extends the string to length length by appending the characters fill (a space by default). If the string is already longer than length then it is truncated."); expr_fn_vec!(rtrim, "Removes the longest string containing only characters in characters (a space by default) from the end of string."); expr_fn!(sha224, input_arg1); expr_fn!(sha256, input_arg1); expr_fn!(sha384, input_arg1); expr_fn!(sha512, input_arg1); -scalar_function!(signum, Signum); -scalar_function!(sin, Sin); -scalar_function!(sinh, Sinh); +expr_fn!(signum, num); +expr_fn!(sin, num); +expr_fn!(sinh, num); expr_fn!( split_part, string delimiter index, "Splits string at occurrences of delimiter and returns the n'th field (counting from one)." ); -scalar_function!(sqrt, Sqrt); +expr_fn!(sqrt, num); expr_fn!(starts_with, arg1 arg2, "Returns true if string starts with prefix."); -scalar_function!(strpos, Strpos, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); -scalar_function!(substr, Substr); +expr_fn!(strpos, string substring, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); +expr_fn!(substr, string position); expr_fn!(tan, num); expr_fn!(tanh, num); expr_fn!( @@ -488,15 +496,15 @@ expr_fn!(date_trunc, part date); expr_fn!(datetrunc, date_trunc, part date); expr_fn!(date_bin, stride source origin); -scalar_function!(translate, Translate, "Replaces each character in string that matches a character in the from set with the corresponding character in the to set. If from is longer than to, occurrences of the extra characters in from are deleted."); +expr_fn!(translate, string from to, "Replaces each character in string that matches a character in the from set with the corresponding character in the to set. If from is longer than to, occurrences of the extra characters in from are deleted."); expr_fn_vec!(trim, "Removes the longest string containing only characters in characters (a space by default) from the start, end, or both ends (BOTH is the default) of string."); -scalar_function!(trunc, Trunc); +expr_fn_vec!(trunc); expr_fn!(upper, arg1, "Converts the string to all upper case."); expr_fn!(uuid); -expr_fn!(r#struct, args); // Use raw identifier since struct is a keyword +expr_fn_vec!(r#struct); // Use raw identifier since struct is a keyword expr_fn!(from_unixtime, unixtime); expr_fn!(arrow_typeof, arg_1); -scalar_function!(random, Random); +expr_fn!(random); // Array Functions array_fn!(array_append, array element); @@ -565,9 +573,7 @@ aggregate_function!(array_agg, ArrayAgg); aggregate_function!(avg, Avg); aggregate_function!(corr, Correlation); aggregate_function!(count, Count); -aggregate_function!(covar, Covariance); aggregate_function!(covar_pop, CovariancePop); -aggregate_function!(covar_samp, Covariance); aggregate_function!(grouping, Grouping); aggregate_function!(max, Max); aggregate_function!(mean, Avg); diff --git a/src/udf.rs b/src/udf.rs index 69519f499..8f5ca30b1 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -23,9 +23,9 @@ use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; -use datafusion::physical_plan::udf::ScalarUDF; use datafusion_expr::create_udf; use datafusion_expr::function::ScalarFunctionImplementation; +use datafusion_expr::ScalarUDF; use crate::expr::PyExpr; use crate::utils::parse_volatility; From 0c82b3fefa844783f5296bbeafb714b187c63ebb Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 14 May 2024 13:31:56 -0500 Subject: [PATCH 139/413] =?UTF-8?q?chore:=20update=20to=20maturin's=20reco?= =?UTF-8?q?mmended=20project=20layout=20for=20rust/python=E2=80=A6=20(#695?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: update to maturin's recommended project layout for rust/python projects The previous layout leads to an import error when installing with `maturin build` and `pip install .`. This error was common enough that `maturin` changed the recommended project layout to what this commit does. A prior PR attempted to solve this by altering `lib.name` in Cargo.toml, but that did not work for me. - [Prior PR](https://github.com/apache/datafusion-python/pull/694) - [maturin ImportError issue](https://github.com/PyO3/maturin/issues/490) - [maturin changes recommended project structure](https://github.com/PyO3/maturin/pull/855) * ci: update `ruff check` for nested python directory --- .github/workflows/build.yml | 2 +- pyproject.toml | 1 + {datafusion => python/datafusion}/__init__.py | 0 {datafusion => python/datafusion}/common.py | 0 {datafusion => python/datafusion}/expr.py | 0 {datafusion => python/datafusion}/functions.py | 0 {datafusion => python/datafusion}/input/__init__.py | 0 {datafusion => python/datafusion}/input/base.py | 0 {datafusion => python/datafusion}/input/location.py | 0 {datafusion => python/datafusion}/object_store.py | 0 {datafusion => python/datafusion}/substrait.py | 0 {datafusion => python/datafusion}/tests/__init__.py | 0 {datafusion => python/datafusion}/tests/conftest.py | 0 .../datafusion}/tests/data_test_context/data.json | 0 {datafusion => python/datafusion}/tests/generic.py | 0 {datafusion => python/datafusion}/tests/test_aggregation.py | 0 {datafusion => python/datafusion}/tests/test_catalog.py | 0 {datafusion => python/datafusion}/tests/test_config.py | 0 {datafusion => python/datafusion}/tests/test_context.py | 0 {datafusion => python/datafusion}/tests/test_dataframe.py | 0 {datafusion => python/datafusion}/tests/test_expr.py | 0 {datafusion => python/datafusion}/tests/test_functions.py | 0 {datafusion => python/datafusion}/tests/test_imports.py | 0 {datafusion => python/datafusion}/tests/test_indexing.py | 0 {datafusion => python/datafusion}/tests/test_input.py | 0 {datafusion => python/datafusion}/tests/test_sql.py | 0 {datafusion => python/datafusion}/tests/test_store.py | 0 {datafusion => python/datafusion}/tests/test_substrait.py | 0 {datafusion => python/datafusion}/tests/test_udaf.py | 0 29 files changed, 2 insertions(+), 1 deletion(-) rename {datafusion => python/datafusion}/__init__.py (100%) rename {datafusion => python/datafusion}/common.py (100%) rename {datafusion => python/datafusion}/expr.py (100%) rename {datafusion => python/datafusion}/functions.py (100%) rename {datafusion => python/datafusion}/input/__init__.py (100%) rename {datafusion => python/datafusion}/input/base.py (100%) rename {datafusion => python/datafusion}/input/location.py (100%) rename {datafusion => python/datafusion}/object_store.py (100%) rename {datafusion => python/datafusion}/substrait.py (100%) rename {datafusion => python/datafusion}/tests/__init__.py (100%) rename {datafusion => python/datafusion}/tests/conftest.py (100%) rename {datafusion => python/datafusion}/tests/data_test_context/data.json (100%) rename {datafusion => python/datafusion}/tests/generic.py (100%) rename {datafusion => python/datafusion}/tests/test_aggregation.py (100%) rename {datafusion => python/datafusion}/tests/test_catalog.py (100%) rename {datafusion => python/datafusion}/tests/test_config.py (100%) rename {datafusion => python/datafusion}/tests/test_context.py (100%) rename {datafusion => python/datafusion}/tests/test_dataframe.py (100%) rename {datafusion => python/datafusion}/tests/test_expr.py (100%) rename {datafusion => python/datafusion}/tests/test_functions.py (100%) rename {datafusion => python/datafusion}/tests/test_imports.py (100%) rename {datafusion => python/datafusion}/tests/test_indexing.py (100%) rename {datafusion => python/datafusion}/tests/test_input.py (100%) rename {datafusion => python/datafusion}/tests/test_sql.py (100%) rename {datafusion => python/datafusion}/tests/test_store.py (100%) rename {datafusion => python/datafusion}/tests/test_substrait.py (100%) rename {datafusion => python/datafusion}/tests/test_udaf.py (100%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e03c2cbde..239b1718b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,7 @@ jobs: pip install ruff # Update output format to enable automatic inline annotations. - name: Run Ruff - run: ruff check --output-format=github datafusion + run: ruff check --output-format=github python/ generate-license: runs-on: ubuntu-latest diff --git a/pyproject.toml b/pyproject.toml index d35360519..24bd29617 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ repository = "https://github.com/apache/arrow-datafusion-python" profile = "black" [tool.maturin] +python-source = "python" module-name = "datafusion._internal" include = [ { path = "Cargo.lock", format = "sdist" } diff --git a/datafusion/__init__.py b/python/datafusion/__init__.py similarity index 100% rename from datafusion/__init__.py rename to python/datafusion/__init__.py diff --git a/datafusion/common.py b/python/datafusion/common.py similarity index 100% rename from datafusion/common.py rename to python/datafusion/common.py diff --git a/datafusion/expr.py b/python/datafusion/expr.py similarity index 100% rename from datafusion/expr.py rename to python/datafusion/expr.py diff --git a/datafusion/functions.py b/python/datafusion/functions.py similarity index 100% rename from datafusion/functions.py rename to python/datafusion/functions.py diff --git a/datafusion/input/__init__.py b/python/datafusion/input/__init__.py similarity index 100% rename from datafusion/input/__init__.py rename to python/datafusion/input/__init__.py diff --git a/datafusion/input/base.py b/python/datafusion/input/base.py similarity index 100% rename from datafusion/input/base.py rename to python/datafusion/input/base.py diff --git a/datafusion/input/location.py b/python/datafusion/input/location.py similarity index 100% rename from datafusion/input/location.py rename to python/datafusion/input/location.py diff --git a/datafusion/object_store.py b/python/datafusion/object_store.py similarity index 100% rename from datafusion/object_store.py rename to python/datafusion/object_store.py diff --git a/datafusion/substrait.py b/python/datafusion/substrait.py similarity index 100% rename from datafusion/substrait.py rename to python/datafusion/substrait.py diff --git a/datafusion/tests/__init__.py b/python/datafusion/tests/__init__.py similarity index 100% rename from datafusion/tests/__init__.py rename to python/datafusion/tests/__init__.py diff --git a/datafusion/tests/conftest.py b/python/datafusion/tests/conftest.py similarity index 100% rename from datafusion/tests/conftest.py rename to python/datafusion/tests/conftest.py diff --git a/datafusion/tests/data_test_context/data.json b/python/datafusion/tests/data_test_context/data.json similarity index 100% rename from datafusion/tests/data_test_context/data.json rename to python/datafusion/tests/data_test_context/data.json diff --git a/datafusion/tests/generic.py b/python/datafusion/tests/generic.py similarity index 100% rename from datafusion/tests/generic.py rename to python/datafusion/tests/generic.py diff --git a/datafusion/tests/test_aggregation.py b/python/datafusion/tests/test_aggregation.py similarity index 100% rename from datafusion/tests/test_aggregation.py rename to python/datafusion/tests/test_aggregation.py diff --git a/datafusion/tests/test_catalog.py b/python/datafusion/tests/test_catalog.py similarity index 100% rename from datafusion/tests/test_catalog.py rename to python/datafusion/tests/test_catalog.py diff --git a/datafusion/tests/test_config.py b/python/datafusion/tests/test_config.py similarity index 100% rename from datafusion/tests/test_config.py rename to python/datafusion/tests/test_config.py diff --git a/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py similarity index 100% rename from datafusion/tests/test_context.py rename to python/datafusion/tests/test_context.py diff --git a/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py similarity index 100% rename from datafusion/tests/test_dataframe.py rename to python/datafusion/tests/test_dataframe.py diff --git a/datafusion/tests/test_expr.py b/python/datafusion/tests/test_expr.py similarity index 100% rename from datafusion/tests/test_expr.py rename to python/datafusion/tests/test_expr.py diff --git a/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py similarity index 100% rename from datafusion/tests/test_functions.py rename to python/datafusion/tests/test_functions.py diff --git a/datafusion/tests/test_imports.py b/python/datafusion/tests/test_imports.py similarity index 100% rename from datafusion/tests/test_imports.py rename to python/datafusion/tests/test_imports.py diff --git a/datafusion/tests/test_indexing.py b/python/datafusion/tests/test_indexing.py similarity index 100% rename from datafusion/tests/test_indexing.py rename to python/datafusion/tests/test_indexing.py diff --git a/datafusion/tests/test_input.py b/python/datafusion/tests/test_input.py similarity index 100% rename from datafusion/tests/test_input.py rename to python/datafusion/tests/test_input.py diff --git a/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py similarity index 100% rename from datafusion/tests/test_sql.py rename to python/datafusion/tests/test_sql.py diff --git a/datafusion/tests/test_store.py b/python/datafusion/tests/test_store.py similarity index 100% rename from datafusion/tests/test_store.py rename to python/datafusion/tests/test_store.py diff --git a/datafusion/tests/test_substrait.py b/python/datafusion/tests/test_substrait.py similarity index 100% rename from datafusion/tests/test_substrait.py rename to python/datafusion/tests/test_substrait.py diff --git a/datafusion/tests/test_udaf.py b/python/datafusion/tests/test_udaf.py similarity index 100% rename from datafusion/tests/test_udaf.py rename to python/datafusion/tests/test_udaf.py From 1da28b5019ce2e9f2a563d0088c75a41602970b5 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 14 May 2024 17:09:06 -0500 Subject: [PATCH 140/413] chore: update cargo deps (#698) Closes #672 rustls Closes #682 syn Closes #653 parking_lot closes #648 object_store Closes #625 h2 Closes #623 tokio Closes #608 mio Closes #597 pyo3 Closes #642 pyo3-build-config Closes #627 prost Closes #634 prost-types Closes #637 async-trait --- Cargo.lock | 112 ++++++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b4568b96..9b7ed560f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,18 +376,18 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -1036,7 +1036,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.60", + "syn 2.0.63", "tokio", "url", "uuid", @@ -1231,7 +1231,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -1299,9 +1299,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.24" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", @@ -1781,9 +1781,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -1946,9 +1946,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" dependencies = [ "lock_api", "parking_lot_core", @@ -2089,6 +2089,12 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -2102,7 +2108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2116,9 +2122,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" dependencies = [ "bytes", "prost-derive", @@ -2146,15 +2152,15 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +checksum = "9554e3ab233f0a932403704f1a1d08c30d5ccd931adfdfa1e8b5a19b52c1d55a" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.12.0", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2177,15 +2183,16 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a89dc7a5850d0e983be1ec2a463a171d20990487c3cfcd68b5363f1ee3d6fe0" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", "parking_lot", + "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", @@ -2194,9 +2201,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07426f0d8fe5a601f26293f300afd1a7b1ed5e78b2a705870c5f30893c5163be" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" dependencies = [ "once_cell", "target-lexicon", @@ -2204,9 +2211,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb7dec17e17766b46bca4f1a4215a85006b4c2ecde122076c562dd058da6cf1" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" dependencies = [ "libc", "pyo3-build-config", @@ -2214,26 +2221,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f738b4e40d50b5711957f142878cfa0f28e054aa0ebdfc3fd137a843f74ed3" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] name = "pyo3-macros-backend" -version = "0.20.2" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fc910d4851847827daf9d6cdd4a823fbdaab5b8818325c5e97a86da79e8881f" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" dependencies = [ "heck 0.4.1", "proc-macro2", + "pyo3-build-config", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2437,9 +2445,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", @@ -2616,7 +2624,7 @@ checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2650,7 +2658,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2772,7 +2780,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2806,7 +2814,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2819,7 +2827,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2839,7 +2847,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.60", + "syn 2.0.63", "typify", "walkdir", ] @@ -2863,9 +2871,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.60" +version = "2.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" dependencies = [ "proc-macro2", "quote", @@ -2929,7 +2937,7 @@ checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -2969,9 +2977,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", @@ -2992,7 +3000,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -3044,7 +3052,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -3089,7 +3097,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] @@ -3121,7 +3129,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.60", + "syn 2.0.63", "thiserror", "unicode-ident", ] @@ -3138,7 +3146,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.60", + "syn 2.0.63", "typify-impl", ] @@ -3266,7 +3274,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", "wasm-bindgen-shared", ] @@ -3300,7 +3308,7 @@ checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3554,7 +3562,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.63", ] [[package]] From 344ebad1989d6e7c14c63be63dd7132aadbea710 Mon Sep 17 00:00:00 2001 From: Richard Tia Date: Tue, 14 May 2024 22:33:11 -0400 Subject: [PATCH 141/413] feat: add python bindings for ends_with function (#693) * fix: resolve merge conflicts * fix: wrap pyfunction * fix: alphabetical ordering of function * refactor: update arg names * fix: assert correct column number --- python/datafusion/tests/test_functions.py | 2 ++ src/functions.rs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index d834f587f..d2dafc199 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -533,6 +533,7 @@ def test_string_functions(df): f.translate(column("a"), literal("or"), literal("ld")), f.trim(column("c")), f.upper(column("c")), + f.ends_with(column("a"), literal("llo")), ) result = df.collect() assert len(result) == 1 @@ -573,6 +574,7 @@ def test_string_functions(df): assert result.column(25) == pa.array(["Helll", "Wldld", "!"]) assert result.column(26) == pa.array(["hello", "world", "!"]) assert result.column(27) == pa.array(["HELLO ", " WORLD ", " !"]) + assert result.column(28) == pa.array([True, False, False]) def test_hash_functions(df): diff --git a/src/functions.rs b/src/functions.rs index 4b137d90d..8a7d73839 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -407,6 +407,7 @@ expr_fn!(cosh, num); expr_fn!(degrees, num); expr_fn!(decode, input encoding); expr_fn!(encode, input encoding); +expr_fn!(ends_with, string suffix, "Returns true if string ends with suffix."); expr_fn!(exp, num); expr_fn!(factorial, num); expr_fn!(floor, num); @@ -473,7 +474,7 @@ expr_fn!( "Splits string at occurrences of delimiter and returns the n'th field (counting from one)." ); expr_fn!(sqrt, num); -expr_fn!(starts_with, arg1 arg2, "Returns true if string starts with prefix."); +expr_fn!(starts_with, string prefix, "Returns true if string starts with prefix."); expr_fn!(strpos, string substring, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); expr_fn!(substr, string position); expr_fn!(tan, num); @@ -652,6 +653,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(datetrunc))?; m.add_wrapped(wrap_pyfunction!(date_trunc))?; m.add_wrapped(wrap_pyfunction!(digest))?; + m.add_wrapped(wrap_pyfunction!(ends_with))?; m.add_wrapped(wrap_pyfunction!(exp))?; m.add_wrapped(wrap_pyfunction!(factorial))?; m.add_wrapped(wrap_pyfunction!(floor))?; From d6c42b4dec452f75305f37f937ed1669dbf5809e Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Wed, 15 May 2024 09:18:47 -0500 Subject: [PATCH 142/413] feat: expose `named_struct` in python (#700) Ref #692 --- python/datafusion/tests/test_functions.py | 26 +++++++++++++++++++++++ src/functions.rs | 2 ++ 2 files changed, 28 insertions(+) diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index d2dafc199..d34e46b33 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -50,6 +50,32 @@ def df(): return ctx.create_dataframe([[batch]]) +def test_named_struct(df): + df = df.with_column( + "d", + f.named_struct( + literal("a"), + column("a"), + literal("b"), + column("b"), + literal("c"), + column("c"), + ), + ) + + expected = """DataFrame() ++-------+---+---------+------------------------------+ +| a | b | c | d | ++-------+---+---------+------------------------------+ +| Hello | 4 | hello | {a: Hello, b: 4, c: hello } | +| World | 5 | world | {a: World, b: 5, c: world } | +| ! | 6 | ! | {a: !, b: 6, c: !} | ++-------+---+---------+------------------------------+ +""".strip() + + assert str(df) == expected + + def test_literal(df): df = df.select( literal(1), diff --git a/src/functions.rs b/src/functions.rs index 8a7d73839..975025b6b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -503,6 +503,7 @@ expr_fn_vec!(trunc); expr_fn!(upper, arg1, "Converts the string to all upper case."); expr_fn!(uuid); expr_fn_vec!(r#struct); // Use raw identifier since struct is a keyword +expr_fn_vec!(named_struct); expr_fn!(from_unixtime, unixtime); expr_fn!(arrow_typeof, arg_1); expr_fn!(random); @@ -680,6 +681,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(mean))?; m.add_wrapped(wrap_pyfunction!(median))?; m.add_wrapped(wrap_pyfunction!(min))?; + m.add_wrapped(wrap_pyfunction!(named_struct))?; m.add_wrapped(wrap_pyfunction!(nanvl))?; m.add_wrapped(wrap_pyfunction!(now))?; m.add_wrapped(wrap_pyfunction!(nullif))?; From 856b31014299f34f1e2191f61273bb40a791c47d Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Wed, 15 May 2024 11:36:42 -0500 Subject: [PATCH 143/413] Website fixes (#702) * docs: link to examples using full URL in README PyPI otherwise renders these relative to the `datafusion-python` page, so when users currently get a 404 when they click on one of these links. Fixes #699 * docs: update project.urls in pyproject.toml * docs: update README with apache TLP URLs * docs u pdate docs/README.md with apache TLP URLs * docs: update index.rst with TLP URLs * docs: update to new branded logos --- README.md | 34 +++++++++--------- docs/README.md | 12 +++---- .../_static/images/2x_bgwhite_original.png | Bin 0 -> 60451 bytes .../DataFusion-Logo-Background-White.png | Bin 12401 -> 0 bytes .../DataFusion-Logo-Background-White.svg | 1 - .../_static/images/DataFusion-Logo-Dark.png | Bin 20134 -> 0 bytes .../_static/images/DataFusion-Logo-Dark.svg | 1 - .../_static/images/DataFusion-Logo-Light.png | Bin 19102 -> 0 bytes .../_static/images/DataFusion-Logo-Light.svg | 1 - docs/source/_static/images/original.png | Bin 0 -> 26337 bytes docs/source/_static/images/original.svg | 31 ++++++++++++++++ docs/source/_static/images/original2x.png | Bin 0 -> 66517 bytes docs/source/_templates/docs-sidebar.html | 2 +- docs/source/conf.py | 2 +- docs/source/index.rst | 8 ++--- pyproject.toml | 6 ++-- 16 files changed, 63 insertions(+), 35 deletions(-) create mode 100644 docs/source/_static/images/2x_bgwhite_original.png delete mode 100644 docs/source/_static/images/DataFusion-Logo-Background-White.png delete mode 100644 docs/source/_static/images/DataFusion-Logo-Background-White.svg delete mode 100644 docs/source/_static/images/DataFusion-Logo-Dark.png delete mode 100644 docs/source/_static/images/DataFusion-Logo-Dark.svg delete mode 100644 docs/source/_static/images/DataFusion-Logo-Light.png delete mode 100644 docs/source/_static/images/DataFusion-Logo-Light.svg create mode 100644 docs/source/_static/images/original.png create mode 100644 docs/source/_static/images/original.svg create mode 100644 docs/source/_static/images/original2x.png diff --git a/README.md b/README.md index 4345b52ed..b1d5397ef 100644 --- a/README.md +++ b/README.md @@ -19,16 +19,16 @@ # DataFusion in Python -[![Python test](https://github.com/apache/arrow-datafusion-python/actions/workflows/test.yaml/badge.svg)](https://github.com/apache/arrow-datafusion-python/actions/workflows/test.yaml) -[![Python Release Build](https://github.com/apache/arrow-datafusion-python/actions/workflows/build.yml/badge.svg)](https://github.com/apache/arrow-datafusion-python/actions/workflows/build.yml) +[![Python test](https://github.com/apache/datafusion-python/actions/workflows/test.yaml/badge.svg)](https://github.com/apache/datafusion-python/actions/workflows/test.yaml) +[![Python Release Build](https://github.com/apache/datafusion-python/actions/workflows/build.yml/badge.svg)](https://github.com/apache/datafusion-python/actions/workflows/build.yml) -This is a Python library that binds to [Apache Arrow](https://arrow.apache.org/) in-memory query engine [DataFusion](https://github.com/apache/arrow-datafusion). +This is a Python library that binds to [Apache Arrow](https://arrow.apache.org/) in-memory query engine [DataFusion](https://github.com/apache/datafusion). DataFusion's Python bindings can be used as a foundation for building new data systems in Python. Here are some examples: - [Dask SQL](https://github.com/dask-contrib/dask-sql) uses DataFusion's Python bindings for SQL parsing, query planning, and logical plan optimizations, and then transpiles the logical plan to Dask operations for execution. -- [DataFusion Ballista](https://github.com/apache/arrow-ballista) is a distributed SQL query engine that extends +- [DataFusion Ballista](https://github.com/apache/datafusion-ballista) is a distributed SQL query engine that extends DataFusion's Python bindings for distributed use cases. It is also possible to use these Python bindings directly for DataFrame and SQL operations, but you may find that @@ -120,23 +120,23 @@ See [examples](examples/README.md) for more information. ### Executing Queries with DataFusion -- [Query a Parquet file using SQL](./examples/sql-parquet.py) -- [Query a Parquet file using the DataFrame API](./examples/dataframe-parquet.py) -- [Run a SQL query and store the results in a Pandas DataFrame](./examples/sql-to-pandas.py) -- [Run a SQL query with a Python user-defined function (UDF)](./examples/sql-using-python-udf.py) -- [Run a SQL query with a Python user-defined aggregation function (UDAF)](./examples/sql-using-python-udaf.py) -- [Query PyArrow Data](./examples/query-pyarrow-data.py) -- [Create dataframe](./examples/import.py) -- [Export dataframe](./examples/export.py) +- [Query a Parquet file using SQL](https://github.com/apache/datafusion-python/blob/main/examples/sql-parquet.py) +- [Query a Parquet file using the DataFrame API](https://github.com/apache/datafusion-python/blob/main/examples/dataframe-parquet.py) +- [Run a SQL query and store the results in a Pandas DataFrame](https://github.com/apache/datafusion-python/blob/main/examples/sql-to-pandas.py) +- [Run a SQL query with a Python user-defined function (UDF)](https://github.com/apache/datafusion-python/blob/main/examples/sql-using-python-udf.py) +- [Run a SQL query with a Python user-defined aggregation function (UDAF)](https://github.com/apache/datafusion-python/blob/main/examples/sql-using-python-udaf.py) +- [Query PyArrow Data](https://github.com/apache/datafusion-python/blob/main/examples/query-pyarrow-data.py) +- [Create dataframe](https://github.com/apache/datafusion-python/blob/main/examples/import.py) +- [Export dataframe](https://github.com/apache/datafusion-python/blob/main/examples/export.py) ### Running User-Defined Python Code -- [Register a Python UDF with DataFusion](./examples/python-udf.py) -- [Register a Python UDAF with DataFusion](./examples/python-udaf.py) +- [Register a Python UDF with DataFusion](https://github.com/apache/datafusion-python/blob/main/examples/python-udf.py) +- [Register a Python UDAF with DataFusion](https://github.com/apache/datafusion-python/blob/main/examples/python-udaf.py) ### Substrait Support -- [Serialize query plans using Substrait](./examples/substrait.py) +- [Serialize query plans using Substrait](https://github.com/apache/datafusion-python/blob/main/examples/substrait.py) ## How to install (from pip) @@ -172,7 +172,7 @@ Bootstrap (Conda): ```bash # fetch this repo -git clone git@github.com:apache/arrow-datafusion-python.git +git clone git@github.com:apache/datafusion-python.git # create the conda environment for dev conda env create -f ./conda/environments/datafusion-dev.yaml -n datafusion-dev # activate the conda environment @@ -183,7 +183,7 @@ Bootstrap (Pip): ```bash # fetch this repo -git clone git@github.com:apache/arrow-datafusion-python.git +git clone git@github.com:apache/datafusion-python.git # prepare development environment (used to build wheel / install in development) python3 -m venv venv # activate the venv diff --git a/docs/README.md b/docs/README.md index 8cb101d92..b4b94120e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -20,7 +20,7 @@ # DataFusion Documentation This folder contains the source content of the [Python API](./source/api). -This is published to https://arrow.apache.org/datafusion-python/ by a GitHub action +This is published to https://datafusion.apache.org/python by a GitHub action when changes are merged to the main branch. ## Dependencies @@ -66,15 +66,15 @@ firefox build/html/index.html ## Release Process -This documentation is hosted at https://arrow.apache.org/datafusion-python/ +This documentation is hosted at https://datafusion.apache.org/python When the PR is merged to the `main` branch of the DataFusion -repository, a [github workflow](https://github.com/apache/arrow-datafusion-python/blob/main/.github/workflows/docs.yaml) which: +repository, a [github workflow](https://github.com/apache/datafusion-python/blob/main/.github/workflows/docs.yaml) which: 1. Builds the html content -2. Pushes the html content to the [`asf-site`](https://github.com/apache/arrow-datafusion-python/tree/asf-site) branch in this repository. +2. Pushes the html content to the [`asf-site`](https://github.com/apache/datafusion-python/tree/asf-site) branch in this repository. The Apache Software Foundation provides https://arrow.apache.org/, which serves content based on the configuration in -[.asf.yaml](https://github.com/apache/arrow-datafusion-python/blob/main/.asf.yaml), -which specifies the target as https://arrow.apache.org/datafusion-python/. \ No newline at end of file +[.asf.yaml](https://github.com/apache/datafusion-python/blob/main/.asf.yaml), +which specifies the target as https://datafusion.apache.org/python. \ No newline at end of file diff --git a/docs/source/_static/images/2x_bgwhite_original.png b/docs/source/_static/images/2x_bgwhite_original.png new file mode 100644 index 0000000000000000000000000000000000000000..abb5fca6e4619327822eb03a71a2b19b4b93e0db GIT binary patch literal 60451 zcmZ5{byS<%@;0SN@Zb&!QlL16;vOuxv_P@qR@~j)X@Ekp;*=IG4Nj5bQYfy)i#x&j za_+r7zjMB<#mY}O^_Gdo5@O#u&^3L6Cl1y4y4qJ@HjA%lYQNCSX@{ADfL zED8CA<*cahhJr%I{MX+jB`u~S@94+|uAyUJ#Inu+jRv4ROi(4k`+8!U( zULc0tN8P=|uWyv*Dl03utzY~MXxqBE!?u-1iATATX4we(Svj(?gzuq7zX$Grm~W1P zgTQFV_)oHWQ*7RVXBTA3dDmj_vlPn->NN*MO2b%EZS25EoBr>UKMnwZ^L0+z9d$dS zh$NR5IwD80ewItGN4FlYhT;#mR8W*NmgRQ&yy)nQU#CA$^V}m zH`dg#Fgg0D+KxZ^bO%G%EW0M@0csY-5hm}N`7 zI(tK;FdWQ^`cFBpyQKh+(5ru?EMD_$z$IjA;S~+PK77H^UvFZ_Fgk~jEl)Kr5ktk4 zBFM-0h^{8yK-F_q|0w`PN*F1DYww<7#=T0!>utrSw-bY%V-k@hvNTv-@EUg~6`^zj zTOoP(jA(f(gsLNqjLllHhX|sT4noZU{8Lo0Rxl8i^?}_0C1v4m`V>_`KKflT&(b9kx2C9Pt3vV zfvpK&y1HzxoaHce+j-c?Vm9FM$z+lXTtK*Jw{t>6#@^t;9W}g)`zD;ocuXjfPkT#q zPDE21=h44JN`t~Dz_sxAwWCYQv*dJ7bnieVgO8vtSnuCAS@~IdXAqfl$UD@i1AMG|+YgpZz>Ez?_Lh0C2t3N@@u%)Z zK)iiRAslCwH z%ZPd344k;}7*sBxE`JL8^c93yorTReXgf)@c!<0b6e{{Jp*vInFqZs$BT#QRTtz%VWcxPRtx8 zgeO%}R|EZ8n?J(d_`@Az=L4d^1JzQkFlEqztW#w76uiyl**>G;DqE#=ALA{5wv_cdqT zubIs$hdgeb&%Al4S0lKN%Z;d@_MdOcHx2=&N}aUZmEH$6##)%bTMNow|4QmU5%2~S zUM)DkRt$1YZ_2(tLW9x^8pP21f4p=<2JYkcIiq?>BsT|( znb%)H`u8We>5+_LXW#vvDV6sL{!J+N(}9?#RJm*Eg+2~b!z~Z@cb41!`us_S6UWw? zzT>#iJEe{9NgY256ffGJUy8~vJdg-`3D(jfE@jJB*x8lx1^?|845U}gek~ss@3mLC zh=|gBk(B}C!NZ-+tHXbuu5^(o(C7lZ{EO7ZGR9ek>-;2TWWHNF7${sVzMpih2?S;Y zQ$P9M3U+e>x%`6s3ty7`LE%i{_q~oa-644E?Dz;v`Kye$IQq?6-Jp?0`t7%!#0I9ER=fEv?N7xfG(mBw7qo4ZJFdc0l31Q7b4k*}4^*krX0auQ}9)P8~1L zq^O{J6&n$?ZmcJZ%m4l9uF0NBoQxPH(6GhRrN|)m^ZCr)yj744XxMc znq>$7+m)fgKq5vxwEkBN(c?Tiy%A^pc%~D%q6JZjL$$hOOLT}3{`(D8*mXAvRD>o? z;HZz!Lru2z$!CYJ6lOV(|EW?m5=+fwF0P_%f5nKlNB|jkpd>LqC%xvRWTh1n$v94R zBfB==+N<7U-0xEjv@affriS3IyyLKgzEq+3r=ST&Hq;b_wz|0MhkZ|T^ea(mju+t( zgxiM2#hhNPhOzl(Tp!Chko~pKghnfkXIt`|Uip_Ccd4 z);~o_wS)kPT&~-^gg1hgl`PIeY`*+5E#Oja&2g^En8T`_A(F! z_DX;J`}S1v-rz2)b&f2z=S0a%Ln-(GJukxa0u`Ko$~Uybt-iJ`U4u?_L=TG zT96hMp?;gCpN=Ftp@kld!u3{og!#tfV&X}u*xfeizt9R4C?CCCLmBo*d14 zv?jpxP#s0n9<_oER^R4<{)k&;1>?D;r@rG!k+QYm;AQgo(JgfxE0N)8(zm4cuT2l6 zM;a8p5G0{AyG8f4FjEnu(gQl4qXndV17&&0;~#2C9{)MQpn;*v3Cr|jjG21Q5~|W* z#Ey%I!2Q(zuWfv<9Dnh9J$LsZ>w}(5bmwP+cT#w6 z+{`?#dbs^VhK`lTMtK1~wEgh;f5+V!GVUJfFr|CA`bqPT-z7PyXfSUligxp|159Cs zf=Q(%%y9iJPBeLioia+8uuDonhsV0NMP6H4S=IZ0DFdMffWciUdUw>iZ*>z7jMqf9 zabz?!I9%4=e|iucF@&rJKSxai2aGd7X`jn+vQt77QuPnR_I8RE26z8eHA#IzD#b?= z!;IVtkci61lARvW0A2pEAV${H*;$yxl_6?came7$Ubry&Mf_dU>Bioa0K_QgUk(Q~ zNdd43uSU2A+Af)d%XNE5uS2JC3yeT597^zrncZ3RaUM6`kf%Q;WHO({w*e6Wwae6` zDgUanK($C2DNBA;45L3y8E#*IBdI%#(+?glA5>2(AxtkxSG+$B!F)bu8nbNC{&6C|uuG#!6&@HX970MY)iORUPwbN5Ce`~6-#I>r$Fif!_ zRs&eo{<(#Mqfx<3Xbm9o=56o^reTUYqS^@DY4Cgsr<7m(dy1Y(L$Z`ZpRfaFc8<`N$FmjtRPHvj_o!MWs>>IJDZYlhrEEvsxbQ#Ya8ofz|S}*GwTkiT-tphD&xx#wDLlh)-vnsIP|e|W0_rG?Fc@q6;raLPuNL%S zS9|(scFNb6J}{9p@0-vo3H?>Yi~k?wz61lc%zbrD9Qx>+o*GT(*@wIIyVK9im*OLX z)kIGPM8jvI=g2KI0wjH56~k}^pI3Ut@~>@=S;`}T66B2e=n>`YUq>J3=f2Oe)nRYz=%cz~qR=;MOn?$3F z?fRTAcau3^;1%<)oN}6)7WVNa--L?PC(^6`RDUP`HYpSGQ$1V$j|NiAgj!Ig{3mBi z-dr0&)8!&RjEuVUHLN>7tHP%#I-{F+g0c8F2ciJsDkg&GRjtwMOKOM!t%7KeC7x4k zDDAWNDAA(6ziLT%oVQD}ryZHTBs$e#CK>b+Q&5S6Ot5d8)Fdu!P#MEac0D8w)v5D* z(!ah5h>^!WoWITFJ5+YtJ*nIH3>Td!C(#r7*s~=i9(bQ*mG_zC6~+HTUxy9=7C@FX ztq-ns)Xp1txi|Jw;VwB$g&Fhu%eR7X?VY}Iw9cr)Y79CfD-&l32bpa5<#W4t59$zA z)>_Qt2+S3$Y!}b|`6?qf!EWH=zPL`W)5S6!+p z&DR1?25=<78gwmHRW~cdJN3^O>C%ERBQlI!35Rb^3_p^$`Ap*NoO49j1ze<`QeIT- z8?a*eo1M60zjs>pcQD%D{qO2d5V?m>MBn2fKAz_9Fcp9HYw(S#2!FQ-dTyAEPsk)v zg#TWqysI?JDJ8al;BY_xFU$xQsyjJ2VOV-DamNud*A+G*7MF>cPtMV#hi9BF68wt$ zQ{Zcd{`nK!OV%OmB%D1IeV^&4S*K|cJ1yf3fUdA(4&0-**5hr}zkwqj4Y__G__@-+ zg9J~78d*DQF_)T`uMBRNS{Eg&ViHwmrf#y0#a7gDB(`*;E}iDFK1z3*D`uF_Q4Xe@ zbi=!_9w21zuEEfHWCxRuiAU^qZbZ1$7gZ4$NZ;-;EEE!cMh|3XJ{g^F2z<$wOz3gTaALmc>=IemMbu#y+ zSlAHF`13O>-LHk?u#pLs98fi=443R?$)@26p|Ubk1Z5Q^yc4>cXU7G2%AJ?(D`Bf?|u1gAwVrl zjOU-8XipuX0Xbv)LeH5Zh*K<0)PfwJFMu>l$|J^EI9<7f4q4iRoFjrM3ZO~!)UzNS zg^hd@uZWBiMJi7s8^8&1+u8 zMEbRqxbY4|lfo>)7eb}>P8Kv*&CcZU3xc>K8KK741w>OD7}Mk=hm4=uX#FJ~V-LJT zTCiH?xmQruAp_|C8$7#^o{2q^Y@9Q8#vVQY)sWabK6q^?^d7ra-dV6u-#M&199eNJ zTH3az&?FTbf<>PLQCv*FJdR~mP2&0}xV}SOM)zw=DQKq1gl}OG^w*+h&?-gfRU)Z^eICQ6NVi_fQJPA+epOybQw^)J6kp;HSraT`X1(Df5p?Bj~m2{!{ zJAq060+bUy-qpo4t14n~rznW}90Ta0WH20Ka_iJ*l0X$y%+@dhkM9PWB5^L&2~Lcn zX}!3X)_-quhhk-=KuI^F)?{BwRP&Nm6`mu&=n+$cQtLUq_~oiMiA_2NJ=Uy%sA_Gu zE#S1?&Ol-%fc0cZ1CeAl6l7|&u$SF^xhhH zYAl3-Mo$GOMvAwUyhd@+5ux$n-?v6fcPnp2643VPOdpjPf@2X5i9zg$cc}R^FlVoy z^*VU3dg1?zjeHR#TY}o*2%Plg%5n3HaBKaX_O6GpH9J#?)G_&8XJztj;b5PASPGOP z#p%rxJ3Day#`c|ONYmcVm(fL!ka+%r&_T17CGMhEWK#}WvT;ogc5iqqG4tJZ;0n4M zhNvPh0l1@z=!|snYmb7+1O^hgR`=gt-@oxMab2Wq;%}u}OV^8L)zw%XN!qP6Ea{FP z`_>!o8424}TtUQ12Fc%0pip1~?*1EJ-APS^GV6yTtPjm3e^6_)+$B3Bg$)T1?Vh*r?eXoDI)y3Z|84`bfNPmzXlZr%+6# z10nCOPdHfJj}zM%65DGbc%)&I^$Ib>we7u#TtroS&wR)i?9Dy&i#999?1zRRI|dAW zd_%PwicS_Xc?nmTuh=Rh@#j1Df>)SQ$xsCpW}kd<=+Wy1kHL&`{q7B1iVCg_H2kB} zZvn;Gs#IL6Rf1p$)*EEL*{Ho}YkonXSo|Xoyb%kYUOu7C^0+SdSU&tysC;A8l!^9l z4|8o_bcmtLfj#W24cu9gYdXP+Bn_pzX!OaJvI9=4o}`=%RGxd>8xz!qlbV+>ag*b! z+~k1lstusa=BzKXbA<^PL7K8yG#uV7@b@Ppwf;fy))6588|GnRVKH9SIMeZ&l$L$( z_8;>Ye&+to@r~^yvtDN#Vxp=|4sfhH-#QZ})2(R6*rK^a7CT#Q2COm@?;mwUIoMB> zXl;_7ioG6+_$g*%@^s#7Pi5hzOVc#1giiTqGMoQ73WHTW%NR?%1LGdzLjdp5!0y{`7zX9+oLOB4PSOD zGZVb7b57-ldf*Rb`DD!{Df@hel_6H2?>ea!`@UwXq&h$=x5~hdx_q14ITz>V(z-Y@S z!S&`T`@4Bh^&{`mz{N!)Mel-ePrDiRTSstEWttM`et<+ZUUWPWIIfjb_w0Rh2Ib%r zo9dg>=fECw1YOCQBwwqOn`8`g{Q9T2*TjhvSv=o~I#`gy`_{zRq`M|SSHudj*gD8d z<^TjQf7>9tn0jmbs^_&kNVww26HmH3PBHc1d|XhEGXMd8ELiBPYP?*c_IOnMjXnSH z)a&z=bRpL=#zeiXRTP*YK1Q?1NxQ9XCngREdR9H9)RkIT447`zw16MS;7I5uH>Npp zyWWowLb7@cWV(c!7h1}{Bj7tK3a@zY#4D-4`wPi`On=}?aQnGIvXD`HOtGlPb^31Q z=5F{Dwnp4e=YTx#x_o7hRS98DiZ=oqCqlG5-{-Rzc^r! z+Z2mC1txe)Zg#ju`$oX#@w6(|-Y?gE$JuJGq`wz4V;NXCPkQnvbY2TFu!L8iGTtm; z7~Ia(411_F5FZS01Q(kBNUj~sK+Sjjn~~W^gl??H`ulL)NV1yqIbJQ z3_8veZuZvlrSEpS~NkZ z&FNwRHENcJ9o4wix%#bpNmwUb@4Kw_$0vk4GyZJ(&Ht3l*{VvnAmp_4vjdZaMNR`O zd$7-EmQe7l={6A$7GRp-XMcO|g7%_ocKL(r3ZJLh7Dk-lqMER1ui@RjN4>;D^WsP( zf0?!%Ms8f&h>>R!+xCNjXWX98EU{c-EhMFIgb5flKr3c7)X}#hW$Wu8eEVbk`;iuU zwa~ZY7sWlo$0#CQH~bKB$rg2Iv=`+^_GKocuf81{a3~%(-i!SKR}?6iPe$=z(qZCZ zMjU2-EK8zFz-Kj#V?r~T%q#6HH1A>*U9Js9STdTJE?fGtt>UZ_ybH)!3 zH-ZFpavBt1zS7^P8)x0+cF#!TCg(TgcvJ3XikE&+mOU?gA6|&ozwD)TK``}eiaW3F z;ZvjBqT`ODGKDbTY-jZD8$v=lX^rY{ge=6F4Z z8204fTb2t_8sY4oU6oSn(NaE%qpN`_x(AFdeaBO7h7f*yt`wopu-sibc2xi`rl>GO znGP7OcsrlWxy7J76r;D<7PNE427l9*jyzTzJJm<8{8_eSF&R>snY*!v30eN?71Pw4 zgh#mBiAwA$^ihc^M7dvifA4vb32V{kK{CxbLRJ`wS?_q2Z-uH)o&jd@#vkg=+W?%) z3x0?Yk+6XfLq=LvB;BRUrV#ZC-fpnycuz2u{}qWb%d?hMe%5f`&+d8Qx`qX!%tg;y z4i`L~1T7y1iRA9>>t6MakdYP&_(XEXcZ)&UAM6b-tI?A)`pH8!56Jm!c6n9{_Ye#o zM@$As)oPuDl2B67$ZjH4Bz5Fyv^G!R>hz{$$TW?cN!KjKL+g&N$9L8=hP)kqk&~E_ zmi(ch>g>4aUL)^nk=JT2-IFJU__PF)!DUY$aL2ZRuSDyYSTCu{1f+-3o~?H%xP5aw z;ZNLIJw0M^w&6F@Xps(!p!L?(Aq1+Un+JrqroT0MW+Cw=`?F^2UkkC{TW z^CM9On_`x{g4HhtiLL=HU-e_?g?A0ltw~}S>I9$gIXr`k*?nE4SzXhH{r(W16jmYX zotL``06TulJf}PGj{V&&XRM%@W@;L)epc=S`^r^k4)J_d(C74=-LyLn{cXtIjNi}! z`<`c!cj6N=Ss19ZM!%Pn$whHPj!we|N{f;!aQdawqcrbjEwI*dcT_XWm~ntjN*($I zDEc0{NFMeW!EuttxbwNikMy(nu_N@Uf$_}CldmOpe~w2e(INt*GBA{r-*(KQm)(g{ zu9_hLhR3VXqCk5*KKf?~d_UDRQ58RjCxISI2J{&NKJsshXj*@3#+QM@WGoy!Mg%%!bs44k& zG}UcA1sB7F7;=?HpI!J|h?j8{i!i<`0)RQxV5pMIW{&uJy=tuTJc(L3;AhywN@H zTEXM9NaZeyGHWgR8jM|*#F2y#O;UOkHENmKBlS*q1Ub;(KRN+<@5L#slq0DwHz_{v zkLoiPdA2KN`N<{LLkN{A%IjGfkUiT*-RByKg;grYVc5h~djVZBbZc};>Lb;Qbm>PO z8HFgl<~6|1N)k>!3ln1nJTMmcNx^rbs3*fEFb^b|#2mT)^)q;9yC_sqgmz#4!oG3} zEk|Q&JEd1|bFdF9txlHXtW6U*O zL&C|Smxw*CC?;X1(I!`j^o#nd88SODHwl3u5kdFkE{k88QU0NLDeXVIw|ah{p}mc) zBhwZDVt&!gQqQX`te+8J;4}^Yj&DF=>jkZj;7kNO$XcvGVpw($s~Nu}zLUu;uTxSf z8~H^UM?ae($oeQeBjPwXG--H|JF8dw+D7^E@+}*zcEtYRf;DAgWok|I4fe*eqgP(M z|CS{KmuAur^OpsE5M`At?-HZvwi~vA{iW9?TRqn37cQLX$@&sbRpz{qwi~|U#Ac}& z29O9$#9M44_z+2Laq0tBZe(qJ&woib?&^Nx*mZhB@5le0_TUReLYGlh&uarQ*9>1NKA8&C|%;r$nLM6i;Xk^caK))dYoLIy%wb@u>51|oGqe}X6 zI7%%4+~D!lJnM2z9QGS>d^^$I>yQWUcimC*tk14U25z?-p>HVhkR1|N?oWjaRWnyL z!=5S)5l?k~0amxZec!b845&UjUa=99fP#mQ^=P8xzCMnM2wO|A3{WeT5_^PC$M zD1L*fl%p0sWjar5*MlkZBKq>ehjBoya1p#cYAmyQ^cE{zp**a2;SrCayMl1>w(k>h zt5jY5Kvf@<_BVtU6s92jBe$i?x2ntzqA==k(vsSVzQR-tp?=!PhN{A zCnc59yZqgJ=t(v6CD&gz{}mg6%oa`g$JL{n$Ls84WK#UId7ly6RCZ!{j3q^9T+if0 zNY3n2?uW`rNcUJdm7U*Nk;Cg)3lFiv3d%T&|e?QoUTtw@qQ=3M$2CI?5zh zkmCXmGI5MRJFG{3o^KbHH{5lLvU~ zSC`{zFvZ5ASVJxkSFZl`fuSl(U92zowGLxqU{F%;CO0|E3qEX&{A2MUk1ag|1L3k4Cw%ExP zVbC8@j5jB4Ft6MyIs{&iNDRZXRn_s{?+Rxtp=bPgY7)~4KL#AWr(SP~r1dk3{uzEo7r{LE0Dil0#uS8~jB(-QJ^p@NzwX_e{ zT387vBVyiuG9YXg1FkqUF&Mu9p8v5e_%gXbbG-c?ymQr_K@kADC@X#MX^H*lK*Y~- z=NlIPMOHJ6m`RNR@U|`-Sa@i-!eovoalsxlRC_rTGv*P78xi{>%acrivoRs-VLEKW zeIyN53rR&S@;R7A)4R2~yk)R6Zp)=2UzFe8KHZT)8!JkZ>}@ZwCqVWz+0yyHYV2I% zl$lOJ3({p>(sadLPN>^HVOq8AkQ3rChkvgUa7xr{4zvWO)b^oGWk2E+~;$= z(xKuS!KxTJaz)ZO{5A_)~!#RzUu&7fDXOdR#RRaDSKry$IrYS8FH4A z^Q3Ug(UIU=SEhOJ+J`NG;b84@h1l|(FnHh_p(aOoH*I3^(|V@Xg{{E?CE-q`)>z+S z?2msOBc(%uEGHNMdErl?`1nD}i1t9vZeu%%AWDIU;mI1e5E~&f<%8V|=u$B)CMnGp z=Z}zhobs4sGUbCWf&4>px#kBs{v?Dk%RX5xB-!u=W5-B1 z?|RD6QNd=C7}@$38R81g7>Rp3UlX#LxPyOn8dNaByX9?`FOPo*AD~3#E&6!$ZKJ6d zN-RUh&9R8LdZ)TgC@q%Uy7Aos%?^{`An2yVo)xS5Qo~NkUzpGdNdj`GF~#_9kmR58 z;{LI!JBAPo3`O(#O;`U3qdkEz$ibI&Dr2Y6REq*^FZq70~?eN>HBETreQiWGMJ zR*A%GhS1EjlQ`4-wlinra2IC`XNv7S#rwNi%?%^s6c6U-QAZ?0ad&&}V>jdtRei6T zX>QF5?q(9-;P5a@35!TV&VhZn#ms5DQ@H;Sb}hJ|WRb(C{b^o!UxtX@v9Ksr#Pj)> zn^#H&HjPic_9TZzlDA5}<{#RR|z>!`*D5RDj(&JF8>Itx(RqExl9Ee|OZS z{2-XIj7ifz_N!^bq_n?R=f3$|u&V75p-tGgD#7-H!biRM|6QK?QVuZFLxs_V-{C+( zSmu2$?zWp;5q_4Gc~$G%ZfQnp!#JYGoOEFIC_R?uUgItccQh(3<YVMI}}y&_8o7%mq{xAXh3lJ)#<;BMoFaa+|9RQuMMOF1B0&rH$rrw1(e z>H(mGk=tstgX?&ZLcu2XI1h7KIf3)l-M6rq-=CrxI8cx5MYS~9KYnWQd!aOcd`edi z>~yyEEtdh{JexOJq`V=z?$6^mJa$^Hso)@|D8m(6PX&H zE>Pn2M)AE#B#Hpq=TpikZoDRpy)hEPW-qTxQ1%GpVLyDT%1;^Z3K92==r#hZn8UNT zhFtt8aQ~cVf#gs!nF}MhP7GLD<&jj5y#m^74u~cyjf60~zx5(jG+{9fGk$ijlSIK# zG+OY^D4vphR`1g-^*rlMp1Rfnb92Ky)}^b=2wHKL0qe6hLuj7=LG?kBmL^xq_US|n zEyEs0|6Nng!gRvnb(k8Jw?8_bm|qohG{;NS*h`ID^Ew@|sS?o~W#l=a*IYGIK}jYC ze788|=GPr0SFP9=53Nhr2M|W-9iDd}JNmTsG-6zsFS6Q5!^#;w9FCTcW2{e3?e!A8 zvsr77Gp&^qcUJc88oBGrGSD8`A=~TF`xe$xuRm-q3gCTeIFL(zw0(X>j)C&=yX#uY zjG{$Q!m0NU5#h(ru9go9Zf|nhI_5esTqf&&_*IB<%L1BxWWljz7#Iu~Ap4YX(9HB>OJgf=66?%y3#(hr4VkhZUk5xf97FcD+EkmQNE@z$I7_J-L0k1!o^^+WOXe12)7ly}W7iOQ1 zXe!vNx4hpWCF6VkaLJ3>MZBTqN_L(KBo(*|w=Kp2Pk93Q{+t22BPN`w`w${!htkb7 zi}5vt@|g0~LfN|fxy|8JTSKKxDf1ho5tzJ-;7$%atT%5Z?sa0ZIfBpb)+{G(76(}o zzRl;C;hNWuy(R)d`<`}fZv-r#{zj$5_l)89w-pvJxd#d~6mNU}eiyrK_QB#O)lO@G z1m5j$vQ&C^Ekh-Gfi6<%celuS81GMLsrR>dOjL1uvb9GA6P+6BEE^_`N#FWnPJZzg zD*t>$flYS${N93CE>r79xd0ajyit8{Y!lS>2cuE4Oe2zW?K7GN(NTjra*y*QY)}P; z4P=xUk)4U11yY~%%GAXVd$AwjF3O&LoD2Q?{^9Gn8CWm^0=f+1I8w3=_jke9O!&MiO2G4( z#SU?BaqU6PoXE&!8yQEF5Lur6A9D)6C(8gtrHjUJ7!stRKOdRN-<9?xl-30Ku`uH` zR?Sl-*VBTnl$tUVi;>Jfm{Ghp-c?T}3%Xjka$a1GGJZoa_o#N*TNXWrca6<$ppoC| zs_O#|=({4l)mjLNrJsX^5Jk3eGnj@rik=Fl|E!5!cl>yyfBEby&|N67X zyLA>k?q%JAmU5QzY!7|QXMI_v64+TJ;fECyY06OhWq!!V?KEgc^=g_9IUD0b$uv>C zjY1NJNMJ=QyknA{c< zqqC}f6|uneI}c?fiO}?r%UeauVJ9ET~PF&IAV<`kWsu}#HaG!jOonIlIFrz04aV1bgmoEZ#I9zT!?Fh zbm|?Z4;aYEUjLp0fKLc$C)zKp=7OdzVT-N92;*Wsl+5~fSyeuYIlZbVSOu|@535i+ z3B%-9ycq`YZf z!Db6^3@gcU$qnZy-|pg%K={5AFG>rlE~UQ7aIFQ5N4oCBT15B&qb!5w`h z2K^~|vMWurG~U9e<+c+?SHdvJe>&-Hr)x{O1;$6W!_yEd_HzoyCJaYlV|!ff`zg(b z$jOM#H0EFnwf72mrP&H*DlOlW%vhz@<8+Y|>o}jMU*lPt1dg|=U3ePE^igdP*$N#a zGZ9Zx0Y25LG@!pJ#Mk9P=&JpwAK40QlCX*iO{YDF_<0|5v z7K0@0p(y18%orrPL;GAwGt(pvI80!aqB7S({+Q>B0w%(81>ur(%32Z~5!+PLq(mg* zv9PiVIpc#v@YU2QTBH@SnkFz-ll;&E$ab}%UDkTpJNCS+ma_%h|8(gze*_GNwzE! z$uY)bRp9d`GA8y9w0=$G zi^@}Z?d*+GLxLRd`TQ}g7gu4AW5(A-Y5s)6Q6g<#9VIm;ch&@ZGWV`Zu2qYEw5L?? zdmxW77PfN6k8?PZy*zY>kmA((-)7uUc2)8A#wC2Qk1#>l1yzPzMG?6*-|Kx6h2t^~%Kan=ZjL{LX`gi_+x zS{r`iSF@--jXSV^j%Vgkp_kn(>BC8g<~n7qJkqBW+6Z9*+F%Xr(N#~Q$x z_fZwgn_@kvd2kn#Fj1<=x)QTV#?W0d+N~zl5qNcjzjL$oo(=A2&|$7fE09KIWumwp z(;pxy4@J^w8b6r{J}Pn?;H1#IH{_Nsj0{h0`yej?z2|#I#&*nvJFbwcas`L`+^CQH z$#8Uod$m70U~h=d>B!)>%#U^uxVZZc}ea zy}!Y%-Fq8N9_d7CVRlte9&R?%D@?Epe2kH6Womfe=GD7AmR*%Cxys-kNc-hmf7SB3 zx%gOKv(A!lbKTx$h1rqyoQH1U-?s&V%`R!cccY)Y(jlNyNg2FL;agcgt$sL@l))NPdAphOIhtIYdVJ4MT5kA4 zRv`{Y@7NTa$B^yRw%N;}I%a~F>G4P33XIFk3J|S@^uGwG6VcaD&s=Y0Vmm^k62Gne zyr6A<`Xuzo{O_?%j4qaN3zHvm_22#RYx9M2Y{hsR7eVo{-=Er`)Qno@_vuq?$-EpP zObu93ntKQuDGXEwq_i$x;QF}6{uq6wVqWP7Kvm3i=2l$Obt;V1<~nOnqIZ8K)7LtI zAUI`1#Lba?tjJ9+AsZzepD*^6Nlg^882F`2rRlcm8mKP1S`*7-cpI<8d4u!U5FPZV zw7zq%Jlhe9H>VN#2CRfdX+m43Dp99b&T629p93J0rzP z<)6yTN(D>n=1<116%LbNzrH15G5nn`wVWf8LpVpvy+Q4l9f~IcCDdWCa&?-V-+CNZ zftVXX#2mC_l{^fOUBuhkJ&Xs=01#ke$4=0?=Nx=hW?=uM_jS#w`5vi#+40_>V!SPw z#}|nmm!}&OUoPoN>AT(T)&sAY&I7z8Eb~lc>w}HJJ&c9^v;eRQwAUP6>k`==SYw2? z8PHd$mLL6cDPNTqK9)!`n;QB#kfqwTAqN1TLxm3mI2o7t|8j`xL|p1kXxtb$^zVv= zbd>O9b42Rw#kZwjiHg33soUy0m1ttI)zsZzJ>TW9MT91|vBnNrsnZ(enm=jYP?`%` zY~(*WB)B@@J|DxHxLtPousL`Da?z_bK~yW!@}*>P1twE%8gG-9h&H#x4%VWjEO1v) zXD}ybtr%4&cHA1Ak43D+^_~O`UBaKGpZfhEHq)|$0f~El@s1Ydd3kPECQl-&R3YuC zta)i(pP$-_l1+C~piLKV7Z=2JzC~@{d>?eHH@qlB6}Vqz(56CK2>&Y2{qawtlj28? z!|iRC>~R~PJTb>K0=KDfaObk!ZO;*uOr|a*I#{Snb8{#mFf591e9HopgDFx)TKE;B zZv!y*O?=qS#lP|DEgqKs5v1IS|mx%@b&vUmdgc7mEcCm_!Vd}q&9uy=~k5&%ZsEE0MU zCV70*jNk&=W)Di;X1C#20?`!?kqtvbHC=_`mCUn^Gj`b1gW&b*>kv$&ahS_p3mJoi zrl~qvbdYA*zeZbMkV4)(>TaWNkops8@!_}+EQn1mK|hm*!i2=ruwtTR{x_{0tRCxQ zFbh{v;}c`x^nod;c$t5V182+i`rQY)U0~t&MAvy0buWEp)zx^)q&IZFutS?)ybZ)2-8dE75s)0S!1| zP4NOjLSscmxks?#5wzg&S>){wEgi+}h3a(r7YAV(h(Lqm)wJzRg(%y$v-{{$YgcpQ zD+93W`=Oh0!>s$p6~tJz3B13TMs>M)Z7Oxi{K>XB8HdhMjGR7MK*LtCj*BM`%-Oe- ze-d5GX%(|Qy;z>&rTy~L^Jg4NuA*gmQDcsIRK z*q$Sl-~U)1Dl)6^5NqLO9Wq1M^5*45k%UCpc&VW}(q#qTk=bUi)?1lmc7Dtjjdsue{S@yHLl%jp*^* zGPJM5efW=lPck76T#{n;f+`Dd4~Aj)b#k!#vo}FPNMnJo#6A6`s~NkE zlpyy{JU-BXm7YAzJr_k)dWQ#k4n1;~Biu(#kuQzi$5slOh*LHr@B2oK(0ceuaOE`E zv>n+PS3V{Z;+-tw%^+|$95z5}h6tbOh1@66!h&2QVrvJgMTxvp zW7nxB_rHfklHH_nq{?g&MfVOM+_kchI(4IyNq_?XC|4%nsej!}>!<)Itl<8F0(xBh z{b?o3irjYN^4^8Il9$1c2=yb4>FWu`UP->M5#7B{X5sa2)cft*G6rb8(7f3@-p!AG z;LvW^P`n%zs8p|Zlt^xG-RG8hJvW#G_?+_BxL9lc`B!-AAvR$GXD$V`hQ;DpBPifJ z&E$VX^M7<<&5PgLz7Uoks*$HVA|LR1pxMi z546MQUvd&Bs$1|&*!i28|6)|Ua8ISkaEAZg@jxo_x?4|1F@yK)w-Up_H*;uN8_={8 zmYS+^FW8K2)(K;^FmxS_FL3wj_i4RY_6Bb(AAXq@4jO6&50e|Jt`<@>s1jw44i}yk zQC41>9ulUyw)<-}3XeZHuobh6uwGp>u!N7@@zW%{Aox8aybXwzPcP&fN?x9hjGK`i=8r_l@?G)!Y~c}n!rHYMJh|E(6qpCdwh@o z8h$hgORiCt>{C%Zv#-XfQ2i*NRMg^El&<#0@*#)tfReCc)*c<51jeZcFroPNrB;(X zyDgoDboOKA011yn@SlkI>ze{M?Q0?C?qqN~u+0uPKaZpdMOG7Y6{)9Ar(4ABGS-$fHRdi|7rIwK4GMH_b%VF z(Zc`7)mH{Y*{xj=~A)j4)GjjSwHq_>1*y9umSa!MAc|%Gw#wl*^yk3$phQSX_(ubJt0? zNXYOW`$#!$qc{sHXMVGik0kVrV5X2BvGOSXbF|JRf{Ln@>MRE@Mlwg8v{VSkg7$h| z+3bY&MMzGc!E=$orE71d_YPL6vVjEg4SSYP;z&aMqU>e;1sd4Sn$f#+N;hYMA79sb-*A`~9qd(^6qooywMI(M%E3ywj|1n~yjfbQn<{ZliOJ<=RigFfao1L-^ z76$Y_HnitmZT*TV->2lrI?_j1ui%3y1oYgqTC$XPuzgJ=WG(}_nrv{^UYQyf3`T6B zOlUru-D%WASes1D(`CKi2}O|oV;m4I1~>i0a$-(04!6mvi(Sh$*~_otDPTgFn?iH# zpl?qSwNlL5A@~eOkp)Z}H&*0iuo9L&E^FHq;d5Z%m&EG3hp)FW&YipZjBgKbDfO7e z|CEdpNL4fLNQwp_+{40nOf#?P%N=d2etV-+l-mB$!+P8-B`{n*J8(On z_6<&Wr%qze1uyrJ6OH7JNCMOrpDE-MweDV_?|fa<t5F>qHkgUG|QQ!BbBA3 zm{a+T?m^vuI7xra1IGItcNQ;9|AaeGgUeA-_6!B=V3P+>PCPh|^iww&Ce#`QY4! zSrF`>yI)A0TX>L8neU@Nc!>Ce-G|Qt9aw1>*piA~==qYtkjq3apGb9ods8XJU1C^R zs22q9aX}Oi-6LMAdjN_Mzj1gV2`zu35 zo*q{JE(-iLE>bnUu!!XCqFISHiK9gQ6a#BE-OPnp%GzJrL-Z+K#4pq9gD1t};kB%j z6t-y15-d&#Xsu1pgxIPvSp?qL_ccdKVUo_4IqHHmg(2R7kWq&XNh!;TyrNs(OxW|B z@&f;JxHerb82bx+Si4(N*!RGM*0y+6F;L(U zy*@O~VMB0Tt?Q)7%%xFIlA&_*ud;}7`hqRSN?C8XOASn$y#OSIOPV z$G_7*b%LzJ4Kg*cCN8t<#2c0kRjfRl_=8M*#QyJr{{E#Um?85*EObM)sk)+zh6lm< z)UyfOFTFWAa0E0-T^L$3!<-6ZZ6^rD>eLvKgP1~+lVl?Sw>+AU-*510I*AMLI!A2% z@gF=&J{7)4tOmA=tsL>Tksvd@z8k}U{MUVt3lpGvY7GhkYFM1&MI^2V8X=NO)S^f7fc)x z*X}t+8#rI2lwS`q{bAia{uK7tTlvqgO=uFRMJRqgQn>s+48)U)lCMpIUoJ*yQ~F=> zW7yyEYjLA1wJfMrc@h(VXQKeyT^?d>|_jp9eh`d))ye6f}X;y=Ry|3tP^E7tS1I5ann@ZpFhuUi)ym}bGfX};wM z#PEhJptZAo)MD-=rYEb7ZzsCWF2(0y+?0jkJW!H4ysK==heniilQogJ%$BNN?!km( zPvq3U-ki5kL;fsR_Qe(gn3b%Gq(nxQe-0_c$lcN=18WuM^Ik+Fg;01LM{^cwLHA!B z8UtSK?Q_M#xSjald4|6Oa$!6MOKfk##L`~p5P6^a`DryTSFc_%ecmrsDWxem z&=f&c^!Y}E6)bI#} zLAGTh7BZLjqrN_75{dYyS=Qp_F!rOjC9>5@nUPK3u5onmmj>te;FtI@guwAag@_3h zUI%5%wzGK;jD@|3I4bzj;o}c74XliQ<*bjV0|k|aaU+bsn-^4MeA^@bcIInv*XVG_ zqqo0T&^$`IZ2JlqAFZH;n*oXcPjvX#NSq-uuUXL`*iUV~SR(!DJ@D3#pL>5pU@KJd z{VL~{n9hFm#8O#xuI`bMU{R#O`lUg2ftZ$srWH%${sT`7)^UyC&+(vQT)tQJ)z4ez z(sCpd>Cuh-@jpf;9CBMy0rZ$IW8a<2QPRYYopzJ_a}0eZXLi$j#)@E6^*=vA=L!J~ z2w(~meU_{3KddO=E~BJ(3H1_tw}V!hw{Q01*_#?B=U&;Cc&<1P+||FMcg$70IFp*) zD0V~Dgs#=brxNY~GDNz}|$tJLy`G`LNqk z!Y@&=MQ_{gaeV@0t<^4UwylLWRvvsK|U5 z%)52UJus1;#1}bzC#Ati3E@BWy}FaFT0Ehed{emPQgxaSQIJ2WihvZ=Mi%SNK6Cjn zK;v#W2V3{-pCOi}ml0-Pn5q~otn}c7#)&)Wznjqjgchjl|3hN?JDb8K3dMR(cJEok zvzkgynZ*RKB&T?d<06e7;0`NRv>SpA-{s?RKQhLi zDpF3G+q1J3$(4!dG56odi&!^pOIFFLZHm`3$;)j*i!+CA!`}5Yd{>f?HU?|(3U|!} zIk;!Wd|Td2s?y`wa=g=(fAU$*u7Kfit+6U=+2`;u^}nP@Gz;ay&gZoT8lU3?cd?2V z^9pJQ`R~^`d7RxmE9cw%{v}C4-@>5@{_x z2ToY?V&ZP`@OXazM)1ZR%N=3=O1ZA|8*X*y*AI)_GIq}Tig-tDY1zRa05fg><4D0z zvXwx2bV6ol1`c&$eHXw|yYiv_ET3{D)ybff#+`jwe6X3L-&9fkM|V<%PXBMlj`-!+ zg8Q~DL?4ttGd=#VZ9>u=0ZHhk7n}dSNm>_9(LjR#RZK}rv21Nt&=Lg{R8XdQKCJk(xt=SlVCWXZ1y`n8+Q=uKo6-4nd8PYWgSp*J0=SY!s!F@1b=1j;PS@Skz zsr6k9tRZ*7XcL=mw<@*9dd6^u^8Z_okHwIVL*sa-0Z+m2w&Cd=-pZ8T>c_W>%%u$?A68?&F|!6emNIhkLUT<<0x&p&tLzBW=HqDm=8 z@hVFw@R1=HIgzj4Xs^XUl~o&>jH>o5wrEs?DSk!Vuy7)arM zorMm*D7So+;T7IYy@`8R5-ZBLiTlq@9+x(zYxMBZR}0ujAm>|wHW8_y`7}(0Qh_+f zK&iDez$4<7!NB(%qSGBO9MC7(LkQOy>_m(vdM#pO=9P+mg{qofp^pqnhq`l@29B=(=r0Z6sur($1OLz)ZSzOT(2thdUoUS3B8R8$?Ci3>=1Ekp$)F6@n8UzMn8M*sjO;7 z1XSEc)B)_-!L55N>`brFT1-7>4r-tBi z@6Mmkr4(uJ=k~gXbN|%r$`ix^rFWC_6?~G^SFsjm59!zMzqOC}nqpVOCyY4TJ=S2O zf$h2_6^&K%?7lzAa$rq*P37@^ODRcfq#MEg?=a8$+3DVI)YtjFJOFq)JYV4!xRdR_ z>+Q;vyZfTC+}9#}UUldEaBOr6}g@tjBrZdYQFBi1-FH`Xr$R ze$>!_{^aZKPk9qG|Ml0)Pnl`L7jN=^-=GUs_*nqu3*V7mED&^_h!;^n@>c%s(4+ia zUJRoszDFtgpL{>Zdx+)FHtp-)B{CQEpx-=R>tOAc)xyS)q1t+^oiCheu1#fQW$oCK z_7Qde+|W!Tu!{?%%d70_)9@9(F42{4)&wck|V1uu+=K?ED)d8rgs-j!x zmu+)GXvfAS7%bVYG+>gOgDdv$t%$v(u*fgiTA<_JGUrT;yU*y;_O)#0i*m{b1o;o3 z&}D+v6;*n)#reo!gO&^TR|mDUU+GPDMB?wtJsxK{CW4!@_1{E(M9vC`GjD2j$Lv=; zr~V+@zA8oV=<$S>VHXu0YZO5O&oc67YD-83nQOy7-lt-lbw=_P&AYrl^bU=}*m>DW z?lD>Fvw^Q}A2gSEE&W+)b6N!kikA4FkzxBmbiH|Y2yD`50q1XOWAAvb!YlHxRMX^? z!yNNj*ed+%jMURU@d6|!2ehXK7FWf<{-C7!>i>M=&;o!tsP!2D8~&0Y#f^%lkE_#R z$bB_#QKXzqQN#^U69KE(A|hJ|0D2NAgek{zc=lkricHxw#HcV-WbR_9!A4=9LG;>UhAKEq5lrx zw&|GzEY8j;l4*$qXsA`zLta+UbM-)lX-TN*$8QesXSP239$R#NiHwk^R-NPzI~NC` zG&B+*kruE9tL9Zzt8c8s#vhfG%l!p*UR=UIXV5$5sBE!C-pX5(0k{18?}NQANiCD* z-bsR3XnP<3O>;n=2`)}jyKPO6Z3b2`S^Ub_+p+?tF8N+-hwK`k@>EB$ISpbU=WUZ6 z$#&N-sf9hYVosjJ&Av#^(Eo_TbP<<33MW?JBfIcLYU-}{#&B>XO@J-^0rd1%ep=1I%sk!6I-{A|sDV^KDu(&Cc8^ntW)Gobat z6@AclFV{soOExq7;CaApmSxCYcr}IGqW~c@(r2!E1^T@%g-h*cKrs9q%ldTCVqSzGfZ{D=N2dhNiX6*kwolxD^ z9Xq3FmZ7HpD>k#%(Yt)Y+iKv!!a)doyYf-b)IZ|c>rTd4U(S#q%&?&ctiR=UEPNT_ zo3<)o`=8EoX1LBi9${>9@z|QXcB??*Z)gjDi+fJsm=0S+A#V@^rA`HQuT5b+ac6`C zPeQvPe%`B=bBh%Q0*PvwdYxfU*iEB0y8to9-B?#f8sv=$lu=k&=C*A;+&Oj=mpf5> zFoQ1ULk{U(s6I}@irMq;H_>k_sKw6JLX59=n+1P7xq4)-K3$#dgOaQ=h)egMqnNhF zx)WD1jKNDd>Tg@X=ZV+`(8nQ5nA-*Vd^#VgySBVdIx9!}n~XUA)GEZXN~;en^kx9? zv4Ym#s4uzqE+$zBUX`Ehhl$6AgtuQ&?C+RT+EjmH4pK3!)QV61KzN|J5jCwmw|&jKdN{GBoG0j@})9yY`${y^mJMz?z_ z#C89}Yo{C!RW*(+G_w&086vQe!0A&H*qx_|vNitES?oC%?vi{w@DS3h;iYqH397D- z75-HVyg2g<>Q#Z05=%Qq!<;G^4T-kymPGV1!M!dfy+XMKM+$ifP=j;GYb6~PgIQuP z(OGDo__~~ZpyLM<^C;=%XB6Y4kBEoD5>p|V@(&hS3Ty`JR}~Tib1neA+HUn45V>+U zi_9rP@`qHA+wQ&6(Bzc{xm*6@g!iT$SJwbDQbBvutl8t$T$6HXyx-d5m7|Dtky`_r z<=p?y`Yz{VG0`MdM-e9DZzakGzQR^Mb3BcG(x(@M{Y=K#TV^}PWbv3O>)qeEe7o37 zalNd*dVYgQ>~dU%y+2ZU%~ut3`UFdtJWTn-zxgAz%Xti2Kz(nep$-|fQoXT^`fa+J zy5B{R!f`bvrF6luECZK^MIaAgp!D&g&(C*u{V_bs)RJ^Ky3} zth^k2HuCTOcW0Fxz?=Q>nkfVQ^-l>ilPzE5*wIlX=lR#^EN|**^zgPiwy@0xA?eQ{ z@s;Omri%;Jd6EY^an+%#ZD^MM3nutYOnSJbQ3e~r`F4k6>_) z31f^dj2Zee`kh<~cic*g*eS>)s9E4_nSS4$I4Yf5A!3+ib$l;Lv+#&S&@Vig+zlcW z`1vFU?Q!&+VK8ca%8{#_FcSz`bF^y?-)dLK^@H;C5i&o!dSP<71GhM!vHLEO^$#F1 z?QjMT@QW5>31t-ZTO3uqp@FC2BfdPZh$8(|4*C4WX-btw*lJLia7Xk>p`u=51Ue7e zes!t74EpXFaqy;7m(^}cQGMDoqMpnoZkPs@Ulqo&?PrDDD5VX!#COS(h#JYwQN0lJ z8@_$AaLC9$80x%|J5==XLaVq@{m2=+f&lQ3%>uO#00|-tU5a;cA>}c{#`Jy&(Sep> zYNbmi8swLujRm)&-BDhy``!(v=`)Rye_AZ+CbzE3W#%BSYvWkNuLIWM@S#ZCEtyqoc1c+gP9C6QnScQRQ07h4b&Iknt1y2;nMNG3 zKlw01|2K(VQQ{*61GXWPg=x|6J{u{@?P%XbwUsaHEOu2I;ei}Hxi3p;sf2w&p_1X~ zsO|Ho08T3$U+Xa9E@WfD%)8Y~Rym1$n2Gh8fy&b)J%TO$;N3mEoJS8R3m;w5Cqq&s zyU4j)ce|f_lZQug#{pV=`jtL?=qnOmelzt8879lxG!YsmpfPf1Mym(2_@ljp9|xjol3wmV8YfpgUu9+y(0}3FyEK&&)wl5Bb~IcJ-~(+F&lOQpb?otAfI>|>a?#$cH4v;$j<(te z;rp^V2e&dSag{_;Yz&l9=ue042FE2&ND>T}@spr*2X8ub?-DZpOx>L(&cevJmhGqHcoWZXLyxQxTgJ(%kJXo0CY`*JUR!`lk=tO; zv!?02(5+Nzz01DT_$z;c;UHi84AH^oy)nEVk`<7%N!B|6^?&(@d}Rh4DkS8zC=V$i zF|&*tPkeP<4?0mFlU;ZyORPW+(q-|Wq?S@x#N>{5U-037ecp~Ps>(IOO`y1(E;MTU zrVHEM)4jfp|;1ZDGoBt{>Jb1?mL*+Gc&Wto*K&Ff)Jqo)UTU zCx{9*Wa+CtLcRNd=lz(CGeOOL1Xk&G#!r!wUEx|{hH*ONrzpe&R|4`zo^DppdkHYdbnEJE&D7KYI;CZ zAhE$`XeE8<;Cy`G+J}{+XICGr2TH-dGe$0N(B1XnfkF#9D4s*e44JteB+y7tMYAm& zyf2YFSmnR9h#5*A4o8-qd!2uvAh>-2zReI^$ED)~)Z!Jv_t)X(Zi448C<*w5icJ>Q zsXnrG)w70p826(M*7;(Q4@T0|!nPA0OJd~JAskd~$qI)3K2GXDjK+^yW(t_B{EUzx zd=g>N2`C0qF)&g9K(XrEf}elRXDbcE>IYCz)c}=baAcK*Uz`L3q=CQ_5LRlHUc`;2 zVO;2P<5aeJ0!SDwO$rN1LXxShCaFdw=Ra*Mo{e0K)ftL`@4LX*LuN%?uEVSxJc%h9 z5wpl|o?l^XCjC>bXLz5u?i^n@#jh{yKt%!{JrcI>qfbf-qz6^uKMwx>qA}Heg89pH zPCJcUS53`M98?yxN0@c)j*DJ5iG-D8h;d2q;<}rnZZ(-~ZIdDGL3m)OlbUC>HRaNf zF%cyuC9&;*r_w*z9ul)$ADU%u`NNbpzV9v{w@sF9VGC=G`uaS5eT6D|e>&HkW&ijEm4ivrj)VDFD>eQZ z9+G?s>~V4Gn-+A!jh^1JmXZt3WaI=(kU*|#JQ$6+1=Qs{STP`Y=dqkKDP$}9%J-)G=_dV@TA2{#<;HqY{}4|;e(s1U|>ti(gr6|2gs?CveqnX{G=i;pww%z{d+_h zCX#3w0_5AVndpo+#8u*=SGKnSI2X53ekp{pf!#}*}j{?FT&j4;c)YlzRF1q z0jxynUI3X1ZW!hs!IIPw5j%!a&#OJcDTABv>5%6mu}7Yz`vY6!NOMV7VEmfK)+3mLq@ zMEYo^Uq}Lz@xO{UbPYSfyPZ#(CYHy7Jmd>q<%4z2-w}V$P>qi4M#yiOx$dy*M?;Za zm>MhZjpjhY>>zka+$hbT`g$V}7vA1x^r$CtNDm}$zBD9Aq?`fl_F>%c-9ym+=%&2# zQ|Su;*mo(ANl|U=34|JCCM!Kxj!~r4t9EL765rj7kYdPpS8G~N1RqaGMg{UIh{bN{ zbAN<`!-O!&@zrb`a!qZlzXFPMR z#vxH6;{HiE%f)jS-M3G@bu~O)nqE{y^3GZ|D~#D?Vb$MXgO-6JrCR>CP}vhyj8qPf z8a#mRTdUjtD{VyHa(|xDJ2zR`JzO$=kFZdCEa5IymD#R5b{Q~m$)+r_bW}@|kR{@P zkaQm5aYXeBs2>voH`J#)Z!m=I|246y^Y%fAY&!6Ds9**ASD(1hMV*-+^Qj{iEMSqe zy~K51yBmLwt)4p7ecXD=?I%Du2Dt-O?QY!zxrDuk8JR~GQ)0?&82jx}`q5Gq69%%cl!ZqCv1+xT(z1Z=-GAc&mQ)=0-oed&)~Fb_>`KP8pXT_+DLX zzgOkW+`x5i4)b=yB{}`qKeA2X1z~R#j%vc9-2T^&uEpma|T4A0PM4p$T8c zYT1mFaDl_xjiJWzWCfAAABzZ$EFN>&<6?|F+`jqk?y5CQ(;@`qtTk(8Nai7CZ`omS zpG+XputA)Att@4kAmm1-Rj$7!fTb|ujg1wld|1P)ZvQgDrBqd!SeVWM$T17xPnfsj z%9}@DwK1}UGa5`xn@(u}r5jym)O>dV9)eWvvv$bGZHy(T^s{~}^!3!$xKy_0qYZ-x zxZzXyVf!Y-;S272IMHm)a@un3vWM=Kn!#{uz18xs3)Q&YVkPy%;yOZ0D=)vpTMK0I z4zf#q0u)!)Ssr(~W?1vwLjc^*vo{9M+`zb7b@-xt<4w;N?wXQoDw|g+7zW<1?>YP# zk`-KsenU6BKV9%+i*LB5F9ip-O$1x7O6V253KsFmK0E`Bk)WkY$otOx{-OeXEOaj| z2bw6`D;N!EDI*d&z56*zDcGB~boL#~u(VITl*b8N&i`m)LceWxG@c~RT8UW|q(GcI zYeNsPn7XK#2V(3r0+W7`N+YiWK=K7s=Z}shhbQBu_rH6OJzv)(0w2lqkb;jiNO9d4 zgd6uTg6q^Mat0`i+F#vP$O3BX=~u1v=-Tu1rBpxM1J_Pe-!|TO&${pOKH}3pT(?Ch z&u14|#;_IS4;_~&F)U*%*nnH<`-F^I3Bo-4u<4KDT@ACIR46Qh_+QdLLP=pKEn*6JTw*!VtcKmYTI*7kIA!>Fhi0hbuQ3iCFV@ze0Ko zR(f*Y_~XBtNyCTdu2ZPrqasrbWHDB`(6Kw{q@#RsK59_1M0AD43P_RJ-T6iA@7q_Y z&oTO>`w6Nk7^WWtN(4?+lp<+u=f9Fa1dtL+q?k?by$myI1rar)*RDMd<9 z3FPW_go7( zjyuB{d+Z%Q=t)uK%>Do^4qq6Nbp3$j!b-Wp9hL0xkkoo{n=rZbx{52qgX@d<)+-wu z_Iw1HjnAzyVTcS5D`9A0DBSuv+}B4BQ!f$@RLt&~lvMp@3axMNwMhXKntL%ppmcYhiK^w0?WUcAFcFMm@nz23I1AO)U7jWM=r>RdTF5kRJXK6eF*oXj<9 z8gthp+RY?EyPc%$>bl1@vK5@MJe3?I-WufGXdnJ~B&ZMEc;Y;fNu=uRFOYz z+@AY+sWT}Qu8}$QjhlYC^9x%1vUL1MF1Sm9;(9p>?DHn+MbbJFI%nwIXjZ2dzfL$GZiJcfdDlj)gx>@?9rp)B9Q>gYi1#dfcwJzfSon zpU4Wol>GHhGh~k)xEj>S&FXF$3%8zN_L770*>=D22NTs2vb%I($}uxnty91=^^>&{ z49MJ)Hp$l1`8Y&oeO~YdUVVmkB&KtsRrHMYQyp%PINtfwUMlv#P9)tkR)E{r(56E~ zsytk8P|3a-I!V@H+Xd>i30Xu!Rk;(%xsP$Q zN|Jg=+zD8_`?4Cx$f-H0(KRIzBKI>!LWeMH7fkmqnz(!?DZdaG%LRnSKE8}HuNI{9 z(^%0t3D=}~&_ni$P+dLuLMzuu}3>#~;&(UOrcsFuY%1QoKf?n}H2{x?G)bSB37X}!qt!D%p08j~czZ_Bt*t}~-*Pe8FB8T5*El%^GMu;$ zKcG5Q;FPWvA!oPyAbF91o$LcycN1Ntmfro&M4H-?d^JjVss|1DZoX-+zp~fwr%96% zN-{be{`CWPv$SF2;9UOxN8DkXE=Bc)n@?%^oix!ZbqvZh1*9)~$Nhd-HOk3H_S2Ih$%723;<6Ff!NlEoE43N)fVMhQ+J6H2+TkM3S z*C)JXD|Y_Lf4o8%%{CMmkkz})`GXF=jLf3RB0wW4{ht=F)%m87+$oxUHbqZKu4HR! z&n%Is-^eoi^j{ngdRhLOR~95K!6b99A0N2_0YQX2;@l#%NaCn}{iYG_4T}kC+TSyd z4pwUKlmb^V^J<1->lPd&PG!*a&cfDf9Z7j6g~=irIZW>nitO zKIxim#@OLYMr^WM@^tpE3MpvHd*$93>=Y03$cE$gHACOVf!30EMz_{$#GnfAxvF*t51dZMb4ekX) z3E{{$mRt@@LQ`eWnonL1F)+qqheqjvtI_)lu>>6Y04&!fX14OKoyUEsRwb$ExLmxq z{5Pg&YH1&fgwsMSnq7^4qUfDjvCVuxYHd>~S;DSiht;5hXG46jWv-$hP!Qpk4(AdG z1J4Yn9;n2i)j5R|)cz`qAVoA16ZPdL|8?M$jTb`1si`d(tVNYy{6SDFq%>O9)>Ehm zV~1a^0r{y_-8x8zTheXOE~yR^u)838xODhG?(FCa6D6m~7!mGBqkhGp$Q^RwiqYD! z6Kgk#LC78-tn=k^li-nTY~E*@w8C)H;SE3 zYpKg#L*o+OZLa@TaNf8M+<^qJDYCbWXSapN^y)S5_gO)zTEKExU^4onjIoWKQ|NDS zQaZP#<|MR6(i=#Enigv>)Hr{>K&xWJxaNb5(J+<#Y+` z+QejAztdTZ+xMK3o_mXeCixk?`z%WP_QoP@p&CdxWAh;oGIxp{fBc~-5g+PWG7&yG zP=OR~LHj*)Lf)91e6d}*{~o{!pgQIS1`2Qw!|_`4Da9^Yq;4zR78rSkw$tLLeXh(8`!YrO`G<4j{ycLW9h{v2EYaL5Of&_gNY)AyZ9MIN>ic0CLHnw2Lb|K)2kf)Fk|zvoDfW2cksj zDWGYW?ZZ~Pi}t?q$kGNN@5qtS_2J@YF4X`dcBvrZ$L_k6%myJSU)dk2Jj_JyZ=~Kb z@Ke=U5-D=BYz!oxfTEU=q%n51X2Ea3`QfObum|*C(4o{oDOc}x{yPOAw&T4`<3Q|Q zT`>^RM0z+S6eYedMa1dJ(NZh9wR~qN;s`1m&q@@)vVb`gS8Mll*_`cke5*f>e@@e;N|FvO0jSm6vxm)wugv@5CmEX}=DyOrC ztg`=g%d=+ux=6go>$kd+=0W8?75AGIA1>4JHAKbto9_yOX>s37@iC>+s}1!@E@*!xFc!rweNG7SD02LX~>nJE!T^HF7N9RfC}KPh}v` ziZ^uD?;U)YXnY3iZxyIll>fE(ri?2Z=IQgaEU5W3@vRna=jHHUS3;#@l2c+xiqG)T zTRezxg?SuogOar=AS)={ro?9{}pl=K#XJeIt*{ofpH8au*1cR~4L(0`ZEF*j6Y z<*j!yz39z}iYAdKH~o&1+lShqbU{pY+Lg9CTh^wgC$Ag1Bj2>Jx6aV7hS9MW2#(4Q zSmHf#gl7uk4ko160BA7%yHbv{g<3O<(_ioLuoc}4SGu5E(XiH?7hO@)04{Ofst~d= zqt5u!YbkK=?4<*fIV{gbS%WCrY)f2j-Z$~rX)ooSwKc=VF-4KSab6jnaZ_kVqx=J&>j+cPN`mC;Of5=X8-9snwH~Ytd4+}{w z1NuFG1gE&XmyfS46-(-}RUgfy39yY^!c7K567z&$Sv$d4&C?PiqbhR?^`W>q)p%ZA z^&J?N-s;AbbV>mIKS1KnjT1vx+ukbCe@V%bFXLZ}wI;WDy#MtU{ctc&Uc&ziS-bNlg-#M)^&{q`t+%aUykbsc7BYB6kUqMC7FXGCblMs3`O(;UE z7v0x!RyzC|D4qzH{qLPumU_IVH06}Oi%S=`e)p;&re9=V>KTb}kYckp9Ff_gd2Sd# zFXfJE#oIqbK<+z!uW!$T*v&W3(AAJi$rcOzNn~Qr3K4qHGw-yPGdFhEwpOY>H|t-> z=SQgO*|;D1#e8$0n2hC}Jn_a8BWxS}C~cs1he02&^I3u(RU(cNxR|_r#Y{2YOeER> zuY3E-Rd5<`Q*(s7?-z9hqjkrLP>SUdklW@g-$t1Ou^YrtWFVvMLs7ua(>>y|GyEaN z9)EIO9)?_exYd$|wnzARb@zvZdzsJ9gQ#%-sF47_#Sa5?uuodwQ@y_8^7`6(^|XkYXgqAY!#1i(>S27* zb>&PtxI+iR&k|7148ZPU0_Wq2y%Z*Gk`|@dZ)7vQAng1Q+_dL0=0IJiZWr{^CnFDS&^kUlZ&RCWI_7{OXu4{(4p{e12eGj`d>&!=!sejI;IF_qb@c{Dx`7iZXkE0%g z$yR?bx=S7nX)x+bsrc`H`Ko`}*CEKL9~j;$BE;5OZ-5|ko4|J$J5cXev%@>72HTcX^t{^N!Sy`Af4xVqSC#G~vY`KC#g!*aX!kGW zb~@J9+)l;_z^(Qlgj!OHn2O0V-3aFK?pvySxp&^l{m4uP(9@s<=O2g$FuM4w(-x2T zTu)+y5MaU%o6g(_??Dr=CoNHQOd62{wU*j!nBsXYJ3$3ReBfWTicY|wg^=#wGSbFV zx`ipn?ThE7p1gZC-AUFpi49CsSS5im;QvnQ8t_0Y!JJj!b3_3Bjuq6Umf6@P5BoRE4&K*wS!nhTF)b*SfGnW_?x}xfs@d(*-CpJpzE;r@%3r$ z)r7>xR^7vXMyo3pIR)V^mIwzn;vzK&ckHCZfY1jNTuJ3AwaE!{|A>Nbaw)IUC4zY9 zx6NH|To-x5F{k~{I~!(+4l@^*4?SoiHEf#^gLNu&N8oh>5M2DP1uq>J0Qx&uX%z)( zT&~7fOg*XU`#WIc7u20$&YB6~?U9iFpyGLet5FNdUf* z;#E#siVovNz@ro&G0E`mn@03?&4pdUbwM3Hu} zofhNZbY{wN0WyEH2|8OoJnE=DL$0{N6q$2j+)}kx@2_mPbR9Ss*;*B#Wg|5LUnwPQ zAF&tAG^Jov6@oUQPlTQEtaNr}M%_>0Irc9Of?7}1b1jnmXP#9UlYi%LM*z%!`1vfU zK2ZXdGT;;CF-I$kK7}xM+57j!-5*qqm^)KbR@>;VDA3J z8g|qLssv>Fy%FF6&c`e)dl-PYu9T8K-j$T**w2!KSjG;Ehj@vF!s5@xl_JUHA~Bg@ zd8Quh<84Yt6m&YAbK#wL+)v{K?8zp;R8!p-`|o?qSo>Zq#Xw26*qkO0jT0KVQ{C-kruw0bu69_Vd}*a>{JxaLZ48rl zca`1tM}1@K_4^>`qYR501PXj-)~>Q`3TfxtzXX(N<%)#pwxzHSI?+KLaf}YbL^81S z*A&c?beJeDJs2Rv2cZA!((Ra$q7GOIU7BAa?reHk)rEz-Xy6XgUM-e*HdAkowzH3{ zm--+&6bml+!q|q)Qw_0~`%RD1^QBxdIhSa}FuS+fPdYXtYXrQ7FRjYs5a5o2gzRzz zUA*KVA|z|w(nJkp`2M9;#f!Tqken}6-5963%K?$q$SBKkM!jJI_DrEb;mG&j8UV=S z|7|UeG*orW2OP450mUfwkQxWq`h_P;0NPK8O^r@`o)@^fP}+L10{86SxYge?TWp?E ztGshm+vto_Ny?Dg`zobWTs5Rb-=I*Jm+u)~6X)myhaXk_91I5Tv z?a5ltH(aL(gzzT^@gToRhaaC<+MfI9nz_hqT4}a(vxEbxd7v@2YdSh#VT26^LP~iO z))5BAUcH3v8uKahr>jkV!@gz^-3_^?VGiHX#2hXthNOf{-S+C0l3=)7O2o3-^xER+ z;cS1M1sb27LaV1zt>xUf<7KhwxD04mf6}nVWESA3|sp71xTK(80yOHX7`nlM>*X?^u+ zNxMRk@9Bld>a#b?F0^E2zo5?*G&(CeY|ZmE?$xu-+@@5r&(Xv_INVh4vc(y;RUb;v z)ZL5E#;KNfdx0F|mg=BVQf_}%JxDELn6OAJnJLp7T}z(;5z@4G7bCttpzcT4y$zPi zV|eMXAIw^{+eGaAEJ&q7E2KW2{_gT2Ig3j^SBj(;ZuCd$84T9KE0F^epkpGdtBe#t z`JldD+q$G$_)s-@Yxydj%+;jj?MYMAz zj3w6Bx40to$f&2s>0$S+6xDvv?$d-YaQi~y#R5JuY#1OD(M{IRCv@?M#J!V{XqMhv z`f1yDyFaUrWtMYdDi_n`&~Sf!&eM^Ji&FPtFx)u!%S(&f79C9M0uS!MyOvD7+i;n{ z_=RcvE*p*fX;r3Akscd+XOW=dOaqpfIxtH2(3qlymhWh?Hp5%$p)Syl7w{XZ2t$h^ z9oTV9~fzMKM3lci4;~JIG}kByuZ^2PKva>HYMSAHdXBW>S-yK@42m;bwGdXV@BRWA+n$PnGhkGgED!>VWjVjb1kc|^l)1vaW} z_T`q8H(pk}h!31joy`7&2DZbNb-}8iv|7kr)g(lLmMuf)&PYgWAVb)I3tm;>$Wg(fJ01de(;Q5rkN?%eGX>4?l%q-x}#Rpa(x3B zwoOM}mY;GA^3=L452o=6lDY-A`Mup$D;KGib6>;JnU`>nFEQPJ*y1gdtb^ZMF(xGh zZ>6Jx;Hc%Q>3cn@*ft1W1bBbaN1*A@7-UG66HjqZx{Vp9!^1R93dG=td)VL98I6s3 zqC{J5utt!QnkXYVHga6-%ZmZp#`~?KO)l-xJ8aUcEf?MjKwUpbxVQJc1oE8swrYfs z0iDV1xtun)70xd-bg(7V^i@r8AXVA-tCn8Uj z$&g|m?Q+xU%x=`%>2qUrFfB<`ib1SWiKxNkDKl6kaURWYyk zqDg>_63%;?7r76qo0>FQ3DZf55_cz6Li`0kTvyiVDI~{F=S}iB*UQ+q*R8**c+hn0 z>14v%r};#dSxjG5I-?`F-(B`G$S}3aX{GtZFF?*x)JB|1mFF2^j2XpM%`@XMO>s-kgiY2bQ5`#7_{zTY^<`Hi0i*15pUO=C@Dd-o z+L_0gHpC-Dw|{+UqL&aZg663qvV0y-dYf9LBQ0XOH-T=PP|oUsTqasLpG>_Yk*u>N zXH(^{AS&=jEHrQ&5Q@`Kq3h&w+iuBiL|T*Zy()+#R*Yu%T5L@ZXNw~i!eXCBY9>o) zNhc@tXe9K4|5zZv^r3rBr;(qLe!BN#jiI&4M}ZKTzorX4b7O2LL~uXpdUq*jcmH>T zLx$ia05kYf=O2&I%ntZRSEEPi=t&_r&NxFK$*3@FM)WCuh$_7S_(9?vfq~lYJOB4@gzTaC z_?%N?Wkp@Mhh-Kg|ss5KTf5MN5EohEfaoMHHw&i|A(RuM@YvSB6IqMMEsHarHL;fvr%9FC>-~msdhl0Ha0=Nx_6wTc=0kKxic*m0T)9Y8^8z2Y+HZkDU{9n)> zUUE=lQEl;epUjXCd8dra)s_b1k_Q>Xu@ zjWpM;v4ALeMS%p^D4?%iI5I=~c8-`%GRm2E#*Qu8ysS@*wDnMVRGJFv;-%MG0xQMy z%pTE$s155GT{0qRgRgD*yiB>JH`w*^SWJORSU2;xX{+on-vv*C_5p3mJ%yb0EX#ik z;^srwL(`zhHgy91sf><#w}WG&DI8e2^_rZc2k*3BC{yk|j`aT+wUTj=5BDq~H1V?Z zZ=RyQGE%yz+9;?g+%`HBA^}!AV~EXs4ZO{Kc)-kobr`UYX#OluP1Gyj@}WO>Y56bb z0tUt$nj(5=jA1^uz%C?r2;XNBm&a`?OEj4*A~jLN2CVcg(P`SeT6=3tVG46;4iMxl zv;Nzl-5wIoTxSpqTmCT_js&r8Vq75QK8#4;VC#GH*hYa?-k#vd?hkx{tKszzfQ%c& z8|kR~L$sW8vs%rVtM{v$5D^a5rx4PX74+?zsZ?wFttIlmEGhdksQbylW!ht+0X8s4 zVKnyygHb0m%K61)f8d?RdIB0ydYpmaNxLWM<)ym#9_}KPaw@T$_V#K3Jhk{%TY`b2dI13aL7G4#&xdp?xk?I!xTgtmRb; zY6WL=8I-1%;3{wH;@!tg8T;@22}SBB-A@-^X?&|DFP|6bp-lb)q@QDoxgg)XOz)(g zdA#CIRu=_R_szp^{!ViLn(mDJG;I7cL)=Sl_rsnhov3s+zQ~n@SkNs=O%&A4W`_nG z%!*g)Ay7rlO);!{^t8)WoCfrCbHK*+Ocof#wif*;95jcVvIo8Zb7mL_9N>9F;^~v} zf`EArkYF^(l!@6|*${B`Z{qM|E^E%*`>_Yu>1}xEXVaHsyZ-YlZaKBB;rSh};+VUK zAfQ)upbs2b3L5)i;AYIeP2V)0y|I@(*FZ-Dk!NK)8)6>6n##a`qrPm7J{DIX8>gF? zSwEE!DOL0B-&@5*Q6#y@QEs>2$^z_8RjMq*kg(V?uDyQOmCwf&A%GP}ULS~#(D|%j zm~7(gN0Y(t_6Y^1$b4n1DM4?m=eG*p?Uy!giBVuYN%H(K!lQ2TX1c0U-3f&>E# z(}aYpFNI38cD)x44bS`A&`zC=X9Qg7@K??>XL_gqMg1o0z> z4|B8xRZYe!sed7pchksH_TPI;@}RI+&^T{=k7t&KAP$Zs@M^amC2t5sE8N#prw+5F*Kep44qfN>&JX9kTt<3#4KZdd9Y_4{_E>+NJAwKZfE zdm_5UU!!Q8Bl}W-h>~)z5@QG_93=>`nuc{|VyJ~ZAfDa4O=^hXBfWKLj&%$Om0g}1 z?-L4-iRqQ|@gY4-_dcrSdFND}UN$i7(9FhC7y(A1)CYa~!?VE&z1%@&)$QaBv9K_c zp)0P-q*^~GR$8ESg}@*U5+Vg7^=Exz%UU%_1Infk3fq&6#T(dNg4vG3CJr zA&RvA{lzCv30ug!*CRt+9e9@XypkmaJ(L4|G%qEiT4NyD;H_&_U5l^~{c7mq1ZMZC znoR=iqwmBN`|`(@n5LORT-?LcRdq75+XFnb$yMc#ElvaSnfcuv7vw$Xd;3mu&_Q7j zcu(42bgx4~E*Cm2e~8TDNg!1|_%wKL`2YWPU5JKG^B@1z3eB$wCWzR$cHaU*0~4HG z2h<$j6@A<`z%_GqhCfu zs5*n@8Get{lgsuv-$RKfzby*jf7>!N-x3gRK=HKkbX-7U8Iv#V*`{{N_&Rjr$;*hyWu; z-Xi-fg%Ks_y(YE1KC#LyZxwKLNRFzLk$U_K*+c(+eenGAGJq8gn|oHaoMej>b397?w8xKs~oV<u@85txLg?6jX6T)x zJD7vFBWa?2cygCrGUEUg+;mN$=_kTq?fRw*c+w@pg>4{MVRQOE@T(~yP=_x{T0J`l=cOCccFEz z;5~BI_+_$aaFndCC0rHZVFGx3#rRl38=RN-OG%lj26~2uR~n;~&7VBilt@!Ed?Zl9gn_ol)4|}DSpxx$bzZ2SKw2(-s*#`D9|F^%bj`bGwY0qJch@>a%ikSd3DG#cN6 zIrqpi*oX1Q#-=#xB?bZ=*~KEADoouzva^n$Uf=>z0~MiKi>InScl3nLCs! zts?o;X3N^!mlunXv(;T8?&~A2*sUHG*$kUc@=`D<{a|d(^Hm4l>G8r+)iX^^J;n#Y zX3fs#rI@jbqe6GXeG+v&go3H_glcC>nn& zI2#65pPOqafH+?)JRXMzu3=u7s`8a!_!?I+g+y=_A5`q=km$tSRh^0 z?(24DzLV8RRglVV9evlNv9+vP?y{OTa)Jv$<7Z5TgVIF`1D~Yb+0)I(c!T{gbpS3! z#2sQP5b!QgAz)76)373h0pQB ze+Fv;cdjgCfQ)j<6!U{p#ho2H=9>-XRdW$4Z9QuOo{YbXUWkhck-}xo{ExzKA{NQy z9~c_N!EgiE;y9;Rvnozab#aDtnqMe#$98cyyb4-2W9|f@9UngbeO|%(_qIZ=YnAR# zP4rvKY)|nl_rKTnDf#o|0P=gy1vN~ zRpn0{=%9>V0Hyz$Io6yD0_lJAzU04k`o}otn)JgvhtA=tikXx&Bl^^r#q}+Kx~jw2 zJiX+TWEu#o3{v&2=6hK%q-rD?FW#DS9}ZWY9Y>VF)x#h&PhyH3l03fN?3-&O$k}_mndGRb6^4%Fgo~d7Otx^}Pv(5)RyAJ^zm#pfv|U&P3C=sWU^%mrjWSU$F32 z6_JHBiOfV*)&?5x?<(P{JI}4@Z!$4R@RRhh(hs$KI}aq=A+5DGqOSJ7Ia>8Mq z&Sdgwz1Z^akYS3XgZ9kt*T^IodepvC{xU~<5##Bji2Wp(GF5W$nq+DYq3H+XDfC;vY{jMeUQ;=|Qh(BZTzC}M zaIFzjd`VbUe~@&)UnRQ}CwTSded+YhCFr3guDO=)kThwH;bI&$!As5J0a%3Iq-oLb z9;Iit?8r^TlzD~e$VQ)2T}>S%oI-KnSXDCto8=|XtVhU2Y;v6BqFj@lq97PEqcoBq zr~9Ktb$ za05zLU=HiAtw;_CN|s!QaJ&u@f#~epm6lZBw~Nx>F{4`Ipk`B*j>_ud=QGh*67~U3 z<4H;o!Cw*oS$yceqs!Lh5N`RkDZxgsV>*h5p+1l^yVAMn-4!);W0X2hA_L)8-+f{+ zFkeGR3%5OuRBrltY_2!k`)ltrlv53G7ybp~p5Y?$B%^k`d&zQ^+ydsc7~WE6zuCH~ z3mbrFm7lD)0L*K%*1bZ!)qz>(!%+O=`*c#4kqd&tE6~^r!w8T)14~57)sk3*|g7+8dh*UA`4Ql(AR`1MA}#LpR^B$%5`qHydI^mIg={30uur3zw)i8U;j;ULht$ojMNTF zQM-^*Qx`ab2j35f#AQKqrWo6{?0qtl|L-wuypN8?!Sl2inSda{1p7w(*Su`%)sC;t zZXMf|Gnh}DN80I99Z_N5-GqJ)a9)rnPa9|OPoNYh8kmS5yj;OP{e&=>D-`#ew-{MJ_H#B0qcFPzEzUU-qfmuwt3))Mlz7 zY#-LyKK`1B^d?Q73(=2tY|x@3azb1hUUE$ivtgoWKzlDyh+FufZ5f{Tdh@%84< z;c{He^k9S&8>}Kq8M8U@(*<_AgTau`dDEOE)x#kF1X_%W0Dqe{)R!G_!)FcOU=E(m zJ_>D5q%&hu)vc57x7}iPT&pU>K|hr_yKcDzjLbmAG5-${r0Uz`@36z6Q&HB zYQ8;E8v~fAyyDUZ;=xW0V^q+DSk=g1pT+*aqM@OQi#r=gxrC%v8L;Lf_`PtaNxK)R z!q!y$V+QYwXxr4?zMWOcVwyzC<<^{(sGC>55pUa@4E!|~eU z^6}8Bf@q5TGd2Xm;)%UR6|ey7M-EZjXubs*L$yQ|K31~@>LUUNR=DO2iqy?|Vun0D4yPKfi0^)1dEzloG} zBGPFH7oPW*X8Zhkwk)1wI4Ho$J0Xl*Ob0M9wixlf=bFOHkaBq3t)E`WqAj5zMfcr& z`CULFNFz{U&iW8oHb6a{2cr&{!jI9Uxjrd_Dd_}W`X=y5sN#p#ZD;7Vf;qVYgN1_+gplROru)h<_{TtV#hu^q4K4xQU%pSit4 zYw+b>n)tY#&?s#LA^L}oz7)%=i8Rh0!FO8>kf8wMnYTWNe?cccGhMG#9*YVD=@pr; zA;BcH0Rc1vucr*Xq@`2`sG#x3nM@#Rq-3=0A!t>7Gk8P@@6zel(`wT-6ZB;YkFoVl zZ>#2$|2HK6rXq7TTpzr@t|Tt39^G8eY1U`Cj=vkcFL!AZ;i+$^ku zv*GsQB~98(AQj6O1gEpE`HxLril&Z`z`od)$A<&rPnR8vf=)F8Yk>-a2ZY;e^YW#_ z3fUEjrfFoS=_}$Z%fGwjEZD8og<`U6d?36(a#Yi3-*yEQ!Hu}odEA@TN%RHLQ@Pdw zvE^G(Zn5%!mhx}}VABBy7)-1LQNp0m{otwVN5nA`$f5n_Ry0)yisvJAiWo~p zBnRx(4RaL{TVhq{(BPLT6j70w&}y)~08Gs~q>5Q&N9v&XUJ{cD@YRWhI>#n@Mqr2$ zc9E1!CO=GTt2u?R>7ZD)O2y&?YSOlO*a>2a$}Eub-Wsx1g&4m{TgP9BdL3M+q6ERE z^CYVGl(BntJPo~sAK!*GBPIp8A|(CU=$apay>yC)-(EYdL&V`7qmBsc{eY`D27-$^ zXunWcDPM1T9{^q}S}J$x(WTF3dnyrmx?&1Q5Y`=JIA}HfTJVfaInKlAU(KfNtuvkj z4^ZGIiy9JpOS2_)3c06Iyvs6qHkkh-+ep(7henQxl?!A#RSbc90Mz8x5kfsdDvl@A#7q~$~=S;fFdZM()?!X ziv%?*_QZ}JE}c82u4$V}ZVEx8KU6Q64_qeB8?TSnqM_il3aUO`W(Z&C%u-HypW^Lx z!ofeXt0;fCcm9-8P*80?#2;RrXh9rjJx_Dl1;v$1g}JP$v|r=jn+j$CZ~IZe_W67_ z7L0%3eKKrBcck3?d~oDlXZmAZt<}&CV=gPkevBVm^Reu(PuE5B_fzIiZQo0bicj^D zZS%(I4Y|^3o?<(@Xj=a>n6B?zJuf3q~=Xj2=9Df{(i%jdl<;Us$)fSTd%LY+Z~gbne}i zO^F+)_Yr$HEpIy#T}-nka%3z!TeHc=+Y>9~J1vQKg4PZUGeim{-gqnoA%L2k1}3_2 zmx?2zm~pPg41raafVPe#p3Khi=jyTc7&MydIS(u>gy0b43j3X2+!a{*V<_k~ST=uo z5oU#} znldPNV5|82U98f|A%^7B0?{J$>8y~w{GV;riDS00A@+)2&S~q!G9o-#?gzGVLza(& z9E)84y69V0rD#|mh6iHXv#0pIh5e&{p6EG1V`))|5u7r#iBvkPAg^SE46S)+tc5jC zc2<6Y1#qBk_k1Ysa+xBLp8W_Pk^b$)Exog>h2xSttW}Y5JTF+}C{@HC4VGK$WK?je~>P)9yDQFn~E5|szT#V){@BmIkU%J_+sHaRLdd<2+ z4|51yX`zg~Q=gmFb37%jARr#wb2`D3eiWhkC5fQ%M<%6gj#_4 zyT3K6a|KG6--Lk)oj!hWeoq8J2EM!1;z}*ej##fXS&&FMLJaXZ6SwOIS;4U^GljXrY8M1*nJ#N>M_t8 z+ruzaX|3Yh2$7s?jXvDa8dsBghQTP8W^GZCUSZlhYIsr}{cYm|{K#lL?^kt3qMASc zuVQ3wE4``0Ct?+Zz;aq$5<+a7p5;k2OAb^cm||T63tr-%Ag}Lu*DR4_PynA9=mCfj zE-0%7P>^K?a^HioF|dF~6jqG+g%Ojgxc>=06f8WQOKGJ{Gxq#6oS!3C@LP_?6*U!U zFiW0hAd(4o_Nl8?1t(gGRhr<@audsok%pGdq(w%l!V!-D{UrQLc<+#i$Pj_D@7dFj z-_b+7TXjZzkwh5&4YT6J5K+hx_xbfcQn3(i<-_J@yXD*?=74hq#nhxzn0nHroxE># zA3{*dLJ59$mxe3gZak5{ro8?+D&Gs5C|dRVss%S_R`!}Oc=kPS)P@aL&XEmMR88Xn z^kzNbXt~_QygcCl+_{tzE;#nYFDm)JtV8!(>k(WnRXv2H)2;e^?U0_1varjc?XWwcM|%5pQk{t5L9ifX z9A6+AhZ_IZ3Zt(>C!ZTpR)gwLa4>3;#P)u7R&d6naS@O;YZI)0#i8geiZqo4{5UbG zJ~UgYrK6#{s6qO_SyGSNZjh2!Zj!DtvH~mi8Fk^7k5ZcyhNjdMZ!sn)FZY^K{Tq zMPOi76!0lW^+j?M?ppemShF3wubpIuZ|oH1=AD`?gk+f*3P2eA7b$1_4=Cv+@Wv%! z7W0qhhl#>?^F_iBgo%j8yo7DJ&Jj>R{!X_ zHg&F)nrVkY6sYMCmIZ0hV#mZ5InLb}yL|X&$93rJUlkaqtGhI#L-3>0#?!I#hdecy zR5Wavaba6?dcMa>(wKf1F6QJ|WaiK6oX-zoTilQ5-F&`A!Kf2?SZ=`?Tb7&8i@4!- z>OwLOvU*P})J1^tv_OD0tm*j$A|K)#%2Xy_LY?>)(Ayng-3sLDq4vS&2zGEGA(2) z5%;nrdh&|nMbz=;LWr*%y(l&2sAfJGTQX%#^Y{^MnRssJ*^e^y?SvDp)X7KH28)IKQhovvz6%6!W`t_|DP}2hyq_^VDEA3LIr%B}Zo1aaQJ(QlNzc>ehS59?J?QNx@tugG zv%*2^M>k%R4ro$#46#D52Vw*xq0yxGIGjpTGw^S{WgP7GJINb_o7IeGY@YZ* zi%@lVT%mcTtu6y`5^o!EUHJGRhu8f1zb&G3n!Ak+Vz}SCvo_KH0zw92d4Kuyc%{wX z{1|uPouu<|(gVMI{nju=X_Lw;5SCf$?|WIjV|!k4>^rwD4a?2PEy%8dsOu4m`soMc zwp3GyW2{pgzR7I{3sfKjXwtWy9eTfq$Y5^T<=7kn7R*nO=9WdtcrtXsn%vh(E@r5CWt5MEcGPPkb2oIkPt}7;hHODbRSK zBqz0J4d6-_*9OyBS;-9(LeJ|G-Fg070$u~BdU08Imn?3KF6I55_$FM@#M|0xtJI=q z+QaM8Ue2kyWHPBVP23o~S|LKBJ3)U4m<>g#n*o$4gjEpbx_60Ta)wB6ZUYN@;Tm~9 zVvMd#pdS{3KyuBfgs_X(!Y3BtZluuXPGPDfh!UIOPW2*WyY}cgTm8@PQ|_tQV@`$0 z-dmW;Z25fo-a*A#SJxRezKjrC@y*qqe10y`k=8&sE=PJT&J?hL=S;A+^}h#m%&~Bi zfW7e*Ov)V1;t`WP86aX+Dz*< zR#1kLtFs8|r_&%pr!PWG-1;D4c(b;Ww?gFAG>)`^$HhFD!j;%F?Jf3(7kcOeUwY6@ z|0fm)Q*vUv{L50snRsSYfeq2PG2TD=e?W8t+a1Hh)R^!4=|QG5D;74ey%i5wJ`s`F zB&YmN%PMRJi^$H&_*bgbFBbZ^FHg^Kg`B458ShNoo|V|mHLdnL(;A;n2Y%i#iL{mN z;A?(VA}B0z&hhJ8o=Bq%d-uEKmTOQ%ws;g@E;{lKA&MA^BRSUUaV{*(CdXB}i+3w= z` ze>;S&>{5|PIySc%fx|v3rqg>bfrW+~fqs2Hvx9>e4n;fR3(m~%3F?Wid)~2Zatth# zP=DBIcQa^EO~c>%tE06R8seaNAMytuc1IzcwElm0&w@Q;BH9s?tF3=3LCKzTbi6XR zxmDKT+;Pki=K;C=xCo1&rpPmn)S(f9C>h8lTC4M9CVGX1fPw%-<`Fu48u@8@eV(&) zggkDMtZ8T4C;d)=otbM(pZ0ewLGQGJX_+ad1(BRcbz6)`#;RILuU~8&!L0(CJ;Xrn z7paJRXhh?WtWAjhN6t{f16PMxUIm#bhTT-o*0HppMr zx|K0ng$~L#hynS3(O6{&a`s`j3V(-H)Bao617zC$j2z7G+OV&kC_>*r+6yh^XNBX)d2@OX|vxt&r@l188gWrAB z*TvxJl(t4xqMeo!?QGq^>x8L~_{Z6j-WRfXyLdiofk;XtJzT$u>sFS;;uH`c@sWe} zVG!0Y)cwjo_M5^4t^Y>T&DinGEW%ABHKf%lD*wSH{qWvg*a0HieY*|PBzhw;hT zFiJA3{20(}nlcrUl@sbbViakW)vyD(RTJ%mbt-fc7 zF{OA2Kc^eadrxfandV@eGLO9Fo7!gq0&f#%@(W@KB|L;r7-u7OgAAQ(xmACPG&Rc+ z<%OqS+%4|evT8HLSeq^<>uqhI(dzpF!z>lU#BUE9qzJdv<5)u$l^&hD9!1S!J$J$} zLiz#3VS-8l=94TjP%#i|obbi|;*9UDKbfz$N9M(bO<-pJihXvj*P24_+a2j)ivtTG z*0^G=uNY&13ofQf0}JbH@f73n)+YGGa?ob`;6_J`w}U(^K=xn8@(!vlTZdaa?|a0k z_k;3X656~}o;@Y}{9$w~K_gN~c+3ZaZ!93X>vU6tf9{IRGRMxF?*kT>+m@1S$M+me4I@5FgE!m)#`n}eygY!|B(Mglrtep?Z>0FA ze!Jru2e}D|2kP!7-Mw#mY-8iynAgF4RWs_eur3`Xp4$>9=riBC_OUhn`_m@I_`_nhoZOJs>d&mRjb%cY+wvkTpFuw zWLl%)$U;|rT9(fPk_s}(46%Azni`+koFgH-sB?mkF(%2QL|W#L?5yM2swMgl%pAH8 zCCw{m*M?Cz;YjQ;x`8fnO!xj7c6Qv1UMOtneex5egqCc%-dRi$R&C4re1(T)lF?&f z;?^4vS4UGaE+>{kOg@pO_d3SU2deax2+q!Qk_eO4md-tg+yR4eUc@{tMtgAa{_!i@bP_l%G^16Zr#F}^-va0OS zyDXC;*W^k~bWx>ps$$~)jI;otSCUb+Qwptuyc_$ zScLJV$|aThxCw)uv3B@rWvEL+J$!svCubvyR&hf2r7C^+Qf=x4e%1B4b%t3dupvt! z(4R?nb)i%O{qv_H)7b9&uw*T8;WT@>`*KFP(2F-5XkzAQBgK?p3g-tIk6%bb$`tSK zIh#Ke-tZ4$XT842_$^iaW4+`P?*vYsG>|i4*A3SEIjndz%HD3>2CapJl*T(5@fk`h zlS^uDSuBq~1+Hcsm{I2!v9C0+=c&1Sf@%`Ns+TXW7|v?&Wsq~4-za@#|$)IVsm0v{|rarWy6wdM9CzO8_h)Gh{_?BUN zmPpyWEY<&XS|acP$No<_Xv+0c%oK$Or2z!J)FQBf?3r>IoJQVcX>p~gPBG-wh;=CEV>r`(qTYd@oEP61*zwgbHOTOR`u{-3-TSc42`io` zU+NTPAlbkXUGaXdy6uEoQ5%628#goj$a*9XR6mU`=jj)!^b~GdlCJ$ZpOhylH9lLd zaGEP2O=o*>B$0a{Tz0i@lH;4w>B4D^tt89a7yPuK4z(z1qsI#95n=p?NX=Ch>(UnT z>af|;Lb{6lC1>hkeL;q_eiaEuUDQ6T&(;b8s97mveYj42`*S9<$~Y>;N{IxWHd0F%^pbEg&~M!y-Fk!^(*I?z7Zw2N+HgjRV| zMUEQW-pdHS@x9;vsC>itnSq%xy3z2>VcNaXdyKmBv?eHEUxCA`yX3=EC${i;2(L%* z%orymy+yYh-E z=W5>M47=gH{?w&E>x0?muwkXgNmp;~INs(Lp;wMfh!v_b$K?l{2ODI&C?#qB+o)3R9d& z>&4=V7)TCUf`J1*;5JWF7CM|NRUy&(4#)XJ}o8 zrSDWAYb{x0VaiK%I1f=Z1wmlm`3#|vqGS2GW!y;Jb#}v+R%yf}>ph_%yON`xB*8iK zihBjd?B)gpo(9q_A2S>C>YxWdKm5J>p!-s6t()I3b@L>Yh%W6c=DjA24r_%Rm}gbT z^RvDNWKjdC*5k&7BU;&TWRnpR|C-XmHtVDdvx`exS($ZQ;;{V5{;6W)A~{;9bdcf} zK+}iv7!6vllux7|t9>0pCm$q~(Yt4MI=E?&2yBWQ={iMM8~@4r$qM;9Fwji>A&@tE|_%On}#U z9p2IjH0RLuylNt*xRCmTCw)+Vp33O(3H1)#Y1b-f&nt)X7w0@oT|CUz2wJjQ3+qqU z3w-nDDwW~v@29VjGgQk2olcu>CF8ladj%!l=+i}2B>@0deC&?vi4_)6Cz z*S;g){OtF3)g1g?K4Ldfc)_K`pMKwL45;+bXKMij1J;}_gs~~#AvlMq5*J`3u2(jH zE^McVsiaqG&+%jCx2g4zvCcJpNw zBJ?Y#z8D~U=S~RXc{@GGe*CZfbA?PA!1P&F~rZkZ& zSZ^>Fj|)F2H)^`Q>HT5xT-U8!Vpu6Ij86n&s4w#Pl=p&D#1w*kDK}fgZiesVB@YvG z%9G*|7xl%XI&r@;!@Lb`KaoZq< zOWt?_PtXlfJ&O%XB-)G5!{rCy$&id=nCoG|cOC4wS9k4MJWcam*+ld2C@(nATa8cg);dC8$kN0$$E(VGGm#_AIMp~*_ z#a;JB8lcTB_I8i7_~ASpS}!Li;o7*7y!#$Xbedg$+=<(%lIpQ1$w2Csz22rgTt_3U zBToo$UO30}9aXm`AEMe~6;OO|G{dF+lb(Al2unjxZ5Oub3+)nAbD_=?5}YMg0Gc>_ zYu01LtiOkn%pi?%f2 zNq|{)_mR=Y_c+Q4!j``8qCCnfnIwpmI{dib^#!A=!Fo=yoZFF%5K96aO}(A|Jk8YS z@BhJG+zl2}{5ElCa>GWcOq_Q_&3*!d^1P<~;rAilAN&Dsw)t6L8m?)Yyp#5w4|my; zjs2om3_FQ{W~C9=y!kalVL*NxRfb0r<)9c-57vc`t&h+edN!XQPN?s7H!X%lZ-hIr zkhe~)DwG_7fd3_IoQ%b96Cx294&SCr`ZJU%njkbVS&FybMNOzmzHYNoKNV4DV6a9| zyEW+B;b@4&6}T3`ZT-;I^c5}aV1WJmYdHcxDZXr9zPt9(o)f(bw;WEQ z(>>zJ+_yv17?0lR35^c>YgaL{O#U$OaKvHXY4$sswJ?9KQeEwbw0x)j02KBXep|9Q zoZvX3l6n{wi&-iJuOoD7r=5-@=oRLIppYT;o$~F*7u#`8c_ee+;|v^VEeUjy^o!?s z>0Wkmu%FhPVw2DE4S00OEGO2W%hLr6F|485xm#+3 z=#dDURg63{JNXDayGmXuXpIh|mzy}JBcG66pH4W^zgY&evir8zsaaS2J(Yg?2OP#8 zabPx@ClY;AoE#da?~2I*F9%z;CVVOg542*^Qm=sVcY`<;^TXF--$Iqycgis7A^LnI z*qzu%0ddYSwOJ$lE;c#_0J&)p8i4X1r*8o@UNu-yikIFRtGfFy39&K-kJzM=qwSKgcS)O`oW{Bqgeqp-y9vf`&`JSd4bm~}m z#&cJOwdmlMve1Oc;DGkhpjJwAJtoylt;2_eYcGaFWia{>2Nt%9Wm9kvw0#_ z@;kES4E^6=hVbp9Kx0>2J>6qZ#5HHkz-jD#0UD; zQX76Gvo4_j=m;=TCIrhQ*&nMroe()%&H*16X~bF5?3!`N*lbRZ%vQt?fzsRf#j*Dg z9H?_de=0JSZXV4tLljA@K3Ziq3TCHgQo5;wmb%N z-&Q5V%sSdT-!6B?ct~g4G`{5S*i~ObtBzRv?{XpAh}8JKuT;8$ONw;lRE3vZzU7y5 z!H-zDfh<7b-x4`+?JS8`lCpq>F3m$ZoI^$WE2U#%O3%&v@A1TLy7Ga)Q;XVwZdHGl z*Q5Q+(Y@PF#(=LZze?XHn>=aTPCX>-#V0nfo31`_n8e%E-&Lz;y>LPGmX85D3rE`* z&{wp(X9)=pu?aovZ5*B1kU-!xQp^t{ zvA4k#prg31Y^FBk*TAd1J$Dm;Yd`1-3OZ}OB1*eK=;88{LKx=qcjD-D$0gG#mI`4c z;Y^$LSVF_o-YWU}Mb=sx8Z~D~8+DuLYzVA(*zgKpp$X20NGH2q1NE9#?*hb}@EJse zLYzCW#KogYpQzA8I`hhE^g_HWt*g;}LQx8d@u8&o1cQvK^XCb5^onIF-RDmlPh*>d z%RVzMC&n{AO9l>4-`EBSq>Y3Oi%;0*wFckxR8Kywl=tDTeUhc;GNaP4{JX4d9~2uU zw^5jEF*uN}kIwhx{%>zs(oca@(AH1}mMwEXFYu^cHLKS4wTT8VKC1Pr z{c*&bG3}UG@;L8z*W$?DnA{=VWd93>e^dcps8jRlt*d4+Z>g&0D_FP)+j%T0Ow%5p zxYTtKHo8?iOqH^KQY0{*%-z{rQ3Q(v5SGIL>Pzv3SOys))10q11muQr>#h^IYcj)m zB86dr`mhXj4yx(ALD{4MYa7j+hU@Ly)L%716a;Q;q2Yk{(^{=_@AdC{aRLqum!fQ9 zh;R^|Oy7?aT~ba`%x?jeRq5nu>L_`UTb35_;wuCmW2FCA)^|t4*#>>9vHBteQFd2v zj}kq))kTXWdasF+1ko+BSbbL~dM7%O5M7AgLqrfnjUL_Ny`S%W&v}yb%{hB^|G8(c znXAm)Grx2FJoQX8Kd!cTwyX8hk{3|5iaey+bTgl0 zswtpyru}gwt4t8WN)V$FT~##l5dly9r#0Oq?#+7j!?;?sZT{}A>4flehuVu1_-3if zBRY@)OU1LfR)@Y;@vur~`mPXX7FO+-(;VBshV?uPlU-qbEuz$7nF=kKX>Udk&EHY% zjsc?8_L)_uQ_^H74BSCjv!_=Bt%hgOF7!S4HP!pwYrwm@Mytxnx6fOa4;zfNM4ju` z8pet@A^sigtwH;NeK1dx{ifbPkeT5oJwg7KYD%0K5E<#@M!Cr|n9l+#5I?UCb0QQ8 z{6l}Z>)Li?+d2bWT5MueO7thQR#3Y`{CK_14^r2XTtd8+bXWb|OBX{9C!I1H`Kas} zYSY52tdy@%73~W8w=Stoa`UjZW*1xjd_ww7hhDv|nm5VdfU>)}5|_PdZ<#O$SG%Vm zzFgPvHu3}8UFWIVpZ{#N%+ttMVT%U{W-Sj27zebFDwvNl`akR3Rr68xf+>;jEBTF_ zta*m;6Z${T(PHwwo+KKEYLh`-FwuQ=UQqwUowvMa+k)r-*^P4-vEDQ-Mqm1n8Yo-| z7U{8@7?(2QB78f!kUY{I|NOV-8YV zmpS$RsG7j}neEm#@Pz6<2Yx%|s zf-7LNH|BoV6(7%r_DyKqK|x}}r~?Jf;kiLm+92X~tEUoP3;O~<*|>~1mDa&Zuf;Ce zxN#t>I8ced6V931Q@*A_U8T!7%N1HtS=%VUW8(rJOyo5&HfDecBi)i8YP!e@?gXMl zaN+dxEPMrQPzra%Ym~_WDc4uafJNxk%Wq9nwMTAS&8Nf}NAI;b*?-%*^Bnu+_bKV# zZc4nT0yy-{*HChd`#qfl8vzAt+=eX+jj7!v9EJvIsW3cjh*G*ulMBP`Ns3ixp4rC^ zI$2<9D&#Flc}7l*3taL(jSL9Ymk!z6omJ;|9^!!j~gDSBr4MGL)8hfU(t0Fnni6BdqKX=i#TkNv!K zd}U|SguB^Q?fZm>Atc(qxJoGKyATyLKcS^Q_1jnbvd9FF*{~jP8!M(Vk;3esxO*ND zYH>Pl%p*(AA^t!a6J1$zuWsFnhYnlGAwhL9BhK5~=vwWWVd^KlhbFEy1T@waf*|CZ zUomtZ$dp1z9R@d-s3;Hqwfoi7Y-Tarh96~x0F~-R1R35+_~rX1mWO6kj~Y36_1EJ0 zhch1(*{=CrvG8J7JqH;1qn>}R>AY~-Q>|`K=WU;R%8dhiVH`-NaT(ahA!U_8Q5Qw0OlRivV*W0lsE9F*;^X;tM&NMl^P1rwnYt&T-YZeN z7dr02yktv(TmGVHcZ<(?VRoISRMv||NwgRuf-Ue8y-3^=6mi(hI|xuHPtR171BsBR zYgtD_uZ;k!x;E?li2re9%eVT-BSSFB#LUU$`XKT4t%;THFQ<#MWcNWjuYSBCiXviN zCT9KoaGXXVN^MGh&KAKPRI26`6Yz2Ij;gr4R&r&l;jFMzq?|3$%??#DeI6l6v=26C zCB~ljRu~D<6%SrQ&zH9}5(@ zQNkiokl380jCbJ_7^8*{>GFKE_CU(^c6HkoFiM`P9K3oY*j&k%Ks}PZM)QzKe(Q z@d=Db{qjsJeyv?_MXvYFaZ&&Cej9B%8TPz*d{KFDn;40hZ2C@lJ+ex<^90j~uqnvz z@j^j{aZ_!2>y>xP{8dq;{-_!RsM{FErdv^h85Vpy`endAtf+<>=u@w3)Eu{D3D`LJ z^wlsWU(I6Uo@ZUB!UL%&`6K4IVmj)_ggxd`;f%jcb}qYnPecavPGT{vmK$ zRRRuZ5yPJRXX=)3x~{9NAksIOGrD3=jq!+7M8k~GTqEyY>@Jea|3eC7BApyg;ZsV^tTEr z*LG@!va~`rg`eH_TSjCWgA!`VW&ULnUhULqSFF|f)zPv0_uM*EM|O8eyNnOK2He#? z*mpfE4`EtkmGjQ5?HK*+G%P2)@6Xfjq43=GpCqEux}yEOa^W&kC}*<^-DSk{{t7BO zRdF?3Rw@S#`gA4=9aG=?zm?eeyK5bx1)Fg{Y(~_&VtL<8_PNcwS6Nn?_BJXaZ8VRh zj!MAc6V&2U@u5?Br4zy`#H>AR4*KIXa++paKN+8^4f6BSbBwhclC*497Yxh_1f z=1?!SM^S5U^GLUMrHwMcls}Hyz_~S@mSSA;dMtIq^P^XX|2bN9_(>WzI6I)q9!2P_ zx}70@qW{%cIPV9|7H_dt>0XEvc~7@YZ9k0yNj6UC5ak&6{_8cKob0y)EcRC%E)VB^wxoK$m<*cV^nK=4~h2FXTbHUx*1*O4~0{eWJ*eiL6O$+I3TE%jAU-ej~@%2 zo!~TZ4-Y9W5MB z1dOe^W;46ismW50MUxuQ+QKJLNpvJYHkMp+DR z<;_s1whlIT^g~m{&=o_d?P^fm;L3ZXM-DY#^<`Y7XRZU7Vq3I`Z8b?8Co;E_n;M`Cy#x|*za8R0X9S&+IQ|`DQO*- z_~L-@?wu_T>0f>7?zJMh+}S(%u|Knr@f#m}Z6Y?ZRXOP7{UAZXyR4iTi1WCibY~N}vxCP%I?}mA`PPnwdk~xUeG@w8K zJINLH?yx<4c9695t?Wwk+dBo;YWVp*if%$RTmF>G67GUy*Gb&>Zk;uS)N8!BnF`k? zfpX?Q69G!Cp
0wT@6K-XnfZPm3=!#LmB4}5=`b3SM$_okAa^S5ZzzK< zfw0ZE-z$VU{FO1gf4JU0`jug;=afrEgHtWFLdU1P0ViMc%}*X*97h<-ut9ep8TO*M z;#yWB-Kq&W-VIfNo$U;r#5xT$j?Uu=P=(JpHRy$_llOT+Rf>7-M)Z71K^g)K;$7;7Jei8Rjj>|GH6FEtKC?sjGc$r#~B94VLoQz)(SuIgg6FCeI zlP*Ra%boI1y}U(Gv(#amQMoPkgE)_VnoKb7O z!SOUX?lPZ6#a#+VMh8FK=~a^$`M8_GM~oAmb9KJguaau60`z5Cz{@rc(L$NRU7_ms zrmV4k`KSQfjy+0M4pOD1ZXuP_xX-V}jU16N8S8IgC^D>S#q>?B1NLXb1lQZ!2~z%3 z(uI8@S4;~Tp)YzTPW+)p}{7jZ#IG6LtBF1Av{L@nMh9r9fSg$G6@{oXF z)iBFuNvB?Oi#mC;Y(&|0h_@RC*|06iUwja4-a5$Ud=@66TcpfLOc6A5w~g{e)yi|N zWy3JAU>Z4;V4GaJrn75prn)t6)6{QGmb8$OcMLIUaD3U(u8+lrKW>!^&^F@kT5MSB zXt{GHAvmlGH-;xW$~jj_%lRzg{e#8kIM9Zwa(XMFNF8I6;<6w7^s4Oj;XwN=gHHL{ zzpJi>%*bL2b9m>7Gd8g`Zvdr)ZPO-Id^a-xJEooJrbB;TXFUpOz zlCqCljJn9IAJG#K{;k7^D?int%Sq}mEc8G(LR33mpN@~AjdC(T(Xe=v`ytT@Zkt2K z{F8hJyk%)ng$S{%aACSE{D<$@_kCZ>t9jZj_q9XXQ}IBn8UE22f4msN8@G}Bm!VUP zi(cY15~0I&uG5~woIkcY)#6O_YO-)}cYpAi zuTA991%F1L`P`TF0R(}C#DdTci(xmdySYQ7>LCYxQuQG+Ru8)mC+BeT^qH}|wBztY zy>dU3efXMUhRiA8EF^)I{aoeOS6jcNcx5(Oqqf7XX+!M^s^ChUZ~r=_S{}Qc-5ZEf z@b{v##L}AVFZU$TF^w5t9ncbVwB-hJ7=f3n7(#!xj1Es<7|ZImE{6S{x#$D#O2?6sJcOT~eHi?DM#T3`D7D2zT2<<-?`7;%)3zJlthL>hcZ}}1)U)#; zCEeVO;$_@3d?J8QjVwMrVl>xQe(f?~qe@?+fQhn2vG?B@c8ayIoR)ih(?R<_?bes2 zUpZm>4JHGfqRiWZynRc=_)P4J`%|j+Lf@T2K_>EQrMK?$Q~0cB*29g!&Yz|azzQ8k z8<)X|Um1A;Opcg(oUXmcJ{CCsq?bGUzi~C~9xy!+Ku(d4&#LU)oRfSzr<71-p(=Jq zeK}gBQixePOnFew#}Y{t#V>TxwIX`h&g)cKB=@zR)hf?mC~bq@Cw&f`dDe!PLey(P#rr zJTL3Z)KLJ(7&Kd1b34U)VAo$6H5T!SH~u!+?f8rbSrRZlcGB&Oj~-=*uf|>~GL&3$ zz1wlG3IpS9X|B9u8Y)Tm70_5=TFDC>RlmH}`Xy8JZqeBy^#f%+v7=p$rCP@JsPEHW zf8uQhUotcseH+3_9@9dAJP#O^36ReE-87k8ZMKlJICYamXna5`g6nx$-yK4mM zl{zjv4g6Ai`Y>bkW9-Qg2wE5ybL~m9({e#Q2QmG{0votN?>1Ovf`;n20#`|%Uh$i3 z75DF7JQ7{@B4I0tXjS}n6GV;+kir1c0nb&zkn74TS7TcQA66;_A{ujFp9xz(ANvJV z#|=-``Pw%Iwe^fHnQB-q*-)&Ik7=hJzXgvSG{Ms$*4M+fP;eI!&N;40yPe;^7{orF#yrI$uD;lG^j#fioH1oaO&lNz`B!pNp6g}e zBbyt8TJexf*cF@)ZU-?xEU(5Cm;AE5=LfR1u%*9Q&I)kRx17J@$M~``U0f?C;q?8m zz=7l_fXeP=($=k%YgO>$sf1dhcw#=R%;`vHahkUw6|QD|=cU*eSx{Ns6XPDGJ8vau z{`Ws|rf4WA+DqZ~_AfI?~#Yj!`BcA>L2K)cM>>}|(n(uAq_rRHho@MBLhkRsDDbne$IW!Q>IUTK< zb7ya2yMJtbm-(MaF(wowfK$ozjxGTYvp*vo;G6zW`XFE>gJ4;?ZzGxRq^uLA4eb5; zN=%cU)a8~z^*P~>t6_SqujXH+3u^LMYnIEsh4KeCjFSwW1gRvw)^jmb~7U9-b&{f8U?A`4>3+hXNe*0|>HE{c316xJ0?Na9V1H`BnI%+G zA9xyeSw#S*^PF}i{%WLd2W%_}A zI)pot@J~g?w*WmgFPcqZ(fB(KhaqG{M~;i7rr{P)CGtIsK*4^AOJZ#@Gm5t@?3e8Gvia=b_m+F zTU(;@E>jrNB_{^HI?Lj7)c*1RI1oVvP(R2zlAq*Ltit(o8igrGDq=RjAZLDw!y{=% zlh%Zeg@U2+oAp(8Z$@J=li0&J_f8rGJ!*i);y*dWN219(IqFR0-z$)NOhfB{ z^>gfB_M%3>OL1XfI(SDmo3~rZJYQAb;auBj)R;|@etE@}X40oWCpcIJ=w9140Ekvv ziYOhRYGzqE3ukRk9B7|&h|~>{MM*chuAA#Dca#Cw z9sMT|dB8AwGuT^?O##CVGOW@WtMU?v=yei!@ct*Gcxu&GWD_HAsyP9)VMr~*^%5`R zdFt_X1msec{8qO>PNtdwSGL-J>VRK|gn*F-XC0nTFZ1`5p)}e`W}V8>P=|B=WZ`A< zM+;23ogAqOWDNLpis6VFUg@f?E&ZjAvMImbRN%&h|3u*$I2rW6@)Zi~lUm!%ve>1% zQo}NoZ{5?F_l5OQprDa8I9<%F@AN!H>J-mP8*gUrYcPEU2#)`dLtrjwG8Q~j_Hs@dh+2#OL^VP-TYF@Hg5GKa)g|PI+D@F!$OA;nf)SuUeT^dptl)6# zc61!aBKv`yJh=KEU7-}eD~6q^WG%7xWdg7Jkun)E3JxYoisx=sWgSepN7r@dFE78O zK%!$FN`83rt-bLRBiE69abz=FIMM=oJ9*58*S$^WeyX^4dPUyeqE2^wrDw0^v(lqH zmZw+SN`H+6?GE6ngD3Rs%dYbILPdSyWO+f>K>m+ciI${D(3nEUT1 zcRyC;VYpafAIInkyD$iuS*F&V_V4gNyaUV?02j>69h$uH)88OmCS?lml$cFt-SRlg zfrbVKh11Km`r0g_&h=3bcfp$luRlOa>Z-&Q7Z$!IvgZ~)XSMrZ5 z!x>=G8}1Isvul{hn4G7By66L@DIk&gUUcCbp=G^e(G)XKeWLog?{-LL1zLl&x@ZF6 z-2QKLI5-Nnz=ka8V}9MRlA%?K-3$2~f%1F&!}9DJ;x?_`5Yvtnw|4wS$DiML9?e}H zi3kggiKhM&(>{vUDpQGu4&@XKCR-M|MQ9pOi(Ltz!r9^jBgw^9R~=O$ey| znqx+05R|;DYE@!lKQSbv*_MEXBBPBy@;AsaGe=8bru2=;x?p~auDzBhh!D*+t=DsR zx~OBS!HQ)@P~}nYR;D)4#3Z)${L@}1JP1-0tW5b`OL@^U-6_|N;#s|?+%b?PB{JAZ zM!%J6K_o$5F2H!1XA_@>qxgbRF9t%>ZPb@q=S4T`-jwaHgBAbOm&*_gp0G-@K}|~* zqFfXb2=KKIgnn}&O&tgKXr;LxZmxJ3_E&USj^z63U;7HiwzXhXa{^_CaB9gE$UJ^P=LfQXEuI>L0?1iGHG5GVNL%#nb&!%pH7<#s52et=_j+Szj_pRM zd018=VL2I3X!HxQ1Dv{8pA^*`pk2;9w=!oSM*3;-PG#b4d6_qy=0P+-qw+83=K2C7 z&ox}wE@7TA%fE;*W{G0LK(w12nI6FA2CsgFC7=sY5b z_ojGs&tkJ^VDAEe%d!k=Gdu)g4qRRCdr-a0ZbJX*9UM_suo5GElP&rc!e1&@rgelG zlu&n{zp`_Hj4F=>mYDJYtl3o*W~r*x;$AMoeJ9I2TWErIb!QCw z$;voOLH>Ht&=wu%p|atbN|)PNd9X||(0k7(*pCS(VLkC0mRGm13V)70w;xc+{yeh6 ze9*lRcu7h*%Oxb_=?5zouCT(kMT|6zAv(t8g=EyNey1KEf=%?hR?|dt?ms}dtKE5* z^$umCBOCvxJ{+LN_@EcaW(uLzw$^Y*fc1_(Jc?>Mdt#O?-|r-1Nh<$IqG&Zwe(AA> z^Mhz&7PrVM;pyQk>bv=tNdCX_M;t1!QU(|ZYi;cb0Dk|Dz5PZKMZIWpau&vnh3~*a zX_i;Dx%+lujmVjf0jS_7}>gUY)>rhtvO#q`|~UG^c9vU~|@>{c*xh zx3w9yLwT>!a>S)?jB{0O6J(OeE@t((v9DvWus+&EH%nI_M=BNm+3v5Pfnx?i;s9v& ztN}14TZo!QDNR^=;8YbW^jXXCIIk6TFS7x!6R*z^68x82=vhHfh@IykCkF}{zU5$C zj+W)ator$;T|;VDhYpSGoB&T$j`UCYF%tak1%C=87_97zsyHE}VtI_@VZ=y8FtXX4 zvqdUJO68=VFX=`PO{;Vr7NytA9<0dlj%hE2RDBxVYgm_&on8p{9jLMgTP81as{8z@<$KTFDL@-%~^9A zek;SIZ^u@N?`D~6{@`X3O-9bpE_kD69uzH8ar^$2?dFPO5yX9tY|$N%rhk=dsOZ3} Il&nJk2THRYC;$Ke literal 0 HcmV?d00001 diff --git a/docs/source/_static/images/DataFusion-Logo-Background-White.png b/docs/source/_static/images/DataFusion-Logo-Background-White.png deleted file mode 100644 index 023c2373fc4949037ca1e11bfb72e2e469a0bda7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12401 zcmX9_1vp;s8{USQHZvU`j&3$RZG4(B&2%>#Gfa0Kd^*R}Fx|1~nC_nL&Z+ zujfGBv{^-#oEVzuM>Z3=G;n2^y&z?e`!yY(m(6^^Xa?+BI z?jOD~oAP2GJ>sB)ONyzv&FrP2&8d&xbt(|=tFt9AOh!|W$bi{ga^IOLGj^KO zDlTTOcW-yAyxLC4hAPYlQ{AeYJ+Jq@c8Y60{z-mqeZBrC#hXyv1|DQPoGmqbcV5%- zvLfx%g8P*hh4oCOrQqk&&6wEmh{fN7&UHtfq|SRgJWi|KXs@!~s_o`U@?+;3K9%zQ+x#XMe%5@w&kYbGu{WKnh8UB( z42do0Wq5HZMTmU_O^UUv+6fK5Ja^S8Hwj7PG!7(sZ56jUUX)O~o^0)3g~@!jBrbFN z@9YH%3W^V3&&7fV`>SdRY<|f*`#Pt!_VhO%U-%ckE2OlCzGT+5#}U5#VDZJi`N~Xp z@%HZez_Y>5Jl)+a#JcnRbW|xNuM;K+FE?xpMBB^E&zI>>6V#Y<>SJ9VlE7gVLqw4B zBK%{lURjPvMsBpfP{PL%Iuy0|(iVt?(H?cU9L65l`O0;BOf=kbq0v2y^UeLu$#O7F zWc2-Ql=qDUJ^?{8eOGI=3H4O1<2T*Ta+ZnYOS!l{;A_ zA_7m*L0VSr%4csh^HQBcQ)7S^EEqk_&NpG~&3va9&>aVMo?Wg)k%Hl^OTYXiR?1|X zRu@CV!ldGv)K?du71>OecL=?C8VEG`x~(bj&f|Yxg8eZu>WyQ}GLCITqjwPe+s?u( zBC;55LbN?u%HHL)U69wiIP|m(?vVOU~9Xto~9SoHR`kf-#z%%bWM5xe|F(Z@X~^lo`k=J#}x-EYRH(1TwaULCJ3FZtmhy7*VZ zxj|Ku(^>nL0Q4-Xb{XXnlZ@-CNquXXV{c$XTq_ZlBt%Z<~TIh|rE5y{S*i#g+T7?K3& z#j?Uyze&BQ^h3MKiZdbL_WA-RB4qi}hQ7yfr8A&YtnDlMk0(rO`7jaZ)fnks`C169-2(30SG-FOHLttG_bqxL?1+Hkr$0J5z}ku~jiB+@}rRKyM-LxZ-`g zW1u3^@cC?8SZR>q5UCKHSsZ_j5PUkqrOr_qWic{`P3Ecl6ne*>0ZTTi&0 zk^fuR;}n5H`Txr>+8g*5dgwED`|j9lmwclW19YeYg+2epDgW)^0dH1gmBhR5Tkfw# zG#1g=yXk4tw|&1L`Gr%#yN40)Mk5ZIuS@))HJRS-Q-(oTC4)q2VB=o(>}pxv(<2L? zKn!ATVNKoMrcIbO! z_9rG;P+i45s={l`6~c~0ZU<3Av; zNW?%W06qP~!UO}M^@m$|8G+~5zcJXmQOEiMO_Jv)5?Rb!)uI7ai;qoAXYTr?I&MOf zwql>FeSAI859TW!jS9{25#_-ERWjt(vxHP3P(;7UK`ZEMuJ*h>&nok}IqD*o1_4YP z=|06Y0n2IgTC2L*iB20N?bIskNgl%A(I%Gu7)w01GitM5ZQMn?a}Ivhw``g}VFk@X zJ#=uc>7!t+S`_qybFTz8J*)JRXS$!}T^+X&k>r9CCUq4)fC@^DHBFM`hqX+MQ?1#R z`^|4rm5~kai?wIW^Tl92ho6<_xrG~jcTYB_%D&leD5u)TyS>cSDCMKV+Lc`1LiX4# z>ryRDw2wvxB_rmlkv{CqH`Gkr&ISy5@ljAuSVvvQYv}h7fvKFvZ#Oled{aN+IeKRD zNzJ`LLmkg}|6`fGB5#?s3|5(SYk|>5>uARUF}dW?euU*wrepD~YBQJ2I}q5vylA<9 zb1G^C%XQwu)qnPfRD;*+{`O)Bv({?7u(=RH9}7VBZao_2C}c!nCw4v|w-K8R`xL52 z!ydCrRJZ3<>T`3E=_jdGV>gevLImPFoqug?(VMgNm>Dw7eDN1LZb~O=Q`VFE+FI4r*r-`lai}^O!wmcW<$i$TVw6iGWt7! zhK5bVA$tBOX!6hV{z-*sXvGQjsrEGFx-J`#0J0ZvrKT1+0u!@tX8*zQ4Q9IvFl&pSx0= zhn}<1hJc>Kz&QH42%9AFCkH?EDhlD$;PpV7pr^t@u0KG9wD4${2anx?2dcpDb>033 zaFyh4*VjVmrk!{3%gSsfd&6#dG~F7Mt38`U2j5)iZZcvm>1u~Py~sel?o53Y-Zg5aR|dLiUWGY%tL&uVzZlQ z2hB4hG};8MxE4*s5Je$Z_Qy$9HQzhTWo#J#-;c&Uom?a++iI>)J`lHRcH`6Sum)m( z_r$_$n*QeFIgrAuWakwuarmU7OX-|nYl*g#)3vET^y4k{aWe_E6X7``N2~FZ^%5lq zVj3VGCC6{$wO`N;w3=(LVD!xp{A6bqodZ?GCx=7&uH)*|yu7>xU8d^;j!j<)6>@yt zxqivU?Ef$+p2LKb5am2$X^AD4Fs@QB(d93AwU07JUaH?Dur(RR#>1q6n;DsRw6u}o zjVNN50G~d!ME^}%uVbSAbEUk$eFQ5QNg{HUY}xio%KgoBEG#iR^S^QCDiLWcaX zFJFq}zCzb~?{DNgb7vtl(khC?pnk-Z-wMW>p^&$dqCvGp235di6Q<+j`Xu3zS_1=~ zl4KUcP5qmr_7}t+_Qc?;?q`_-@3O>E$}EPdS+6BT#^nC;Tb&EcY93LKY^BR+;r;!J zO5xV(d_hL8ClQ4G8nFC;>jInZ7#S|w#%-O(#mSX)yqNza2g+Bq2BJe5JU*M%YZfE| zUvWGm+GPChsi;9Vgn>KJ>H#Oz#9kp@dz~tJfvsIy|Ks)0$3HI8(vVS!1C)fXB$?ET zwJP4u3Nu=V)WA~zbwlOPf)uHTk^Mb(qv(F!21UVRLAyCVR%2>1pUj_04_8lI(sK8C zxYp6w)^jGVs@-@`mJw6V7TMC3dN^0mB=C-g0I$9{;g8d7D-6qrfyMvpVyI$f`%0$^ z_Fqex<0SKR_*E5b+7A>trx9qd$bZuP^q1bldBPxv(Jx%{zCD}NM(c{0oB3(j2Eey@ zjy_-S)vWvzB8i5&ff(Pu4DY+TYDZkk4(iG19I**sQK1S>t6Mh?mHw^aZ$X&j&1Kpop$L9E?9ng?zldnz+i6pWI`afIlPZxj>>=EC;@Ay@zu~(>-jjpbU;8tr75luKCMvwI z7YQeUNxiU}K&bw`%fyNMY|KQ$!(Kqj`ebwo_i>25fo&Ixid8ymbA2fzc^& z{AB}bD&7TxyAm;VJs12GShNxbC9A}Jaq$@W)|2E4^pFzHes!}UyO{}I z1CA%mulyQgn=)LyoWTyv7CKU1yVE_*_NhabvSKq=TNCAnHq6~%3%O$U(7M9350SWN zxE|)1YNu}dFRg5VHQ=|9qk8Pgs;lPY#i_px=Q__OW|+$X!PR2K%MN#10~wd$%w<2@ zpLX_SGX%ql;V4WPzrFKZsNc_NK=G!xOd6y@MN(<3Xfw3(PGHlU^p@NNF*$T zN~Mj!i=uzX#m@;By|iHExKQfF)mm-Z)Hpo;!gVW6;c1uKu zJH}_Nie{2*?GCQxXg_RR!9}4CN)R){6kAws)$3nA!%Vq2i(b7Ng2HfRr|P>Zicxgg zd$c%NsfD)u3e=8%7J{F>*p$}=dFf2pwdXU|NKzo=Kwfk7owKY?B1cc_{K{7&grdGh z32X0gayUu+W=?8hT;tOh<57QY?25GfU8r_b&?;(iUz=Unvn*?hf2k7JFH=YB6dqVxmbG?BbZk-C0G#O?cR_~BQ~(<!bv&HN z|KMV`VsL`X?@$(uD^@GkRwr}r=S+W%Lsj?w!QC{;w*U_Z#uR zRa0=1877?ToALMP{$k6~?_8QzGOFxwt`p~a#~zcG(-AqY=UL1P@QE7Nor&ds?!ma6 zA|!n{9&W{o;Mw-LoP5SGH4|4qk61x9T3T;kS(UxJV?j>n#N~&9x10s{E$x3zu|ecI zzEoJ7c_|HPAErlk@P&XB4V=f{^POc>Q3UV;o}Pmj2&8ekADd`pf2%j73Ug5bC&a+7 zBudSsrFVu$>5J_PA-r!RF!s~jVu?mdgXToIKZ*Q0=)H>lqQiOST4vtAHel?r^D3rh zY+INqQYqM1*zG-`033|?JX=ZM1FL)o8CwyIvr(4+QLmEn+L~aKx84ANX9-FTZr&Rrw>toe0FT|*cyEvs)>de_5?A(+yvhXIi!4MOKr#-`K*>1*qyL`)Ssk-(j! zpi1h2SBL18R^#hZJDHIvkVOIv1YY>Qb0&fOKhG> zMd^Q77`i-9DmCD7Mvf>_SOX`}6p~-X7-Mvxzy50EwvRNxtFM}M&SmcQrc2Yxv1EZQ z#|;jGWw5$eBw#-tC>bBr>b2U;;VG+eDF&WXFAh`3KYf^a z9Cp${Bl~&K2NyoULK&`9Wc%8`PuH6GomOQY+#*&QoO387b=;XK4g|;@ATqPpbHP9RA=?c!wbA!cMwS(hW7?`u!0^_~S6tVe=dXPGuWGJ&4WT1`Kkz0W@4Xn2 z8$=L72nkLX1gOJFxw5w~Ti{NUlqnrsFV8iQ8r{1V7v?hd z`GKgbKGZ@9G)i4YYt~yj_}BIPZCb@iy~6PUEVvmOy!r4qM((-9U^x}sT=}3j(*wm( zG6|{peG5{@T*=Ej`LS2d3KZ3kiFz_6eNst7w|DcbCrgBNdyLAU&Su-RhBiM26GUcm zm#F4laBG8X2)~DyL$wRq^D(q%kXxE*UMu(_dvo$1QQbjo6u;!n+y#Eo`;n%v+&yT` zVZn~nqStr_MCZfRwmOSu>Oh85pqd6l=HzMW-bXo5@Z&Bnea4q^rw$1;kP0pVHiJ3u z2so$p*|)^O7;+}w@!K6Q+Khkkrlj9=erT3aj}7C0Y8)z*Yd?=ah@Css@ffz8POG_$ z&xr;^!?kjmqhbLkJ>of`$9){bLV4e;NTWn!KNP z^9p6lVTK!dW+tl?6U}Q?J8+FU>SXXaugmk7SeMk%m2gh^w#LC|Mk-fHttq6qV=#4SK8wV{n`Ts6^E%WB1CJUSo% z*h(;CB*%C`uNdCIN5ZP4lghW1Mn7SrIvzp8JrAdbdpu?9VR$qX3zbWM;1>k<3sM!n zuectC0Vf&}Dg!xAB3pR>F#Bw0ahcwyZFw9t(TmKM=m~0cbtxj@32)`Zs;)C6)>O6} zli=Il@>oKtXSHZw!)*G&5&wyUQzwOX0_Af8*P)|Hz=dM70vtCWCE?D#-1JZ5@IvLz z<0noy=WUUrFPD?tqe3i8W&6symn2!xW| zSA*zK2F`=*Z!{_Ja<>CY)v|^$Rp(RFT;T@WSsu$hQfsr5Ugpy7L6@x&+U>HV;5_D_ zqn)3#XQ53ml9>a9Ksfj{p+a8KH4KkrK4%&|+C`A??V&xVHnVkh=YygrmG7qq5dFK&Q};MW0Kj|qx`AlaR~hX9#j zH9e9yn;=rS!53VF(YsWFIpblrHB^Y9_o-g?Ho|4SMGQxOFCl`7&9KAzg~jSg+gVw9A&qoyx9^(Y zqGP5>tjS#s%@{l-?I>RAuYT~EUs}OFH0dED)Up$O5_BJRD-M}`F7v}Y=Gp5jPt#r* zJWNs^d3XDfmHi-D&>CEgu#n+$hU0O_Ri@n<*~b0g5R^4DD9ZqnE;?FXZm;Cg0s?ZA zn8s8qwp~mfXWO)a*yK$24f07`CG%MC(BgXHiT5+i^5OuW;z<0urn`ST1vEWQYrQ?R zk_QV-Rd(zxH|4++39x`3PKwVH`_(}L(Jy>O{_byWemp>U{{Hh6fbQ|sl2Ckw{?AmR zAvV-Qv+lpS@fb4|eW$7jDk5-n?dN;mP6AsSfkO~lSc=a1BKo5pX)>&wfVYKefw^)? z%6b@s3%)+!QU40wya@YSmgQ+t!xk7zODneJd%G8c4pzqbszLyu=lJorQ*JvpXqPxFR7c>%r;rmDhzTpukE;zRi2?JClReQdB3R2?& zXsIlezl+fR0|4j{zbUPSzIx7UJC+}FFynm}yPBKcDgDgXXu|=Nr`M`q_Vmt&JFRuM zM(4z?A&FCk5wZ`Ef$jNo(I!9PBb|^><-|3oEXI@cL77Qo;4P98=-gnfXUA=9GhV1} zp=;7;eCB}4MptLu^trC-{99`@;86aBZ|6rvWqYq)EZ{2B{?3KoP8t5oVLsaAb94@Q z>6eYPPm&#>FZEiG^Au7DY`~iv_u>%J8Qiv>qv~0^*oqWb?S0){TCpmn>qn3GnKF2M zTEY)QmmZoP6AV2P5R(l)BWgy;#RrYWjPPjfKW{U;J{_wvt6?o}SM~qE7}vEu`~LAT z%0PlDW%n=}8zrB^hfS)KSC*aA8tYgUoIr^Ld>~`S6Q1^xg0FxiCtL>mp-i75uoN0fuqgvuQ;6E0rWBy# za*B;3vWvn){_f92Y^8=#+*3V>1)$1W!6M~BK3`~)t#6IpTPffj2*m(T!8c# zkH950|DQF@+En8}&MCU?&8qSXzF*@PoOy)?EzM*|XED{%E*Zj}tBVY5HDX-95D@t2 z?qlgv+D-8?*y7te{O5Q;p9E}~T($B*B*(+1en@8Oe^Pj(;Bj#TC_aFth}sUl@Xq$m z$K5i!`Fbv4vz2iTgLrRx3Q}zRrJm+vdxj@R4yafjZ}oJwnQT>eL}MTBJ(JA(Wc|-W zrhbY6FvChLWH&?ToGZijyfYhYS}4i z@U5cGd^@78od8DajL{$d=g7}oA#MjC=3BCn&i)<3@Zu#c1dm#dcdy zf%g0=_4&%vAbM({qWNdLdH3y7nEvRvm#`uct1*T(^W8h6dfU{mnZc~3ftZ@G<(P`Q z+**P>S?L}Z7<>t`oF`Efw?!wIvrRUV8_Y|VC|4j$GV+|;tlz=K%EgIPo(>-B>q%{X zsFlk~Cd{Z8Eq~lm-4Z+@B@vNs8}J+}nq@L@rWYbh^4yATuVx$HHMndViW~8u6;S`T zJEUL|-1{1u1Y?yG_V-~a;I9p; z4L1#acT1ONabMU?5YFek(U%O#+&ZePb_V&uhiCL_~_!)sT0W0{6 zUki-3P|J?$^Pi(rVe@KN6U%|80n+J68}W8O=g)uh_9~1JE3O+K%@9LPLI;Y6cOH9DK5=mu~MJONXuj6qR6~!J0QYIoR8l9>jw2p@X);`V#U4LM47|PaaJ{&Y}UF1uBWVO~}e})dBpFULlv$RTj zZUpelby81bTj*g-&$A*8P&wVG7pgr^`s$#AC^|Z+X~arj+ztx1!2R$k$=|-(NfB2@wR1z*L-)&-yp+oMpVA+tyqbsOEb<|@v+HwPjX|7D)-x9Z=L zhk-f>Pns?|FnPmjG6iL%4hw!{jPh8QZKJE3idwKg4FN7}7Fn!PZwTuaeQcvx>yU5l ze(c;%>96a{e)hV%GEcfGd(-$@HK&|M&8#x7a@ff~Gv*-`i8f;|KvQny!d(vfItZn< zMiNgK_r^0PknomkCw+w|6-I+cn&QYIz-*N@{l)a(v(iG#XI?ezScu`*-nx(th+ z$Gn;qW)bn0@LuJu5yI8bhs-gy|6`aQZxl<){_y!r(Go-Qy8H|r>$qvNCI`s$B znmp+qC>pSb1RT(W*n=+W?P`gZ-k9X^K0g9`ZIJWAv?zAx@Jg%9C_r9@3q!nSREO-i zLV2#rrDqWTikQ4v6joQLt*C&#bHbCFSn(_4N&h!578~ z_k$eBJJrBdXwXkvF}$fKyTvVsVljBRGZum+=gf)dX#4sYX>}@9l4BXgsH5G*T@z9X zqznn8dnFgMT!f4b0gB=0rgY$cHuJ!Hjv0DIQ^m*)0x>^T(U_Gf*Tw-E`f8i$Y)poV zTtCM+(&*xvmH<=i%2Ldb}pgX?%1iQD`1{9XG(FUdy%u6?-&h zSPH)Xo!3a!dp59<)mlFwVV&n<=zisG&% z=^{OsKYe=t3F)A)*m9*|$|AB5r;8=i(5?lXkvMey!rNgOY{LBP!-9HMB+RT zQ%e(alv_CO*^JF~esWJ3?^LcgMN-<|m||RR^IMTBDN)3`8GzBhL`W+9bcq2b13c0f zDdQJctX^vJgiZJ1$5scn)@vIPRnf8b z2^I1+qEc@bs;6A29%`(X6!^9LDSd#A&O~on{f-N@mzqgS8)Rq}EyyjQf#ld{^gd7N=eojTy+n&uBE-mpZ zihHe}lY~l7~*<&G^p# z!s3|zq(l;pRgQ53qMtA3oR#`{IpxI3H@4z|t*E?$M_hR#Gw&FT7kSF;&n0q!iQT=7 zgt!-|t)H2enV;rn8AmkV^Eu{A#l4Ia z&^%91U^q&e(3hjf;xyCeNRk1ESt!dRfyYPj%q6wIUH@fhQkndgYJ?C{&Qj~ipa^^X z5ZaCKaW~EK`K&!57Rf8+2_ykqmKK^fDY0*y-#3vxsF>< ze)1tGLnm!iQ&xr|x^hs-^eQ?Yux#OXyX>*)l%N*iY1a%1q5UQW5p?!SW4^ zbD`^Kj0R+t{dtz-&7Iw)LwsFFP3E%{v}q1zDL*I9R*8?9)(zN-=N&HxOW=TgH}>yy z6t9@b;UfcjeBdf1mrMRxp()^}Ns`W`Xpfslx<=A39?SU09x1@g%!lbY);uY96{PCu z`C3b9f89;YQt^e?Fxq+W4R7I4VFKYkb7|wTx%dywbLooqw2M`*>4q}$K6)RR(#wZ| zL44@?mwzFGW{wdX1avzOP@*|0L7$ z)|9E?sh0uQFHoAAg}8I|<##T0-k-L#M2=YWotW*#CRr~Kmg=yAv~4`?9U)B;anDRP z(=|l_JAI4(n`zj1W@w+J@A0e9)3yUnI%3pkez+5*2JC%~iC9vFKDE=Z3<6f|V5ky} zCrrmFD%(Wyx!GuGaag6F)Ug~83DcQ z4g&HW4ptOuvoMIRs6w1`8=3vtVxST0_@2U0@jMXtwiG3D7hMH)Nu0(o!_f+JCHIq) z#bBbL-vP-=*|K9|0vcviD_v0u?C1kgrg(b&aKhaJ;4tLguQN59Ox#TO#?0cAKPHB^4#oDmp(~+M z8Y3FFq$!oGez1U;+g}LQyn=?)(F2*cF;j_@+>E@ml4@^W+0ALray}mju6aa3-G=U4 zd+m#YQ5g|%_}sMvYKkyN@S~CX^vqi7MgQ0cv8K0+$NV1U_mGn$^2b%JvKlw(ikp6zPAqlPa`Jd6e;;=_U>GJsD zQh5=6*vgLvU3*Ul1Rg{V&s)1gOt988^?ep2u(kMQ18$TVCLbr0*(j(7k7BsR%z0kS zKb+Dx91>B@#Qug5&6xA{Kn17a^=U2;_<4=Lapwc z#7!f(oPcyH-V6Q*t%fDGQH;zStw5i+`RGC9^BZp&WcSBs$zFp%8zaf^vV+II*85a>EH>pGsIwp*O-$zr3tx11U{0V zn(sbIW59Bn;n9%9qn;-Ld)@K0Gt$^%9i!YOFB+Be&wfCCur8+#o}n2XbjwD#+$U|N zb2zZ-7>fzFqvej}x{Y+Xt;;gK4hFlC9JrQyDRj9qa?v5K%y+X8>CXwtIO8JD%DO1z zt|hyn+G&zPi*jw(==(D(Ko{#l{16Sy`x@=4`R)pyw{2mIqnfl>j~Bk=HW7nQ9R6+l z?@yk%Ou$B#keB14Em+K3Onjy&jz%S;(Y@0(Th*|1Fn`FGi+y=eF3dc>_ODs^%)D*x zvceI|jjTk8*g-l;$nB8mVf}`Pk%uV2Bon<|!C-G(x;&_5A;J4ED900pJ?oA6hq_~E z6t<%NTY#8%DTGAT&?A)}p&b13L+vAw6TQDm<#+B6K!d^UI8&zk5%qtlJNFR6D?D;V mju(Mgq+Yy}fadHy(vAE|h(j5=67)X=kKj^@k|p9FeE$b|>@&>( diff --git a/docs/source/_static/images/DataFusion-Logo-Background-White.svg b/docs/source/_static/images/DataFusion-Logo-Background-White.svg deleted file mode 100644 index b3bb47c5e..000000000 --- a/docs/source/_static/images/DataFusion-Logo-Background-White.svg +++ /dev/null @@ -1 +0,0 @@ -DataFUSION-Logo-Dark \ No newline at end of file diff --git a/docs/source/_static/images/DataFusion-Logo-Dark.png b/docs/source/_static/images/DataFusion-Logo-Dark.png deleted file mode 100644 index cc60f12a0e4f5a3bba66a01f377adda7a03c8113..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20134 zcmXtgby$?&^Y+p$DP0RoNq5(hOQ%XIE!`<40xn2{)Y2(}AV{}#NF&{iNO!*vpYQwo zgUbuAefFF)bDx=e?wN-OEe$0+Y)Whp2!sbyme&D+kP|^5Bvnjw;C~_$F&Kb`|3$A&ev zsB8%2)q2JX8I~*m2e1rp;eN6#JI0H(i(@ek6U1Ee*Ed6Y(|VrMxGQ;*l%G?5er&k`^~c#~aC?WqY@GkSC-1ora@`itoF_*N#*{6ybr> zib=n;?Xce_9Z&A^;kI~v2Zs`j3vnZnqs-Xv-aEC#=1=0bGcn})DpnY9`cP#ds?8Pd zqsEph=Y(XZ(Vw2Bn}rgAo>3RZ_;e~`g73E&O{ju*pdNJX2vC-&iaJeU3{9bzc9n_99`NU91l)^LIhWy&scxbMu%eV3Uhz=J;0VRY0zi_ z)w2IhM%Fv3wb%zKA9kK(eBL!-`v<}$Xj7aO1X_-cC@N;?t$rVT4{p}uS0 z7jI4$6I>v7(0o=It4@GbBTzB5I3?F?)&%F8Yl0h2J7D))^U6)GF@GJbm7@PMtcl|! z1$d%f)c&8`99zvv-COZBNB_E&ty3%VabgWjuw>yMo=ZO5Z0=5;*WtNsxSjb{1o2w*aBo<(|&`Vx!Q#tz@NL|%x$&Eb_t{K zT$ZKJ7_77d{+RSGNtD9YIZS;vaesui)gG+01=L`Df^H}&Uo>1J?_tccOR@As^ru>p zYgqa9NIG2TIWeASJ!=RupL)E9O7*JBD?n^(-7gp|X$*w9_#=2EFkH$j=>VV%^1f8- zfs?x-HAnvQFH3aY zkbSDmbc2Gy&fFG`F@KC<(l*O9;dBw`zQS~Ycb$B=BXI%iy}POa6y?eY6Wcv2hY9ok z9ySjgifl+?$~IVI_Zzq|SKVc#d{jAvutla+dR5|f+S(0G&twxo;oYy9m@T?aR~e-L zwM6&HiH5Ebaob3i?(|Ii<3Y;_E6iW6M!#{vgBNi}+w(;L4@mlT z-X;%(?DaLx_O-YY##$|%G1Z|X0QZpxpPLiaKl9zjDs?>LS!e-{>sxii934t`dIR$N z!^tSgw0F4of<2QVEc@qyepNVd1l}{K8uGMhoJ73GsGKHTgI9jcu{ohOj+QY<9C4Gn z6<>!}%uNVY*4&9sC6U%lBU$av#bythrQi7xtq^E`1pKTVkRjX@eJHUpZ`sedlR;?v#l(eDOV^s}i-JNN^6ZK@ffOz3j(^1vc<$9`?)*?qfIK zESHKddqOBCFWfW|_G`r8(y%%g3%F0BXB{!1sYs`AIxMYRz;GmME;mGg3*w|c_e*H@ z#eB~_@YGcnhy*+BBYqCAVXj$~*%s;+d`Ry91No=`z*gudAB|f7F3sVe!?RuN3;n1Momw z`1Ej7JO7=CT64zvfG`(T5%<_rKp`Es=Vddi`AsS-q|aWF26Z#dJ=#jgTU8=m!^P*e z*#D*?rhqMNlW7i|fumsBT&J%IG^b~r;2nQ%?;qPdy@nPj1NVNiT*a~DiRtHw6ijdz zG@PEfSai?nvyfil2}y!FM<)*mh(lpiWV_WaS*O*nrf;7z%O~(>82fAfumT})T5y;M zinY2}w=K;s3ERwa5@3i9g06ynAJIKeJ=fy?ruJnU@G$+6+mwBqs{wz2t);FMKM=&t zKEsUMege*Up{wjROAu)Jwdv7^!oH?Rx8TTqYBv)SO7NLHBo)_fD*7@|Kp@PL*tcQ` z$3NB}QDas3sun#jEdVdP)k4pTU`N`Xa z;r=FrBZQyaaj5$Z(JqNniHTAPTW~R`2b2Z!!Molc#{{zh=RTE)^F04;9||CWk~O^k zoK|$twDVB4kx{i-Faf~!h&W7B#FjeZ0Tn)HCW+2t8ooLPM5bi39nQ|ty-^8hnHi=z zUu+SYfj;0VW^CukR*cSA$%1^3$W(qyvbBFdc0LP4$Qx!ch+WWebh?apM2mr2Ec?vI zJpd~hL<43Zbe>e`(Z!X%AO_+f*YoLU1~TVbEQaq4q8-PsmpRcEnQ>D5sJx;Fb08Do z4%&9IQ><^7ogyzYcHP#c9P3&5Hcq_r}DYO7^K()$M z8Gx=OSt)!R*PB=%wEj^9kmbh$TUXB7kh|7rA7JPfk!>L*9LM+LWM&FY(SJ=ak|#vS zZL{b+7EW%u^8Rp4?o$9KC zDtR6Jz;J({g-fb!R1PP?D;9EgPTaxrEroLLfCo;IV8at+%o;5QfK}vI z_nv_2jI=k-W++sM>D$}pNQ1AeR1it2F42Rvi`PPw2=CG`E#D}R9$1Y`HO>YCCB(SJA5Yf8QK3SF%RavmFNi#nP7ppOP=|bBa~%nPUt-B&ymOT; zb+D+g2n4tp6XLc~ragw4&&xf5-`;c1$#6hCogBy6eMeX9iOum^2ol$xpSovdbsFcc zZiNE7NA=`=ew{S#Ua;1)1wR-GgX zwu4(q-6ICY-s2KL9#dZZ&)QmWU=O{luaaDu&0KXdHyQ+Y1~8%Fp`PaBK=HC-Qr5P2 ziCeCP#*qru0Rb4YA49J~Qeq>U-6aWZO4jQbF=%tv8B6gYqf1^0pdyuL;s8H$9pPvc zW$Cg5Xkd&^Q;ZH$uo9I}z}pYKOoO_Xf9fj)S9=|GJ7ZD02MsQbs&k2m%tVHsz?!7) z^ZasqSzSSAJL~0eag2`cg%p!quMh7otLL=#U-EciK6^i<1Pt_;Fe6zv?4KQ{LNT`k zrv*Nal(JR3FfoOw?4M3Lh6;bOKF;1iN4#FWa05O(fp*56jbGnhrPC2y^iT>{8cJ{Xp;p z*cwjk9WfZt3Lo=JE2$C?`iAZtUhxL6GniUw(D8i+;8O}Fv*ma+7kWNQvWk^Q81XWU z2grw_b^DJAzj*VQhioNA1epo6D70 z8iQt52Ad(i(Lz3me;eL0JE0NHP-wuyT>6rEYKZN3*3XuR=32TygrY!btOKv_3ajTN zN~Oc|@uC8WNaYIR57|rcSDxK|sEJJZ zxrAtAX=rHf^Ra8P*RDCgnZ!7faj%;IBAR}B1qLYVN~C3g#OnnsYT7=6>v;84-Z&1r9gDbX9I*QYxoD zdqk1|MbtJUmV#Dmh{G?0CHhJYu!`xI%S6QCua~uv!VhAflFfAam3AlncZ3mYrH_ja$8iI>s4R6SO*GXF@AV;IGbo9!1wLCwEuR?oD3{< zjb!@0w<0;BkB3gMF@PyP)v(Z{%NvG1F>gQkMm|0`g-=w-W9|!K2mL$>EPiMn-ZA$I z9_etCT&zCT&La#|&g?KvTZ?1$Cgs<+xWW_r8ieXqX7NMqIZxYxe}|khRfzqBlEYcZ z+f`nRlP2I7{FJeP>gWjH6`5uyeR^66U4A4zgQXi%x^4vhRzFc5LZh`iOUJqAkPNw z8D$(40TMRbXP1XoB?=%bm5`OK3DNm6V}ZMHjY0TGS|pA#sG3BPhm>&-`AER7*u+Q@ zK=Vkm$WM!>%F@Ta*O62`p(qZ__IfK<0eoArGG>1f%Us-p3@>nObo%ReWG8!-8{a)6 ze}##%XZ6>nTVH?H&d&sAh}{6>Zsq&D|0eq+RazET$# zxvyFl*o&VUq~ejA3RE?61o%eAcz1o;b_S?ejoyz}oMKHLH2e zdusEOY(A5-2y3YkLE#2!1ST^d0)LMF=04!PWPtJ+R5~WbK`x+A2M-7Dzg|Pma7>+kUp3f4eUZxXA%Bt=!1shLzuP#s z2+}74D6YZ_$MxpgS9txldnCtjQyc?1oX{wS`!sMPg-Bi;#8Y!Zo=bFJuahCjUI>P^LopmiDiv6R~+d_8GKc+yTmjs2@pS+JBya}JrL8L2m zCnAlg3;mT!Y^S~A?EbUJEcI#;`F3u8(tIvwL$!8N;%b^zt0nq)wm`cbPGN5m7Hk#N z46CAzgaN=I$5PTDQZ0JE-u3iJu2$YD{#kvl;%W_*tvTc|Jp_#G!oeKKh>cE38sFsp zJ3Hh0{@PtTjZz8h)0XAh0e%u0zYonYv2#xI1*I&eO^zia6WEO38WPCxU zuB|gjvm@rmIEpNR>U^u4U1_xW?|G&I=5>&E{rrAkM&6`BYT}vNu$|wKBEi^(@ExsE zJtst+F4iYlA76YIDI-`86Pyf={<{9~fgjW)cQjroVMsQ}>$}s_@0|ukK7bOr&01(-ZpI3UuziXV2A_?)Oyt2mCh}{A zSX*$e(GL=)T%9g+qwfu3Lq(R=IL`gN>Zm^fCZx4F*B}|I|EvsYJeir~rQ%27QCOQ< z6w6RY_%LF_%ozY!#67q4>ygUCIlkA3j^G+c3g!p>B7ioCiKs9kbVrb)9Iv7v#@_|! zG$W&FnUZO_J6#yoc%@9eD-FjMnB3)ioj~+`r*BCWiXN|C-y;g*Ac{{HI)Yp87T!_( zey{Ez$sP0nRPc>LU8n`q8%+^~0NXzBw$jud*dQ+I-gxuqF8#3g_|SIqpxI7+TpuBt zhsfiXU#t4DqxpMQL!+h=s@(Hu%S0WOVc&(5dGjreIq`zJg!5^vKG(C|>z~tSqES{a z5Us@WtFPBvWDVeCCz$c@>dib{c`_Zk_R5mnBjoK#Tbf!MpMQSSZ(LmY8kU*wr772j~WWh9EA* z#+D!g>3i0sD5>Gb8O$JBF9#K334`!l!skP1&QQ?XTHH=oyu*}d-bMiv4I&i20MkA9 z+|7PWTkgc53w-oLZ1-k6E&teRklFZt#s+cjJ8G?1$4j%)w3Cq-q{5k5E7~MUKp2p1 zm3?&Wtk6V{+2B$i(*$FI@HZddA+Lwb|IxAHSX*;7?~mLSv3-cT4O_lt3A%T0`}CFY za=?>~4I=$;qGVKCCU^moxZlsdy*pgEN7{Zw28wLvwG8mN)(&+ymPqIsHITr@H*VN& zvIqDM=7P%!My~YD`@?rdKn$82YnB4{&aFfb@pKQrHtAr}kNe$&>}^4-u@COwXQzY? zC2mmzPui}gY_fx#uV;PspB(|A;&W9rt7C9ACL)=5+yb)Pt@Nm1`FoU`f%awRVrza< z<%BCK3nSUsal_Q@F}}@^z@vt1{7)&x(|{L9^3W6Zpwn4S?tjy5e$L0vfd@@Z^|t9w zfZMrozb-7fCZ*5B`Aw3sGGpzsrTZ#2LJ$ISfmu#7Eaj20r5U|nFBt;FJVu82aJgv; zY1g;j(}9WX6>D$|GopIn|6cqF|EqKmh}%xL^Uv)A!a^XcU4V1qwS+~2#@+kEWJ6!* zNy_7zr2p=cw=~MiE^v6H6-S!kkQax71hk?mnRxdZqolwsJ|l=mj>|KaN#}mW`Y^Dd z39SS+DQ5EWt%l0MG<&j{2w|*8hO_sw_5gpONgO7_cSOeHechwW<|SRr-rMNZ=8hpq zdd>k%U@twErRcD$*FD!S6c5VRu#4F~4+JMP25r$l5BLHjw$td4qhHCNz5};hIW`V^ zlz(!<=g!!4udJV^1U><3=XF_}ZCBV(XBYdh?Mi33Itl;uXG*>|MkmF@&sP53_1Cr) zey8umAdv}MbW>plfePvM?tWHU^p%12?;gZ>;51A8_MhGTJI7P24m$HeIp}lm@{51P ze&x;)=}lW|G$I5=>hAs5OT5$AG=D*UslZY(#Y zC`y>G|91KIVc<~uI4u@5=IU2lvg@4!|8c%aN6M1_#iF4gPVGc_j+OT;b3_iy8^eTZ z+~)OAOBzL9PY1w}z?L*bbL9*WO{4`N2-}Pwx8ip4AA3b1Nwn`nBN%mY)Eede_L-zh zfeiLi;d0H=!`qb)TZ`BG5^)&dym`6)cs(Cvx975z#sT>`BbJdFvFM%Md>Ww?bmM$C zC0$Anu~-|uEU!qFE}I>8=W_RvDe;?ZwY(~X6hMJXHC ziXBWsCCW6O`R=ejujZXM?wCWe+kqUlNMG5E#P6Kn{%hH$(+e2lif2B3;8(LgsMoJ^%zf z#f)4_xm-!&x2au&D|d9L>9T?+*cWN#&+-+^(L^NM$`|+mjDoy&j@cr_4 z!{#a!`nZucR|w@=?Xjugk(Q3NGRh9XgNYRhpabB%cJ78-lqKp5x?^Y-OERV7-)0et zYdGb`t5xjwQln4&uiTK{kFf}9DkHlD?b8LG1_37X1s&uXDx3-B2|*nT1gP7~xv+7) zg;Cv^u__T*12yBmZ&gak$FXVi?haF18hbfa0yikW)f51#zz(c z3wh4WlO9PYhT;5zoMtfB8lxb znEe=H3i3oZUaq&+Ijr+JIcgm)-v6D1rIC}Xoa5cH?3@{0so8|ib)q2fVPKtmWMcbq z&+ z??jd)MO%Hs!j9c98vJU!A>Lm3Q$sP+(og+#f&|`iIkzachJ1arLdQSzh*X{h6Z|JL zd~!OuDj5p6R=$Vl!0NKulB5l^l}*7fs&*ZNw61YSMuX#`07FWqtYx6(yPp2vEC3&8 zs0xk>PHCO+*gHfS_{kIj0FGb)j`l=5?3!{MC(W-%IUHf(10ozg?Z)EkU(NjWFt&0WJSyW(2M&fjfF(M1Hv8Lq&|KA7H>wzFQmH2 zm$4>8)JsG0?2D~(;0xO7r7A_`+HqOg6U{;FfuGD&2=8`WZ7QZk2Bx+^-Eb739Pr9@ zzm%{jhs<{I;l{c)eag)~o8-_?gX=G|iATih zMwF4n)wQ=&uDC50Ae2;BX``>RZmwUDeA5WCjEZ{8F(K425K`fRHfYy; zd@5E|M{(PJ8+5CC#JhfGzfKG~f7KS@ofQDxwp!ZR^XVcdOxQJtn=_GqRKw59km%Ho8xXETOq0QzsZ_OX(D8aiu*M8`g z_>vXWAPKebrdv?w;?RFJ$40spxzyvxo7)AI)_v*wVdOWyH>*V^+oRk*qlLdQf6e_+ zQ-->|bY7$;X|GsLcQOcHB|MACc04heGk6XzI&^u$e!aRh>#>!XV4v#)eksD2Y*=5 zTm0|52lXy$j(Fh73{c$$i?)P3RsWd5ds`dR&j;G>R;>K*He<^jPG1FX)=i*^y6?RA zkK}XO@x)<-@-6-di0Nar)kSUpXj=cCN|LVA@`G}Ury>r_8+ekU=@!JWc5~YwlEK_LzaDN{M<7X*;QCNd+%PY@+?*JKA!*jK&1$7q*c$P4)$qm0 z95T1ILc4y}Deazf2OS3#oU#O+8nwkft{mQh3G|)oD3+eh`7xG2Mkv9ynSgZ;Rq9Me z3y^fh6k6LNwww|-;M!H#fPXJ?+>tC)aqtxu(YvX;3- z-h41cfemoOZ)aA%L|u=!9In~4`g6B5%kjl^AUrDj znVI|F06I3vhp|RI6HPQ)e<(6eLYcF7!qJZ#LOd$YR@NYE*ekm{XYu6&u>9?4rbx#E zZ9>=q+7Z-`q@4cG58K0+CTOCK=5+)K53*paw|g#LSS<#BOy!EmKP=GiUaV64*CU=D z75Qk&iVl}h_!q61uoDRj8gf_a7 z)4Ba(e6nrcJmWL?2@49g3UUDN4{Zl!-FuGJo#~zD+%zGiRCiIue!R?;*a@xeTHYZE zs|OZFkt2Vftc=69n~9odT$^KR4-eb%!@=<_Pv`1m+mkfrXFfh24YNdkV=BzXmN?!@V{e{GU`%!q(5WH&7o!no|^+^OJs*-#~8xC?UzkO z!Kep-XHhBV8}Vk}Qm_4*!tzEz!SU7wLWy*vuOti@aO3)vE+D1lA;~P8CLA9^ExcY* z%<1bG6VD03m{!&KC}S}%yP?i}l=JGW-oo6ztSAS_Pi{nx93G`j7G5CZXS^deIR$7_ zVq9y-5ms>os~@9^Lztc4PN~%$vqGkc-Z@F9YL!zD0wJ0fk7+?2sA}-zh%-N*gt$>@ zXKA+Eir!nf^0^6mV|{Ba)>AFk<)z$v|3X;7cfv7zLDh{hLAb>InJc2QORb>(AJn~0 z_Uy7S6;i-5NzaP(7QcNS5C?DrE)!lt7i%3vMOz!kU-_}OiDky z4iy3QP|j=Y1-grJwL}%Gyz=?HEg|Ej>qh!zH{TP5_ebObg!Zg+F_1Bt90WjHk_pV} zN!)WfgMNfb@V)?8?StUdZzg8og;Ok7>lmATLKg?O_qS<9iBTe|n zWPmR=79|TvjXtB|6;An|R-J(ChYnN(DDtB2jo#x335{>Mt>vQe<`9 zT~szcs%RqbVF|-9lH^^NHfK2oz*`gqL>Eg+8-;3hwj~3Y+OaA^Qo@O@)93{TjI8*e zyLp3KIN6<9k1nt(oo=+_d$OuO3Qm*R?+%1^b>{_5cpHS1#$-az(yYkQ2YIMbd{EBA zTR$Lrwn@HN^G?mwmgjwshJ${ejNduDbyH~ znTMl`uG6rC>j1`58A`&^cX6QyCjC+jVjlmO5@!U=7&*l}zUwS9xK1i2Nhq)f>}%Y- ze~pU)f|QR)mA%-0}=rvB!`Ego0=8sV=JJmKMx zmuR<>L9yg(EZa6h2aaZlKJomdaD|sT{z8rjkRXu-GBmMV(qr&Ec9~o&udSKO7p!q| ztT7vf+TA3E_PwbKnek@ED1jIjrv{0&iSnb$>Mc{7uw6NiD1;b(=Wr9s1$0M7u>Ioc z79#>DgAM3QG$^y@Vg5SoHZnCMAEWRxG*w>pI>{Nf z^M0XyFVVlTF)kBnhOM3xjj0k1|7O*xjZl>`Pi^Alb?U4R?2>v6HN$o~oM2pO!`c1~ zl)smi3q7(5V?~Eq;YzSgh!-`jtGdMOfx6UmkVbx*(eFn@C4h7lgL?x&llD*%9z|xxtgWu%83DfE>VcyJmm+^i?gr4K zF@${4^ff#kbVl((tjSW*9QJ(WWhW{tSqMaF7Y8mH#+_`Bu5;BZHeQNj9>4Ua{aKgt ze4rg95PnvCMsQv{`TgBO7a~SyN?&rWW6g4MXS=b;?N;x_TX(|4%fzVSm|4EBq&sO2 znEf9nIbUa_nuaO^@UW%c)!bz`VRcMzD(gggJ!S;|FK($$`AikQ@$0!5w6 z{2t9KA6>1NaxBZfb4i4fJs$(>=bgI}(t)HDS{?3_r9RDV4us z0O3yjix7=(vW(eQ8k-l?o26$a3#3WnIlYY>hS{@#bT$`9)1v4W^ixN0>bP&xo%#v4 z#5&hWp1*Ic;qLrm=HD+yv|ABn1N-+?nNT9rtm@+T7}}p-;+}ARCR_&z|nbr4EGyDNNfLUAY z=)1YYVR*_m$cu<~`H{b$Zd(r+O{^l%aArQzy3Mi5Q)Hs$s!4xY7wWbfg7`A9FGi@j zO+}$8X?4@7Xgsvf#=O@#u@D%_x;>8h^t2+;$cU(0QJngGB=5Xvf7WXNJ6? z5g(OexA%q@_e`N#epUae??ud zyZIWTrJTqX-sw9{e+V5nu@n}x`7reYpj~%&lrqg10LM8{Re=F*uk_AppgqN#i5-%# z)UWSx!fcXBmK=h(E!iw3`Z8Y=N04OeJVN_kt9-@QL7ZFrw*odGREY)U%k=&Mm}Iz? zY`MSA`5`YIaHGW`KJkq1ERsFv0EHAJobOMMo`W_$Q**F8HF z-(;axM%ofzWuJ;Oyq-#tS`9dZnEfY3URxG5YL<^hmQ>L+;f;(1g`YL%X`B}m?XjI= z28I_%?P#W1YJC7V)70mj3Dgy*Iv`+n^Is^Sd^8gl{Yw|Y)JBDBc%CjUpsQ6rf8+k1+n%?0Wy@Hxiy{KPz#;1^)LyjJxJ=xVz^TLc-A40) z%jl!|I7cKlQFsGZ#OU)xIb!SvfIJt+?b4gT6LnqO(Qr<56||&aoX!?90L5a-ZyG(F z8jPO#Rj@=Vvaw@8Im5t+T4`8AnN_Q ze#NUXXr;i^Lz3xkzF`&DulI25N!M2Y75%Dmg<0LOe)3^)^Nefp7f;cK&;R5wY3!>qJo$JtAJ* zD|4aERjz1eK;C9u>2abrh_qeS?i!$qh7}g{TvcdRZxHF40(FOb+i$6ecWTa&4VC9} zQUx%Fn#JqD{K}ykA|JB)55}&a`R&@i_)M}!Lb-S`{8MwK|MAM`Ma_ccFLnECU;8Z| zx--Xr=2#UQ4uF0yY=JkHYw3B3c_WN(2$)Mr8y|;?NBDE;)7~!C1ZxXlJqjhNQ~AF? z=4(9>&Mr*ZLj-2Ud?Xa&9 ziUGQO2bw>Ul;ES~!x=lcdgWREBGx*(6%4f%W4p&MLQ@ zE)kf(wkUf5COGPDG9hlj*a&uVp+wy!OWfdYJ6F2yV`Ixf)hD!6F z(x*kD4fA~~iSx-?(>9>A{^p9GtafR>fIu)hibtk%k6oC*j(D_Z;rNrZ2f&>o1`6l+ZZKJXT$iP$1(_ccj4ln7`q8%e^>v ze4YeY0QF{dp9P3Oj8fh@QC8%%a&7b;IOgMpBb(Q#M8Y{T)P3w6-_E%0? z^Q4l*j8HY+r7p$`6s)ALD+oje#p(IiMh~6sM}l?IPIms(V@ff;nGv)T)*O{oY#ZXr zoItmkYk}&#{X=UaFiMQqY8dk24{Z{{2WOL@=akuulOL02WpHW>c9z zSDbCltDitkFwhW=54L0R2_|Cx?os>SA$#=h1JKI znM++Sne7OZ*woIv{OTpcHgY&+f$smJNk$ZM5^<2T|p_(DGo3go(4R+X^rcF1)Sc%0Z+8i%@56U2tTm&~tXv79}tDsi4K#vNpO0fAB*s zJV2||-)LURLy=AEi=W(Q!U7cos9mDJBoXEEY+`p{e4!kGif4nc4JWH^vqi7Y$K>qglDoD-H8)G=w5IRTzoEa1-dKqz@&CYl*G_|+4b!_J*z!iOw!1q$SHGdVL>W=`n0p92}~_)k*Szucrm9}2K9)4nzw+~-=hXC;v$3;L)O z>eC1dvd-ENdx5C}@@W*I*oD6@b^9`nIUu}QR7;}sZasgY0rx9CBw|s31_)Kvv5X5+ zvd_(afSs|pTMW#njasJRgjb{XI?qPgvFKmHvy+D~%2`?IfL><`8Uj%Pr=x1rFC$FS z9K=+?vd3UuE-$W%v-GT6=!9+19DhE=~ja-hVw&(D-&vLjW%$MS-k+TVGj|Z$he( zR00EmNf)&*XyKRxd;x%lb$B=gD48jYuQzt>SEJIfLiz>s=mx5<3;{S5 z!beNoP6Ge3ZeH^KRnG*~DbWOA=@w@koYr7Un?Ay;DYByS~ z;U|*dO9Edu7K)L={puHu2|Jf54-R~2g&T^r`E$OwaONB><%H)2&vK?*fa})H(K)Jg zeMAJj*vZ{(VI~=)W1&<6+B{*j(6Ew79}`ib5m%8^@~HGR+S>7>LFkLSIcR7fg!uhh zE3qGU3?AOR(`E-6u>6+qS0k(NRsc>Vp#IX|#z*se6IL>Ul8g$qF!ZjNb-IJ2e;26U z+F;;a*7K0lrtjTy(VZXwL`tEW&t*lCkCO8z)1r!gK07v62T91)(0bHj_WRhubf}{U zOp=OXbh7qox8+;+j98~1Lc4WCeEuc|k%isBk`)xW8%MKoGAu*CuHCu;il)`~F)3e1 z`mDEP6d@9!DV;u(LVF}T%)TrxW@r{MN!BI~B(9x@0N zV()>D);z`$`Zr%`FHm!f2!*p6~kH8DGrc&-n z*lL$A2fY7+2ck=FGNs-oUfbVI9?soEO6lFRP4q@aDDQEy?^n6I+Wh}9dh2yh?up6$LZy;`{*mive+lDd1QvTrS3N0}I_9mKJ>SEdzbwalpfk2=u856zhV1mLu$|p1 z75R?JbLB+$ZKwYhG!gf1VgqPE%F`f$UK#!UUF^b6@3D>4bp`Cau1+?9`# zTSHMvC-eRw`ypj3=x^J->gB=zT70jdTEj@qV;+H!U6zDF@sb-cZ_{ROc1ck^ z`$-#m$dxC)*0{w%bsnGsex3VhL~I5X#c0laosL58O>>jdGcTn=qIYl8cHw$A{W$1s z^7wFfHPK(pO^|ttoG9U*aJb7L7G(91z3h8|<$u1{_BrS#vwPs?|3>YW&q&EY&EGp` ztG|6`W)xOb&`G2Hxl>M^kZ&qh12?ZqW=+Qffu6}c{at|KKD1lhKXWQ&7U8TA-w5S6 zwNgwq!kDI#Aobb6u&-@dykwaS%I98-!k92Fm(xs*J)%@Mjsu3quZ>?H-)Axo#0zZQ z{?kz#SRQA3=`3L?SmN5G$Dx_yD)HE|d!%(QWs?#%sa>rk#?!@yE#OqA-;jM09eZ?F^q3TC=5l zsaX@x@C7UU-lT{8{pya2nssPhMY7pDOR!0lTFbHa%4y^Kv!(M9w9fXIm9ko<^hEJJ zJ}dJf=2emd;Y|j>Xegw)=Cj|o7RL>&Jb&KOFbIL*H-I{jYv+DB54couLtA-nD)6%Z z9hH?r#Sq|N51@EL_IXa}sM31c(xdh_>c8Idt*X&gN9_I4`5Z8JNOzg-@9uiJ(;|xv z8yBNhd8)9Vg#u0b6uZM$TDou8aO!1}*sZNzzo$WY{>5GXyQ7!-K&${9>!swBl2CNQ z0x*b!rJXi=bjKp?oyc+>8+eI7&s%0?E-QL7i61~v5kNPCOmG(9@nSctYe$pQoYz1v z?BZ{Tggo$K30n!ap($uE6-*Rg4pYETk0ba}k>|JWml)A9sQ8?^%*@cG$#82uvQK#n zCd9{8i6^gjfKvZyC$-#h1A+D%DI*XV*nI!fx$V)QjFxU5u2}4s`Pv!-_!|vpNDWSZ z<@vTNQ_pY?-VR1&H~=qd%i=3+q6JR_<9G*6a5cu|VvLnUcu~wGgAVQrNxLLY%b?QC z9nEj;&exq8M_!xLjyM!?E5*}Hcl+QM8-Adbzi()R?ryFhZy&KHnr41K&wE_=0W1s* z1Qxk9H`NDFOV+@&rUcq<$q|oP8lSwDoKvy#mVO|dwzfbq2hBc&f0p{H6FrCCu-62F ztr_=?VhBt^)5}`FBSiB^mH%H4R~`>l`^86;T{H38$ymmWE!o$>SY~4EM0$}WQCY)l zsW)3SlQ0I!Si(%Cddbp|M#N-DvQ+jYOSV#ms8oLUj?d@!_ndj|x#ynq-20sK{eD?O zmueN?FQY5JG}Rb5IBHufXJ>1Ql1fRgZQ&j=$8q~Q{Qrqwd<3j!e zfHy`IJBMv``>Wz^SvhqQ5Of#In>Jc_t=3(?x^{N~+$9}KoxAFtNN%W>>!1 zlU!Dy6+yXD9MvUuFrLrlb#PfrPSDw+G;%C3zD3~!?X%Q#g7lPZX6?qg=7Xme1!?>Q zmTJUPGrQjsb1h-93OR-N;u<@!1bb#>u=AFe&1fSZjqe8%zorTZnafa&+>G{omQyHz44X4 z-cMPjWRoebb)|>Z6gRXY4zo_MzKPvmyf?3WH_S`79)%=^fz4OKX(M~zS%liDU->dK z6VwEsM^Ghw-z9x_cV*c-zYv*DG2Qp2k02L}w5Odac>#zkhG5$h`Zzd=IGUxB1kC51 z9B0*}+O$vDNKlOZ#8FL2Bp-T3_#mp1u1mu*-T>noDCRmGrP3?)Jcsv|w2tF1Ms2C@ zs2CCn8{3C*qJ> zX|$zvz_f=0N`_LMQ=_ra&y0RiY5YF3ceRl2l{(3fS*(m;{YQ%KsXeGv0~! z4xE}QfSn6%(i}tttC2E2tse7s*jO1`v2IJ$>@915Zz&|iC*jElOmW;TErh8m962D` zXR3{N0nSTxCVvrQ!A^CCB+hOqYcvmaXSpOkk^*-Y?Hr)CpO^EZeh-hL-fPN&IMlu^V@>q=zwU^Fg zG@r{Iv3lbDXna1*nCSvG-kA`lx-r>5$2-=YFlK7JHcU{yniwgVJ7m)H=KnTA2=-Xm zP*&;{gH^uy=%xsz24HL|55KpWJ=1BqY(zIj*J7FS@JB(W}eB_y82_<(fQo`_xF>uaX zsf;7ZUZYQq2}y(asAW<<&=)CH*@C}atCt9$Dpi49;_rR2xU$d4S;OM1FosV#T9r z(Fg}qh}M$_n}`q&@kLJIEz5MNL!IB-E3{EW{6iD4CxM+XlIF!m{k+U2GM^jRpb~8F-f(z9 zUiuaFZ-Atk_vXfQG$62q7TbUUTM3~$if2ZClFJio);2Id>P#;+P)NM!yyKJ7$j^Vl z02Dg|R+vrg?0{G;@n0>lXWq}qiQo)%n z{PS!t^?il#C1nz8#QGgDT0i(!T`CW@g|>W?L$2ZFP65kf;`HGT#+vqwB`5sU$qM20 z5UH!L0fx$K)kHFHZn>FH<4IJ~a8x^oBZhOpMZD+hIKh3yA_LeKc-SqYcVP3a6FZ(Z zj<1@z7`;bt{01>Pn2D|e*2MGR915urLpeK;y2|hvhWBX;L9evUjZbfL;emPUxui@q zK5|8v6JzQhX>kjMn=DcDyY=(%4?k>zkX%kM3R@PNtgw4e|2y8O0T7 zY&>runm>&=8Hs#}6#g4=20}mive;~4+ZD*I18(cT4^+yI?hp`*c<=I{ zBYR%DN6q`cZjxT9B5AOIi6?9UdRvk0hQ^5f)S9v*Ebsrg7y1v}WAElui5rTE^%}Mc zIT6uvfmGR#niQy?f{e)AR1ky z0QXGNR4;#J_n*mzi`h#DKE!}ngZ#|U>u

xThpJ{@XGqVFwspF`u-1bu5j~)W-_Ynt9txI``wwhEhL^y| z_k5yAlIFZ5hSiyk-{fq}&1KXRV)By!jUzxW01c;ifc!{!8-{Uw5fVSvMIwFNGsn(e zZ0)XkFsObE$I~?hYSds)f?7r9K;Z(^)Qnc=XD){DEKa@C>h$DS5E&e@D2-)g@5p6# z=58_RBZq>DyS?su;qIjCbw5eWA+V76mUW++8H_MWNEKZmftBP3pli9u79T`lD5+&2 zUh@)*_n~HxI6qw98sq?yrst4xHbF zPde}|OMq_@J}CiFNHTx`%yyGMxerDb-ZM+5zL81+SkBq{+r|4NHG!`P(Rf&(?%qER zVIgZK_Kteng3$h-hzuIX{ho4?xRo;(qd};vFS{Sk1Odx{`jkjm@S37PqQ#dHY0M{1 zarDeY<+o-cV}&7F`&Hlz0k1&MpTTE`3jBe-Q}jP0&_E7h5p&=e#^nG+OYRP0ww?!i zrE^?_Kmrl&fX^Cz+7|&jhLTNX%nh8!nre*N;n)ya_|1+IFf18RH_Gl|jrMzW7HE-l QKzI&v%+|@K7IP}?e|Xv(bN~PV diff --git a/docs/source/_static/images/DataFusion-Logo-Dark.svg b/docs/source/_static/images/DataFusion-Logo-Dark.svg deleted file mode 100644 index e16f24443..000000000 --- a/docs/source/_static/images/DataFusion-Logo-Dark.svg +++ /dev/null @@ -1 +0,0 @@ -DataFUSION-Logo-Dark \ No newline at end of file diff --git a/docs/source/_static/images/DataFusion-Logo-Light.png b/docs/source/_static/images/DataFusion-Logo-Light.png deleted file mode 100644 index 8992213b0e6072414aaddb574b2336cc6beca66b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 19102 zcmXtgbyQT}_cjtD($Xy;14x5(DIp*o(v6gK=Kv~_(lvBQN_Tg6cS%bQ3@}6UyL`Uy z`v=Rl;NEl2es(>3pM4EgQIf%aN&XTE2?<+HR!R*C2{|4K>6!eC=fH2QlqvOqe=r?2%6o+kVgYeY=U|m8~Q) zw|>XBFDOGZbV)DxP_f4GUdy}?;q|9_#!QBTB`C(wck%k$u@qh+3JV6boXrqfhYOX1 zk?XZl$(L|vB<2_2>A%U!r8V2um!P6|3dYnynhl_A=l$xu2CD|MtIGGj8GptxU(3#2 zL|G14%ok+r-HMElt*%l{$(z6`k5C01Y8j?Ho^cKeQZvxd(M`Dsdw6)DL!u{xk>I0;MA;$j~~Onf&%}B&L(QKcD{nd z1=7!8%*;zcfpx!u5Im9+2!N+|ZEEeD=CPTq75pHm=j2$Yh({0|ot}5;;d3M(2*DK3 zUmFPZ@C$#RNqxtbjrDFma@rpi`_Ol`zwv=1v5_F}REhq@Kc&tmvHKo0YJ2oMuBFCu zx_6j<#70d@%6p{ueKK#?g}z_Fel4ZPBB?T^sl>Gq{R|50EE-{w=9m3i5Fi+0?r!OmA*(M)DNjThEAVCzSpCY_|s-?Q38W+l!rV2TniJn*jj{Dgy! zPIOB>OCjCZn9TvIk){#w@7(sM9GQAJ-}NJt31 zFIGG!v{awuEqxHz#+xip?lHdp_mVMH>0H-&M5<$zE-=PDt0WM93^j~bgSpl-bOym;qELtBupP#<4rkO1q|V}`L8g6 zgYL^`q4FH@F<;f+UO-X#@;&Qg6~pY$@f62mKHqi#{&@I_MGe2V`5jtAUc^PK&-96o~0HOZ%NIyLc1;%Ohewo_YiL z5*X{hWs;;5>Q_s@#N{jpCq~iV6A@EJvC3-yZb@pl5mcw^w1^vC#0h!D#)%tP$_>1UwPhj9Py% zDvkDP1};->i;~o&dR)Lb@zSDne?20MAuhJ7_6(^~X#H-nNt}Dy$x~q(llQO;87V1| zslE3Yy`ppjg-#^$hsQ&EN=O^@1*>LK0F#G#q?5wcMtd9^nMX2eikB`3(LC!Pk z`o0A`HKUTdje!{{LqtI@qce?Ho?$)pH_tV?ZKQW!C zo|TGKSk=c0Au|vL|E$RGC-Hyhb%vahMo%{-EQu2@#zv*l56DLN^C!9$le26JYpHB{ zoVfEuQUp2xtL2MtRsY`nQ^z%*+=dkP?KR{->S+#Hm~@^M?(Q*ri=chN$H&E{Re@&* zb3@wiFlmjo^Y2kRaK<3w50HF!_#rjCh(B+pb{_-mQZrfGE!`5936^0q9W%1 z4npxV{^Uu}#nLYXieqGDe4kjV!VcWKgEEw*q{wsL@7F!~{S5CipYo*HZ5w)1NxiZk z$A$a>2Y?LVZB8-)UXRUsH+V5@Oe#%mfNl4C2T11w@`T_Fa+F%ZHqmO5T3j7y)KAXE zKpGj(qQ12&o=HjL>pZOuD3l67`xO!8yj)v*SDKk(qGs;g@l-Cgll9WGUAmiRyAs~N zuJu9!G9^eR>GagIT(u?vMPrV)+NXD9{oG4(j{tTF(e*rG2E4?Y4)SgvX_}`oct2Dm zx2;xF`nk-vDFbQ7>B79d`M}T(D;4;PdBx##Z8e7m4K6vc@xlVh4V=e^ASv?M zz4v5lE6XJH^S??X2G>h_%=cAtugndse-_us$HnvJC$9|((a87sfXlM6FeBT0MsEgl zs39Z`lWPHY9TYkIn(KrKnrRy!(>I@8^QO_-8)AhmcWs|zUq&dU&t3@7wNB2zCsP4- zsGY}Rk#0_C;GQ8tg6>h`s%Bj%uG#z^n($ z|1OtYi~GK1z>3ork+b?KCvmw_N#Q~08~q)H7*!RQ1=XavYV)YJO&FQn!0-8#g%`<) zzWT=7xPiYwfPH>7FH8nROkLY%#!4{;6A?-^PJv2jO_W0M%ee`Z;% zFGcxn9!llSRsd&=%Za=EL+wRf!69zKQI za66_gO3zrAJJR%j22XiI*H)5@v7=eAUvkAkxZa357=qdF&fdv^r(j0nH@J}$Yo&*`BGY3ZENZ6vo31IbFU)&Ks*47y@pmb|oMe|ZW}0XIb_ zT=fN0;kv|&Pi_6XoEWlbn4Y>agl)@Ylpq}+Xf#I+qI$ptkZ1n9U%_qQd<{_AUA|~O-+IyY5skZORHn@UV z2YjjOFVgUJ#R@UdA*#*t3b>~jpq7H>H8cZW1y~~~Q$K`clu&)8ZM2HQs3UV@Gbf9v zb}@w9i^s$?;Qv9cuHo3C%s{+VM)9|i-J2=TkQOB67vWWSXA5)dXi0*B@4y^~3|h!L zq~~-({Ur`pzm5}E8oF8 z9%?>|GhbpPd%Q6)olC~%%yzo=5VR}(Tvc0!Y~JBp5H7C};+r(e8>=(UEGC6^XaKMq#6~Tgb?=IH_db8v zu!yudd{bolV3d><+vApjY78i%RB*uxE9WqENUs4O)s0&?I``}f^MS|>A12Xr)*>|v()&wl) z&~Qk8By((D@r!Mqk{n-a9YeV$o__~5GByM#RE*;IW)&I<{-V(lq0dGYWo=wcDWVhS6= zT^RikUp`1)Hlq?2BHMaG`Y~OH@4fpfCb8SyodLO---9*8H*7yy148)%V4E?1OD3e< z?H_gz%TRVy7-fEr?7dtuPQ00ReveXZZ{mWFk3VnDDB@1V6wG8wom^V8h~XtL3^)AMK@@MBEmNF;ED?PaK{zBw;_rhjgMEF$05)_L)&{_|JPTxw%z zVer#35;rcU?$$vb>&IrFUQTf)vrpz`HY2@^r=#2CP#mz9A*ZByv)(S{<23y0i9(;L zR$R2QeB;ic*+DImR2bi8x_-ba{HrKavHJ_fGSjDV zrl-fYP^aZF>`@#0OW4P^z8mlR%zHEaE~7A}MP3{9^$%L-&6>i0!p%e9%saee=0(sd zFB+$b*@_bd>=%zwvjED~_YRob=p zk}AcV6L>%Coq`1;pJ-o1#w^CkV)4f98MPhMtJ`cCo+ z|Awgj<3{t~KjD@I)Pf0v=NJNcb>M%5UNfv$Shn$+~O! zHy7gW+~VeSk*-o_9Ofri=m9U{33iV5_Nl9OM~M&l9_tDW9eiaHGcbdh>_Uc{*N?I? z5QZvGpc;+%R_=+eh&~H`N1>m{xLp|RtV#}oT1GZfg7RdZO!&@mq{U+5G8>w2HJ@dQ z4$+nRDYqZ^6b@(-8UHCb`uuCg=$t}0q1!o5!oKeyDXT)_iJuJSYAf&>M~EF4Z0Xim zJchUZbFjamp49JcR|MIq8qbpnw0{Y10+Zg9xoD*u@834$_uGA%Io%504j#dK79^7C{@{ydH3CqUNmtNIfLbp%5)A zUSQ;4HsnjYC9hy-e4|sxZ{8ND)RbVjZmz7Y`M9?{JxQwWk^(MWYOyz2qfcqGSQJg^ zmFDjYY@XOWt0&6gWR3)^Ek+TCQ`8uJu>ZcczHArSC1=b}whd_Oo!`6Mw7$O(JH1VY za%fW_EC1N~uxqAJU9Q8~+JjlXy!u;5y?hn#r<_wXfDr34It~}(p(OEP^^*IKeIg-c zv6=8jD=(wN`cwMAb(8j7#SYT|F_f<5X;tq|0%PQDtLsQAIY|qw9tv;ndZJ3*s6U*{ zN`RtbW@vX&A04}on_RyMdcD;rzlBW11{IRo+n3j~_V_LSmi*g#kqUh%f83e%c<6t0 zr13!B24RxOG|TWTud4=TnsE1A6gC+`-x9H*4`uy!Z#eo~svtj=QbYHL*Jd4Z$=Jqx zp7MVE$jkyG_{zmpDc!{TpZ-Y*vkNa;agY0=`>+Ek7dswk?hBmxK5GW1`xxDGYc^?JeJHaI#WwI6z9KVOWnM zmXP^yA%las`mz10qz)5uCi(+C#**@ksXK|vd0P)>sLxreyy=Lpf9kdpp5siQAL^hv z3B*K|L@uk;L~jeffXjat2W%hoPo8Z;b~?;8B?zlPja>nh<(f7 zh{nckFN-I7q^6742`pW&F0b3wC>kt!fuqLnt`-qTcQO9gK8X+3g(g+MUPSr)1LwVG zo^n8k@XzPZZ@5U$xS6QYkD3#StXMGWaVkUao1Z4I`3G(qzk4mQAIzTx+eE1p-J>kn5kkt2X5}ZrkoKR2%NAI{$a%8j ziwZ9n`-ldSM+arnH{NulMWg&V?wM}RbN&>@Roy>b;huRdRWi1k5ec~Lb!qo}K^p?y zF2l6%&Q13=lDAo=n)y7u;HS{+2)pP9m{%$WIJN5we1iIP?0Ekl=?fcO)>eD26NNCr z&KgUg)e-dJn_8|v-GNT+M%Xpo<}^8VxonzmWbC-P`$1zzw;U68im{*j{se+xH~gyy z-G0KaLX(ZV=BS5#Sa#Bz$M!d;Zgd2E;~TS&g2oTJhB_(ZbP5&`0SE_W84j+_spK`w z6^uAenj4%ZyFN}P6?=yzHx#KlJXKMJ42CEIE#I`NsU5YP?OT))xwp9j`bsHwu^wST zZNfU;bqeKQ<_aGEW$Ir&#QDS8f2?->O_yDA%kZUSh^+8=7p{*T=LM)L2>^C$_4ne{n&sn6 z`}uzRsfWYcz`+^n(>362k$wDx&rr@IqSv~rAQi)B4yG24*cy zOYiypV8!CMOAolLfk&@QxLJ-%vHtLlsm;L76NrE=SQ`rg6Uw_gmd#l3(0rePF{tN? zPv3pick^cni77j|+PGCiJ}ULv^W&y-Y0Z)I_`5ObP@I$+H+py|^^`u6I}IL$UZe%NKQ@U{ z{e&!yz8(I&iWKD$WdWe0qao8wsan&2yOqA`zc0Qnnxe=_;yGTlyLi0MWN-b#cdE-} zW`~UY#{Sn=Ch0rks-3Af@)ox3{EKOdxM~ zD4TuWH7kAJNrfyPnu{WBexXxQm$;+&WPfD(^&gP??`oABlf%d5*Oc$v2R0qQmR`mK zh=zQ;Zg%>i<=|HTymDW&w9Gu70-3Nw%;hFGPbJxr7u{9)Qwc|EeU1NkOdNv`~QTxp?1& zW@38Hd6~q{_ZQgUd4QtzC@-J9j%$)Tq31SnV$Pnrj3?@Tvcbhp;?`Jzw#-p@dE_!F zBvLaBTxc5FHQ9aplg*tYUflBQwESV(8sLI{De(>WHE_+4C|FQi=^-;q;GpYskIZ?5 z0ST!4EpFP3CUr$q$79`}z#*B7;xD_jO%r7ifV}=V1B(Qn5o9TW^x^DWJonXgWd*>Y ze`K(s)s$3nCfEAB!{l;~9J(u#w_hllNK#V*WBKp{WY_yOYVmhh+OozP-zP1KXZ2((w+gVtE{v3gk zo*vX>o^GRQGe8}E-Xupnz+P?TMGgY%|62Lj^L6}KURPU(7USUAj>L{vcX(#~@pB%h zVfW)P42vYXC+gY5@oncAy!=~liKMz|cAbFx1J7a?2m9HA_2`!b&V5O+Ue8yEty^;0 z%WP-dke7AVEyYV^iL1tb&!6RMbXiCvVS}}D0d13{W(2aVP*W3nC}@jbe;o0#H9aDk zET>oKIAc+AINrAsR5+fQ$mLUXY0K0W z>l0ouXpon95{Iwb-$^|p+AyU`NXq9iLi~Qau*Xaku=C!}6x$7El$dcv012^>%l7cr zRK~b2CbNCTI}y324k$8QQF*ea=%cJdsqN_167B<`!(E5+f=J(RO1&E8{FSyF2A8Og zj`e*8foS325tRGZpFe(@>Hp6H$XwR-V9>ftt|KO$ZRP;o-VE33@|<;Z$?M&&-rjLI zLVf(LQg6o!{pp5abPv-hH0zoJ2euvsfLMzW19cuJepPc;ncK%WwSzqGF6^WMX=wNy z?9CH#@t^(8eDvt|kHaOYo;(zXXI7ry^mz%IU=E`wQuy7uNrVCQ1c@UpT>c;BdPZd5 zC4Zj!-a_zU1m^v48erMtwrdOeqK%>GVmIw;x2ZJu2!Hd9g|W3%-`#V6o7L3}M2kIl zPdlgFt5ZA1{R=mYbiC&9V|?sBZ$)@7Q%?-CYb3p!|561UqpD zd?NMT1f`aDFwVgGo4^kRqLQg-M7L&ZYJDc8`<1musMi-f;&J{G2Gqq zz--S76P~z4M6-Rs6~lhIzM)o3bCmtnsa8^Aj;~qf{EzmrvBTTPcz2tTr8Mc)dQKhp ze?C+1)tfXe<`9xYY^36(z}62C-4%DQBBB^Yko4o}8NdIJDF5?4lE`D<^=Fx4cf#vd zCYl~3aIS18L4}MBYsiQ|XPHM?d22iuTyh=2k^RX|_2co!CWa36LL8*Rsx zKh|UR8{V#oUlWhQjUy<0kLYe86_qWa9b}`kVRfKJB#8FfC>-9Vt>fbMLsUUDZ~H0Y zgpbCOuq#;Mi7V7KPI~y=i~l{@{jt+MZe`EpxH7Rtyzd54U}*lQuf1aK>LYY;=;#21 zDD*F&vcV^7FtP=Sqbl-B+?b`V4bH5}@cUlOW{%w!g!t)qq*arH%KxcCMu2_W%4z@D z8tgZ~6FrrlJnLn=TK&XsAC3QEeZy^E`2(YT@U%{a;(SsswwQb1K6t1pr@&ukknV!7 zH6BpDkJ8ug_Y)p7vhpJN{{@KmUYULuXcx5f1_A6OFgKw@WtCot-@g9|`2sFF5 zvGdt$#!a}7^JGEECQp z8pa2bUne<^pj1^RwXFj2pspDRB?iguA@<591>GfE!6$jBx7pC!t9h37r~cXF!gHzF z8`+Er7@U95nP=Ga@BUO}9cVwEsDd2S86SehtdM`Gwp3Dj#o0)@U(CD{)b^u^Vw2+O zRfS5LM6ZwHNn$ZDTc4C1-beTE;&HAx1Gp+{_C?w#%i%L^2(=6(4mK#k#tE7R%TqET zOlTO#_)|b=i0#3_5mPN!v$1~ePF3U+J65xK9}eI18*}Xb($rrM+GBlLNpVBXbsF;; z*h^e`AswkBP0Eb$ZLb)fK0~%ydQAqmL#&e3wXbwO(MfaEe)7n2rjOc&({0z+h2~1l zhAr~j`D23TA{V6qO63(j;Hg$CbkTS8bBwK4!^?_;xWxvD$O65$k%^J*QhBe726p>I z*TT3CDFJvo+uGj(iec$%OMEpV$7_-0;ElY1{wZ~>7;+~kWUf@>*80&bs5;M-Sq`^m z6whHoMO~l36yZX4#4hxd-Z3kEs^wQ@vYn%2kIxf*4(1w{0Di@m2?9>+O>1T*h?l3Q+`1d;AeN}nDXfdA$!K*r!KYo=va%Oc#5 z%5g)o`^Cw)1u?l4!;%Q49csugh}0Iv=dTnsn`9UGVc4(|tB8BjQ1nq?#QmKWr{p?- z@_p=_WFpmrl4u9kkA5a?_#og3P7&kak*`sE-~-{s$ia0q8O)^RP&Rwv+)zQC?44sM zPr!T@5Nsc3h~An|vyPKlI7qGkVskL^Xx*SKE+0nMfQqs%q3t0jI4-0ttz5xf-_@8n zaQc&svq8^)WhA|L!b)9NT}K9+65=*1){%DA&nuc=Grk+hw=9}6+ghU}R7vZw>!A(f znl^BR&81MT=mvaF&~|YxB)A4G7}dyJ@^@1`9X=uG(=r{*jhYKLQBpMKYGl`g&#n+V zX&b%O12fXWx$7%LAqjm-H*RMEH7~2^qvz|MGK+h$-ODBUr?)+RC2#1OSB+K4jF5cR zs<;SN#5G^hpdVPzz#sV;rbs~_HlLyaVO17yKGruiqPPys2yz4}Ws{J?@-l9qcEC)@ zG2Rc~A@6p}SLCIOsej!*nz2q>Z1PHhqhKzjq;Gd`{;;WoEzQ_SmvFKhArHK}(a_U! zy-XbE5R08?G8)CUnp}5ENn1=JcnH5!`k^FyrW+@XNH|`+P=T|PCg7R!BfrZDu5Ke@ zk`A&&Zjl2MxU2a<;_e91y^FIRZrQ0xnp*yL%oUjcmTAX+Wo@sB5Hyc=%gnIf8hd1b4kW7wfc- z*dlG^>jC$dWPV9x{F+4-@wU0*8uL!~mZb}8HdOY$j7u~WAO6jl0`aZfMB*@aw?EO> z;K=V)m6z(h1JZitwU~iDLr1wwES7SJ7tQhfN*59MfyJ z1o*D3XrBRpRskrU;s$P-zD7F>Ak|ElQdGXuePJQ(3Hx~Xef4*;UAa9)ZKyO)(>N2m z1H{yuK1p(i9!P>6S*zD;Db4&AnNOk)TKDG|rY9|3~&RSjJ{xZR5aO<8rFf zjc(-BbIr-%OS~mdwi3bi5JsgBu7cJ($3%eBxzgsW-3*D2;0T*2M$K#SF+OHx^HD|{ z9g4?=^z!gM-(rG4(&CuVd>SU7I+nrn>`i$o19?MdNqUjUU&N>_Hopz>)Wt2zh3% z(GJL-YZbB*>#ewJoV$?x9OU^kKGxQEQOF)V z(q+d$#$gBJPxdfRZ6nb5VJbXpJST2$&uKdahE&NDR29d$Q2G4%57J9c$Zy-fp$jzp zhpy3e^b5jC zrU)pTtu%rA6B6ldT$3fzY`mC)<50n8r(;u6(XiX zg3asyo%7%o!E0FJ&&4^(6eHzH{>omvA7SHe?;pOTMq6~tDgfs+`7wUj4DbRc6g$#C zuC9H3V9F(F8KCCTPn;tWEj=YwWnLjH4*p+2}5CQ4m9z;86mTB0(Cr1#bd9YzP}Hp1tMmg!ZX^rnbIvMyA{Jgt@vI)oMX_ z*j+2duDcJI&MJPk>M%(X-G1toK;?bY15z0SQcTLmYDvp#CST~kQ)IbsQ*6r*-wwi> zMENA&Pi5fU3jGfL=YSpnbkcsUq8;&>{I+^EW_FM|DO%>Zp0nyo&3PKCH|SVDIwt#a zV31n{mmg5VGb)5I_K?x* zgU#C7tPKmXmhBbhV{f#VcfQX^E@oh4V1Zdwn#`9C*)LVj z;J^c>YP#^{NPA_OVOlPSS4xX^M9c1SXZAwu!5dABa5@jo-~7f&3q~U1xjW(XXrKJi z4ET+aY`vKZfZ~oXjMbq$W3O=l246f?eTTu=lldw-FWXfF_Plzj}ZXJPWKn=Ryp`$pSr#&;zEDC2@DX_VGEAgn-P9P&H-MR z8VJ`mSaxM<-Y@i+;-JOBeG$j1k^5YG%UK+ADCg?#1(}QodUO)rwpXIpyc1Lc%z#f4Gh(e+V$$?BV%{hK;kt7A=7HL zT|^J~<{GpY*0EXlhxuTw=J2_veX-$C`aHsOYdoEurmZuuQYE73U_O?yuHiU3vCb-Wbezxr4}Mi`QLkO? z(Om2oCKtPbs|#HJ&~&KY;3kKam=znsx6bJU-o!K;d7Rv5TvjMwQ9Fa$q#)2j7`Hlw z0&}gRKbORjuTc7_wd103T-8pGJK@!xpYbz-pIE2n@liy--=47uI zA?Kw6<`aZnYce1az6>ua`}8F{$$9Tm)DZXK#&mT%c>bx)uzT{G!m40F-2UopUQT&o zxb?mA=}{|xa!7TH^e%{FMtHx4RNdQj3j=$kQzgqaPEpn?gjld#J;WfwU`2sPYANYL zO=$`$eoyc3mS+#!^1q@vJFE9&@%i^*>U=h_&Qh=7){i6j(>nY<60nOO-i%-_liQe4 z4XIIofZN*(^_Q{J*%nes66&tNU(TM$jUQ2uDg_kt`~79~JVy8by0I>HeQdPZUHUtc^4>euy{qA<4rM-_UxegGzofnG=dg;hii4%)(~-$hmA zp919YMpJpEI9)tD%_wt7)?tVT(UpK)6333jH66O=`W_1Z8 z2Y#kr_2FMSea>WlJ*DKYX~Eg=_t{h5H_FKCv0Se7z~Fx}FXC99^?trZJRFVRk2nK( z8r|F*)dMPCZlKbQe4#i|scKzz4eMJlz$*-Lo)z0|r56?vLcswpY0M9* zIuxdy^6JZ&@vpP1DpzIsujo0Zm|CqzrN}E^;tk~HWbYINdSmaTKP#Z+LXNd2yUBdMIg^;n5o~MS5<0Wv3%hb z5Mdx_iOVPsIFT@~+Z%y9)M3+71rNP6f!PArHo!++&xDCm-mD6Dx_)2qWNchVJ-=D? zR!c(8qp|g2?ktIzLu}-~jjZ%P7nN*{E5oRuIEhTP4yx}fF|24C5$rpeJc@ND`QrxH zXkyL%t)fG#V3^IYcf;NzCN9z$|6RwR1;_y#DbD^ETrU&6Bt||%uo0axxilP5Z4cfwa@4mKjhgK`nKmY{M2t#!5PF& zP)%2kW!0rN6HldxU9auhHcp$6Xp92B>yQA}lW|p{JSjvOx^≫uSMcw~$CUKE7o& zc60k-R+m_zNHC4sv<>I}ervA2=stG7W!1=b4fBmK?6Yk5 z^4MFI8}m<%WrwZ<=K4nJEiNJE-<%|tqZ^sKPCNu98M`!^eteoEOBBxx%Z5GKciqD6 za)0tQZE-dXpU5Z}xM;&T6@x>~RoE21Tln1`r38(ezl?uiUUmc1oaZ0GR8-X^{)!H@ zJ`1*j%v0Gdo$U*}v0XM*UuxZF?aL-mxRxz8qDCt+F5kLQD-v5?6 zNqZg))cf~@YWCS1t|Y;<$3cz~jVW@R%Uag2g7qYhlnW2i(}XR!emQ*RZ&d14NuK*` zPmXMa^8x5_&Hrrqxlt;y1Ly_M#l%LPC+)KgB;N0Zwbhe(D_hUIC6n}0=hxe-nfd6~ zL}838e_DI|_3-%1*F~W*9EC01l(OO`Wv$8HIsJ`Bo8mRWY#gBGos_jtnT%1nHlDh{ zPihxP?lo`^L?Z;5Px4~By#*}jF!t-MySO~HnZAq}C4tKE*R_J$#XiVm`C%O4A6dq~ z3+$Ju7M?P#BOiNAnJUXt^%)nf8M0RXVL+ySVlWX@F{Dlj2i* z)0>cSfIZ}}S_*GJ0EoEDU1x(J^Pg{CW$?-UkXiihCOc=*eNP>ON01`kapzwESu1Em z8+NtuOAbX!QQ_q`ar)>;PogO2h*4p*$$E&NU*}xq>YH-yt-o>BT=luIMU;waEh91? z68=yHH8b`@?*xhr-3>0jM#-HwZ7CI=F`CK9d5?5|eh!H$j4RJlkC#!IDG7YGczyE< z)DKyjIKtrqzJ8pD_d^{9k34SK#IJwlXePOgMq`~z*94@rL|D1E0=J2ryaVRP&&Gj- zI-N3C>MCtgc(;8+;V+)-dQR=$EWRTMFpAnrt8229%j4oDOMqj+C8cUrFa{k7^7)wx zue9x6a?u`8{V^E5eFZXc^N1^Xr~M$d>$b5~@_^^^P2HW31=1ee5ZUL+?$A1>djR5> zg9n$XX<>l0TYY?lQv6{4cdJKdPCxsPbnOd$?H+Iak0O3#GVA5XH9ez-Qjd0pMOByL zjo!d@6lyWs(O$;iQ7Hm_vc~M;{fhj^LSi-x7hwTD1KcURn-r@ z9~g)A;oeCt1}oO9{rXd<{M=mPx@qre+gHyVRL5hHtSizsl}H6U?q~{T2;OvJEuvoG zgK(MVZTPz`{sIHetFAi;jNQ%GY+8j|9^N1QzkV+k`r;(u{1KfuZxHW__7v@H3qKf< z`Z(N$7<9aHszS>?k9B*~{bdf0o_Xj0h-iOIf81U7yFWn?3x`9&4s!g{Zro`un*KUNS%gicXte8zTyaba=?#x4s%bv$4){`7?T@@BZ_8 zSy_DP^eVq42JNOCyB#^a&%kzUm^rkTQ@GqZ7-wyWkC@6O`>?+>9a)JHv zu6oHSYKcB5GwYp0x1Wo;o)}l;lcRWZt1y3{pgg3$+rBJxWagLo0XEqKz2qX_9ShUO zic@wnzU}%FJ77|^_9d`adefu*aycB`@G;}aqug*9C>c3<(#%F4ud6na$I9P4mLJ)G z%?uDv*G#w2`@94dFX(wYQU$w)7`l=!#MA5$$4%tg>U-lU^D2Jc`$ePu%{*YX|5E}6 zix-@;(^!x)_XB|fXEO21V}RFtp{xfuQDevYF`=>*`gGe+nJa z(fFB3-kTDU+&(uwZuYii%;@LP_#>OU82=*y$BP7#j>@3=yl4A9)pT#~;YA>>q;Fnx8BUXOFKgdPm|FNUk@cI-+LSb)kD|TM~h;4Y=nW-y~h~~;*rNKm%6 z!d31!h;Bbli!xOxyhG+49XX%C-hC{018GUK!r21_9(X>1jsrVVA#$I4+om7u%N-tX zlCHMv{jE0^TE`!eftioN_uaWQhk>ya1WYP8!x;JXruV7t{v({7^c~~0CvIb0wAXfj zjz4Xtl0fDw94^c41S~4|ZqTUj(}0p|TCo!r_lWTKxxBP_+|2x1U_u2H1^%JR50lsR zi8$@y?tZcEf${%*tFYs1?WgI}WwrX(S1A6LlUc;}kwz&MGW0?^?m(dT7qyss47-n> zzqUl}LxCD-i#)?!|AC<8uV`F+QdOA=B(ChS4;P_0$JtRlHzoz$^hejnN`K6K!7 zp(43ccbOyB!hdBPb`P_ex6WFrGZwD$MRuBxe?S|yjs7KMCwH3<;ST_hjAtE4OqF?t z?*A0+DL32=Irfe`T=?}GELI-W#6#!&U>KKuItavb1z5_%w%+63-E*Hzn>;7BWRp*^ zYb(Gwj73nLuK-Bn{i;O)as!6dDQ4vEtUXx@N@d5s5g zh;kVW1q}zl6RoSxVz0ZdZ_@28M>b1NFjrQ&YjJKN;DcvZWB!-F#SyFGSLrl;nV9*Q zBZtE5l}E}wUkgRL%hx{bbD@1&+z#-1vF}`zLUvANKq&AB)5<;j%q~|nhU+FhF=177 z|M@vW=(vRUcp#(3kgYME^=*blNJLlMDruA(y_jcxY2^2(ql|_qfr;;cQJ+Ea`1- zvGT@Jpjb&~l*(441g^*322P@1vEDDk5EZ&n!sgVZTT8kOZZBfT7tt#t_#ozR@;>cBFSvO&kVzxV47fm( zx+CB!FL3eWPm_xG#bFT}3%o94W>@5p6ZUZC@XeV8bE73Za@Ud%9kLFt6Y0HFAv0MLJ*gp56Fvs!|2Lz+#6Ip*$!C(>$4$@l zzM6s8NeRZPD9A;+j@R1pehwO{Jx%_LgGPdVz+?yZ62M&hCAMOO%g9&dJPu)bIozku z7kY5fN4noMv{ESbnEor)GpZ+LJA5^*?v)@2e?h>FaIxo569&4h_~KuZP@TbrxA>zt z*A|vq=-6UW$}ou?FyY%g2B70icY>K*-s<^dIgY&pOEMT+tZj~S)FZ+_oBN>YnHuiX z3iZd;3&Mh(tJx=Y3Oq*WRJA7Qh248flIC^60@w*#@YHJ6010yr;8G!^!LDl)c~^1) z+cM4;H!ob&Vc5sM{+N7PvW4<3N3w&&Mi|EG|CMn5@l5A&9H$*|=QK@rh|c=4{Fuz| zA6e04ex##Q5wTpmEeXx$N5fUuwo(|H&Cg`$I#x#^LynyM*1}GHtjo_e!pO2*?)%;S z{rNoJpU3DdXoIevSO&@ID8yoe5a z9>BAIUjoqXuT7TD%0BM+@NFO!MZoxXmZ{7>%p9sRc&c`P@TgJ((Y5AB54CH>kC>df zvCxZni_#M<(cn$BynV{3a8*jDr0@q5sD9g(r)e;9NDe>n*-o?2yg3RZ=~xqxc!+oG zWZ!W^V59|4P;S8_Qsstcg2x&Qe0;}A)T%CIJ?vfIKxMSDB~yRCmBzgXjUO(!m;`L& zTN+IT%vomf0~1-mb=7bw_)wOG?Q zi%((mLtZkGN)17}*NpUWsr{!t3uSSnfOMax`caLm72L(wp_C8>_tsW0gU7506DL-; z$cGE!{0;WY=TaWn@1TzVvfa-L;1#kdK*K_b=@{OhB8yZu3uV6_5UmqI6TJ)(L zhacYNLh20}TwdvBgVq}0R$;u)oJ&$sNItEm4+zVa^AyQhu|#XhwWr#>xINA?UCAF%E)#-|KNi5}AD7F(4V&$u@OcHnd+FvNfdBO6yX1-d9|zn7!a zvMI*lXGE2P042+Jq$j-xB2nUk-t&DgW+BF7-j^vy>`zp;Rp0nT9!O}Q}8Ywd2_1@9w$au z_vw%GUztUMB@(y0@U{)aPlMm1U;3YF! zG|2AhhADgvvMhCMX(i57pf*|GM%C)vn>+RUpSst=q0+#)B$)8@yvdH3v>0>~wTkb(wmK^20>_El5VToasS& zy5cc5A?g4MhvkK;mLrkxYxSF8fEVdS&mZ}yN10cZ_||#*ihe(6(a2c6^}ik1#le;c zPc8D?j=>|6YmvS;J)FOxZA-`?&|4S^=XkKegcEG(8F)KQA04?hw>gzh5juCd>Y54k z@F`p2opK4ZKIMQbfcYSVMQDQkd?F;nNVwVQkpH4{_FkJk6WzXHEYx(7qXp8P@Ue;E zk(qp4zS86DwE+7y}%rTcvFMm-=PFQO^w~DTqc&6lER$#2m ziMXCv$}aAAki^JO_y)N*Koz_;d#_%lEy{In;}63EVqGR8m!lZeSwIEBR5OB;EX9RZ zkE+HXMd+~aLk|L64?@U)m>o+Uh!H!3CYPN8`|Fxa0qLQd@2BV68Z$I!fGpf&2kLD| z{dNx*BZCWQ+QCW9P_+%~KD4ULsa*6$XGxXt9WAxHgK0p_&3QtOuP7_PVuPzQBqdHi ztvseEW)7~Z)kG~oo@Yq@HHaV89T8#(YjQMU|Asou5veQs#*>pqk8O*}R*JQrL|&7W z4QeBLZ=zv`ChmXvh$(X-J=Q`~9)}Yco*Z#RL*b-J`4<`beOVj`=i&rwXk>dtkCj;8 zz`AM_O-RhEu&wb)vt6`W`6G8_vHz7tYz3I7IPq=t$Aklf{WTUq^Xj3}tS7Wj2YO5?6{?_)F$xtL}R zWY#g6tAWnu38R#6J$QS-M;#aD!O?FGzVF4ugC!|E$n}6Kp?CS=5m6>wP}T(M0P#S67*R= z5BS$wBA!g%!J+;)NxrF>jpW3bB6fi7`Ik@+&W%2Rnes3LpDR|$*IaDataFUSION-Logo-Light \ No newline at end of file diff --git a/docs/source/_static/images/original.png b/docs/source/_static/images/original.png new file mode 100644 index 0000000000000000000000000000000000000000..687f946760b059a463901537403ed81a1adfab69 GIT binary patch literal 26337 zcmX_n1yo#1(=EZ>-5r9v>)^rN9RdV`4l=k5?(QBSxVr^+7$iV&4GzHx`ndOg?_cYT zuGOczYIjxdk`t+>B8P@dj0^<@g{B}ctpNoEL-qdsJ0kr1`+fVi|N94$v%J0=6ch^1 zKd%o^SviF7e|~V&kOM%~Op_kH|A4iTRF;H-s*gu`Hiv_P`s|?~Eve=8;jG)xnbgi> z6QqJ4fs%d7Chc%)$^Dn(#QdpHQcekHs)vJzW{b#PHf@PP?5tyF=`!+>eu@GR9+-zUwR>~x0y!#Ml4hZsIR$a=D>F| zU0**GT2EWwGsl!)Pd`&zUq4en^WZI)B-k0z81-t#6?0Ek3yM5%{*&qv>2FaqQ0#BK zXH{?n7Uf16h@=i-doXOHW;|@T^J>?xx;IePLfA{0rXMlA) zU3`_6d>|$Lb+aGKthgez?EfY!%)(&>v;>@=$-r@__Qd#B30cg9ySRDXx&JOlP@vRV z5$@k9#x4rI3s{I`DhyEk6itZr-%xM{&PPB?v}A$gI^{NpwG0dr5@fovFAfbF3#Wej zE(wov@!DMCKHewJq)Mp??I)n_3oO-@8i8DGuMf`uX5R+8?MRPK&#f+{hFM2w3Tf;V zdkg6s?Ej^VAI1N&7GP|q$0>55;I!sVpvR&w-JIFy{)tE-V3D}ajb37$I{JPmzSE7 z9-7zor$#=0Ck7GgKhOo}8W|a-pUZIfzZzLU|G@Ss8Jqu+2&t^)N3GM?CV6-0mh;hY%5I7Q=U^7*&thrCiiQDa1Tim{15gaQ+E!;MCx0r*;`}&#n^Ve#96+`wu-#Fx7Af z)$XsmJ|+dU{4(Ybx*pU=l=ASscjMgM*p!ol0KEm|bR|QnM^n4e%7&iim)jo*|FaT8 zi?B~%hqr~*Xj<;izt+uRBRt*?i8+&)XL3IJL)_wFP_QVeun&fD#K;e||8Kg_w$h`! z?A=5u6BV}kR9U_l!e4EUypSeuL!cr#$+7AHgIK4&XJ;h;QwpNgxY{1rVFsZ`sJU0g zbR@j(Dpe6syL@-*_$STeVaPKY67En7OP8dwv_p%(vXU9e9!B;5aZKRE%}a1~?3LGf z5`02srqYuQ(jx-QGrurNWRSsKHj8uR#~3pxug9Ned}X(+*Ms2XLa;T`f6hW`NWup&zVS*eDDpf`S}Dy@6D7k~k`6Bu z$ieQ*X{1T4D^M8P8yrrlaQxqZel^ZIYH&9tEkAOqphFY0d|A=F74R;7lf6HXvoO%j zh>dQf;lj;BAgAK0(PPsJu$wL8Kb&O!a8*v-qW#R}AkGc1Z2^L%(O;AL$R8Lxf z`&V|bxj8I+0>iZv-lz~pOj%$z#iAwD5iSAOmM;89)Jptmm|yxxI#9RnkB)zHu`z7ATQVks{M_{-ZPZ}%ixEc^@oyJtWf$!wvVV343E-cEZ@C=;?Y_PEW&D{9 ziT6I&E|5%Ll3*G1CpErpx`i9(21hpzKa8#D0lrbm{ODzUAmA(1N}=! zkYay;K6ytWO~6=v*r9j=Ov7#W9#q8*{m-v||6C_R6%kzz#i?p68=*5UvXcOkv@mJ} zyux7I#(yfQx*I?mnE@&EP@wMdah-s5mpJn8bLvXrg=bLTf@a5#H(-uJ}|GIs7!i;0R_<+4BsAjRUUD?BoZ{Oy5>_3)-&!S)# zW2K9b5GQSl!bPu|V(7REj;4Dmr;@AjoE7HYi7Znaw+zYgmzh`kBDc{0rY(G7JRNQI zcVtXZ%t0%h@w(Y6?tr6TJM`IVS*b3Bt`XX>a9h z(!vK&ilQa`-*_Hq$-M%U%~g7(iS0j`8^J5hSP~s5yIUMgjAuf4I%z; z%TLvTHcqph{PLrhuWOXz%YNP<9w7q9A7k{kf7~@iG&bV#a*nrSxcFkEuu1RHW6$k# zEDGn)cBhcK3xd*zk&Y)I)@yu=Q!j;^W6($m2aY(piAA>(qEtIa#cpxn$B4T3okVCN zL=DgbiiCCRWBk$P@(N*SkW7L$z04E(-1f8o6)Qu(t*aGfdfQ z%$R-a1|?5VuOvQoDXOveL-mE_1Z>Y?liiWH<0C-eoDg-%2eP8JZaUc$1+uE~I6=5M;SHrkjuzCIVMJLPRO zJ;BkeXeMIO7F2t}a%xmy`YJFvwQE3&UBJ$s!WV+g#at$z-=ZX!E4 zdTA8Irz;7(BdSW!K@B%f$k;$Gc***uezUb2>Q>uzr&1jhvnavVUr^{0t5V@jJs8fW z)_O%)vHHDB&RyE<$ItSj3;(jpUhQO_T8yccwaC}McXigg5n_hFc@guYtF6nWMyuqGe?5&ESlr_zk7js# zBEjg3+tDSZ;QUSF9J{Do9P6Nzd>;xI&DS4Fa7{yW-chGo`gcCxi~x+W-e~U8ZMKp&5Z%a&pdasX@?&-N<=0RXmeT}0QjnE1@5t(#Q!w_|8D(r5 zG{5PBK03>j7)bbUbr5qnsuf8;ZiM+Sf!ENb&$s6F0|2L~w}pK~+3|1@b>sw`g-&`v z3?Z+{(w7G#TRPg&?gOTO$%$Henlxg#q3EyGP8FKJ*CPGot8*SyrLwD{DjSG)$LO(B z;nt!JocdqbM2?f$yLc7T0x?`Xf$5J4)8UZsY13;0r`uj!0U+eAwUmSH%A3WHv7?3Z zVkFYY+Nr6=-Wz{JBF+tTz8kp(Ve5rZm@8lb_n)&~q47lXV$o%Y z^~LK0TV%Y&?vqs+sy&V%dL3oV4!XkPMxyewr}X z>$`&lA-=01y5>!7G4Y_ zhA(IXt4yey<7Qb{*7u4iw)S*Z0Q^Fc76ynpjcNQ*P4*JE!%<*sX<1nlw6=?^QZ12m zCHn%mtnL(|c-Nz2qs}y_uo*T$O{q$q(49dA{18e%d5Jl&bxG~Cf%cYqG+bj%N?w8W z2+J02EOLiu0;*m$_jz|Y03TaDu!Fy2R*aU>PwV96Dk`Mya;=J@b}CVQp{Lceb{D4r z{}p!ooa(!#WdtCI%AhhogkIpOa(J~bV3_33L)iwZNVVzytbs0cYXXgAL$_&InfSF1 z(zx?eXl+-kj^|U9qy>{fri9F=eEABZQS}Q>r6=b2wyeMY{fF`0+5PwIO?62A$#; zQC|z;7q)xLxH5$VTwBxShMLGW;06}9gAa|+@>8yv-ObYgHY%6O2EO8xXRtWCDwZjEs&(h4CfR}2)-UN!&K7s5<2I+*y=A4c; z^DxLG^Zd}@F$(o_fgmaG!s!20@nkCx7I|GIWj|Xl$_8#^yq2ms$ly`7==u(xh7e=~ zQaZ0lmATD*eI4_63$VY0j$a+1fA-B}%a8Xf8cRVDY&=ley=VJ*p5t?h}Hrt&M& zlr5#O9TYn7C5ST8mE4@oL)s8Zw3n~zuYU&K3C2TOH`Qn)HHyOL&cCRAyJ)eU_tAiZ z!_?2II)77fK3}*GpJF(>`!JFcgEp_aEH8dPZg2Ra(N|7>z-DA1ZlP{6aJQgNvwB13 zC>36A2vV}tj!3>0Fz!%T)m%*#JJeCI>S-a%i-g%d!hR<bED++Qtg$>H>Nb# zs4E(vO?3j1_o{4?h9{yuGpwUceWdt->?ao8^zpo49HSbtyU~V`?pC6C-mPNgVw~6jz92V5JVh)kGK+d2Yr(I zCEjFa^IT^$t8-C-M+F&58U_8sEMW`ny+&ARF-D6t9GmzfBm4CNrIFD;n;a)VZL6_x z&(-oS$YLKpnC3D|B3#N_Q8{NC8JY{A+c~QQb#bZ670l;`Lwiw5Nc9hg!fKjmD?qw&|a(W`W-gx$uGI=h1?a-RNwRet#r{YI&!D9auTU zf*In^vw9T=zsxx0Mrf<9A5S1~hmFMp+|7tM|9G84OgV1=`P5&w#&qW^XZXOII`zzueCJ6Oujv1E67U5CD$rFRCF$gE?z7 zd)xH88!o?iDuz3B!!;{ZNs}HNKA*jqy{|WcD~q%dx$u%I7mrY48=m&k9p9MiVOg+) zZr@LXyWJexKys|3vJbp9Cliln?6)#Kxd{*QGO z?j<1p%FDgG=cy`%lV98K2Oe#Zp$V}{@S#YWoV-xRJ#o@`|3U){yb6&Ag&P^lzTWyK zY!ZpKz25xl6A2zEaTgr(l__NE{u4doshiKZ^6l=-qV8Lv5>142E?=$0aY5aSn^UR~ zt1LZ!zq8H$I6lA48c|S93b!>?!@W~*`eEH+zlMg)mLCl5TrJspwj(<{~?n>W6t)MV`e=_F3(uCL$6vhjb zUg=E$MI`+k*um@dQjLu~%BRKsaQ_CWo?3LJ0ayHGD8SX*zA$5L>ij^dfScz%{?7S= z>FecwW6IwRO^{)%Vd;A8g`1lm60jE<#W$hv1<3zmQTGC9s%=eyunfmu;@Q&pS z%Gz06u3zspWKvNPWT~;Limtu>Wv--E^R?xs-&n;*KGbu4ES#YYQrD?7i&x)4<2*an zhn&TFUqi}7YA+<18CCHlz6W-kjZl15*o*4}iQi)H2D$w(@mjnQl!CTO>=rW;%aodz z_(#iNrV8l@jf9S0#36I;OyP$^o1Lr8-|V60F=vtDv_LG}V<^CRq$641$>fJW=;U+r z)UnQvAOnKw=i{ejn*iR;R3Kdf(>QW|-g59&|EZ)!Fn5jBvmsUiJ{Kg?e_=H&*z>eh==JgD|O3=35xx_i%8(n`qt#^i{1 zKKo;(``F_)+F0$_Y-AT+j1^yCz2J?={p>z7^{RwoElTxV6{yy_&ps&Rt^!sRe>*uy z+9a{}!S*^ijafPgz~nvY7NGb1q2(NmgUax2zNg*8Hb~bbs8s!SBRM?s3>t{ZXQ2+# zjW>wP*|l~PO3Z)n%HpqC{{5YzL|5}l6=7th)xL;3T*rGXH{0Tr z0e~L+2rjLT)~R5Z9#yBa6x~lPfQz~R78m3@hqIutK=$}5cgF!jAKo3D!o$FM_oZZ$ zoNS5rJJb?kacjiq$-4VRh?xJd5rLRiLE#}PTFn{+%E4_tk)-M-e^_Jb!uty5aqCtp z{i5XFkdVX*lE=G)vKIzZFUxh!Yzo-y zpA0v^Pj<^J2ZyP;HYf#ws8N9%-8FeFgw3_U9G>&S6>!uPArLWi7uN)RrZOuk!H02VVdhaCfq$u8Y1{C>)v3Vwj?5snIBC$`P)yYT#`*x&pij4VB0ayiI=)y6Lc^Px9BSHfa=EQ>G@%qYgm#RP#ceT#RU2!;KnV)vc*wC|~ z%#|V*(>IMr$(+6LWO@1x)?W7y=xeeQs$LH0CR#Ezb8s)5$t(AIbIPWJ{#WJr_D2Nm z0Js8Cm0%26;>Ei9_JeajBVH5oRbHnGU}3j6RmsIRFaMOrmTois`upiR8HUvVCt|TU znWNJ49@#YB$2tE$F4E#_FR85rC%pzve>OBbI>(N(gj-Iu2y>M=8k(nYa}FDt|KStw zML6@W@o&6u%mEGx6*WW&5$$d+wRhu2xn44-#dmf`pZ?w)4mG|VTe+9zJ&bv#7sMPi z2g`rJ_w?T%?>4AKOzRd_Y5WV3Ii&H^Ri5H)TNuOsm#DnoTP?OM<}r{-H9`StTMT2% zT^Y=1+@p6Ch2ETPCA(_p_P+W>1gVP`0c7;q6(X4xRqssLRouL}`<8%e4N1lQHXDSH z55U9*pC-J99WXiDyuB*NKPLNr$?pzH{ZxoF$(5d$Xs_xd_*A^D%`CRVc?iC=(;5Z0=KOhOmH#G|GM|U_)Kvwseg65Tz5$~{B_jWS zx_4HI2+#h^)IwlP9L9%oseOn~{33uc7loXcEk7Q|`ZxddZlTB+O;I}x+L1#n&|3Gh3%uXIo%bwu zG_ZQW+p30BYMC?H`W;QbORqReeV)=09?^42PX@eP{?&t2Gawp%Pc^&^v__){`M=F^ z_dz@^_N8scD2G@D{tOowq+AnIQu(x|4R@8XLD!Cxi7F-c>)N&b zJZ1Q2rDPsSYl)(88JCnqx;f0hdc=gK3QdQZA&y+(&2pQsA=dVHyW=J!pe{o^JJYJk|;l#g)l>z(@{COIJCE>H5d z=|9i^&VevX^{I%eJd^s{@W?Erk%*c!vn3&m8r`d*b^%sSPPvPw1-%hs4g<9Lx(nex zB`5hc>Rpt|TU3rzo}36yLCn^PDTW)yf)x|qC;ya%nu9+o@ixaZTV3+x>;4ZfL|#HH zjm0+gRHVGPO3BHAU+d6KkyA6uPfwj~(^Ge&0zI`0%Q3Ru`wcV3GheH`ES20%{3yWm zkbA*=o5_$Q*&Hh6It2@J7n{FlBoNsyEXE8~#iQYG0Q^5LA^#b}d*BaG1906Hw*Rd^ z{%3@CAR{frQBi&|`Qm9Z1))tPS&XQH|F&_~>?)he2jKgpFZ7dS@QAVmL^Fgl2W#LX zyKr_2#m|At>m3l8Cq74f;F(stwlf?Cfj|6XR4G!9i|x$iYHz`Y`ak2t{K7#Ez6OZ! ze=)|}yg*eXo<-+-`_#gupB-?(tyv+`5T_E@b<;_KPrqc;AQ+8lT#ge{!$Uy7fi;F? zhpdG$)kjjl6L&3bTRYwMBG4KB3um!4fElr%l-uVRlgs}9E6)AT4pMB#xb1>U?BMom z9=}E)KIqDGvn}mf_I%bVZ&A5H=%lV_Sc1uO{s%6I1(XRc@+jTt8d_no`Q4G=;tLDKpF-m z@u>+SSIbM)k90%}2yW}64f)9)Y4Y$?=mq%C(!d?G)L^NG(InKVWUX>-vID!o>49kB zYY1$99h4+v`{aJf*xR*Z{T;7Cx$m?2%I!y&$V*4Q?ABc@M6?iucDd41mj);1I?P{- zYl_ltf7f(EcHQ7KJf&A;P0^v8!Z7)Dt2>P*_udZkdK-@E zbOx8CA0uKmei`gixlfl*J&2WCVo)@P1q`H2{OC6QF|pn5J;6ZG&&n+Q13A%X7cl50 zS(lXH_%!hix?y>DzwiGVH?XuK6UM~5NRonx7@`NvC4)f%*S=1OC8a$LfhonuFFi^GD)FvSV#EiM#BCWN0lL8x|>9b4xK;eL7>v>1}8Llsd5xb zmRZ07WTqrkb`3@#41(awnmu2wo?2h!KXs7wX(vUUWQkHg$wo{n`4;v>6j*=gu1VM{ z4#}0rUYI4F$^Li4`31ir>)uZ)yIfm$D7%0IRRciZ%m3JJ^;oYglNmJG=|eqpG8FquNEK{QOFsm2YNLgO`8pXHfSLAhXzdN=3}k3Q z0or?qI~r*kSZ?6jzY~Su&1MGJhgLvbd^${2Jh z8AA#dUyi2+vzcKOUQ3_5MBk0j#OGf#PGiRBb^C755tF^@7Q!9+F~1YYw8E(e0=B97 z*Islf`1J*;j>~Du>`Qi~WCE=x2z0Jc*c2DyNZ0Ic`hF)Q6-4Fz=d4DH&dO4=*d!GH zaG`zjk4I41SRNz@2Pc=7S`s^nQvXZ1y3*?TyqIw~{^6dOA34jz!ja9Az87R3N+|YL z>nr4M&s_C&cUsKBn-M>pL*?B}F#?75uejJX<_FQs6M2rg+rxp55r!7B2XZbAPi0}@ z2@Xz=+$zQFTJ!OPVU*-Sm%A{dy^r{mw0cW+7r%qn)3b0Lifym+vOs3lM%VR8gmc8t zQlVk&{ij$USDH2<001l-Q0gI<5MZVY${fS{cmi_c$+-`ai@>gp(Ec7z@89OsO(_N> z7tfQY<%!=}o^6cC`zmmMwZajzlR-j%=vg4c6VC$Ov>9kU0At8q@kF#9bg>t3e3cUB z`rfOs3PclfLCv;?iUSRduupip6i*uuzDqya?hxBA!z|{d5@LBlB%a203HP`LZM-YR z#g@KeX7mVw?m#%`#iGDjmAUdV8kk1uLB4B*ZjgZPY3+D12|nMILaFmA^z*Eb4oMbk z`oJLZENfaOSuZO<3z&4`Z6Q0<^S;&LB64(aag-+oTaVuye?l@|8g)M2?d9uk6C~MF zPlTE+#qs{ult;xb4~u4TYLLF?K*1bVtwSwg5Za}ooWEs*4{+t(4F(n6o=z`q^q7}U zsPT-QxWG)@m`5^~6>1F%!RNnzw$oiNK!He+XBJRcB%*-`)o?4Qt!SKNSwr&*Wzn;k z+Wiq@HC<6dP5PJAI`V3SF#u5AAu#fS_8FK{vYB`1zup%_rrzlt1yPP!GZqgd&)V z3XZ8ve)+0g)FkO$PHhRK)$-9uObYSn6xE%gmVI)#29ziKq~=SiIf@<_j1Qv|qSsMl z%-Q^Voa!!wycE+&p%I#_X?&*e2V9(lGKbnj)Z7@nw*h4=f^#U+22Hf>2L+h19pTja zPIB*!0wB433tzHnE)#$i34Aq29`+2=0cZG$)%(SxPFute$+61DbUlxk{FEeIlWM4j z{|XgOBd{YbbXOC_=BP7O8$?4sSG4v^$p4X7E(&wCDGiPZB)^21ld^ZGir0OoZ}vjY$Ew-dH)a&U3cv+a{LYVRNU$?)pZ>@ zku?lUek=b8E2A`itb&GylDZLE$NFu7M5lxB5a`|;fLT3#22}ngUuHYtT5E%w~-BuK*@HUr_*vG zC@QW5bN$$Uq=)lMq3G&`A4~94okYFUJ{wWM_O2i8L_#&Gl^-gEbFxRw-1xHXkDQl) zOC{1f^#;&b{8TVyat@*RF%zakBZW-th}7#Ck>vwJEa7%Pqf1T^9>TyzcMZx-ZpHx1 zwj$^cO-QUGYKPcVj!uDtrF8+IhT11P-I+pHLqVM>0==a zDRK|_Rsr*Eaij;6JzVB87znRoaqysXC=PAQkhHeuw~{}CHOh%A6w7RG2!5qKRu3(W zGrtmtT6$vLt5y&Qx9-jl9aRgrwyPbbOVxJ>x>Za4!aORf3nm;Mjw=02)|PlFRH(l; zh>3B-YKxy+#jBsMpuJ@Lv7Wz+vV~T%r576#O72LDW(ahDv5J8?;7O4U5a>bzjBGvV zB)4tI5qwvnO;&Vu^E{GPMH37Il-V_h%kmTQu_Ge&*uP}B%~Naou}O#Vu|!`NtAUuy z^~3!c(B7F#a{@`-PgOAwJj1V%%sGz_wWb{(lr1VP+J6qAC!|2IvgD2`ezTz!I>CZS z4_qf!;*%j~(DGyyt2ojijigq^WfWVAy~ws%M~amgT7FLq!LWYHxf%d6nZ@@mN_ton zccifB9(-ykvNYbFfitlkF(LJ{onaBGNp#P^uq@(k7dky=QnhS+kL&D!m)p|aUzY2M zH@%u;q?lPFQhqA@MAI!OM!ot^9mB%fRAX{w*T@||ZRP6ln!MShaVwyK@CuIS*kqz% zyTUtYSV92t^lNf@_J}Ehfu36d>uE4`T;1VLL++Z(f8d;pM_j`Kxy(Qs@&?E$XTj)Cq!0;&LzD5~5 zumk5J$e%m)L2@c29qX5;eS}`c^Q;>&tLKC;5=x@KvQEDDyC;p}AS9fDCpvDmCvc{b zYILyiJ!!uv8^N*-oO%O!CklV=#Lz2VRW3-z(OyOUpgB3N)5N7Zk%jY9jMsz^e!Qib%B7E!r4mD)+((ss%H@)??VFuvx2K7D*t5=F#!`r3iMLaoH#Q zEFxkHRzKF)aRa`GpDFo(xV@z3?^3axOhBH0s-wH+AJWT^@Azi&1usT>!}E~Ohohlp zckRd|?R*090TQ(sJab!GPcrHfCWdi~9|8H22M5%~QcI67#K-6J=IhoNd7CvfFRuqt z2~%b(a@7a9?!*`Y@eDeee<7NJ(Ud+X0hqDO93T*X-cQ5gi zfsccsDHx~!T9m4+KpvaQ2^PM8ZjWLG$LgBBS4;-~_|^+tv@;a=NiA)$r%WLNwr^fjxq$5>2 z2Veh-TqS{l^TJ-Nv#Yx8>$ERoB7F!tN52mWeXE{N-{o%4?pg-08| zx&FZNLj(E9JGgI={?1to_1$H1r6C=bR~TYBn~xLoA^@yE*Gv_WEWUppNONvqXXNrd1&R)dF7)hjwF9;*_rNz*z+mKR00{isj zBO!zwt8bdTjT&^BNnkgKrb+}U*D&s3{Fx1@K`S1YLk6OZDH-DhEsUPo&{3xKM1 zzG*D2gfs*AapLIO+qr=a-z8XIjK<^%1$;*X%et_Ef#z70r*L+<^~gOl?%p>WH@Ogc zmM+#|P;GQKDFWzASc8EMtK2(SGd(S%k>72TdHV_8E;{+`O$Q3~t-#Z^4777-sLaZ! z>msyNSJ+62zm8%2qeQ97)$gM&D=RnHc@SJ{33S+7cQ7nCv15Zc z3otlq%rZ?{_cw4I0(cQ6zN4UE6UPc~I&xabDhe63v?8S11^XXHaw$C!m{?vUVXV_6 zW5 z3AHI<6+P@@Vyd5Av4S&;lcqspWXJmg9`7!zIDdwUPId)fv8ro2PRO&ac5pWT$o-8w zoCr=YkZqooJ(rR+1XoEg%mXBOc-IR#;~HA?f&O*wFi>D3?&COyZAf}2+1^1tc&)UP zdf^=mPJX+%yzHix23(o448G7Z<+iwMM}1fHufe8n_bT*p`k|LlFMN-Dd8?k;EAc-H(GlJ4st9lT`FNs zzti^poONwNi}&dLE;uEXZ7{5yjbG=3#k;K<(K9dhwCrH^dB-B^M-*UD|Ar@`%h%>* za&p70c<9%DD+Q#kxVV7>2A@l^Tk&+<2*%4hcYt)B--n4h^W!vfVg zfLxLM-y{W`28UW?6JmW4L=iqTAu>6^88BS*{{WxDLWmPdr3w$M>`fRBwm6%pR9UWC z7@{CjYkoZTR+8lOElq*3yz>(5Bc>yMVk(El>I!B)zjO`>02P-!gj7F0SggUs zqgon?fA>}p!KIjSF1{%n=PH_oUYV!3czIr35+VFcqw=T~q-0cSK~WF=zMi<*?M>&x=PMKr9RPi_+`EpcD}-DepIyNvdGdj=;z1^RLpF#q%B*g@qaC2b89fEMDpP`o;)YBB~x&m zmY0{Y{Aqk7YHmgtL18ZCQr*>*#HCM=pXZ?zu_w5&0*JX2`)(MIqWm z1vb@N0Ecnljk4kyk9ERu*UgjXK|+jap2|ry?Lh(4Jk2@+Y-M`8SCvtI<-)5}t{Yuk zZH9CJTqJUg)Y>v!()jvzX`&mBsedgjYl3E-4puBZA6ElcM?@{`0eL8-NV*cE8sB(ORdW-VCk8Pg}#}Nu$ zc=$&ZK>`xGQRzc2E_O31j>dr;#eBZZK<1QB`pOn^IaG?zqD?uzzI%@b(FCR0FH>GC zgbnZTavypw&j}{-ajZC_8Vbe32HA!1vxS9@9=D^Ulji|Bce()8-tCz^m%-o0G6sf4 zt*0gPxns=rBN6}B_U(V{u%}}v)i~y}vbySBV}(e4R1ucAH)V2KpKJ}S=U|ny*wtHo zPJ9&?eEd)ob=&l?2!Gb`V-nIKguwu!qko$-rT?IzU!nq<0v0&Q6Uu7FoP8!6yIgg5 zapkw8Feqpd%%J7Jhj&9Yyr&F5Vjzk)ryNlm1&6Vi=Qe9aCgKp4hB9fIq9(I1=BH4? z08JhsZ>Yw4I@ADHJv{$}E85U=cYzw$_`9X7O>B$wKWZd19jNyOH!qz1+&H z%JwY@$iJYciMH!^QB^~BIKP=*oJoz;omzJgGzn{|37A`Tg^>nf(cIks+1F*O&SNaB z>UOwCniP{*M_O}d4qiNxMPHR&tm7nXsH;l!qDI)jGd`N1olZock>-0_xuF5l87UWF zl2n~y)Nc!48Zl@j8T8Zu(!2n1mQVRze#+%s>GFKylri@-7T96d#dQuA=%SAMxRbFj zr#@e?vVLWLJX-9ZJc1StZoGN0<_#SVKGGy&PmM z9V*U9$A0Re))J$xs7ItCK;?+n(~ww}kpwo;r9(per%OUtICf$@C&LF@eqp(M+y~;T ztI#FKmg0I7(ZeLetwAdx3~GWXLUgw>cVgTO!03;=Bg{u!a_K^0+e#Q+m-`z`Ap|r& z=rX3YVAQs{s%CQm)mVnTHfJI%UYkSe1lrJ$95-~u=-563K> z5=;@}h6|x%(b8C0*Bbx+yL_#DUHd4<@z-l6Ka9It55kf93GF#DIHRW(z85NV%X=8ZQYlbHB7VR8uEqPz+p#xRkyK5t4 zrI12zd^mT5&)<7r>}Q7qCTCYCr-*^(#rFrpnuB;=pUcaYV6;v0eEaO~=w+W=S|2p;Nk-qNz{2BLVlWic~Zq zlXt3cuk-A^p!mCiu9_=8iQD7_R25|g>t5H6hYeKxPP~!wi%u3VHQJOrM_6j(o7#NR)Pvau%uKm|!3(L>!V^Ip5|Ffq_ zfR{#pW8TYN_R_>-AS|NF&j@t1)Tqx#fA6qAkeO?-T(1DYdFNm|4Rc`pGyBrnIv% zS$7Y2X1Q3=TO2BjR;*c4gG&palc3d1iCgswATN+Xyh7tskscM;MU=hHwz z{TIg47~Q?pWHKhUCinv(Nsk#56IBNcE4?&1&rsc6wxDM<_*>4Kz=mMq7FC5IhzE%^ zn&T3z>S|oL3tXuPYv<+rB2716jkX3bFHb2F?{^paL}M|vw6WbjNp;G`xj5ko!?j>U z%-P#SkyQlVk~Uq5skFce%vRoF7UC!j?_KHGdO9Win@-t&ZvrBbs3wwB=AmdD;n0N3 zk^nS-J(dd|4RUJMBI!E9*`cwEh1#=^LoT9$L?IKvURs&|bMoHrxB^*pgA25)t<@>A zWCuS}iTCFqn>OqVpE7z-EISFF`j`RaiTgcw5iu92Dm~IphmmJi*JlN2=K5dUA@HC! z>Dv71kBrQRNI4V$%O^t}BT8DjJnXs()UR_XwR0#msJ)#rVi}{_+^{u$5b%zujhH}b$JVKU``71)<+xAYf4D?sL5hi;P$BMb9 zUvloq#a`F8v>^Ylr?ZNRBigz(PU9XN(zpf)?$S6U5ZpCba0$|Qu*N+=@Zj$5E(y@M zLvZ)T16=Z-bM9k3RE@EA?HXg(n)90rEb?M`{&{2Um7w{F#uX$m(Rc5yhDM#b#Op2< zW+dow@sIi_6r>1xPP;2Q3xm?CmZelDEce~Lo*xD=u~7Bphp%26;#tVldK;;o_(Snr zbV>>7woXJ~AM5Dvva#F@>bvyiANP?mmZh`erD$Zq>id0XSG6U0_42}Go9-mkwRn=T zP8*}HahZYWzb?Q0KuO>)@=(mRo7850X_JG&t`7rt8{sp<35sF18Z1kY2U7Y(u$rP6F z>4mO~?2$Y1%Nfc9{jA6vPMeRWjCG4C2wafL1HJwh&Ox%4wKtgrS_BajRt!M=5$H1s z*CaowQYC3j&*l4v>!$Y>tNTtwhE1mDFa3q3K!^kwFW`PbEUBD$2FakrvB?9 z@rT__b>H(izSuOP8`)L%m~k!k2-D4^MF(tdGg^RiqgW=7u6K^CI`7MOnlB(?DKGY;dRzgH4-K= z8=YwKSO`m4u*(uSHl!mT6&1_A-Q1wZcD-Eh#H)iXekN6vS-uf~jjC^#4n!=xg80>! zB>s_f9CVYK;GK~0cr*JeN$|BWMPXsx?8QVBs^_?JKUh&iz7LA*#*LB_UnW~+s)bdR$XcxyNfkL6bs?ybb08-e zLd{$!P|ySvIxJ$HizK*J7awjL=QEVEpdWki=%GUTy26!Bh*W>$!E7QV$}0$iZ@Ux{(|H7A zjcEAG<(+4LCTO4sC=$E3MI1ePuN^0z`R94=dUVcSGMTW7Jje4RF1-@_N8QLQlA3ER zi%vYJSbA>#;s%Ngl{s_-NqA>iC|KmwnmnId{s6K`;TfA-T{x`8Lzwjno1;<*~VZUhiZO z18X}PVk~PMUP8Q9{!q6ehOad@dK}ixGd_@v|BG-9@3s4p=$SBHZ*M|5J+)quV~qO0 zgTW(l;kwQ2FUsk?7KPAP|B>&<-O3o}DW8O|9g~zn?rxO<5(({U>u%!K7(osKSBa-`@kxw(@=-MwoUmu-Ln0)zdo96u6^ z2FRr7Qj?$Dh%&VPt}`A-ELPwXtP&YB%a`B$d@$huD2cW-{Lg^<-ZRp|u`6InNvvTi zpS`+V z`~B!z>_)XNEn0h&X!#Nr#H5qpZsc;w+MTm-Tdg8wq8)fE12~ zKxt zjHD(l6OtlN(2~IEM8(yJA})qNaGqore+ikSou$}TwS-wl(*>-(feF50vbiFc8(#){YV$r9jR#I5+PT~BQzEu1s$iR9=h5sda)v$lN zzN(;aR^rNZ8Zk+jS=#JCjjlU$q1aGIf06OxA>;fzai90MgAFHAs@GnzuSe?VrumTo z*j%7eW?^2rS!`GY!OOs=5z~FlNTe#~WO#K0k-!Im7xC3uvLR5&ccC1*yQKfDWR1 z#&9O`$+b;$?zr9$A(h)|eJJRzRkBe<@FVr@JB9r*j+KT%7Wddw)OHHW~S{(MMWvP0_)ssD&RHQo6Ps$M*5T zr5|n6EsM242^D-Bn@zZ;Mo$dtlqpSA8cWu>G^3a3B=ndX z^a6l=^!(W*giFvNQz?rhkM}&O>nEIZcZ06pn11iSJcHf0uPwDG5DQ^R_jQp>Mf${Rb7=u+dj|8 zHrdO*bDt%#hE#t&w_9dWA&~fa+rL>Ke6a5CndN7vc~4VAQp3#ITFU0YM!MOKRYwU) zTyp2<_}NI$oG5+25R~sWs!I@=aG38ZckpMXB+{zu<`s$2=WxBfQ*uP<4lSS4C+M1K zC#Gjr@#jj_w5W`WS1hSIu8R(*<|q1%glfz)Vc1;1CW`}@PD=kS_DF_F&B)T?B_DyV zAvqsun~>wsW49I;79@a z+#)2$c!M*WZ+jowQM|woG(#G`a|hTrV<$}3PT?3%kz3j|0YGoQCqr-(b1DsqmHFMy zOf|!oH!Yn?Yz1k)e4Y=!%=yR@fR&@htk2)2AHb0IcXyS>%hz%rQDv9L3pD5hsP%3<UQ^ z$SJWfoF%IB(FR*0r(Ddzuk%FA@rJzH*(WS2Mk;A3suf7vv@1 zX7alT$S=yX^jkBhVHuuzM{OuN`21Z*wdXGR2~INj(%g|~A9UGneq%iRRVH}fEN-3Z zQ=|nIR%Dkq%gpan{D*LgKbj=?F&|Q8@}H_skhPtk2{ZCsKS%7|!&2p+yLy$ivd0>|~=?p~zFEXh~Ufdjk6Ps6As1t?9dZ85HQ(H_0 z#y|-@GzZ~fJid&9&=L>Mo`nius;h4;Zk4N(|1J*irZ-+bm40GIsx7fYS7Z7jb3*UK z`@%(U>#pucUfa46zn_Z1P7VOHC_$uaIXhG43f4jKoFnR9#!|n#3#%d}#U3@r0f6tO zJ*|}42e|Q>p>3}f(GF&h&70%;F8=gTVjf*Pbh$2btQ#8E$utm5;?!Br4Q-k#^m7`V6s z`jq}%@tb_g>X8?KAF2ipOSD!jPy570nuz$^Co9pbY>dguz(u}B_cXfx`PR6_P!GSA z41Yp>n;>BNnkhzhrev)a=TsSfu%0yer@;2wci&YR^;$(`g3`ar7JV!2%S}dE8_iAW zt1W@dG0DN0gk|X}7bz$rRM3TW%{Jxj5j|}Kx4cGhde4!#Y*P?@Uu2tOz(_I*7sL-_gZl#hFs79H!K z`&P1Wd#`rPXcm!i{;(e9&52ifiFNV$p9Yd6w( zKj=WnCI1q*EOV4HymuPLv^dJ&wX`gZ_s{$NoUO)95#~WP4DgoU@0bj8!Q}4EoA%C| zU*$Y|c@-WZ9c#XtPBB#-ysdEh%4EkQ=ffzAeoVTkeR$(f+&F>bju9kJ-eAE#-$n;5ub6r=HA z7sGobk5x#*N7)lr1qM0{5uCC_DVEV+=#c_M(rtMD%6xqeVW>o!+qLqX1=41wd9X4V zX91VT(*^O`!p|oAB%+lYm!0T3V_SnHCP)8C9>;vVN@D+$r$*y#f#XBc6IYyH^St4) zbY}xB>}|%Isr8?=H=e>zHrRmka;eyzV`SE^YNLt`Qxi|=p#zm+&{iUMzsb%{ZuzDL0-xHnIOLCzr+%`@z`LeWM{P;$*Yg zPBtFDy~l>qH0%SJ;&`9?d&L{6(bsQn*F2uH*i@%M+W%pr*5C9YQxfMFncx?AFTMym z={pGf7YIRxIXRELi9RTEKpwlMn%C7Pay*^0$f*Kz=P_}^gYPY3eI>mHt^|u*NKLNV zQt0QWYo#lr7{uH^=p9)fw++|ZA)=&IS}Tm<&~6RF=*;XXhL7R`J@LU5rU{6F4Ho_Q z&gz-V*u1WZfB4$IklC1mvK;_Is4CAJJ*C?&vCxMRL>B5NS>ke=&-VQ6#L!KfEZvP4q0Zcbxe3y~FV7U9H( zc9QhJo39_!UN(^Gq>|mvUnX_KcyLhKUgYSh=0q~pr5$2d#=!H7xjqfFE}LX9{Xlhs zSzz@mXKjNqox3V6iyAiihz`7rTgkE5LWyoP@j3GGtlv0n{V7+ioeYM%0DOG?ys|uk zvRQ9w`aje7Imqy8^frGX_(f2H5C=$ByyW%g$hwgt{z*h<6sPeKpkZ;vSoSrGPTm*I zcB)8x{R7Ij6S~L;{_~Y$zso$wOMQO%V(EIHpqqm{^(>Fd@?^)gd^aUc_QuG-Dt8-c z>-^)yeb3}hwd#t|PG>+1w04H7SjiD zn>>HRVDz7NERs{Yd27}$)`pnWYYjsO@^}6Zw{2c(Tfy+%fRG}c!)VyJ!&h06)_B*g z=N<8h(2IbZa{Kf!`-Q*M9BFo^{cmnr5~~r%qg2@Z9=+fcu3}(TLd&T~du9U%Ik}rR z-O?P^#pP38o(V{9k0M^vZ|=RmkNoL!y=kg2t|>V~m@VY6Efu;Ybr*3s&AR)pi(dAzmc(3af2cAq{QSB|-eh&e zk^Xd6qa9Qx;QVUZ|8myt2~k}%L@sH^mKR$eVMsqRRir=ijHL(aR*O7ypJ8dXG>g$C z!f~N}uL!;H+@ZP#Pf~~xN<=cLAahEOjO~(qSuK^UM(POYs88J>$@05dG3ZK^U^KgK z3wNl1$P(ED`VNJH_L0N6LQhz;(}2?^zB<8xRYb?<`yfZ01c*I}kdXm$EG^s;rR+RW zyQ~pxJ#2mPErwgD@?=7$I<0g)?~U!_x1xw9t=)m-d^>IX#6R{*X{{@fNhSf?tK+sN z&&Rf`tHaaFs#mg;As2(o(+AibWb`FLbmD9vG%)VYUPAGod{$Aw{9zW zCcCey&&6)o9$q^s*cI(iD_c@tTUlz)wx)#lVjgC2Y(CqTGpb&~dbK(p*&)fs=40Vv z2_wQMtmCPZP+G8Nc7jlJxXfw5O$!WE+Lf<%-WeOF5j*NxtpAwmKL{d;NBBYdlI9^3 z;D)=jVfqGsQ_BTQk2az+ojs}6V$fYlm&|%ntW8W~wMkz*)O4n7!3vvx6pk8%i%C(Y zlCqAAf`xValF>-{KA}a)J7uyF-Utnar1K{6D=(tG_2&iK3^?X)T+sZlz=i=bd?2QF z*AGnbAGEeG5z0Bpp6(ec)f8P=iOx(a-)}=BNj4MC6Cucp#_<(YK`i2es`)l|T5N=& zm=*!cd_D|C9+hNq(Q*vW_ZeMxjho{c8!dMI$qC3V60>P{@N_;fh3d*t>F1OcYGP&G z=P<=5GZMBG>?Xh2ztWiJKG2a=>1lPG;+!jG;~B&|1dFnSx<&zJ;|a(hRrd?_^5 zi&7@7@;6QBnX#covufV*VoF-j!}(uML~u+`ct*9TGp}pr;~S6}STJFa%-TYsFeWh_ zH7jc9oB;+{)?iKC$PjOon@}*f3-jiW3k6KdJ4d!8Epfs zvQMsQ37~#nKnC(caMg>6OU( z3*BLU_@=#2V8t8uy?;^%H!OTW(ze8!t2-i_0Xtf;UsrFqWgzQ{iPTY%0=6RtsOWYU z7NB=~OXQbq1f&z@qPM77_G$OGWPWQ<;E)M5osx z&q2Pk_FKJnK1xMd^wHgk!a_2}&}Dg_MP#48^<)GMf1X&Xz{{rpozrfC|I3DpFp@mG zmETcNH22F;7^~IuY?I}925Dw~mD_Bvy$#uzqL+o)8OUOJ+O|hgK=(n^Aq|gIwwt-w zQ>LA+P3*g{4v~97Ss5xK9Z@jB#nzj1u{>>#57v$@$Jr}s>A3Jyg2N{t9{u5)7~eJ4 zl%3Pm!`Tm=7^QIXJigUulHkMy>Mm(Le)-{^Bs~^E(Rb4;wf#?`xVeH9p#fLZ083v_ zo!f<@2RUHgI&t@FH%pW~aXsrJ86H82d-%Z8NJhngfGaT^lb6IoQDB3>gnVETf6zgI zuvcOjZNOe}jpD&_L`rX=hgiY?!`~cZmMtO-P@*b78P%GDlHBlfuO$8Y^b`ol?5m?F_tQ^>&v zUdf6W&VcXkvhHfRqseVkBdVR;1%mFsT6mNbc(g~a-IhEX^Zo|tkCKavx6E}hZuIj< zIc;^(83gw=&B%{4;(3JJlKXaPPX;Lz3osn}DU^B^fY!_;16A3LO3gZ0-oF-ZA!wBETIHa% z%cL9&nOe`Ym8RJ3UlUvqqjY!4AI$zSdeqT#!0Jrc_~>5fM{qBBLb{1RrT@PE;gG~d z*{jifAIE$>#Xasl0ArkW6U1-IQF#D=`KeIRdQ4R=3Yuex)5-5$+3>*-Dw z@MeETL;GcV!?!?BmiEg)<>g(hKJ{Fm5qTMlHJqEyCHGKcrIy#3g;^s zn5d5kT%C1^9I@Nz7gtX!!z2#wTq@E#a-YdiILw zZ?JOyVZ4l$?OixME3}7*SE0QAq?78KDzP?pW4~FjAM5;kKxe3gH=9FW-E@Hw=|&C7 zhRsW_Z3d=T*$2=BcFeOolbgG$^H9#P|Eka}1v#CcWA>$|_>i4+j zD(_7CZz=x#e78bS-nWsL`E%ne%g1ry=Y0Ia6m`1XY@cN|=-bPvl@8>tK5`UuErpR6lvkLf zR{Ztin12oD|JN25$UdnbJJ$8yFrTBI#Hw(}i>AiBKlyZ3&69JQ;|WCd@)ivjC&GQp zT1m9s|AxPdmopsppDdGaP&Uv8TKGQ`v`p<5=;%{GDIs3JfT`i$C-0*tRjyy?xNMj8 zXyhOW_W8_zGhG0^xZD~v!IS2~22*^B!4U6l@u+lCgk4cuX`{2Wo|ik+^*_@0X7ryi zn{GNL(?%TSwf_s-n>Le{I+drsQqeb}ICH$Mit1TJw1ec|6zGk)v!4w;ZBF_u_VEpn zkI+uneokQhZfvH_acwE%c>Z-e_kYm#>pF#Ib$T-OB5}lu~HZjkmygK zo>)M5!AQfnRvuWbSwtq=>sQjoS1(p~Zdh-c|EegE?!P;zEmgv zNI0GeshP-Ih$YL7R4Q7q{6SDzro7HN-PsV}*bo)F3t=yWE88Al((m-R>pADYPxj@PD-1sV~eas~}#d$)jKV3y3BFd%ixkUHTF@m9#mISK~lD zVKV2)*A=RPNuvE{vmk8*JJU!5>I8*P_OP8a1F_%IODFU@K^`IbeFb)@-@~e9T^+Lp zN!GiCylBCE!{VY(GaOs;f8g6eH0BW*RQLI}s0bQ`Tr6A9vm + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/source/_static/images/original2x.png b/docs/source/_static/images/original2x.png new file mode 100644 index 0000000000000000000000000000000000000000..a7402109b211b3d69c240f0c6e29892af8a09378 GIT binary patch literal 66517 zcmZ5{by!q=(?6w@%F-wjOC#M~OLsR*cS<)%$63%$%7y5h_a3FEQR=ARr*Tl$DWCLqK>ALO?)NL3{pmXA@!?_wA*SJl zcy#}mWT5S{ir#%Xilp)^akNUrn3xc^KsSiEj!sD;i98F%(c6;ZZ3Bm?v;9JSlOYwi zpq{^7!Kdofnd(#qyKSu&{V6T66&Jhh1u+wQ`DQbFg71dX%11Sa!K0Q*E7(yREQ$g! zN75y15q9kL)UcgafP2*}8OPMP6iNP5-Y`k30T`}RdZPi;M_{rX^r1Vj-epbTKGgf1$)r1WEraJT>z8A*^Z5Cf^Exqe#|Md@US z^XfZ>;dCT+Al?w0<3yg4M>Nt+5p)k=StA#s{hQv0q9<~1MKfjndDNiPg_28flf9!o z*5I&N3I0^|?-rG|*8;*`A`2_#Oh7}dgk`nil@IjQ*tK~Nrau24W27GdNG=|kvJ_31 z^{Y%c9Um~6`gZ&!n#rg-ev}g&X-3(L;PfoJdz*vMJ2aQ?+qS#)sg5w{N9xZ}huUQ# zxs$Z|>uc!Rxf(*5vH#5?g7R|^+GixRS#w=XKm?dP2&_4OzP|P9^7h%Yjc`q&`bvzt z(BP$T%#uxEzLwD+wc9_rPtH2iH>w)uW?$8Q;l}(owJ)0=0PJ6N5JO`S3UOF;LW3nf z+8yRk^_r5rO3j*0{-Gz`3J}1^sx<9d)`<%7bt5d<^p+tX429Y?^CdP-zia<0|Nq;S zL=-JURN?vHs|pJyEk?D8m32wfTHoujXg}Js71FP_u=(it{5#51)`I#e7whjbZ5&jm zC<&sB>ru|Ygr?5?YDK@@bCAYpM#voO$i`N^{X1G8s zC`UKc6f>d8K&ZX-KB!n{S}4pVTqeNttw(4pUgCzZ8WY4oJm4=oBVdKbqnngv^n;GG zZtX#Mjs9D~p#qZ_p`X>ouj7wm--F*RXr$B(kD;!RqVkJGo8z>3Vf(YWKv576OY8xj zYu?&}X8fwK<1DM1G6s}?^J}~j1Yk!L)M@jB7bJ2q0cM6*2)FnJ{U*HI9x2k~_^Utn zyF=c7ZYv!7kZ^Q)!Y^!`qbV4^k>IZ#$%`E-ALrDeO~H&>zZMUNt3H&waEN-_|+VUS!V;)*TT+ZVh&+A)3?g^%Z90nw!OBlnqCkWK&_5~{k#QZzVLIpOSH860RPAY=S6Ss^bOOWwHE(m$J0m*0Q> zSFonbkcURof5VV(S|Zka+v@eDMlNzJ;&*V@m?3jxBAvF39PnVE{n?IBwav07-U^Ls zOPDQ_|3!5Mi|1N`^J&)5TWLIKvK-Y$l#l90hv@n41KM!a`NdH3Tb9`ORHGb5UUp&s zRw9DJTL9=AfGLK~|}{CnQIKt~gg2;apcSIKDi)Rws)MyCD5Wmo%eSW1@uQm3)$ zQxI4#Ki;L)Mh#}-qOteB$Kp57*nfroMjXSJ@f|zQpl28n149!Dc5PHSk5$0#aq{y} zd`a6+1^5}21!3B&O}%mZdP9b0QF!Cyj6j~hsM9=qGFJcCsez#u)hO%d?1jo_CIZsU zjD$S(jB&`lh|h2!m_O&|Co!f4t&D7oS27WKFU@jipYW&F)f_ewf`uPshg?kvI4?C3 z_WRK&ip#$y7~tDwNs4(woP5wJS2-rb6|H@b3eR&KpmVao7}bw+1{ z(#+FaKE~`yGBr*wl|3KLtw7d00F03poVP$@Q=9D&5EvtsyfSO9%D^{#YcF*A0i8lQ z^405l%(6xz_GfH=+3;H;te~rz=iD7}g zOG{psQ{)E)#KealjJv)IU#J#69du@+zuERr$inK*F?cEf{&kt5@`I$}b7~FRDrC5h zo?X5AxW2c(?MK4LDB3Bbr7gjeTWD()^foMKws=U_Q|4QcU-E_@-q}T2;4{R7f6+l; zRT4#e%_e;RW7`791_yORS{jMpRTqHXHzg!#_>n+t-aIJUgK%*3+Ko%MoUQsmnDkHCJH0vCji;GT9LvEA5)`Pf_cPT>IuB zR)2SFp7L)Z2+^DX(7RZ*V@4<_8=fNb{fzRPCZvdX+sa}W{tUag ze?)B&PmADj%kYrW`>V_$$sdScdJL1Vo20ieca z&Q?*Wj-|*f(j}SeL4SoK|ITWL~@pgQw z!`~`u1{qC}t;3gY7CW9jElT-zUi2#j1GP&E_-kQ(VL};0m=TxTFOmMu94k9(kesoL z7`r^<^vjKRr|TU_C57SDWTle8PNitMjf2ZW<@?pE8f;!1flApCt~T+x+~`-aIH^th zM*emY|0ac?LFyK&fuiZ#s?h7~u@CTn=LDdy*_`rJ#%SN;6j<4BM%)K3^E@R*D@{A<-%jHE&o+>2sXzEnJ zckjILR+l?OPev${X`v`Au+7|*sjf=2KH&0j&4}`SVYbc1p^tb=z1uC4#Tu&a>WvZ( z?%y&Z0nH)H`hMO@m_#pKbHO<64!56!rJHB=NZxt`5k=AXz7$yRXLgd*&k72qel{q4j$#u0ORZ@v)9g>Qj{p#QH&7yz?0j!AodS7%# zsy^3?3>|;&oL40>#dijtyfOMfYb(UXF-ye?kXY!9KRyDHYedXkDNVBfn;(Sor&8*p zsDMvv73RJ>_I6c`-y4X^UbOocs$pHzjAciLcr>223`bldG!A*2D7@P1RzTj7DfSaq zeII)JHzhm(=y~-1^T~HoeM`pME2O%1G9If<;py+=d!G-l>q)>4o+8=rkRT|Cy(_t9 zKWWAgfX<0s+MKBQx1#Bv6rIj3(TVrlwb%r-qO^o6T_0CgoG zO&#nn`8QDF>71U|;Y85aB{8+Mcb>n3v)o88etR8%KHpYp2^y#4G^wab4ZT@dof4`AW zX)gK{B)WvpdB+-G>NC=&;r4nrgwy#RU>>vqwT!>wMHo=Bx;W!9*BBoxJa0@l= z=SFkt#;UNymkqBFSqsxa*{fxz=_KvL@TXc44*n@Y# z0;oEJu6_dQd;L6LE8%2pUNQ#dvVOm#rmb)PD}$*zqMhqH0yXto-Q47 z@^QlLdKO0`1-k^4g*%Q=_Rom832-Z6+-t?aqAT|cp}?H~nx6I*JLq{Ra~w;SocG+y zI!KJl^;>-fc>t3qeVpoUFaGfJ;>WK`^g(j07h#;UPI3n`?>a`h`zX#%@R)EAD5Fhh zj~x7U$yvSX3tbE2M4^h<%7if@Gm`tygWld_`*elR9^z-<3*)4}iXRug>aCYJh-LoC ztt^AW_lwnMt=s167?b5EFYWKMm09RDp>L#tUNr)x_$m zex6VXEbMqgF8lZG6CMe0Q1U|dkh@u)BtkUZ?R)o<7Wn?BrUAshQo1+dfHN%~XDlAJ zhF)K6B^rVT2Mf!xJfy*(@R_x|)$1cymQ}Yo)jt6s3jNQ)TpQbh7lNlZpZS1lZhq!% zp=oQ|#qpjt~S1pyr)&N*Pwdz9I;2%eyldIr&lAnL0B8b&e_;%il8HzM%FJ<8&; zX%SUV$)!*d_H$<3%vB^OPNzQ&(EJ@T{dfha{#t~F&ktu`s z9DEYqeH@|-BKm`MKV_c|ppCpB+tDSHA@_n9^?L70TgGX=TG0x1j8_#qB$Xcs8@X2K z$%A)IAMg(FB0P^|P9>|%hfKia%v{B-kywM{;;GzTkRRatUG_#*($WPto{OY9@FPYO zA!RZC5hM=zNnfp3VcNuaA0Zg{qevSs>mnJbb>eqJ&{{4hU$V20_yHB+vbHR&_=Tyg zksQWU?>T;&@sqk|w&elJ-wjdWMdMe}kl;dmUegePouW36IGC6;s`h%kS2Hs1_d>45 zb^}Eo)~j?aYbo0#Geljeq7@0+z5bjph=B&Y51y@8=gB?*){s#RqdFgxmFnceLs~Q2 z^zGSJ$Q$;azorlMrq*tZoc7w)mjxkcyafw7PN3(w=PipqOz}REz_4c=C9BLW1hSm!gyeVOM!E`oGh<- zY+g*><*qW{a$KXlFW9%^EJ!za&5z{Mv2s0(wJM-S7$NLfb$G=(d*`Wsy4Iy9KbLc_ z?_`GYVO1u_*z^2IDGvgE`@h=zNe%$re(h$oLMLB+GlU|Qj08G&&Ay6k)xZAb7CV)p z=@t@AaZ6%gg3Fs@5xR$a7eJf$P-G6Yo%Ya}mK^l|VQB`Xi=|Mvs~;fl7_o@jD%5TE z+`EgY2bVL!68N}|4I(Q_oO-H5o9%8-XXqDq?B<!T%^*XU#{6vd*6sT``RIL5wGSg*oqa=Os_>NQa}s18nm4z6 zjj{3i=Bo`-hw0;pAHE|Nko}KLA12LAL*sLdI`K|V)fwMSg;`BXTE)7X-00p@_9)3c zIv1y@3e_vVEtO$%=f<~H&0t;M90R*kcSux^7>pR&wjH(%;C;P+(~LG|f(Zey!6B_B zZN`bU*cDGjL4xtFfUH6Xp1x|xEZ_a}Z@1N)9JdF0D3eZ~Rw~O&w~UBW#FMeP2KkB? zzzR^Rd*%%#z6O~F-zZ#(ks51=UszmDOOBzT*^+NR)@8)y!x?^;*_re$o8B#RGyJ~G zNy$#Vu>|#ZDuEp965wrU1vCR1*VDnq+^CJN9j!kTwsuU4`3>&O_^Trx2Hit(F4(76 z1b7HY*-rW+xW_!uWkT;}2yy`5)Z|474>Vs)9@c(yu6ItyC;V@)d>AsXdQ#4}l55~u zwuEXGW*;H^^ooz8PZyfogtj5^aqzb&_qN&4E$8y1RyN>Wq96rxZ%qTWw|uzM;za*O z6!ccmPEu0U+I{;ZW?+>jGBOtH3={#_Mg)?s!(W?+Xhi1J+SQrK^{io!N+v|Yvn)~K z_}Ei9u^`K1Nqhwf+N=sZGX36Zaa~iMKIWAXYt`pDKcdVGTFV#iOagH|p*2H;n66?d zG5RY?_HYGu{(U^)(N@}Vrp`&(e4P~#`x^hzWL>?w10#oPx?|P%) zjvcx4N7KCw0I@%C5*l3vpol4cKtHB#C(2f#+I5TJTd?|M<-R6Zko`*WC*bTsysgk= zonmZxWRWNBg1@Pw*N(EAJm)HMeu>~}NMpH0-GGh`f;ww&xrX>NFW6Af(`i42i=7Xs zUe;WCWfKMc6%qiD;iwQ1ls^l3GphI+7_(Iv;OkbWGK=Zldu*X}`D69lPvP=;mUEJC z!^9&bV9~8c-k!WVs0v2i$XW=YMDRUzbk5n8HjZL2EiR0D7JiJ=B{+#`Z(?&ADAP>2 zQ{9}%(=^|}<;tXpuJDR+#pk94M!I;Q#p&G8s*V)04qN5&d9N3D$c#DPOK3B)*X$}v z_KFAAzT#NPKf0_8+kdWzpex#bK+iI09_$DE>gzwOX5;^B)zoi&YT!GGuL~l|2E!kh z5+`~M?8hWakkH1f4D*GN?=0)n4bJCxex=MyG17=!slEMj@Cm?NPl}xcKEa&GAt9Fx zkx|LNgjN(18RC7ztYpB*l><%z)ahA5{Rn?#8wy$xG%nq_iBL@S4mJxfOz|vE1&<%o z*|y|9+#?^4N`Wh|voa=KeHUKXe3H%Qx#r0Y@sEm}C+r!iCF9j55$(+( zpW&5RJ|Sj^PPJ`cRDYjIhMwF0K zv%9m+^EOJe7|hUsb05YQ9cNp~#uJ$h-U>6zOJi#X2Zuw7%dA`f_Qla#oC>}-lnSV7 z&M-PuLh?+G3xkK8;ujxCewRtg?Hsb2Q8v>$%4=XeR7b{~3oRiWdmgcaLzRE>l03Vn@cHqZo| zsS20MNN1XjE8x}&E#L27cfFLP_8~S(mf2aP!IZKek!rPkAoZfQtu3x4z!`y{W5$;;l##-FZ#QpYf_qwhtBaUM*uTtq{n~F5H!h zE>f}5sog_U08?gl9p`jfF9Tp=EL2=~b*op}pf;BF0OEtK5}eV3AvA zD8234UO34=xOP_L0)p)SzLxc^EjaA071F4}eEgwZ$dpsJ$!zs?EwzI)J4=J1y5CXfgXBtn}w3yB*z5oNR?dM7_Rb^GP;3T?PdO4M(BsCqV(tfgO%( z0IIh8B~E0+PmC#`A9qBQ`f20IkD1IPAMS>0zR5${cG0f#(#!p6mafJ68!U=_yo#kV z+E%-MG_t(hf-uNT_9{*7ab17H_?|I(3hj?8%$7csSha<_tSIYu9m(!$jj-UrbxzFO*$82qX606IXM5&Uhg_}S1ja_@d^pzV_{&Z!jc2pY5lAe zV(Q}=b2T;t_if&Z`=W+x)kyKoE&6H?L1rzuoZLRGcamoP*} z2jRQu7doi7&PBDj0Z%Cxpu0uoT$%#2nYnXN61J$#)mp!RuD5&}rQ3uYM-jmYsAms4 z4^8G)>Q-^Qh?xbDdtPJpAx)B4~V1n||s>a!tGqNQYIE{rEz)9t|}tr231K z&lQ_Ga6o7U`^JDHC6>LM#-&$jGkHPfo*#x@+F|eC@vFr-;K>H!^uk><^QXdqwJHWA>xJ zhlU94Jp<0vI$7LO4g&BuUr%WEpwBj!vvz=l?)k^j_pmy9+dV2_V!!(b9w9#gV3DU^S)a+x`0h)f~p zhD&4Kl@G`=_s9u5nVWws3$y?WY49i+$(yLM>rsJgT1duj9q$JO^v}ZNZLkXVOFv}e zAlwIcG$3<$$dxDs@2_wz3LvYF7w8bQA7BkzzicG_r@hGgTpSzJiEuvj5F#0yjgaVz zodFVV8?gqbjujCVr+*nMNRPMD$g(KyFcVMi479yJxeE*8arDffgQYkn5;B0)=Br7{0$(5Vg7d1#jnZfO~)QelEV zjcoe{W{%{Xo_OOd;DBcW!EMO#nluG4hTtE_@_i)<$7+ji4sp zjW%p=6D0S?gTFvnRe=2%g)Ga9!S!J@^ zmUH2ogd~!FLbWZZLkDeWrsx8q=P3>t5O2k%a$nzwGN_AhC1+rjWS2wP<*sdESONss zf6X;n*If0iG26>LST+Ut_A9?RPYsBy|xmvj4H_qUlZ@DDf}rpamhgC2k@&r7TttzW2AWrBB>8($so z06^CKZP_VZC+BL6RjX8gv}i0r1OZTBw#x)wes!n(uxB*pzP1KjuR4$IeEyF8t737G zJkhvAshYpXCNQUst3!C%w7<(JU}2;?XV_QkyaupAK0+xk=~){A{^_GK;51rAQcK+D z9$$QN`cb#}LSOdxFB4zpNUF*44T>yK+{%Z9(|qTQ*EQ+YYgydWNDSVJ$Hy1tIcB7HBxiT)^s2l2rk3iXdbpC%tIb_^0_L{&fvf-u@L@gdU;d@kHSGjyNFxz zB_$+#Di}>1Q1RB*Ht>vhVAG!FKB@&K{JtwoVDhj_>gl0X&Tsu0WwXZ$b|KV1?;AoF znbE$b(EV$dy-?)PQUQ;$-E#5Y0gM7x>PVX8++bC=UNrb;7KJyA@TB=2>MY^^Rj%mT zeh1(@spo8}SGllD6I>4IDpC6A=|Djy7jG4mCKz)><=U~+jN>jYIc?WRAOcGMWjf*i zQz;|l6~N=sb-yI+PB2F>*HVmW805r$uFK$&A&hI0 zBdu3uE0|um^;JY(OX<S>+C1#CS34dy6H90|NXQC z(Cij-H@diA(X{CinZEm3{=jGMW>SgYsys^m!x7PtOu8W2_g@G%VZJTy-&qxma`(8V1>++YSaa8sN!M)JM*? z;FEaU{HUFmL%k;*K>rzhshK@$qpv7zxS(m}x3s}TUj<56Wo-IY`(tWuGhufAM@NSq zjbiG@HKBkXmOtZ)$_YKYN?9fTDf1~MjY8vn1Mf->kGZ*qPzq3+U`O`M#9NQ4c5551 zs8jxlDGQMpv$gp!V+@U*c~3qaG2ye zX}D4N8qm=(*J5=DTWc%Bj}P-6F1MK?wzR6X4i|-oA!GubftuQqgdWk?s*QzP?K*X3 z^Zq7TVNR;W+?I>#`kKeLUN}~lMf$!8oQlFS&Hc@48mr8M?hyvl!GnW%ERuXd zb1no8u7njE3S1a#(%YH?cNfk(1@-Whj`3X78DlzEV#DcI2sLdG`zbZ-w%X81pyst~ z4e5o2Szmj-0ZlookIywjGo0oN=iBd!lmiY^Lh1F4)vZk<#e<|HU*)#RSSua3XrBhc zXEoSft8Pu?!*eVX_lTL-Xt}=2MKK!kzB{F>sbOv>A84Ue-Cx)_J6Hb{T=!-1zG(RH zHpXZ~Qml0AD;ce8Ss-NN5i zVW+1h*+~e zvHFfUAv$wg^GZg*Cy7#bWPqdV<>HcJAte%?0}+{`-n1`=5qU3ieUUJ+l;5d!M+>@dw$F|ASE=#u+b=xZr!hp0FvMOLJ@I=y2(JtO*V(Aihv_=BF3*O8Xv+&w&j z9yis*o9}{&o)#Md&PsYkJat8D-aIv=4$H!Fp#reTaRk#cTOaQU#xzm-;Y=xZSe?oG zx=lMJPg44qFMC>_C(O7@PIOOAz{HtK#@0`pqc;7{cPdYX0$7okwa}0yhh1sr&x{A* z3)*{o;MVGz-+nv2$Kipc(0D(ZLq6}`n|x!F=BimO_Q#{Vt_I!?Hob=4A{nOZxmRgO zMtIlNbiW|>fSxb5l0`O(>=!YOaM-E8)BO>2HFvjSuXFV|6owffVz8SJPRF?Vph0M+ zZP}U{K{k3?9bNOsH{FTOX3h%J5R6^qDVH~}c7uWynK?;bAi5s?XLd+A9dt>DS1FZN+3bVCb=vGp zg8g^^(7(==4+VM@# z9Eq<>nWxv;q`^f7T!3x4qnG)vWB@>|-~To`;>%-|Al=|F)h%3~%^cNKW^KRSYovbG zuFilAdG*L=g^|;r;`<}(XzRi_d7G9)<iZfsh?}xQ*f>{gpeTIuK1v`Kg%lX*f&BL>8ipP_1r8)E&>$`0R z&<4#mT$isQc6%@O8EB}$Sc50_bs977JThZ2M;2Z5{~5**5!u_Ld0N&?@_rWRny`os z2bSa}$l}BGz_ml{@_%G}!fPsEeh>k9bwRaTPlo z_(^N;-6+9RwZmh*J0rr(6JoB&LQqL9HSEpsGV~naJ>|a1O0rM?4g-ue$?%i_iv9vc zsNJzq*vqMV_6;s3sMK-bq7hEE>Y!SiKH-+?*ph;!Nao)jIBJ>KF7tmPvq=iMoLU)m zI`WIjiIEdDhdB^uzLi9ce7rqK-MEKZJ^I1AhT!u0#$(g~V!V@fvd-B!aNNp>CzK@# z|Dj`~|$XW<{BgTdCnEM3Kg$Q0iM6?W)uCslQ@$QGaW)Nr#arLikv!&4Q1TqT zS&Pe2e!v#>@SKG!dtMU4?SAl!oL^-wW2P|Oo88Q*mzB=t;dz~e#<ug;-2Yhf7yOpE|LQ6gANfWW*xp&LUgW(hMjgpKj zwIW@1vGnvo>xGg4*P{~Lo#ik>biu^q0Q;vmH2!mm2JY*jwROCwk6rxzk=X}Tn8#7e zGdCp14(aU-pMGQ$+WGkRBBDOEj?7sH>cDkLJpQ}Suqyd}2XZT{EoO~10(pWa1Q$JU zPftT_l&G3jwQWkD%F*wK#%k;B45zo%4Hl&~PH88cNu_mfru7foS8bj5z4tb7)CJml zT-r+VL|rX!s^y$oaiT=t;V4*EG?PS0fNAuz39*mhmU;!d0S*29&p;YUT~(xGV=Jtq%(_KQ;F{;iH9 zrP`Q71^+HGPbQ+n`&QZSUv(ti84j?sCSVB!plIv4r1;krq2&K8+z?^66MzaKJnj(; zV^KDV_}}~y)yvW3#>21+9KauZ{onG7xkw~d=?r^p6t~JAUB1&EZ;qOm2Q6S^w}cN~ za^N=&+-^aQWGPZBuZld1SeegRCL;NkZZ%iZ`EaX+D(;H(4bmWWc)A@>zb5Picz|-` z_oiJ_|N5F|2{a8=@t;aIjkDaPCab#!yDyim9G&DcpliM=?qO`EgCY8|D`Pdx?qTM% zCFJfVv>(%>mi?}k6PL;!lrnL63A*s&n&!I{LW*jlX3;I@E&*yJ`D@VBx||u*-Kz*q z{&Ny^W%BM{y}@oyv$$lwMybpL;y)z1c$K6KKbS@|`+O!Pu4I<_U)Okgp+`~sDZ4yY zqf0|6&|Gt)67GaE#!$yLzLwCX`&JuEMv1Sgi-!#)%w(TaoPIe#^ zLC)1()|IU=J+elW=+Pc-M(*40kRf$-Qf`)*sV*DK)f1sd++xjjB+#XZjJ`*|&!dAR zkgLz*+>+6$ksbdwVyR6US|BnJ3#}HKOVQ|QrP4QkWNV20E{DdJ-JQv7%hr!Kx?|sy zZvLF{D8%)xuyknGtYM531TXJ^N$8&w{FG-d`H`1wYzf&JBX|!U{A&Df$X_D3gEZ8mm@`_ptKj=_j*=C4-!vUR2 zZQR$h#Qn~k#50ncFeUk;c4r{`kxCW~7EMp;Y3ItJU|`LU5Jy>Xz;RxfdwjyWKnE_2 z+kEMoU4UppdiVreJc_g!e|8@bgnsD&w~`(&r|Ow?zUc;OojV3~{&E^$+Gx8*YI< z1Mz9I^+@^bVuaA<@1k?&bddZn!r($?*Ht=G*Zf(PcCly#Q+`q2LPZMWv%wbAdl zeis$t>bv6s7rycwYvmq0AMqtpdw9m>Mh_iw^S6+0*dhiZ5^^V&!UDtf`@*sWvC280akLer4Z=ji^ zTqeP!>)t4T!VfSr185%BIvW!=#S4^w7UWl+#G*1~3JwkX!MKnvM>Qwt{(K<|ZoO4ads)}4qpmqRDz_)48sxDABd=OfrVfeH#sUvioA|6(-r7BtiT86rziP8mR+bB{F3t+O+<;jKvo&ZBDZEgJ%i4{|TXlTi%;qcb zsbzIL*tM)2RdwAqKDz(iOXKA2|H-7h_0(>53)wSj()M>3sXgr|v~UFD&WmMKYGi#^ zLvg0MG-CNMk2&^(E>4=IciOe`vnjEI$OUakUDl$m!iI9s^?~9}<}m_2#^|M+KDsXA zR)&aI2cIpH3@x{``AzlnmZA$EyIT3x*raer9&@>sg&Wkgw90po5yl1Pw-;$~Top03 zLaFPG?9Zxu!*f@hsqH>v^aJR@HK)qBk=tN)(*ZMN(Dc`Wy|_ua@M2GPf`a{9UnIEH z%Dg6A{6(FvG)~v#=qG^0AG8pRtmq@KbuZ4y+1yy`r0h2s-yX45uZX_BJnHeW%K7#` zn;Zp&>1}`y21byE&U#ER_R{%@hSG*|Jv6m9)po4YX zw*8)*wzYMb8Hnp+4TcD}l@7$>f2t@A=+efFh*WTHqA$}hyRu7zmImHc6KiluaKkRZ zjh42B)QqW7oZo&mx%r-InXZO!Q``xQBu!2-J~hj^FBvu&TXWHvsk#{S840@QZm~{l z)XDxcQ>^#gaZG!9%C#&Li_3+-qyeZ;%o(w@;|#*O{R~fsnGTR|y*YgWyjA5;+n5up z>V1p|76UoBGu8gOWgV{gj`D#&>k1c61rMyQhcJP2E>Ih`#{W-z#v+pdn~6gec+l{> zwcGO4wC1!|QGaZkrAz#3ZmiTc5C)P;sU||rE^6W^pyVtqySl)I`c83)=q@Zzj;u2o zBE+rf0qsPP&5~|^FvtN*g_z@ z&#;e6`XgjNQdyQBqak@*i>FtmSuvU|>GUl=o2`XM^&;Zl{TFu#=hMrhNu^t*@p2>Q_ANU)$$C&z8Fi=fl#I29RsNS5;KJ(H zOnUo;(+xg^>|N6CNw^#K4c~C4@y~w+D$K#{z$4OftN1wcZG1hfJiQ}29G5y~zMXvN z?W0Q*OR&*yg1Fe78`W@fjb7VF=-WIqAk|`Dabk4xu2*g?&dzvBb`0#LT8;&4 zkpJIW;)Cq2u@u^HZ_uxe&dXDFajHE$puUIGlvh&n44Er;XL{ZuLvS#&)evx`y6kZ> zu6YmT#{4X0YZb1|EqkVTPwt+On4qgyq`sS{clXdFizf!M4ThJ^Q*L2>>XF6~E6$42Q`Btd5N8N$$g(gLy z@bXpVkA4yOdA4xIp`(LtnXs8;45?)b!`*9?52I~19&NUEg*T$S%v>~Ln!o3f{lS~%Ldmt}weIPd1i@r935kJOy> zvIWHa&NW2B;pBdGy9T3~Rs#zTjUv# zWRyTx=|Kd(=v4oIdQe{|_db6+Po?ta5qDO6&y0l%WsvCq&U~)@hC7FF;c%2&IsqH! zK)g*iondRI7cv)FxVK)%4+=ORX5l$G7Wu7`%*nUW?lNGE8#)%5rMzoFoD3=m7IBRzJ%^n^$aNs7i1QzmTbBCElO?u%ez$B z_&`dGOP%m_q{vNXZ89Bc#Y2|Wg{ML1-ZM$CK63N(O!{ek>)UyV4~?rhnZL5s3qJ2E zv|xjOW+-u#gdlePs`3-N_-5v+^G2i}{x&XTj|#7B7*eK)a8=#T@;3^G5p_`pgB)MK z12?MQiSXMB4?Yb*y-$*iZhl+hohcG{&u#0OV-5Rl?k`qcDEr?z%p;Rvn+3sVPCH8m zO7=RL3#;UBx}uv@TwI?V7`FBk=laigz0-X9 zz|&qII_!EX{`IQL)1kwmxi3*4r>3j!&EKBmf0<T{q@Exas)O~HgVyLY-xXSKn7X$VMuPZSm?fP9_43p*ts!;hWm3pH#Pl zbY6L(JQXnBEb{xUSCU(UBg>^Q?Kjwolybp-f*hK{yZ>Y_PV3?_$4_-QaC<8mOR?EP zpjq&wwy+MliSd}0@1v1M>Z-*r@PK*ZqaMCvkuoPiQgo(`H9zUo`BoBG7@Hu+=7 zy)3C5W2tQ4s5zG4K%}jMct5CF`F|?^8vCKzWggkB;$k&2!gN=jF>L- zvX!e4_OYexo=?ns4#QC7SxDIFUlOHpOt@=946kl^w-`RGVyD)GTKT{5awU>{Z#OU- zRSsR6uM)J(3Vd39+(li9Nh)gT3OG$uVD+gCLkIrBk@h|epeJEsJ39v}H5zquD|-)~ zKbA>deYJ+NI&EUvH`kh=WbD(QX6z)9$J>|-pXO4m>n@_wHk1SQFS2f99TU&+ z%O=92wa4a^gdQ`T0&CO0n7o&;?9Q$^EMTi$>uCZV)Kh3zt;NA}oM7Y3c9!%G^pldL zV``SLSAgMzYr+ukNWCz)VfSc8?~z+iFo}M%;+AgPLf(}L-lYDP56|Ii;?{!Ee1a1; z&ieq6aUmq|z+yV=wP%xy)7^9gdqly0&fcbsI+^m=G(v~?$XV2Bp3vIu&o0CI|Hsu^ zhUMAh?!#DdyYW)orMSDhyBBwNcXueotvD1b?o!;f#ih92#a;hb_ubv!vo9au3&(L~ zGBZgg$$3WiKkI~81l9NO4c8DaR#hO%J9;-O>NbGh(2t$FGLQY*Mi zQ<3fqtkm(8Vm`Q@ET*cLMiU;rZ8KY8Lb~wwhI1;BUgAK#UuAra#BBZ=P1bg(%EZbNR$f%4gr&`{6Z^55V};Gse%9>ymZxPmne01{ zHhHMQyma$qecQ8#tw6Kseu`n|7iI}A8Sb=8%}tzyngVFEdjKS!&);)(Q(PeDHeM>$ z|8eom!JtP+fgyrjTH?x~D@EjI=|=3QaSwrgH9P3}VIRYcwzOuVHCcJ*%1;EAvKBfP zw=ESB4~(>)$EKzM{->T6z6s$9xd}V|d>jF2g%X%@KuXhjCe(6%hzKegyzHvDD08?qgbo( z^}epAC$Q9@n)YS6rCl&09^{kldBd}!Wd`5&@3ua9_~xnmi@>YxZ?Cg9md5)x68UPq zgAX%hh7B*^Av6nX`L;zbZ$*~%(NA?M9831F)KJOObV>Dkn${mi<1Jq=_~0yBR+z7I zBTLJ>Gis+5g^jx}u8(Nlg4EG%BU8N>xm-XKQUl_R&3D#~tgHO5Pt6TkR>ATwPyEPY z_mf8@ZdLsIXgfcuAJA?<)Kh~}ffJ*VUoerDrM|76Z1P`SWt;ZcLqHh)jLLsn%XxMB z*#~bhn{VoF%wefWT;~(pkns6#ba#3^e?$1-Kd_#llmU;h@8Yi8h>s6;;Li*SMj9!& zZ*3B$abz52EwS`{E}ya|L$kWLu^!_09S*P5uWw{!R=j&kk#^^E;kfKK!j70(s>z$1 z$AQkP6}C<9`cxb|)S8o>mC#yBt`!adI&NDRc&2l)J)xts^srw^&(?F%<<1KWpc>md zeD+LN-9`6L?WIOwNZQed>I5v`N%brdd*(3i;&i-Lxxzp|e06CIcqX6cuzQvDCgNl= z_u;}6_b6*2{V&i34@MJSg^UFov(_GE&JaGXAP06Ev4_m`wLcZ}%26|EvQWJq#q>3r z`wRwQu%K^zm@$3|-16KdLtYW+v%ghOqw~dYi0?h=U}JC<|4VL5HjHG*d{1sx-_%2H zt)av5o$qqbnaQwi8r3*V`H_v&S3gO=22$Jexdp;cnnpuj?Q3u|sULGC6d0y+r8o$@7O%+^AKgfSb`735ZCR5KZ zis-3^KM9D?-6bL%O>I|j>9unpmA5w8b4t%cJ2tkt1Bm?1f@1@#P{c_L&Af~BpN zP2Za3ae+$S-Bo4ZZ0RQ^Dqp=1MctDi27wpq&HVhA;j=jR-_Nb4?SiF&SGV2{2KJf*0i zogUWzG1+3XUxghb9Ii%M{)7pwK_@Y_P@V90}|A(3fmAcSBW_jPe1FG8*nv^&=+2HV} z;E)cYvBZJPg_IvM{A!;q5rV2{NZ{26Dxap-q%=40oS$>^Z+B@l zT#nOndJIR&W-rq`4UREI(p}DPcTBZoHgl<|DU;BkHrw!EyK_o*NbO`6ZrC>H&tJ?W zojr^n)oCT3)H0j?KTyxVH<2LXriImfu#iff*nC=NfcwPR-uW&Z1izTG~ z<0Jom{P%^MTUcv1tlu^oyQXiciR~`)*pY-b ziX@{oTxC`W|I|2tgjG!6XQ8K3G!twuF%{;EEdRTHFOM+!*;9gdkQWXc-X6=jX$N0% z*GYeHwoje_L{R8Xw~O-<7~Q4AO7ward;-Tl%aiLu{CLF<(DTvF67^mbT z4TM&=A$kk6f-C(P2kSO)F#i|r0Bb%ZS;>!aEBJNb-%3|p!+g9#pDqqTYl3QL#jPf9 z8f9}u;3{q`DNPe4tz9x%<6ahQkoGeP4hq$LpKH#2<-ohEr+D8bnV~<{*Z$^BV1TYN zwc0Y$RNgdZP5#(NZGy#h(0M8~pEI+;LGJzncFbB-{D48;nUY#M<+)|Y{mBf-bU^On!aQ~zoj$KU_!0KZWX-!2tCegjZRW$+W&; z0$ks|(T-6$y*}AHYCdv3}zEx^{(>P?d z#*13W4Ah*mBtiE(KRK3W{<-r+3hW(qsn{SseoJjXRY{O~&Z?=(!h4J+TbB6uy62hW zgJ`rqCgL%Qc#BUW-Q2kfYM!3GFd!JW$@!j(skFMWxsDmh=FMnak`KPJr(oA>7C0FW z4vx|^w~wV_Ly2_0esz7)0bpl1ira%OLma!he=Hm#k=muK+Nq!+gw8Sq&VWn@MKwQm zmn0``J8D!;AY~gO#Vl=vDl(^xMCaY=o77_E({AFu6HxvQ@DDj z6f2sB%vAWW?CAS(HZ<%Ms46!Y7#9EJG+p@hRzwa%@SY)}_&Vs|CZKVzfw|sb=1&-F zzu4HcB>} zO(Q$$0LqoATa&q*`f<5jilu@Ky_Q*!=u0id-cl z((zJ?Bh>M7G8ArI8Qx4kIE`>~rr@gjFK_<5EA`KkG{(t+pAS5%rtAu2NX0q!uNI*6 z^mt>|DP|#6Z9FeO$$(C-`=Ym-)pVN-`k(FN&qTjeZk8p|j@%8CR*t&HY_N}4^$M2E z@qHUkuTkS$xxVG!fT@D7;6zj|je&WX6mbYv(!k9!P)gQ;P>eh3Nw1Np!ilkfUrZjT zZ*TGO&KwGjRd?{O%h1v#utijv^Nhy>pbn(`|NJTuu#px$PY!apnk{HBAp{|{!#=|_ z244w|a*~t+Uaug30s#*=-Y7%?^_|i!sCj-Dw+WWUl5f9#(Y| z|E}Cemd$5KtiU4vi-x7FdrOvgMKp&+^HiPM+cpbDG4g+J2U#0B21^g|&pWg;ID{;+ z%PMr&)tA}&;r7*5tVJU5`fM#p9nxh2DkG9$H|~I@@jBU9-!31UI&#dtKh#%Z<`!`D zGGS&5PYTq?V9YQ-gQtgPN|+?FKunE=rr%g6IwMyw7JiWsC-yqYt+0FcKTZ>iOMl}A6@Z5gmjP?!*2kOCNx_J0Ie5UDNMCVlR`G|1u9bRWYko~dqHEzz1yy~?G zjK@@r`3aTS2p<$cLWQ+<3}aEtevm#A?a7!|TJyHv`4(B5ARqNO<%uBxJL;a|Ow;Xh zq;E95=!{B8`j77iQGZ4v{WI^|!jmqHYVGNShd@H#$!nVj?s;4fmqXCV@fCV|s5~aM zKVEdJIq#|;+@p-X*Hm#g=lWn^&V?1rr&L0cJW(h8T&)Wu9B5T{63<*1iJ-FScy$Hl zl>l>SQbuduImAgu@#YsUULp|=XR6-mEXO3Ssg#9(b^C`P!d$zeBjBNhW=6L5zQQkj zBqJ*#<_2VYnX9|x@#GrzzEw)Ex8vio74E94)SKHsstg-!_@AZ&z7N?mB*45!2D2y2 z(-=?Tw%Ue9A2+*FG$4ve!XILiG2|x}yhWRHR0zF%I7;j0uJ!CpBaCG1mZWt!tu|-N#z%?_{rZx^gj!t*^sc~b8R5c~xi2Qs8 z=f7B0YU8_wNgF4o8ccmEn+;hiQ9EG!85+;tk=G0#B&e(w{wM1f#U1T!dOTizN;K&Q z?~GQ~oy`3V^M=nR(&CE^Yj@iZs*xjO1pDVsM~(Ju{{Ix^_vn`}^sJ5bId@6RvjM`{ zEwjqE-0dhBPR=Gy1~rM+j-;PL57|tcvrP+N6$h+Un0R>_T#LM%p?1($X_-Fv=cr`0mVWDl=DA0Vyk~O4EG+ECDz0a~SiXhtM1!rDYl)`_;J74)^MWaR$W=lr;T&R>qa|CeOQcwe{ zKGKSVDE~SnvHjC&gBV*&qT(2{DgEZlh-4-#$BJCGY0nrE*e(Zsie5{5MiC77r_Q8_ z?nnbY_&tdg=0^OW6pQ`w((~yy*4`=n%9I8}dEDqC?3P6f_(1@OOH@VWst)P#%5Hs_ zvV_Wt;PWj^X9P{e1ByPpw*S)z@8Uld_?544>YaTd3YW~P>cl{)3ogvVE5g(h4frO1 z*`Sydp!<`yQp@j>{&J9x$N6*P=wUhjdPdmQAh_C+Y0whmyKgS|q&D0&jX~UVznli@ zYyS6<^{>X;&qWK_I1Z9Liyaqg;W05``eMs6@+hklIpp0%k@6ZHs0!!sRghZ&2a06@+dAu5(*vM+2ps zC^j3S=gs`EZT<8#acawM6qj)!d{J5lNf(a9yydcM5J1)*Qr6O9T!+20{~*T=Tumi4s$ZEx-bs#$){uZ2J-H<$tD^o^B1_7=D?+ za@%JLdfTc=5<_FsOu%aYpIbNsJ!_0~7wzC3i!X?ts0eo@QvZH6d@^1+6n11i|7A`8 z=RTiZNWr@6_ZyA0RLi$PwDyK$lN2RkH)Z5HEu31muV(lQY4%WG6Axb=cKW`aj*3PZ z1LUxaruXh$4GtZo_A2%csJh6@04JD~ycXs40+Mr<=|WFm0Kq~3M2@x8m;a%TyY4Lz zjj9`-!yf%Z)|!E;0VzwRD%&T&vwIroPg?+KB!QmAnoHuIPoH#zp$qeHaK_ouhBg|K z?HZZaM<~}P{H5L&$2WS!z6iew@F|58f>Z*V;0e|CIwk(vyTw0t`IEN>D!RyREW#`l zg{9Mv%)2|k$Y1F|Oli_i+Ow9b|G)j(0~#e(OeJO+0(CAM&y6S<#i5zd?;DXl-8Ye{ zG+~mh)Pytze1F?Ldg+ahfqTG;Ff7vXCl9t1fyYy9Ps~_)t&_E*Oiy#HyrRN^$uujs zPm~QgCwtAnZs|SsT|I(5Xl&^40Q9>UYoQ+rqQ+hz=$hsBmAY1J5z{R z%a+f=H5pudZ^(P_Mu*fhv@ejXo7Wjw&7^am_JLt zAYr&g^B0$q05YlWOax4Ns)A2dl|-XRamC~bNjb>A z@H+8KSj8^GEwt&6w1Rx*J@vg+SWd2KhE%QU7I`ox%hf>D|E+JP}vQJuL?ccXJMxAVdC0_0I|d9 z;5UB0jq(@{xD-SlYktJCTo=AMVDXI&$s&w|7tbE;>X#a2N+w zeCbYLMqT-;gq?S^$>76pv_>Te1pNdULw+_0f;?AoAZIK~%dE4zpeo)HTm6d@(3Pkn z#IJVw#A2S4F;9?Z;qZKgB=6yeHHf}^`uO#nn_Og3M{nkTs290;)s*x@A0ikP8CR|t zImzj5F3B~g6~ziDY_1giUGfs5Kq5rUj>k_`xB}EesB#XIua%y-NB0B!#KcH_fB~PT zADLoyhk3tC4>1$?FBY`@Jrj`u(cyt8QohR3&l@d{5s*SYVcApYyjS7ZBV=Kdu%tHH z@%_Bu2k0oC06diDp9Yj$v*fl)b&E|F0i+85IXJzh(L2S8mO}B-Hm*YtqDK;-tA_S8 zjFd{>Lt;yJjDYK6Ud|dwJF4-$z{j#>spMP;{HCZoy=vk&t5^s;){+;4wimua*OHg@ zYM(>G4FC0{KWBLi24`!UWW*2ms%Fs8%-cYBW9G|gNA;f`AKG7* zn1(dwmX5Ap?a97>F=3p_4xL$l+bPnQWLcvZ68mn<7%08@jdxx!7HUeW+jO0%v=Buk#w2K|1;-dZdN>|-^*#|(p{o+ z|Cdmu%6AZZvGe1jUW-p#ZLW|azKS?kNs;Y0vv3K3)L!GQwe-X81XJ;US!si4VX0%x zMc(w=3z?==ZkFbH4jiPgN*Z0MBnt2dQsoaI;T<>|dTab>Ftu zaUdb+CR3{GMm)lcy|to9!LVCoT+em&h^_)Y7M42sdWDRCV@&cDj%_)DDWRD@F*({M zCsnc$NZDHQ@|k@PBZq{J%7yl2?0Y9TXpuAT)f_$fiVQ_`29LER9R&-X!o9Ueg`$i_ zVn4ou`JOBf%7~H4;_epman&n zaXq$D%@gb`W7=zCmr7kJ%#E-3SS6D2_z~uB3dr0w&e)=76isl)Qexu8EJcL+-x=&` z3*5~!bx#j`z-r<$B8C&@I071+4}-ffSSV#XXBC_!79u=m7-JPCB(2Rqbv_yTpDuuV ze4imnF-3^%j-U9XRF}#i>}~4B(H4y9j)o z3;B7SxC2?#li2urIecbN1TtA0x@|2I8Uau+_gk>>5O|TlAa|b4n8P(+1dMYPgS;GS z7=0}_zCx0JYnb*k!&uQ5UxxkoUt)r6(emVz$0+)`jf% zBgI=nJX2?D`R3}PRS52t_1^8)2F_X)So9}I<)#O6Q!xg3n0fb?(HWK-I%`@r&fU4w-U zVQhZnjE+|a6n5EBDthe{{Z0oiN-yUvC9CvM=80o9 zcujihWVro!sOF)iW~ILEDtGrG(@<6~vAr5aT5E2iL2wO{{CAZh16qHx;|l#A$Kh?=0d? z+IUoi;8~GEJ$5oH&;XVXtV4y#uyM}EA9N++qQ5Cx6{rVK-E#G}Jd(ueY-aB<=9zDA zK)j5;ek>ak5yVF!=Udt;;(=v`3rr00Sheb4Uq5-VC8c3Mun%UUh=g0FJrlIOg66L(7aKfj@Q$524$sk~Zi2ObMG1GVpzF+q*X_yTiYBtw)gje$k8OEP8t#p z=P(K&5;V;FV3MG@Ma<^ypyA1O;+v$imfMN~uWR2U(PoJ^9v1b$aX^EZsb+AmJw8b? z82K^x5c@n3)>fuhCbrjDdJL8`CkArY3}Y z1TT0XGS!l6qeH~sgwE@m2u`%GAESnK+xa!E(R?*VyumbGIaT|JZ#pcDEsJ;#9@(D1 z;6`@gXUF;@3>GqrBUI7IPGNL_u^Y@M?k6Vg5oMM8+OZy94}A{9YAjyJS857VD9wQv zA>;+;p0y#N1)pJceQ<`2&*+ZT0RfScKpUi358(no#A%B;xOLIL`*|4^olFfn4naGn zrW`j7;rg9r^1H|QnHa`gL%wsVS=EDYLu98j?l+s5L?tIR=qYlNi98E#xuER}9nzs~ zIz5wL(2P}1N($ml%Kt-)tPqS(7H;$4fGQVS*iA1 zOrQ0-P%a}slW?ya?%DTTTgRIl4O%lV9~v-T4UFF|XFN^-4D0PJS)BpjsNOya?b@9$ zjc&B0WLpMsOrmPqlZQxm5g>huCfK&y2SfJ8(~O=Ctsa?k>d5jGU zoj}WD%qKaUOaRor!Arf{HgRHCMM9Ji5E;U897*486r(7Wm0Gu*#Bd|EsCb0^2+l_aPxF%;xPVm9*QgG^+#(tzK56iM>8t}RCs2Df_g1f1DSs+;%G ztwgteOUL%f*#Xi>(%)1TP3P5(ad<0;cn zG|RSm+$n<1#(_m2gh`__e7V!LpM1us7ZKj9D{vR#a|!JgR<6QVN5OD|3yu!AUNX?h zgW+_C2$CT`gJM|_Mi(I(FNTdugxKiY1kn~b*&>CyvH!eD%3G+ zO7;x0iv8(N(zqymIfca(x~s7+PczDdT8@s=3n`0e_Llk3(fac`#oRFAouS}0f--6Izs1_3fdSH_K%8>g5%wxpEWkn*q25oOvg4&X4aCCoM zzf?FZDnMug4MByl`u+ngRzwH+;Kcpx!0CbIS)f%r0X)Gw@+d#6OLh1hEcdJI{9bJ% z8}7u0C4W_}s4Z#HD3M9#Iwpm_s+i(&pcUYxm1 zHBNGV?o=>dzhBhUR#|Oit=QLypdSTy9f@qU6_DRZ5;9M$)$;9NRLh9T-S2i|%o+Kl z{qWjx6#WF-ug2(x0-L8ECzwiraGiSqUHnl|6dTHT(n!==Iecal_7jlv%Ya~||F|rU zEA4rBX1xd>M^L&-^z<-lbg6#rIL|dR>SRKJo-UbHeco3zkRaABq8nN5X+~=1VQQ;K zqr3HSBGm6(>b45XxffTWe;o3Y@I|;r$83 z%C90g!(>>@Sk>JDDe?`JSEQj#&JTK>X^s$A>%{sl$*x7zj*sYSVEkViE5JjUDwg>kJZP3g* zOheb{6LNmN{hlyDZ~|+Fa<&P05pDGLQ0(>%1s~*SIfcU93m-VA>Hd)hl0A?r;Log1 zq+#iE_A30f`TZW{kdM7r;6MvRvU$11Cp>r0WnGh|BY>4ycrYrytol_n5Y8|yA#pV# z^$BWIVf7=#PRLUoy4s-f$L|%;5OE9+5q>GcOhzGV@Wvgvtm7>i!b0WQn&`gQf_i%b{1Q9uZK9m0*g-%;O0*_*$gvKfJ_o92XRg>P*AB zG(8CQWF9^wIQzz7(K;Jd4iP}zc-P!^%I{`3-_@89KtT+F<`F{PUv>pMB+YI&;C|9? zT37tucg#oSy7w1+hCqO;Z5B^A-(l-R62`MFV`aNx51jcWpibersUTtY^1U3{8EMc{ zEi9Kr9Uif}oW3yaj1jr{kKCf=bSh$Nx29aUgr?#xzbkGv85VvJq={$u7Ek@3SMk7Q z+kvpdSsFWB|o>qAwN!5FKs8V{l#QsWk zg?}+SK|tVJH7PU?B#6pi+(k2<2`>8{iV@|DRio{ z9O{#@KvF_|B6Nshau)m1i|}-#ojK@8KiUc`cm+62QBiD3^ej|B`t7&tl6O%gU7>*5 zd$E!XNO%JvcX8%ZsNZqe;XFaM|Gz9z&mJ!n~t*Gx1pySIuJ+9>;hlG6}QP2T!RPs78q6< zdJ$RiZJi_!W(OUD8Fh)wSplUcjYY5#ew`$h8drplf7Dtu2+zgOD48RKU;(=@_D&Ke zI&}O5NNpU}Vb&;>l85;@@s^a3+4DH{3sfdF2r3P586478klFY>voWDsq?KAD6o8|h zw3k!KRx-KIv`6Q^vLTVwD*>f#JY@St5Lu*$*)2 z@ilQ_UOWvz+?CJy@)pBtFPR&UkudF|y-OysI1~sP1bU{GJQC%z79aB5=0L3d&jIx|TXOPZs? zF&m$u(zeH!CPa&@i2CtZnp`p@Y1sl&{oM>@V{%*;U|Qf_ivtM46n-FyHAW*;5-E%$ zv`!4fWz{Hc#03w5CozNK!Vbt9EDHrP+XXI>+uuuMHA~KsPui~Ame+_<(~cuYurP`K@veg z2>1t*N4DbS$BT|dupt^wtUpLp@U=2M?IxLxkt$MdV}FZJY%T6G!N%J8RybM9?B-N7 z(86{_F==gRn2G#ynQ9C9UjM}@leGDM)gp{9Bv={X$`x9#1F8_P4@`m>s0yau*r|*M z40Kybll|f`;h_1O+%pQ6e@Q(1m*x|N#U(!$-WJwWQj{DX>CSu`K&4$HxrGv-04Ftl z`jeNlMj%VKv=J6SB|zx5bc_12o+=U(SM>BXJwIEGOu_euc`!F$RpHkaot{a6INmQ0 z`*WmI)I)==gq|Wlf7d!=yYS_3iHV2rr+o(ZdSz=Wzj&Yu^23N%6fBteF5OX$xb-bE zUF*hw+FUH?4-JiYf*lD~e{krJ6az*^SshVO1scf}Tu=eqg$ew8dG{^vB$ypOh{Y%j z$<=-7CnSVqxXQSAB|4y58lU`63q9v7YypiGvEl%Y44~heNX_BR6wnFP{Hnc=*E1%E zdwVv8dxFR22(A;e6hD@bEZcC!E<1L+Qk$M-9f9V@=;JCgHKD1i25TP|iQj8ukyi(4 z>AqcLeW1u@A%eA!7-prRKhT{!@^sX79!>|slX8Usa-hgRg)r;N8yK z($4QZzvcP-YcQ7cqt-kJlu=xr$w)Y^paa0rkCH`FmCr0AN*-gTDatJvoIHfJf6X0d zE$&RF2(%9(zG6-*B@E(@SDBIE~}4PyJNldxyqoxcKC%wnlOu z61hCSeD+7bx#5b;oJE-ykN-AS(qOYV&pq`%d+c(`^K<;)gcVFLUeVp7hC$NW@nEgX z&)Iju`%$^@H0G)i+TsF>w_9*;Gc=Js#@^aX)XcxuHte& zK{;y0nrsZSgdfCX$?aENxRoH{xO9vRa`=U44U4&-3Yh0wXRg0M;E4dXzMa3^&-gUD zyx}gMK`I64(=P-JyLQsINqo!Y1<`(nmk#?K3dBB6K6@We#m(B+B6>}d*(N+*B6=9Q z$m|KM`6#17w1ua~m!tTA=P4Hhto>;ZSnX`umk%OmZ&c$SOJ~12$MRgvQs)91J>t*L-DXqJP7ozuHpo-g4V$XY zS#lx%dI}oV>=H<;GsCwsR=!sG9IT5p9k?$-`I$Q%ZCJHA52EePk4Q%-0)h@tnH8Vu zDZhYUi(od2j4+~pE$3=vLCA{)NxC+xBLE>;vW2IQF*2CoZy;IDA7d3v7eb48va|mz z1&r^oI|;qRQ5YVS!HS&pUwQ%{MokNeo4yA5g0;j4oi)jwHE#|oi5q`vJ`|U&2E7)= zK9{Ck(`n?&AL?!8zHi8%ZV7lkef&z5o%ym52W{G5I+V5gCKarWd&70HT55?CCxHW~ zO`PuD^DS0xyuJ7RQ#XRM2-Kt74%?>xA`wXy9@#VRo~)!)OXMQyvxM_9%nx(K$LICq zNz+(TweJ2+AL--%~GhXr`;I()5?(3FRWBWW;k+)`ZoroiwE-e$r16i$e0Yvh37%w()y5dG_GsIew=U}0mnRO z%Z7t|n067mh^B$D`nGSexWcFT0=0KN```KgusNScqEI6ygpgoVX6Cv>32c3=Cn1R^ z9R^5rDAP71vYNH&G8KPputj>)>TAW4Rxk$F>>5LcBPl5sxb5y~c;*MAi-c2|3?$`$ z^m{!Ndb}0$4-X9-XAWoE;t4F+k5i372!d3Oz1J!0&xB?pgd-36jw4cT3d;qjS_1R7 zLP;1?g3ApRUQZb#H#h^ZBd*Wk$Gps)w13n0#rBQ;2r)z?pqf^_2+lAG5RV^_uVnIES|8frGjPFiA8zq=+OcqjNuYzl8?zNkAnZ%Q8}CXX!m@ds4g26^AsU z2r2zs&BQB{kez!??O@XWtznm&L~Yg#X`1$fw*@0y*4DWBs(orM6DxJ-*Y+xDSnnT^UT2apq2X{GAA`MB# z_Lk4GHuN55aGx1pg@0^Dii4}GbC2HLMtDHi3U4Q(w9p)V&e#v9VE?2}R;80zH>TLg z?YSjU@WYjCIM;db!FL`{CEA(0)aWCm6M;vWf&(l{a$4O%Mb`J=GtR%s)j_vCoADx#o-@+?ohe})D{ge)o!8`oOnr@im9s_`jrgZf8hte zpu0`i{k=kRuD^@J73FAI*&qr5Uf7nDqxHqfiHrVPezW(MZ6_OfbiO$z?kDv{W`9om zW-V8B4MIcVt`n1E$!nf-^09cMhzJhggMvv8{A#gW-HQ8yKq9vm^3+iSSh|`#Ed3>+ zyUQf<;i z3q(!E5g&-zA^vFI`xFQgjKOs-LC-77%!Q37O?9jgf?}+CACxQ-sL?IDBS8o}43%Xj z2677o`D@j?IttB*)kS7u)$g7<{y;5fdI$WBnSOkz(c%#jcU;r}lE;x^iHFWmu~jTP zUe(C5X)nmmsCFfBFR7z2-ajWTxSE@9NOBf!H=2+T-w4N;I-U@7z52!~dl>0UEI9d+ z?Jcg$(qD zKN6f#wGK=&mR8T>k4moV?3dYk(rQjp*olv+q6IMZw&BVUZgGDnFQxgu3|>Zk8c6Na zHG65S`$9mxo$s?upk#v22E@XW2Mj-_-n|a;!&R zu^V38`j1Jrfe$}?V~bW$@R!Ur>Y-(EZw_9$a6bvddpQmqS3H?=9XWpHsYk!O2rNL$ zk*~w*!pK_yznzQ{PcZVo#7bborc=kvlin-|lh6`W(x6ipPpSIuoQmE_lrE!?FS(V! z|8cVA!io?Hc?vE9G6`6!#qNh_i+Mc+V85#|1qNIFi2-=VupT4fz40!D+_1##Biz?f z?$Gu-p!)wSPT{3WE9U87Elh6H6Z6N>S3OiOP@-m;^!fx{0gI@@pTv_md*GpxEK+AP z}u%bB)hS;1i0? zwEc8=?*#A`!W7ddTg=dIwUZBE^LB{ElAer(mGRZ)cXmET)x>p*XE%NIiVpEN_(h2` zP-WGC%MwNd;DnV(S~j*XHC}yu*@)?@O`wb*4qsSq*OedN@&wgAyPfx_Xs^PDbyVBz z(3wOY19=*1F^FeUm^|+=9}`bp8Eg|5DcKCA)%-uJ%77%UPFAWABrq;5Bpi)C!B_aJ_euTF^_^_0>IS^^;~*QV_e6- zVUP*dwY4%fHjbTAC4~8c#G2VIO2|!Ai`5CHrm&|Orek4^l$n36TPtaU!pf6d)E+*e zu3k}4dbZ0i70(e7k9=vvEBAb{e>r=eAl*7l6uUjbkCcsK?ub@SEa!N^{ z1H?Y2=MrOKA60*?94n>47<&DHPny9i>S56z1YT?E2rN3W{XynPIv;*>t2>lXj77>cIY-X{567DCc5M1$Q6EXr2{B32Y@i!oDGx*sDV8 z2VTbOB>AkQ(?N6=I7)<1(GBn>9*LwsY#ClB9c&g80q^zoZUWpdd**3RN$l>Lom=|> zG~gCO!`Ii>rFD7l%7_%}J)XK-)K{Ci>634GrUAo}v3l6i4q9>aj;+!ahd+ZFwh)I& zzJQ0c_%?@D(H->NLfQQ~aoKsg0dVGuKy&6-y6?zkL0&MnW)Pz4P2Fs9z#OGIlc<;fONK=@Z}8Q}(?h{y=RhX&K>BKz&ttvAY+Uq;^$xW9f3Zh|Ys!E!xtl2RokKOa*zeZiVN3Ci#D+5zK#Oh z#oon~7SH@Tr6Go}HA0oe%CJW3Z?Ho|NDI4C5I8zT3X7s6Z$c0HVvBU9hBfL~ zZbr=c(#AiTpUwgrONbA24`6N!*~qgi9dO}N=S>`ZrA(uCTVfwTCIF{`_LxP6>{f$7ci5(^y0K zP_tIjP$1`*Dwfy}i_%TAP)HgKBq$-i;th*9VMSc7zc}dzRt!R9nb~zUU~C;ZXaCCP zE+s$#l0S?FNTeJ!46B)2+iD`L*VaBi?UJNep4L4VC3q?svJ(|w?L2hz+Vcbw`x?G! zYi8{_NFb%WcR4uiJCOY})_%Bh*(_{qG+( zctf6d{Z53BnDLo8!;;7q5bc6KDZMIysf?OSMR&Oq3N-!!lW2+ATtc25`lKp9XCj?~ z%`s?0uuD|~_b9c|y0sSrmglj{g{Eh?*p?6|te8|rRvE+695W#C#)NmbVFYV7&>^-0>En%suaoEfR+S^V z6utMN9t29*bBPVk=lBayC12@buH$2(NiS%c7P6AuUSzlrq3p&ZkP$aaNlAfZ743X+?Co-R%+F$)yi4$(E%BF76BFpy zFD=Qxtv`(R9T<>CwoqQrACJiY-oCNOB-zN2Lli5(mtujOkrJP^V}{jzjRT2DkCxWe z>?HW-4kA(0VWJEgVjdD~cQuQK@+^{=6ItdTSa#<$mn)U`L-m8tw;x02?+ei$rQG&| zk9fa{TZ}}RD^su=0i;?c)9`5XN_fVTUuD^?dYuFxWKh@+G5;2)3yK0cx@7vvU%_vu z@9rN%Yw9$!`ubnNb`t$BM1}(2Op*sf&NPPqUejUB@T88>5^=3p%mNa;wbhf6QS(s# znB~B`PzfrXY1}Xqcud02rW-7nf{k!v11Lh+pR}t3>z%BZ;YkI()~EB(&eZfTmTuBp zBa%9qDD*h~A5Y)F9$D8mn~7~F6Wg4LZQHi(Ol;e>ZQHh!iEaDqdERrKf6%?V_quCU z)m_!!2l9ZS{PfU>pZeULJUQ0Q)ORxE94{R69$Z|BG#G*N3NKxBREF%0YA5+dxP8HD=9cQEzo^~+@0WCUE2Sw%3KDuMPOVhoE#p^-Zp5^ zJ}LJRxO)f?z!yft6!}|5r)SluktSS>5@$OXI~x;aBM;OY8~1EJ z3R#wM>zIRc;>yI3enPYjFr^Idxn%Y77rU-dF;LP6b1(l>Q_OAwxtTT&V}TLJ%+pq~ zfeAPkljjYme^IsB8{XT$o^yl5x3>w~$l_!@5IBU`B6&w6y7pjtVS=n?i6qdi@N#lNEpGsA!UAasjn8EO!3YasjV>3|s(0CwMv* zSaza61YN|;ZSmF+1&Bc@uwqtP3nT0bK!*vX;JT4K-8+mnt8oF5r^ye=(3SD2Lbh+m zAOMDF2*5BdPcor&1vD1WpOo)64VtmFx}Ln?d9MO%ul`f3&^Z6xlTpHjCN~wqV4O?{{tT(~DFC!|->8*@=k}yj{^u2Iuhs?{hwbT& zvWX>^_p3(KmT31ECmSIP_BuEF7ZYm3q-qO8iV6ukT{Zk5&}g!7hmg-78$md!9=Ck# z&l04(?tl(hN7k$ zq>T?wj~X06e*T2K0mTH1G-&6++rFD#>F{x1vjho-Fsn+nH)nBCM`dZR7l007JCcCs zJrz5Uek=F`HYN}}@{7iM&@Dkf7>pPI5E!_MU8Uz7D0Ay0(96Y(BM zitqLtV4Z*;DHP(W5%068$n0nyY*&oc&{c9oT?2&{V7gm%7r(vez<-;IsWBGP)|`_+ z$eRTzr$1dyzfT6zF#6p)OM)4gxN1KU!T8XQU;8#to)sO-LlQqlg#6$g=G$?bmAdFl z1eqzFfPg6>VeC@)Joocf<_<8&gVp zzhsg&;zRB>^PhHkp@V-eJTKU=jLn znHF{GyrF6yP3q?waL`{QohN(zHc}N#vMC+dI4&)rKgMej_zKkwfPLtPge()}7WFRu z=@eGjl+)drj;!@?FNKSxAP0fNE5w>t;l?ifrtfhEVZ<0@j|5=L%6YeKQ9$k){Qx}B zFC~J>8#yM<>>ru}byiK@9|eLjlFE>l!&XiVW|M4-Ik8Z1$%{^tKV5=wLImHXeg7%4 z40Fo%tNa$`ReF^)N|5Q43BK5AsXEsW*(G zBMSAG#;b)Z#tkV*Ce3p?{R~vo@5e9n4u%YP#^`~>*pE!3ykpc~Y=^IeeN&HPF3V7N zGNc+E&+EgsEs*%Ayhlv;bf{c>-)l8h3k*ov6=r_ic|Yz4(QgSuSpP?W5`-%!fKQ@; ziSDvu;{mW85G+C%m0{E)WF>(CO+zdpQ<(M>es z_KfCK5$doNYTy~}WF@A9$FuwAWIa8h@sz(~L}jib$pZU_`&gA<<-2knOYH>o)h!-0RzI!U@pYHrw4BTK}8qQ8A2u&t~?|L{pkPg)_*QYo4{th@&WMSEw z2+sr?N9IS!A_akZ z-V08LrwdXJ`-d5hHrP`wCtr9ojIi*jxpj4@cmSxCF@T%apPg=yh3|n)szbxJ^+yh7 z2mr9i>w8uT-sYXvPNl;JE-^%`*AAT_rEs%MzDY?Tld|lcV9#?X2LPlnqPq*KQKe&l z0A?9ny|LSUGa3tzt7pyVUda#FAB^pQalHSynJlJD)|pI#3pj_Qtn{N$b-9~}KmPNQ zVf5pKM^C7eS|zqRx1uSiF~23Xt)BLU5Ii~vK)6OX+Maz{6Dk397Rs+HI&M&mgRyt4 zZU6#P{JqxJ44Yp3(D}RLn@jzrTFeb=L`1l2jsQnTZ)H0Q+ob2?--r6IqTh)#x){rwBtUPGL^iN)sMzi2z&YF63nP>I z(n_U(%f4U^{6fAZNtx3RCs&V-rG8g#IWG(J z$vJo6zoDse4?mZ>3!Y1YrU_yqd%wzYjS`%L4y-k`Rt?nSWbdxXi2VUO$0q4ZsQj>L zqkXSZp+wlE34VCl#TB4@_;o*e2bxC0%ZO6(<=;STY6(w=qeU^$+coAoRjIHwR3ZD* z`v@4oEI;dbU+h@lo8P`NZWysjAMf84JH` z0g@2!?Y%hyyGcJgaHph_6?!Wt2DHIVJ#Uzks!V~RZRc`Hu0 zD36{-Z0SCg8ut{^nn3C3(xflzgxr%Ca@Swz&i|u$a<0^570}yCbiHsBN^SU9&)LOr zViRF==?1)P?gnsi!TJMcC0+wEtC z8Rar&`kiyFvkqC?0kc)=2g4eM&E*MV)Oe8kzaAIQe z^7m>ax678;mn`&mMLs^svlC3>xWT6kH~og?tCt5GHegt8biZH3KCzHl#s~vFBoWRG zr)^ga^Gpk#_MVmHo|#m&n2ABwNATrf&GRHb)z|ZwSi?CUX)o0$0i4l-Dt7yMM-z{J zgOdiyX;~c_sO|%yM4%#3sli|q=ofG?!4pvCbG3-Csl%ZA3=BpoA-JY-Xegziq=bVn zz8Q*i^=Xa|uM(~SPE7B%JJu3vQczmKgDs&42^U+LuX=|>a6+Xw=z9)op?4^c_x>#5 z4Y`(5c%z>+-R+bT=|ci}GNi;Cx?Zi)h;E8R*938U94=Xs@!4zmKamC*8WNimXI)O4 zFgH`TIECQ=AOP0=7d+hW%~^x)%2N;tfq+A?zVLP+`-or&01V%;%YtWwL;MVTH>ug5 zU0Z~k!V}>=uK0I$>F33yNn@Mb*7wk0${%bx(rl$!awFzrSfqx1e!u%P^q{_)dY7{$Z$Y3$RWDFcYT&S;Iu;>4i6H44j2HojS zlIk$iJ)b$VRI{V)I}VXO*;UiY)v{o-!?*=^bmzwty?HhMP5s)+)rtqkkKIQ{jBm74;PnhE`1pM3htm!`D2#XPF1sPl_F$^|YrF&>AVC2i-@`{`1 zLgKTcFcj_X3Ib4$9tFb~aujp_BZKT<7!jkNM82ix%>0cQ6mFW?AK@iGL$W8kj^%_6 zK72o672(l;*T)IRb~EgaAbEWPW7#UA{(%QcUZxpTGhx&GDqILp|wY*J65-q(EC;hy= zco)mpHgk{TJ9ZR>hT|qQXB$U#W&k(($?t*8DCS9ZEj*V5VsI%5QN30DDOh{-2S31a zQ8n*!2-3fXl-|`v!hg85ObrKpmrNSG4n_Em3kf($AUIu@nig18)GhUu>b4asv+DaL zfK|=NFE*FBFzX<0XAgy`Ua=@pgcSdP!Q^|ZB<~A-K=A5IKg_T# zZ}E@(2#9mhx}k9_6SamL+)AR}um;X9J(IulK!B>)B16J~hcN*91JWx!xn?}-5R*s< znnK}E5jzYuR}%~T6T3f0WLK|5&h`^s9gP4(fGxZ}Eytl&)?YoiOe`A`ut~5UugFq! zK2}v(sbvIj7XpU#(?h~$gUx;vt)z<~zJh|tQ9kPT^;c60;yR>5B--*P1j6;n=fD+) zzO5)(f6zOk^OMJcfmj{OUNq~dc*=%#yk!BMKc)=Irm%D=m@;{ys!L$K5F07x82FZ5qS{5?zB zfi}@7G8SENL1M6EM6bQs+|4S_@?c{)V4)x|!1)?1MHd*_h~ha627vlZshTX{ zlQx|j3asgXtrhgPYIqkDp{$_OdJI_}S>BCbd$5hbEtT;}NaDD0ALlXOZ$-wyC^J{8 zbh@VHTl&#^1QW!Zl!v#ipQFwgaWMsym#KZgy(mu?$2*x3}o$V;PuVACA{^VQOiqjc@_iBi3{LFBkwbMIM9|@}|ir5FixM^Uo=& z23*@z#%ZxU1C@*a&M`~~q_=uvI9C`>m`orE+N?gH=ju_K6R@u)hEK83F_5{++P%f4 zUThRW1rTF`0RcTyJfTjF+Y$FMOg`4E>8}JK`h$q596*flID#C~y9R}3+Oe@`` zS_v~&c8{j)M%q3oHr^i%^j@tEv4fyw(OEkZd;glU8l>YxUfMgexZa2_?na-5e&5{5 zAn{mTQ%yEy8Yio?THee4-objkszUrE^^lAyu4@`S<8le*-;t(nFKz z`o|>Fi$RA^VdRSxZ(f>fkJCt@nHiP^^e8NA`Rn5?g6{Vsu~hN{z+OF5-dFRR!Ym;0@9?BitxPHtCcIcs)i(rTbyff>g$DEI*6x%BTF|SHSzmsR5`z2*sBAfVl zk7g9M0`{v_UR-z3;HG}6@DH5_y*nhrJj4_jvftByPQjTeCz>9+MWVM_vmUC1C^lA> zcLD~aMl-iT7EplH>uv)Xla*}+!wrC8X8TWhso;<>H8}@ti_gC|?IAsBfcU`7{@LC_ z3BiPeM`1v6oIrq1(o=`im_4Qag_ng4gJJ<4j}Sizi?B|TOEg%DC%-cAE5)lmJK^M8 zn5QQ;^0a)7Qy_@iijqUewLJQ3&-d^ojvqt3_xmWfJKq~}256ob2SM|%FPg7Up4}NQ zZf*pEDU#^#Q9IyFI}#mrC+_(#l+{g_33pi~O*3qxn*1x3EnR$X3lY!Xy z7Z<;JV<^e9$r&zcO_KFNrvexO&i~n5edqdq;iCaAmc_OARDB4bQcx9G#$W1!A2tfC z4^L(s9hIqoL&mq42T2FGXF?qNd^0ZTi>qq|Jw!q%h0%P;4hN$qY&ECJ}W; z)->r{z#3W|JA(2|&3%)lD~=imBcXfoyE9F)PxE~53Blo_C^f={^Cg$9uR9c+f9 z#&6&h0t6K7$4$lz$GkuXimnbq#>9Vm6e*?`q#S6HN@N{D285Ddov*!sWe`E$O(Fdi z=*V4oV0_R3Q*UPp6(C&NDtuLt6vBDp?|x721Hyk+5Dg{N^IW9*E2-3En_tOo;&2W9 z2+lPNFHJvPOwSo@yP(4yQ*%ENb;bRm8?PB~QY?hMAai#lo(V{V%$TwLn!+#xEe5e8 z0?_mU;I<lGSphsy69 z-W?=C3y0uBvs{2hD7`Ravxh zV;5II?34r%R8qc?mYqi%M&Tb-m!QW6d0MOyxrdQ~s{u+(Q33bWkAWW;Ha{>mQ>*9y zo=P4M-~8`;1b^+ZMal$sL!!Ij!rwv^ZzsQa!vU3|7Hd{N?mz3S41McE4=Ee*o48!% zA_8pCsLh^j+7;tZP!b3z zIs}dAW_YMQI*x?^_aOeF+$iEc`6S~W0~n*#gnyg4{gVRa#2@Ti%6dvNOQ6!7*1?9U zEKv*~f%N456F7s-57+wy{_K((NyGb!t*FkVBtrsx3-&JR2_fk8+w}A&t%zE`g3d+{ z0}i~1goFKFJepet*zp}&(o_U)1&!MU>GYp8g6gyh6yIH?`WPk?#xzmyyolwdv?i3r zHBi~5RH&z&;kml>Qi{oz6qlh1H}l5dA|&Q$e$3(AQbHpZ_%D5Z?xzBu7ON%ogv z|NQ=ZeD3NfAjio7aqX^|lXSN~sTziwxJN5|?IW?u5WTeXv0&6cb}t%!)dtIHchDtd zQ@c0BC#m(hDCPL-yL}9pP*JS`g3RTY=~HJNpQFE;_$&jSV|nc8T{F08+~G5Eo`BNr z@tsXU*W65GS4OCW%}h-dyS!nHY= z172SdaG0^3z_SAl#?FzE0f|6n>?n~&RHkKRfWC#J3%scN_(>Lhh%586G?_)TDc1b$ zpdcH$mH)v$dF~j6P%n zKc?lES+cXNw*;eX`j)H1mnW>PvmL6hnDaQ5cXQKuR`x|;$IyZq!hMKH4Hk_64}?LD zHm`kHiYA8u?C1&2zDopvbHO|BT6+=f#~Qi)a_)EhcXcX90Ys-0LlS`#2NUpl3H80Z za0Sx=U&+XBf@&GY2`izn`Y@`&$6*s+ zL){10c0#r5K$*e`Qdcw@I=a8Cp8hX$%**ZBbeEY2*2ViX>A?GP_vfbxiOTszPa3dY4Lt&5=ET%bu(2*`U(uXI2!^69m2}ySk ztpXe_wR_{1?VleCytvQR7zxe@cRsD_kx&yJ)^GzDh;QcPn)cjJTP1=lgZ**Q>rV^^ za6sC@4d%2rHVPZy0vV@b$&@9Qy2M%zSvz`0A4gFxs-Eq73eBv`-FTixV)A>k_qd9y zvAtn9%NF%_TTj8Lc4NHJ#-}MXUToB0xGcXE;bcUXj%EBjaCyOtE-7Qi36%ycWHm`3PuH-0_Ov z;+^ZBwN6(;O=4D5<9HPN!{_Ubv=BSX>#NBEG#N?kWrX7tHyc=3v)~KZR4nB%!A-wl z1jriHsPKD!k*{_QrV`7K+2joTTr?{mN;aMdk9W+10?n=`MWN_cxlCS3RsG-Q&K`g%JyegutO zv(TU|YZ?E^6ZKi6{5IZohN@nyUNlKW+6^#~&XG-;9^>96YqY*u*1zR?Kq9gT*}E^U z-gyQ53KVZB^xm7_g|q4nx?3ok9v%SwDLJnq>r5%c@e%Icp*&gHSY8$0m3TGzjH)2qHkUKksYu_wfL`J3gCedNLByz zBQP+bYB2+z%Tb*eg<%Ko7tCCmVe~WXWrT}Cmu3R6P)c#udO>=t9Zx}{Ih5_#avCfY z@m25;*?g%KZ+w8t*;6&f7GWkkx1a)Ia*|p3RVWHTQ5}3u%p=758{p@aw7tmXJP>MY zdAZx{0t{ZXIqZx{sORBz15W}2R64c2CI(khjI%r++kj>KqiOp?P}aO;1ty=Tb3pfC z>#thDgv*5~C;zItR=nVm4e7Jry3I5I+lwz=??6TrbFR8pKT{;%wcj*s%MUleafk*O zz!fZdL;$*+p!nS=bzknkT|N0sf<4-XYJ2BD7ZAeQJ~G=RvnX$l*IwA@Op9Yi`+QP* z6s>eemP$yIbqLy1uN9IJi@P~&3SNI8u!CsIQ-3;c>>3<$`%bV zqQ*bD)Tat64f!eXWb1V#b@^)Aed~hFQzF})PV(07VK2+I_DVohw_`J@R8MY5WMI~k zi!kWuXaWn*m?N&SXu5gP&fhcaw|1L|WuV3k*t{~ddl!)k062_&UhJ%SQ2z1hrG}t! zfCLb3+YTWyaE>~o(Z876#B+$80wfBft=#AXDDlBmxypf7cQVE_^#*E2SzHi-h>AuR znO)w$(>kvL)}maSCnGm!!6{>4>QFu-oYVS85l(3$9g|u6e}ZYM%F!=V8;c!qi0fP& z{?Xn^$`u2gRBb6dozE>iyfZ^O^k_V;>-%2A6$H}a3%2n}`&N=B?@rq_am-T?or3K}8M z+rZU>T~Dv$cM#%Gxj4wMyEZ~LKqdjGbzIQg@Hi}b!O1I(U7WLYi881Yz(0U`G4-IG zs}Q_bXGa`YNSQjLJNW&s6}bDEI>!=6A=%OU0Z$=cyk+n8uyo%< zj16WRgMeId4sA%Yja!+|bX-)0V+Q)KUxdQi=eU(BDMUMX|EyDvN33h^ORY09a-N_k z$|yx(;qvw8CE!xS5}(?O^8cH$9?2)A>-jhGijsD=*F2548)_BFH6yeQqY)%Z`thc^ zOtGpBW^7`{HXG(ci@zEgdY+#QQB9ISx5$nLD2kwrU2~2cl&N@k`?D(@#eUc+)WSYG z&IC5e=gf`NQ?s}!U;_`pTap1*3IrLn)je=^055%2(01N|^Qt|YR0c=>f8?Hh|At8u zkDKdu7?DZ8c*HP@Q_w@d9J4?v9`7@|QxLA-ONmbQ!u1wnPsM`|(>6uyr%NH=WzdUr zt{1vcswTkQ%1$?^aX^9yXDfyhO3B}O7Pdn;9&(^sQ>eXXzLxH5)A zO4?Dh8BmJn$y|t6R=yjVODVjyyY36g`S_5mKoo)>S=ZC{aZVEcHMXk99yA7pxyR|$ ziEAtr-6=S3+H&xGwC!~WRyj|cq;dxwPSND_gdT(Ts2@BQ0IYn!M+thjb*9 zCb6Y#2L0pHF9Z+)<%^~D7GgAzISVvr_&Alfdg~U9vm80=1PFqK!_+3(Fl(CMMH3BF z`5)+GXnwJCP9cp~H&r$gxfchB>DglDsbYzUCs}1K0t11Bfo}igiUNTxm)cPd{^Eyh z8y?*A3zL4&!6O2rhAQIp*;?YF2>Icy(H1jM#RbVB)v*osaFy6xRzMA~gi^{L_O2jx z+r0a5i2T7!!D4ZIj%qkMK95`NO+wG--8i7y9I%9-0@POXRI_8rtF@?E{?*g`?85J$ zwU)KRt{z2?3{+7C#|vNalW@lOz4vGUQQA(VH2WsN#40Tcj(2swdbTR*u#O7{XgR*~ zD^~tfPdpjbKoENZMdLCTG4E%Li^@RpPr`F?3NtaF1FqUvrbHE_h{4W_gysy;P#Yj^ z_owZQXX*wVJ&=>~$?~^vF(+kXeI(AV_VId}o7dRplKAo8?UaOq4P;f6>Z= zo;%)+pzmszUosONn;DRhbXp38bDq*Tb-mjk1OMWL>}50jTz%;Ia})8_Rd%=Tg3pN6 zoCN(w?2kUI!(|^|4&2isD_R_VO9kNpUqIC-me(MF{F=K?lZC5Ebbe{^@oOZ45|F|^ z*JOO5_wq}c2%<&B{QbuG9AsDWrdD=gR8kY{9e!;{(f{|NmNdpFoN@w%kh2$^Kip;b z1W#@8W*j#1gz}+VN9>G^Sy>C6&&N)u3#0kyd>wDvCd%Q0?H~8ug|C3$%7RS-nul4Z z>#eadC*9BC+;;9x25;yt622KP9VzBjMBD>Xpl2!@$ju~ zVA!5`&E~Cz&TL<7l61;!*yF_E ziQ#eq5tFDZ*_@OVr@xay5IU5kQiP>3mM$RoE8!U5;TXs9vg7;Uv7&!Xvz%082)v=F zyj1IC{7h3|n|LsZ)=?vr`3*831uTe0;!-|AVd?r_)aM<-6N7&=GuE~hEY;Z5DT)kh zjDAyKtp0@yr39xKw3jCyiB_)Z!K}8?e6)6vq^i|m!6}s$M08j^Bsj9&yUjn&)41<*PPl8M% zLlSnxeoaN8C5~d#H$wGaYVYXdxGBYm_b--Ig>{YsyEM&mP;OW2Zcj;vKB1E{_nfD)k+tZ2| zPMxZmX}a>>f2@#-swf{@uN&ifrQ1+frh$~&qmDV;3qdO^rAW3rDQDxdB|GuEN+49fr7?I!0eG?RWc~RxW}A~aAN-s)VGEci z<4p8Lu+amUX&SvugUl20YJZlr-eRwp}1MkYBBVXznl>*E>NFIr>gknwnamRo;Sd2Q61kCW$I0} z$vcp{4}WcSa?}U|6E5J$T|)!s4P}zYOGw|;C4esQt;>;}2be}t9k}PkvIJ?d=_rk( zim?urKo+G%WVaq#%CIxlnJvg<8Ja(IB7(!gI%mZTT@Miid(V;PmuqhiHUx9^>rXF)hW6IlG%&%k%}RAhDC)Qb8>>Q&L= zH+5cVrBxn#zdxp$gbLZNDk&rBSJKNVYRi9@j54ISTRUN61`pkZ*yyuSKltxD{U zOr~;3AgXu1JJz^He@(d7%t9JQ&f;^ixjyecxi68JBvw5})iS^ElDSOlOoc0Q_+vy^ zesSIRN}GOfa(nzF)x8N$Z@l}s?rxBp@rRyd*Pl8FK49f0PnIgpvRkD7(yG0*T^fTG zfLs+V?{1Yh{Q>}Wk3xAcbtT%RYSl6_J7s5ATHyQ9FmhrY+&6g&F3pPj*zzIom1jWJ zif`TtX}t5|;XIBr1LQW^ZVJ>NveO+MUC&zpWkhu$3JbJOtPxtNSF14z{;_};^A>&4 z>9$vh=e9+}Fi!%LKqd0k>wGmohq-7D$HL-W@VGT>DI|iEG89l8^a#%RU-65F(HHnYe_O%3; z$>)zb*#do+NYCv!a|D5j@fc3`fkw6m%dD?i*4oFEi)0q(GFx`U;cmX86z>E@S^X0w zCh??LYdF0X@~+>Kr468wGtGRJ>{O5ZvfXIJcQ+wrXTV(zWSv40;w-qG-fHe;o`?bY zK!@m>-XXcOzh6>mpkDn<%N%7DB>>OpG3|VWb!Pjo#Y8CMxWOgVdlsZfX5fj=nGU47 zqYW|@dEklpSfF=US%6p5Dqgb|GDZa5nkcmFP|}1&HT?yRaISFjQO;K8bp%Zrxa?Ma zM86nCkFuI6rR9imX>87KCZ*$i6mYC}80z`jb_(OyC-QGI7EmO5-%VBEP_Yoo8DM+3 z{RZc|wohtuB~sKDB>6muFlbv6ghQ#hClJ7 zsUS*GBJi-I2TNP&)_$ZG1$gDdmsNw%F4a4`+tdyf-!p9k}R3>8UW3j=0c1q>BQUk2HXxq zdDT?+%Qf9}gUZS>ZD}rGK$x@Rkc3r0wE1#5dUkQU>l!pi+bc;^uCS4eliXa3HkIcm zB3-iHi6NN-{yiP)BKK#fX!S%&nKg)}>_XmWs3PNZSTJ5LW{0PozU1? zdF?lwCA7df(s00=txxV;Sf)SAv+?}8>`%Ny7%*K6#UFB{POR~rBJ1k(F8i`Lbj!t zWdjY#dIWp@>yg-nEUtA>^y7JytAHe^dJOSZbjdUn&^?_Zqy6FJ_?|q^!C|rgDeMFp zqbLs=u7e>|RJ#I3JD4)$Kzj_v7gtJL0>K8Bh@TXZ93)mGBc z^LZ#dRygE-rXau55F^uymTiTPR?C|z&t?U>Y=LE@Uda*30?a7Q(HT@33pEfOYK2q^ zBOR##786($4&<@$!W@a!&Qju$T4W&xbpazkRtcj7Q7@uAvM84KF1^M1T&fX^b_aPE zU#4XpnPgm5FExrpqOG~X7gjkp3`r<)G8zzOPmv~Q#ZH$}V_ zPk69>kPCTmCL#IR@i8`|N*p?B(BUo$Mc{hP-I>Jb8dy&zDIS?`hEH(?atAGUqY}2w zEZbTO>+*g*o{e<)IB*wZTiTMGqwDT^v^vhojSW;OY0L&DiQ{X^QdbrRbu0YLWfXK! zv(Gbq?*b|S>1JYKNYLr*rg!WmErRU_>Ab1zxd0Lz&4GXcwFv$1K?${{O94+0)3`twnK&K;~Il zI*Z2ogb=!B1Lu+Bles65!RmW&SRzUlAuI1XXCfeXa!sOKqRba_Gkwut30H8@o_J9hyPaj#(?;ug#Oim^m1|Fm;A=e!aoQ@0D-ryK+oB}71_pK zw?jI%$-{^ynY;$nrp!p;F(VVfCBXEZrzeNU^H=J030tMa>_0A26>8GJ3Xjl(bjEnJfM^p@%RXZ`3fij<+})aWELA~Wd1Wl* zdTaZX+!=DUS(98x_?J^)H$o{FPGBRgz%uF00d8lHlKc~en@A*LzLx-7In=QP9ash! zXXiFVnz{abrvP@QHmhvDg+ei9V_`230X>h}H462^`|JGy6daRuTcwYeA)}$XLc1_F zGAe~p!K!;30r(&hx@b|(N1+_8gn={31!#nrxTLv#K3Lp}jaPl^xBuwTp4R!akEg{D zBtbqX;_v=}=9(R+0+|E2GYP-2XJx4T=)rNhLNWF>;$?Av64c|Rn{nRBe7Hm$ z`biH!>;LdT1;0Cu7sNLiI6kOQd`qc?lX@loC*n?$tRH3Ek}VAiCVTu%V$CZqP@cbE zqq0aLCV6&cWvuu9R}AC$xmv`Q!=_0M4Jrr6*Mx==n=N%NXM3;PweiBO z*P*tjEZsksX9HzIFa*bo66vq#>(=rcNDzHPO_dWGw`KncP;5 zgG;}EFMesWd7Ngcf-8epM8X8VUS`ckJiYhaD`MQk7S6VSA<>*^ZS9wD#lit+r!Qk1 zO_vXt2c7v&#b;l~OHV&61cRzBKk-gWI-iYQkPS{*V{F=ez-2dppfI?m(*0|UxBlxns)lj3mt!3460@d133L5M4 zv99b4S?;rW05BHEq@Z>O=|X#p9r8x4ne9?=s$}D;s`&;g zp%ZxapC`u=mSRDR$j$ioU6Ng*genc(b@2tE{DJOF=Vtm?_+dz zOGsWN=UD9|v8cSy3sUF~$;RPsV(f?c^mDeRp&)(?DJ@f(E4}vU2tpBzczb!$1^8OH8HEA+G{$J_eW|efDu_g(G!c|RCh-TDiX9fQdU~3 zZmH?oID__ATGTe9JTwKm@#e2! z7Ic?kG6g66TTa7cr2Kh>u<*UW^+4gQU3=GCr9RO?BOjV9jkXDdFIsaDC+)BB2(*s7 zr8^mY%JI^;SR0gjaHcR6(QvY-%-VCH9Lj!vPz40;C58Kc|FBp7cVDjXz`|*8n|} zNr<+hLk2?U_DH9Ky|F+jlgDG!4w=tn0K&tjE%%K)Gz69vdRzJJ$qK#P%=`STn?Q;Fr}zE$3k;Z;pC2;qstkrlhAxa=7$p!PP6|4W;3Awf&3zr;=kP#? zqTPeU*ocg6_o!6dDs}AavT#SF3u%v~`q0`Vp&= zEePuT3o)`V7xmAcyeG5Q-9@G(LQlsfdukU|K(R_L8kR4tjNjuS3pjcg2m){d9}CTTVrRM@&HyF68sDdu zFsyfASf!$eh519SXhZ(5ACR#ZsqxlTB$Xh;MFfrTM;Tfm+s^!-k?1QuOkcVNSoa%r zcGtVJR2|JSsFf@I{uU1nEI@Y->MJR4nNyI)+9`zkEfXpIOdAGYiB3rOPjsG$NfcD8 z#gd3#lijEBT60805?Hs$LsiS*tte0NB6*YOXxW@$Wv%2_&ec(N`dHmQch;&H;+>sd ze|snEY`YiIbjrlVi-I#rp|IF3Bv+z8R9H#dgQEbQVR*X6+GNsunRq2(2M{T|4J5cI zVT;GWz>5i;u*($~q+>jCBQo0O-6cW+T}b@w*>S+dr-oubMT7##^P}z&6Hnxu8buk! zN=3O9hd|`mBrVd6ZKdL4th^b1_3cy*yz}%r&h%!!l$8Qv{oqGkhqYnNyB>zr)6Q1J zSC`iaw1MAe^4@<6wDqnx2tv-O6H;Y%)py{B;^fhG3m|VGU6SdPbq@~fz^1KhC`#AM z!!KEq#A9m#2VoFt5mFK7!~S9@0gJ=49R@G9Gu=2o{fWJ(h+6_L0z!J6s zk6h)3TEplI<(~PoMrqc{|7r7GU?@Sd*mbwYav)h_zlt6vxXTHKzH_9MTFRW6jE|R^ z&i)j_ip(dLGQ?DNWxbNzMsZLdHH<^@VDk;2@O~e zqZ0_A-Da$DGU!l{5vK$0$MKsczyl;J;7X|7%CCnnbMty zI3y)aa+&OkvC?C_CB%%wh5667y|9Fqi^ug*1?%O4LeT7JY~vuaVRcwWBFNh%j;$0T z93#7v`t)+I6v)MNqvtWrlt3mFmhCF8xCo{8-gf*YTQa-j{hCbN4Go`#ilg3N!OQkd zrEEX+O~Xp=Sj?;GVZ8^_kc7HD9p$$c%@rTBr!)2F4xBD-fb11p7a!YWymTP1>zODo zn^04*>qYl2;8V8aj#C&ga2MC8DH6}wWZdk(@aHnUNv9EnX)ovJ0w65Nec9e1;2X1c zgFG`Wj8@Dh0UwqUNl3|8#oz8|Fn80E9EChMl-GUp9F})o55fcV`dB7Y3k&SWVEosM z=(Vd^?2R51LLZ->`CIA_d&>tsz^e$!sYh58F26&%C66fh@T^6uYQ`fG6pK(+e5+o7WuGGWU?=PmnICeE0EC>kM~JTP9Qu^?OxwTRv{td&fX2~XqaUZ>G=l>0tmJ}45r z;Eii2baijy^hanDTm}@X&`)}W+8J;{RcvaiIAsRr7*@m@JmGOEl0U5oLun**1letC zTg?#@5{%!@IQgKlwq1_dmK45h$axJ4a#JMSFE}eB>$CJ2{Ph(Ux|%`zG2sTG)1OFT z zy{@_JERxak0r`+SK95uOn6ZC*P%K@1#uIyIJuy#^)hTvVHRDCQ7`Hrag}pedlOzp? zLYdo&1=NlYWAsy%?&pATBXaNG;-P?W<=aJ?H2O3bJQJkY@<3j-5}aq!5XV8YawS%+#6 zhxfAl99*O*t&X1hea>c}p`FpCo*;!d!di}1OtNeWAtWk-ILE_)=BZa?uCc=(Q{ao$ zG_AxrCx=sNiT{%pZyqu}#e&DZ&15GG+!6n6E^7Vpl`TGHG@lf>iNyV)BY&}9^n`$D z2o~aZRUq_>Ny~DN!pbE2Z78m)f2acW z(b$q~lEafkBM7n%gkntWx;&ygMPmS?Y>C`k*uI>rlFDQanTkYs#qp{w3DtR19=5l8 zhT{*T_9NiXf*cm9PxG*KIdMX5Y{h2p{gUlT3QuYzi*`doTm~)VL8~2%XkNxIN(i5- z5P4n{UnDM_w>_)#qBddYADi+qN5OL1h}5_2o)bpik=4(_5-cxss%K>F<2Z^pRseV* zCAY$b*d&tD{PmLvhynq9*<8u=fOalFd~V}r4@0K9%(8)F>wKm9VuFsej_01ZjzFQs zx0do*u?``US6(tzP2!SuXE}=WwL*p8U>(oEtvU{~8!<;-c3n^_9Y|rmEX+QQXE$#8Ru77en{FTa@ z(B+0cI?z&jki2$%`>>7CRj-_q=^$17oG8ba#Q=>_sCc*sSjazC?}Mr zRhVVYe``0+XSk+8-=q95*;9P1_YFUN11gTm5U@jWm zQrX6_*4?ep^c*$~TBZS7_W;So{MJ$M__~rzI%aWtYZ??f&~}`haiKyLkP~`7Ti6! zLvVL@3+@u!-Q6X)LxQ_o&~M&z&c9aj&&9r&i|X1{)xE3hDZ%GkT9f~+=_44D(t#lo zsqy~l19J4}+fLj@&~4CN(Bb^=_t*2>=}I9@CP6^HG($hpD=%ti2BpMfh4EMWndncj zzw>s4rx|ORf|As@YU5<7d@(Ibu@n{ap3(J8>Cobj_fzZ7t@)pSo%!6m%Mp6m62Ik7 zOI)^H)6q(lyz5d}EiS@st5w%}8y0^eoLoNCY>@_eGK90yT&RU6?ra|#Y9`G z45wM*7dx}B6}Mb1+5~9a1_<*{B9EUwscKT~Z4}pZ=@<5%Z{xrK8 zIls_m(*&9+_diBrvnbouYT$Mg4rn$fqI?xdfp4S9-h>rRDt$U%m714zb$etwPHKE4 zH@n>0u9~O_m-dKQ={FxUcZe{AZdkHsK%E@}S3Bh|f-69%%u09BaN?woWwVJE+-kGgkHg7Vf z^-v;2sSA683GJeyCik(c$x8EKvPTs8^E4Ln`h)05-Cu!9mNUlDtb0izL>R3*+HkZg zWpjTtvNBWSdg+-{26)G7HOp+UtS>ppNUeP9R?K!%O5}wCiQe2+zyx)qy+_hgi1kcr zIRCZqd;SGD8)aTYBsla<;bFsSBEYMIJF||`<^M)Wu&Su zQJM5pI=#=nffp&qGJhIMGQHbOHan=1}Dp0hD=mv5dLbF8w zxWv1u>Q(KvKpR-uvk+Y#>D?HBEm)i@{ocr0Pa}%U$XPxSTjZ_*qnQ;EoSeNmY(UI^ zKUL|3;5$vpak$opw+?BkV>q#yY&LENLCtnwbDc^NMkdlgfsv$c=5@a-S*2Bnu6UnE zl^UG&#BZEItY6{jM}iQ+g&giX&w1bc!!&mz+T;^kG0}O#^tO&%~D>L~}`vU}`pP zKV@Gd|DI8x+*N5g)Yeqt)uZojIo-~g>fg#>5;H-Qy8Wawc2Z)#h_B(aOzQG>n2fVL z=Y|54xH4Rr-*q3r7)31x_4`$nqPgMr_}iuwxo>m+=&jlbRUaY_YxG=1B6+HSf4sT} zSO)>gPqX)BN9NHsSgBQx5T09~Yi@eB(Zn`pFm@rVrA#Gyt;Azh^`;}CG`$6{lx1b@ z`;Bilq(@{@D)-`~H#V&R+NT&9G|l0`7ItoBFJ6AO6-*6v;wf|X2J_b%c5!aOzFOxe z@$)V)M{fA0zj)3-EXt{*B;NX+5{A)0Yp~3_ZdXqYz3&E! z$$p|_&xFaQ;*wX5+?9)W+iAnbea4c`nzwT4#dqKbRCUu4ubT~Cl}PTY)}wZAX<_i~ zqT=J?Z3IJ^^TFJ>t<_Yq<>0jZ>)mh`EBNDV2LYYWVwNRk27#qVigQ}Q`HajZuiGuP zZTrSIJW@*iOd16v5t;eBCaf%dcdsp$HoHxL?7Czblo%Pr(N-WQdUYp%H=<3ajVx;zl z7>i$aAzrMIPL)fdL}lG;8SN7iy&QR<8DutBUEOQa=NC)&niMRc6yd;b9T7!%%Du$s zc?lCUXNipz=3xn%%lu5`yawjT<8$N8$4Mp3>_sl29(Zq;k#K~xFhpm10UJKe2eY3q zjhc(*w>rFZ-+VaPvgquWvR$W( zghBD@g%M1Xvj*xB4BDEi0OMYM2{+=aT3zig+Hp(rd>f(FrxuE=Wo#xusmV>R3*WO> zw5sDcRw-P`O2QP24+YBenb&%XJ?KuJJOPfkRG_mcMGbli?|SNyNsHfdkQ@0>Fo@~S zIBhfgP5VTGEI1U$FEqS0_pl3sv)l9e+$@s#`%a5Rko2(^j&U7-SS8PN;(o-<7li-^ zDkBIQ!RC*{xs!bZx!EBcv~X+y)b;b3fB+mi9yEhARMMMsa8w7FGF4Yc>hnhSxCE?Cw&FTViIxa)#ghfQr;>c-Jcg4d5`~m)-#-{zY8@)3lNSBX<^DN)& ztBAbOq7S>8mscN-v}48_6^9>l=%>=Cso(uE4S)1UXE9mC*;|<{`!lRb%tSCO6+vncAf0gi}V3+-UZHsTwhUkf``wL~w@oz#4 zHH4@THQ+pQ#5d`a&WBW&l~_9kiBSw|u28zdyGQ}EXwKZ9Bqy+BH5v$RPYJ5eq9!WY zndi2qj`svrKaT1wr3w#k!}xjcCS2TaBJD^`k@w#plNG%7ATxL)nA%bY8486FTYJ6e z+Zm6P!+_6V^raFdTuct8@i&|M?35N9b+HsP zJWD1+Cu3kU6X#)l8yXiwCU{2I2T7&8F^e3LEtS(%*eu&G#)n{z;RoECGEE3`@3GW1 zi_(0d7Q0Ji^Nl#QqnK++wgS%3{Sy<5os*lMHqJ}kOpbxF1n_}vgyj`H>qKDubfoz|JKR=*BL@9U zoA}&JB>+}y&Qg1`%$^I;fnLKLm`%HXzDDsXQol1v_+qBUFw7`${r$JM=I;_rC7HFM zya0;&SPX$;lE(g8X`z7auVTzNAG9SJ1n8iBv2Q{Wne&t=7w1rmEFL+`FIE)NG*>Mp z8LZ(jNQaF0xv{Vi%fi99kSLm7z_FO>}#x$dB79f;*U z{A4ou&=N>mmA&FtkHDHF2n$UX!`)c*{E&Y?H49+5jZ}y>Wxh#JNjuUj2$$bXd?dvK z4^vk=x%MRE;sG0rT&1?7v2`?P+YAEDxkFisdi7GYh~~LBw}W4ZP83jzH6+Vx48M9` zTL}K=OwV1e_1~`@u$+{_{-WV>>=M35I|OPlz$9JFfkVY2Kt4bEW8cpX$Qa=$oou1x zB5KwlXYco@Fy#cw0Ir)?ug0IsVdjot@_X7z;>`?1e2NyS{xHX#U@L6*p>Vfz&JyU- zgq{5QE&Z&2-M+F@w2s(TQAiJm)Ek0BI^4C%s!PM~;itXekEu*~PqWAmT*Up>>!-xh z94m<)fd|cK6Cqo5L3X=6wOZ2VByA$ax4W9QpG@v-53cz});X9rK@33J5?$Npz9t!| zlOqRyZco{g)^D+lZk4FnM!oYr`x!a(`n<&LInLma*LiQR(JDhc> z`VqgV`y7k!DDCyDZyh$?dq!aB>$SXg{sMW0nyKf_(@Wz~Me;mn!>78=5;ZgTsfHE- zq6pGZoYdls!+}PfQx;qf8!n^erUuUAvD%#vA^_$ib#x<0IOi^2AD_`sFM25c_4KwO zd@QHFL4$16Fbe9w6l9q}x8kw6OsRsf7w8W)@WisO~#XrSbZdDqeT#Sd?>r23^P(jkio8;!#z7F zL2O}I(8;}rPkYeXAzSb&hn9fupy&>>uzFh5`rzagu2ta0O=QJ!8|vIZGf~~KbDxh% z%7}nx|GWhE5CpbM-7WyM2}Y~QtBcb}=aC2R6y^nHnzvR+_HS@yO0R-*vzKiB&$X+J zA6}3u4uMz3h82D!QreJs}=f!kvVZE ze8(hy-OB2!`Gg)lz(!;Hv21(Wo7JcOog9F9MVZ21J>lk)S+mt#A@;qUuLhed?|(Nd zJ4vb%?S$40S1p|;Wr{#{dm?{aSe7F{#xRxS56~Zx0|pWw{~UBAcFRkm@^LZUqjktI zyHpB*iDwq#C?F7#F6r?+9ZYTwFiQhrnwljSvezJEQg!BygL-)epz2e5+XT)u@#*qN z@z)mDh3hb;jI~?PWpriZG@h;Vq+2Qh#+?KZz$M&gV{|_ zxqI{eOXZI@^naXIrj{}%<6q}J-nNrxJD-PD%lVGsHD1^EE zLFC~OMXU2p(0;Ed{8lBBO0Hl{&a~oXQv%mGx1XMl_9S~*II2x1MIP6(9=*zk@D1*= zOh$vgrZn}KnyI3{w@?c>cJv!QisDQ^kOCKyYAZx#B(uo`iNVgCn8zy7f*Sgr3HyNL>}<)@Kp83 zmcg-^L7X*DR?$&9p_UyEv;6~C>Krkh5 z-Q`pOAP|Q~&V-G#_NOiZEsPz7w2{-JoXyPvx{S-j<@Eg98Dm!3gv4Z7(n0#DYCHez zt)f*2fHyf4UT6*NqYF?JkL-)10n!UTvV6@iA!T|)lRAZG_~wTdYo5F>L`dyEx9S3tGeRsPv8 z>k5b=ol(W*@y$9WbQ;UXdoPrQ*JM{es9YE^>uF#1dmY}6(d>kf#j;9|t5ucT?vZ2x z%$D!7gB~k*WQ8fwvx+x=KJ%70%MeZq%lFPr3$Waa1r6;MK#-CA`{1LYJ8F5L`4jA5HMR^|fD%xrcazwuCIhrZ; zH}IHy{~ELCdKHL_IGf1bFimns+hfV!=3%xMbgnHR@IvL$#lpMN(dpdoVR-3!{}_1` z=0*{&tuUr%ikgL4V`ijM>H1diGhRE3iGMOrxcjyVQ($Wk5!%D!1K`{$pK$fJD&9TH zAMY@Mtu{^nIf^Y<98>~x93{?dCYw;6Ujw#uLB`FWkz{ete^_o_UD{>`I)!33>bZEc zBv5^K-;y0JE;(g7QJO)1yhB!OsVfnq567YUy3Pn$QSe#?&Fgw%RXB2O)q}D zYnMix3)6;8?YHb@5kf2enY9QTJlcs2!q=gV*mHl!Hgvu&n4U~ zx36ERJ9$Md7FEI(gJr-<@W+_Z7(u;;wCSf^wt?2`2x{Tt9zHM-6GxQs{a9iBD=(X9 zp8hw?p*=H}pn+hGD?~T;RB4Z)AnHbm`U`5Z?&O?s|D6^=6>YR3g3KwLw$|glpGb8` zup^3hhdrtJn=PG=GGWBCWm84G#K)bp9ztQHL#>N%WN{~vj4b&JEe1`(NNognOzjQQ zt9I;s)L2_MTdNxPCF^e2`rNduZq$l5hI?_8q{r_B@Jkkq?8Tw_Y3ldK3g7GHT2zGy>7$Sir;yYu z@7|u0vjYC}(Y03P{UyZV1Wc8+U6Q0Vuo$I52(X}P1_db2Nhq3eaphrVn$Dc{1o~%> z=?hl^mxZDw#h>pAxO_0L@3ac;dy+7|tv;mOw)$3Q3zXg}1ISM3aKvz$P&k*g!&{yh z7|W?P?T|d09h}LaB;^aXo?%8&h8jqwjYszCJVH#Oef9@aw>Mjsy;Lu^6+WJCsPlRM z8Qi)YTl^u4Z>r38s7Ow5#`QagRwPgE&?z?|<046fCo=+;UrlVyp1S3rAzitrO#+&e zfiwcExY*XX32q>r#v9$>%O>)!;j&X4k4x`eV$dMh(~#`-leDty=J+Y6&Xz-Php$*0 zA$gH(A6i+ z0ThCqS=Bjfawsgl=ebYUCKt5D@f;hxkO6=)%!`1RK~}$97G_X?ZijG@$9d+VbfbuK z!8oeJH7+AR6EZc+6>dZRFG)p!TDW`g6v71}=JZE}NROt|F}{&MH3Z5;kl~=dT#SUu z>R%*=sZV3F*}|RJ6lk`|jNqF;05I;H^KT59qR0&{KcnwiJ57V=O6T|V9(-(Mh~Lg` z3k^5D?#5%Nc3tn`%f!39ybd3ls^pYd3vKAt!$`likx+68^`6VT{t9GFHjitfz}ZKe z*nQ%~E~HMIAn-aC&=4}t0k1mfwnjzpHKo_bcl8;EN%d|P?_fF@jBR!N_~!O$5mxA; zCQp;xHF2fFHByU07}?W+&b-M1G@5l)H^_7&Kv6IfwD0`_e?1fn2O~zPg%rV=ttI7i z(|05ewkFgq?pC*E9lPYJ*Ki0@hrqi`D-ip)|CxEvEiX{gC_|`1Dj0w+s7G+{L=_4O z=m{s$DMh3X;}E>zXd-e9B(Y_f{yH2Z;O=9uSl3@BrW2yw`ZDcNB}o2}NuJcQ`GYos z{d30#THCDJX1anJiZtzb+L8H3IJXG0nGrcUI+EQFaVg?K8L8{7CtHrcv@%BAFHEVE zFRsMkZP=VK#6o)=8H-Rcy%>vQp)pSNZv3Ul?zbz#*@VJrV1Y>z-!y2pn$p3PVNc2? z1gcm27SJ91BSGGVPf0isXn1cad!spL0Id{p=xqQ^HRdhC1t-SmX)qFKvzHXxn1^px zk%9=8QM5Yu{5nTbS@jv=UC~haJ5Nvosiu#R>()UZPc#k_*-|9NRo-{KWO=S?WITRR z5in)wLu7RECbEVEut?G@!Ym;itSPCCoCi9U-#}Vb1-xwWrsfsry>9A{aEsR(?X|oS z>XPW+uxOCbHs?wk%Gw1&*?GUuq)Wem>puMt5J-NQHt*TP-Bx;1X z#^TT{M$B`vu;yN`9GSVxq|5+F72(|1pXjdiM;Rvvoutg{b58qL5b2}gZEYjn+w-G@ zu-ALJ@Ai=H_eLQ%NorI!XWpjhcjROd4fIg%^ZL)X5R>1|p$9IVj-T7bcL? z3Sw5-s%<^Ef~alDmVq04I{f%XIC=6WlI=)+K@v{&{NJGMikbuCH^1#|!2>f>6SPNJ zxfA$!*}a}UKJEW_yBQq4aXIqou|J8~JS2SCc47MXwC$p|d}hiPzc{Nk-{bPa?@;5k zeP@@HgM^3LYP^i+8#3$Xa(uSVN)?XptHwbtHE*|Sso>h}rkNHdlpq3oJpT3JM#vMFpTN<+igl%(OF#D1o6Vh;>i z3{l*I+~gc3a{t#+me_ho{Uvi5`AcDo3|5|vZ!DG~h81(WXdgv`HwQ+vvQ`ml(cv!^ zTo!U^N5YrH?z-HnzAO9(ol=aWjf7tb%zz_GcfTP+u^NZq^EM&FE@YqQ{CghQL2QQ? zUzxI8W=BxJ0c_mHgqxG<8&!c9_1p;IDAF$MZ9x8!V5MRV;bB&@uML808(2Jy&H+o* zW8K#~j>Nq=?!E?1lBJt)lmUxRDin$AT-yB|&i;pT52HNX+BWX`QATGSiyH) z{AL81)V?*$y|jy~4ev6@O3Os|LG2Udv2lAn>*H+bv_w&LmvH!BvT9S6OU%nPfob&H z<*m6B*YWlt(A=*g*1Sj5y#E-D;@x$8sX?o!cuQR#U_e0%jwh~# z`)f)a@kGL2R4iyFOmSBcVzD0wrhrNTd_}cEj#Esrc z013Ba7c%6uF_r1C#Vr<~D;XKA$bBe#QcErp>b4*r88hgBoZmOqJ*#e5cq!^{wa|wB zP9FkO0auFSG=6@pqFDKSB6EH5uT2|STFNn|l8f4mBE=E?-bD5p+;`G36N{-Wv3Ofj z#si(8Grk|72Ug9yYZLdqjpVOE6(%No;e zqoyi-mR;hjXpJF5FwJfGwfZY4VxG(NVTU^DzhEWEczV^AU$jVAT^!GNWyDS`N%XR8 z2Yz;etxi9i`I6GCpEha0pva6BM^YrRiXYG2`x)I?7)*d=L=tCLFcQ47rqW=b7VY@! z4SEokZAHv0#Au{&m7{Y>dH;_&KBr`tRKS*I?nD|LZCHW5$pkfzI?GYh1qu+KR`R|JuvBKq{fY>}qpx^=z;4Czvf4pU?${9$zJ`dL|Id&kO;gazK?uz} zt`2_jf0duN$mZy-nu<==aNGOngLmtAPW+Cv?P5esH(n*J`#(3?U@6}_L_GzyP&~+^ zd>;*p#m%J1BWjfbn3oJS64@7Sjh+(rhNP(5GQx%?QEcTfNRykG6+F1*`7>|jO=!A? z2E{)Mb0*Svs-yoA6?=D+&ka^oHnNgf)rzmvE2}0}ux*O*q?9lS;c7Q^i4-;oC*q4M zINEKXmzzG7?FxzrJ;Y-3_n3bj3ySl2aqdk;C7CScv}p@=f|zzXeHGv0e{-RnN>0X_rW#Gg5FCmscGgW}9XaMZ{?wBUat} zXy75EIBY{>`;~k~W4Ab&rR#;4qjmTh$`I20O-_|&d$)BnumR$|lc*$a3svm-a$08n z7h#NDE5`#%1Qin`iaHUpoJ9H(`HP_Ux-*Lz7jDf_#OZzt1YD>)3I+f^dXmoBt`!;` zOPDH0`kvljMgGwlergugpQdUDDJGa8LwW_S|JsGAav=|+E7`+B&ZEBX)e6Ln zeyCpIkYy~7%r66$P3D9Rm%tv25HcsjZ$7?(q(J@*aC zKNNdKgxFH**-Q{3VKH6X+wj((+PM{d1o?ItMMl$bMZf)L&_p3#%5lGT6-+jE2#cjy zxMxuHh%V6??X3&e#liO500xy(DdW$tqjMlv_&a|@)Xtj$mAD-Su$`UDm1XVjDsjiB zcyJfvVS#tLhlRi-nQOzRx!aQ|L~qA;?-(dE zh3|R(L%5z7NUP816cNHw&yvLMR;|#bB$9d4RQy6U`1pu=7-Nc6^u;ghn5>OMrX5CH z`o}Hv`+{K6vN-^J@p|^%Oq-1JOmy^v<@vD``fPIAf~!OM6RJ97XW6!uUq#Nmzny4x zCVp>$O1M7VTdz1>j!fbr{iexfUQ(ITfQTt%2SD;B63KN=VsbK}F^#k<&#^!8!-!68 zcEl2;sL$8}fyQVaT)3u#UQAyo<)2rqg%zOJD2i8W^!R3&s8zR3NQG${(2Lr7A zULt1i5zpNr0rxjvXu--4CuKNuE!Wt7UO(=@5~*BTB3+z#Wc3_lngDcn4IRk_i61IPRmE0FgCWY1CeIp zE%Y^oWyL9i%HL0~?kVDBNC^iz@3PrUSno3Ffm%pI$|vSzaBPkZ)rpRS@M9be)|LSN zZrp?w^m(0?k5MNvI%c!63Y*xrWDFiS^%~a@DF zy{uU@2DL)eF48&iFsI;JwGB9(_*nQ!+NI*@HwRy^spGtFUM&Vx5C*@;VSq=MBIAgX z#KgG$L_|Z<6Upo1rz51G%uAr0mFh=wJ8{EECxU9#7mS3y&68D+{dB`o-s4gcsYSUR z4AzWAVl5M_BWf`Z!i&os3*!U#7ee*NBT6UKGDLEhg&J<&nEVQwPI2-1Ggr*7x#C9?$`4WplIIQ+5cisM?x!7o@uCfo!Ssty1I^^Zc72sTu8KT>wQHZo7mlN@Z_cY zW>OmPd#fMb+4{XiJOv#WRktC!W}XOtvBkxDGwT(2I6o8?I8o{Mk?uRY5~^Z1V`n?E*Q%TQC+H=Fb21#mwuq z!C12P6XFShAzk1J9G&I|Z_H7lgjSy~Wv=|k0YUz1lGPq|#<77Ba0%n$kB%CA;~2## zFCl0=plh%;Y48ktvBS=N6jU4#O8*W0F&5+1xBepT5NMkbeJ$(6+2G}^CY}wQ=a0n) z6H|a~GJzY#ugr?oj{oQ;;TYCU#mQ>FwC=h z6w4{=%k#lLHDC>-bCq~_!WA7I0tG`P?nqQmnd_Nx`2*8CofGZWX4@T_7c^+Q-@isc z9oOI^Qx9OJFH&3NTZMr9Rc-m_p{~79 z>FE~GBj>yM-@83Nf7%qqLx#$VN{osfZb_CEYm$UCO?*LVz4zoc7owW91q4gah(q16 z6+>KvqC?@nv;s6ASfAGa}yJb(T712z5 z%PQml^H@xXE{q_9#EKCOF6L^Al5t$>h(8;=HV2v?3sJ3qt zr`o!$iuQWcL(q#Q#L7?6bGI_yH)q-Dg`olB_%&5_b@`_Ar@W?;9t}68|wdgA2Wwa++z{rdf3TdM4VDbF!o!0MwcoG%& zA(jmHRfbC-E)Ef4|D~hoETHOVM_*aK_lxfGIpUcYFPRG}*IEmSXH-1`q$2=|R~p;R zwS~;o585i1q0=;U@GwB*?XZQD%CUiY8X0tgQf;VY^&^4{B+@lX-p2g$))CVB3+w8T z$>%Vy+7aBPo-&i~b8+?o8ZrqH^6TaYh@g85bEQ|z7RzhBAAx-%@jfBuJm9c=dfCN? z3vybxTRoz5wf@+TCa)L150zP&N|JFTKpVZ7A6Y5*0HyEf3t%|1qJLrK?N!;3$!@a@G!5#sL|`Bi;)L|T`0uALxKp&6 z2NUS1xgJ^{j7uUj&o`B(xexgaWure6;48?xy1tx=wHyNlG#`c|my&=`Sn?yEb6EC; z@+RN?i5+%TN~-(&n3%$D6ToDmXOn0>NLIGE-{@-*pMU6vx2UQcN6Qh7@_GCdthKIoj9 zyNoRd{X`anc0e_XDnggIsmVTYAQ9LLD%dqyz3kCGke;Zq)hk0=YupSfc?!9POz?#P zOTd-;I8~-IzI*7{NH!@->0Cp1j2`GVXgfn){ftSC`y-7Cx*OH;Z-_{p&m>l2UqTGI_Zzq+tG|+ zNnsd5SMJ6wRUIr6=U~|XGl~D9;`;~^6G0#!^%9i;*y(93Kqbk8x?4Mm_D-~FcE zTAa(74M+#}wg@-5ov}$lqFui{A8(a#*!H45P6KJQNG%T!h;`3_f4`RRieF?bl7jnS zyu`_K4^OR7U>HSd0;EnM6*TQBIheeN10f^7_UAEq1<#)9VLpPv)P&mrh4lZd?0-qk z;UIvpSYL2HdC=B+j}&6N_Vn=~oYE%@@MM=(dzOK;Y=KKI^-KuT&Yy7v+Rc*QR95Eu zPP+d;Bl%w<@>(Q;$`d_LgkP6K>-a-msxmMUfne5$D;oIK2Gn6KpgrnWmSyD%un|ny_eKjgf_LpRt?C7P1qX%}8H4ncSZXC+Mx{6fx4qRbcnEK7Tip4k6?+vtCI z@uwpka%KRh=$Ecf$%wq@ny`n&R@(FL zZ+bBK4!#_@x!-6}hr#{*YWXm%;Qt|pu19J)e9AXX;a;yn3akZN0mJ4cYk8eOhG(9& z%r4)%v*HGv9iH?JRm^1>)*IcPqKdh%mos_VLr66{{OUATF03+Y7Y`dGO2_%kUEs%{g z`+RdWk?2c>BhM{lUFz%K*3d2C9~sMS_zX6hXwAoQLUGmw8D%|; z$3ArMHVJ(rbF!i6SBt{~{v-{xXo|MsT~_cOZ!6UEMfiu4eB{7bd>25@c&yn-3S3Q>SW!I za#IbTbIZKJYnIavs8gcU>N=SIFK$J`X>l@EspULf7mw+sgp)WclLc!Bt?F(BB~%^x zKF??WS~;$HEIKQU%-g_37See3;?!qFo~?a+^z@&+>K5R+CC-&BC^`j-D#i!@4ieHQH6r@|o@&Bwco$7$Jx(%g)F6@sTgePWd zcg`A2;fZiK<)><`PeJ!e!B`m})G-aR`p9Kx_lht+{kM+wt$f9Y5w0#5zvyD`4Ycp6 z+`MlCss~9@V)g$Cd^ps;L6HRNe!hDlz%d(de!8y89GmNe%f>{SP&MNzI_+#{UGL}= z`-k7b_Xr@e(Z|Iv-m%P6XN)!{0AZ2HckctGILHe<(GA`#at-}NmN?ES&{W_zWwgLa)%Kx3}W2wlch_ZsdcIy=b_9e|E*76 znMFBHrxkoYdY{Y))I?G177N?IKB?+#FZ2QrdT zf1BEGEoGZZ!)d#vP4;9VHP}(;#QWoEGRSVIk@fmI!8ngV=4(X<9eCL8K2J|glyRO~ z=AYD+B%pL{d$ard+3`MvSq%O!QTVdJgNJczUiHSEfO!#dFQPqM9n?r%+m4*hq?8ze z)VRf#JtVBEO=GhrM`Iy%eH~(E z{>;ML;L1SWt-cb86Hgwj?cb6Z21*91w$ym-9^BvQVNP#}FUVmx3z^QF>^=3J)bSA+ zqn@+rWXSiftropAO+=^_`ezYJh&>yQ{&(;sy4tDS+Kr_aJ<-R-3xwkESaFJG$SRiq zPyvFaG{`KlXejE7OuD|umJ_b?w-1(i;1F@X9k(p>MVh*W>#358X))NTGPI1q_2eg( z0IlUZOMs_F{5n8Q?_7@#;eQuJg)9oG1+rpubA6DVAB(V;q-P?-WTn0mhctUNRZ_=@ zTk`1(N8wlK_xXtg(54{ICck*lV$4j3hT9%KSZ@s9kQZP_HvktNX=~<`fEE zD7a!}yNLtGmXm8_^5gNw3HslQ`i8-RhheK<*{U+-sN0GI{dwEKwO`%n`IJW3bqu3s z-+?gtj8eFI0yFe$x6Nv?TH*t*|MWjOKipD&tYE0@Z(!Rt+IPqUr!t!2e08Qjw*F*f z-s-_d@%I`~7!Z{q8h9C04y-zAn8R9Ak@kbd0U)AnJVolM(hQ}N{8Shn zjE6$&;NjWca4cMHDMAs0E6ne6@rwf_VKI7Pw z{EyEGB?tgh0y7E0UPr}~0%dw0)42!nUwV@JIvY)n25MDo@UeJ#9U01b$pTySk9&j$ zm*OD9mb3f4>Z`5pTdOnoj7NLNhPoU&r2mnCAjAhj984TLI{v1yhDt>Ev6d6b2Z?e= z858`G2QH4#&PHx)0@7uMrbpB`H@}_M@O@s9<-0rS<++}uLmS(Ge~9@40br<#xAC<+ zAUo{J`xq5G z|5Q3P_-;;}2|@j%jp|P%fxd(X@fs+(5mG*836sF-#F#{`**?{lE9ZSUCiEAQ#vSZG zO%#o965pak@$%WZ9K#NNQUTx_9)-6A(<-+o;@01X9F*@5I7p*LwpK?~u%o7F zI5mo)Dj~n@mQK5fCuyjEn9Dt`J<~w*k20!~$Jt{C=cq>1D)93^oKzDFB0$ScJoe%w z;j+$4B~p`ywaerM#s#KJK`Fo>HeI!7ztAnb-zOlgkE5OzmhF&DTpz97;k;G2Li(Ff bxBdf~+`;xDbWd*ifB#F0ei5kz>-+yd&UsDn literal 0 HcmV?d00001 diff --git a/docs/source/_templates/docs-sidebar.html b/docs/source/_templates/docs-sidebar.html index 6541b7713..44deeed25 100644 --- a/docs/source/_templates/docs-sidebar.html +++ b/docs/source/_templates/docs-sidebar.html @@ -1,6 +1,6 @@ - + diff --git a/docs/source/conf.py b/docs/source/conf.py index a6beee5ba..c0da8b2cc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -101,7 +101,7 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -html_logo = "_static/images/DataFusion-Logo-Background-White.png" +html_logo = "_static/images/2x_bgwhite_original.png" html_css_files = ["theme_overrides.css"] diff --git a/docs/source/index.rst b/docs/source/index.rst index 6cec3a157..16c88e033 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,7 @@ DataFusion in Python ==================== -This is a Python library that binds to `Apache Arrow `_ in-memory query engine `DataFusion `_. +This is a Python library that binds to `Apache Arrow `_ in-memory query engine `DataFusion `_. Like pyspark, it allows you to build a plan through SQL or a DataFrame API against in-memory data, parquet or CSV files, run it in a multi-threaded environment, and obtain the result back in Python. @@ -72,10 +72,10 @@ Example :maxdepth: 1 :caption: LINKS - Github and Issue Tracker + Github and Issue Tracker Rust's API Docs - Code of conduct - Examples + Code of conduct + Examples .. _toc.guide: .. toctree:: diff --git a/pyproject.toml b/pyproject.toml index 24bd29617..444aa8ee7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,9 +47,9 @@ dependencies = [ ] [project.urls] -homepage = "https://arrow.apache.org/datafusion-python" -documentation = "https://arrow.apache.org/datafusion-python" -repository = "https://github.com/apache/arrow-datafusion-python" +homepage = "https://datafusion.apache.org/python" +documentation = "https://datafusion.apache.org/python" +repository = "https://github.com/apache/datafusion-python" [tool.isort] profile = "black" From f12a487d2e18be1d93e7494299b8a9eb6618b5e8 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Thu, 16 May 2024 08:59:58 -0600 Subject: [PATCH 144/413] Update changelog for 38.0.0 (#704) --- CHANGELOG.md | 691 ++++++++++++++++++++++++++------------------------- 1 file changed, 356 insertions(+), 335 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 863b142dc..c25453534 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,145 +19,166 @@ # DataFusion Python Changelog -## [37.1.0](https://github.com/apache/arrow-datafusion-python/tree/37.1.0) (2024-05-08) +## [38.0.0](https://github.com/apache/datafusion-python/tree/38.0.0) (2024-05-16) **Implemented enhancements:** -- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/arrow-datafusion-python/pull/610) (mesejo) +- feat: add python bindings for ends_with function [#693](https://github.com/apache/datafusion-python/pull/693) (richtia) +- feat: expose `named_struct` in python [#700](https://github.com/apache/datafusion-python/pull/700) (Michael-J-Ward) + +**Merged pull requests:** + +- Add document about basics of working with expressions [#668](https://github.com/apache/datafusion-python/pull/668) (timsaucer) +- chore: Update Python release process now that DataFusion is TLP [#674](https://github.com/apache/datafusion-python/pull/674) (andygrove) +- Fix Docs [#676](https://github.com/apache/datafusion-python/pull/676) (Michael-J-Ward) +- Add examples from TPC-H [#666](https://github.com/apache/datafusion-python/pull/666) (timsaucer) +- fix conda nightly builds, attempt 2 [#689](https://github.com/apache/datafusion-python/pull/689) (Michael-J-Ward) +- Upgrade to datafusion 38 [#691](https://github.com/apache/datafusion-python/pull/691) (Michael-J-Ward) +- chore: update to maturin's recommended project layout for rust/python… [#695](https://github.com/apache/datafusion-python/pull/695) (Michael-J-Ward) +- chore: update cargo deps [#698](https://github.com/apache/datafusion-python/pull/698) (Michael-J-Ward) +- feat: add python bindings for ends_with function [#693](https://github.com/apache/datafusion-python/pull/693) (richtia) +- feat: expose `named_struct` in python [#700](https://github.com/apache/datafusion-python/pull/700) (Michael-J-Ward) +- Website fixes [#702](https://github.com/apache/datafusion-python/pull/702) (Michael-J-Ward) + +## [37.1.0](https://github.com/apache/datafusion-python/tree/37.1.0) (2024-05-08) + +**Implemented enhancements:** + +- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/datafusion-python/pull/610) (mesejo) **Documentation updates:** -- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/arrow-datafusion-python/pull/661) (Michael-J-Ward) +- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/datafusion-python/pull/661) (Michael-J-Ward) **Merged pull requests:** -- Switch to Ruff for Python linting [#529](https://github.com/apache/arrow-datafusion-python/pull/529) (andygrove) -- Remove sql-on-pandas/polars/cudf examples [#602](https://github.com/apache/arrow-datafusion-python/pull/602) (andygrove) -- build(deps): bump object_store from 0.9.0 to 0.9.1 [#611](https://github.com/apache/arrow-datafusion-python/pull/611) (dependabot[bot]) -- More missing array funcs [#605](https://github.com/apache/arrow-datafusion-python/pull/605) (judahrand) -- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/arrow-datafusion-python/pull/610) (mesejo) -- build(deps): bump uuid from 1.7.0 to 1.8.0 [#615](https://github.com/apache/arrow-datafusion-python/pull/615) (dependabot[bot]) -- Bind SQLOptions and relative ctx method #567 [#588](https://github.com/apache/arrow-datafusion-python/pull/588) (giacomorebecchi) -- bugfix: no panic on empty table [#613](https://github.com/apache/arrow-datafusion-python/pull/613) (mesejo) -- Expose `register_listing_table` [#618](https://github.com/apache/arrow-datafusion-python/pull/618) (henrifroese) -- Expose unnest feature [#641](https://github.com/apache/arrow-datafusion-python/pull/641) (timsaucer) -- Update domain names and paths in asf yaml [#643](https://github.com/apache/arrow-datafusion-python/pull/643) (andygrove) -- use python 3.11 to publish docs [#645](https://github.com/apache/arrow-datafusion-python/pull/645) (andygrove) -- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/arrow-datafusion-python/pull/661) (Michael-J-Ward) -- Upgrade Datafusion to v37.1.0 [#669](https://github.com/apache/arrow-datafusion-python/pull/669) (Michael-J-Ward) - -## [36.0.0](https://github.com/apache/arrow-datafusion-python/tree/36.0.0) (2024-03-02) +- Switch to Ruff for Python linting [#529](https://github.com/apache/datafusion-python/pull/529) (andygrove) +- Remove sql-on-pandas/polars/cudf examples [#602](https://github.com/apache/datafusion-python/pull/602) (andygrove) +- build(deps): bump object_store from 0.9.0 to 0.9.1 [#611](https://github.com/apache/datafusion-python/pull/611) (dependabot[bot]) +- More missing array funcs [#605](https://github.com/apache/datafusion-python/pull/605) (judahrand) +- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/datafusion-python/pull/610) (mesejo) +- build(deps): bump uuid from 1.7.0 to 1.8.0 [#615](https://github.com/apache/datafusion-python/pull/615) (dependabot[bot]) +- Bind SQLOptions and relative ctx method #567 [#588](https://github.com/apache/datafusion-python/pull/588) (giacomorebecchi) +- bugfix: no panic on empty table [#613](https://github.com/apache/datafusion-python/pull/613) (mesejo) +- Expose `register_listing_table` [#618](https://github.com/apache/datafusion-python/pull/618) (henrifroese) +- Expose unnest feature [#641](https://github.com/apache/datafusion-python/pull/641) (timsaucer) +- Update domain names and paths in asf yaml [#643](https://github.com/apache/datafusion-python/pull/643) (andygrove) +- use python 3.11 to publish docs [#645](https://github.com/apache/datafusion-python/pull/645) (andygrove) +- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/datafusion-python/pull/661) (Michael-J-Ward) +- Upgrade Datafusion to v37.1.0 [#669](https://github.com/apache/datafusion-python/pull/669) (Michael-J-Ward) + +## [36.0.0](https://github.com/apache/datafusion-python/tree/36.0.0) (2024-03-02) **Implemented enhancements:** -- feat: Add `flatten` array function [#562](https://github.com/apache/arrow-datafusion-python/pull/562) (mobley-trent) +- feat: Add `flatten` array function [#562](https://github.com/apache/datafusion-python/pull/562) (mobley-trent) **Documentation updates:** -- docs: Add ASF attribution [#580](https://github.com/apache/arrow-datafusion-python/pull/580) (simicd) +- docs: Add ASF attribution [#580](https://github.com/apache/datafusion-python/pull/580) (simicd) **Merged pull requests:** -- Allow PyDataFrame to be used from other projects [#582](https://github.com/apache/arrow-datafusion-python/pull/582) (andygrove) -- docs: Add ASF attribution [#580](https://github.com/apache/arrow-datafusion-python/pull/580) (simicd) -- Add array functions [#560](https://github.com/apache/arrow-datafusion-python/pull/560) (ongchi) -- feat: Add `flatten` array function [#562](https://github.com/apache/arrow-datafusion-python/pull/562) (mobley-trent) +- Allow PyDataFrame to be used from other projects [#582](https://github.com/apache/datafusion-python/pull/582) (andygrove) +- docs: Add ASF attribution [#580](https://github.com/apache/datafusion-python/pull/580) (simicd) +- Add array functions [#560](https://github.com/apache/datafusion-python/pull/560) (ongchi) +- feat: Add `flatten` array function [#562](https://github.com/apache/datafusion-python/pull/562) (mobley-trent) -## [35.0.0](https://github.com/apache/arrow-datafusion-python/tree/35.0.0) (2024-01-20) +## [35.0.0](https://github.com/apache/datafusion-python/tree/35.0.0) (2024-01-20) **Merged pull requests:** -- build(deps): bump syn from 2.0.41 to 2.0.43 [#559](https://github.com/apache/arrow-datafusion-python/pull/559) (dependabot[bot]) -- build(deps): bump tokio from 1.35.0 to 1.35.1 [#558](https://github.com/apache/arrow-datafusion-python/pull/558) (dependabot[bot]) -- build(deps): bump async-trait from 0.1.74 to 0.1.77 [#556](https://github.com/apache/arrow-datafusion-python/pull/556) (dependabot[bot]) -- build(deps): bump pyo3 from 0.20.0 to 0.20.2 [#557](https://github.com/apache/arrow-datafusion-python/pull/557) (dependabot[bot]) +- build(deps): bump syn from 2.0.41 to 2.0.43 [#559](https://github.com/apache/datafusion-python/pull/559) (dependabot[bot]) +- build(deps): bump tokio from 1.35.0 to 1.35.1 [#558](https://github.com/apache/datafusion-python/pull/558) (dependabot[bot]) +- build(deps): bump async-trait from 0.1.74 to 0.1.77 [#556](https://github.com/apache/datafusion-python/pull/556) (dependabot[bot]) +- build(deps): bump pyo3 from 0.20.0 to 0.20.2 [#557](https://github.com/apache/datafusion-python/pull/557) (dependabot[bot]) -## [34.0.0](https://github.com/apache/arrow-datafusion-python/tree/34.0.0) (2023-12-28) +## [34.0.0](https://github.com/apache/datafusion-python/tree/34.0.0) (2023-12-28) **Merged pull requests:** -- Adjust visibility of crate private members & Functions [#537](https://github.com/apache/arrow-datafusion-python/pull/537) (jdye64) -- Update json.rst [#538](https://github.com/apache/arrow-datafusion-python/pull/538) (ray-andrew) -- Enable mimalloc local_dynamic_tls feature [#540](https://github.com/apache/arrow-datafusion-python/pull/540) (jdye64) -- Enable substrait feature to be built by default in CI, for nightlies … [#544](https://github.com/apache/arrow-datafusion-python/pull/544) (jdye64) +- Adjust visibility of crate private members & Functions [#537](https://github.com/apache/datafusion-python/pull/537) (jdye64) +- Update json.rst [#538](https://github.com/apache/datafusion-python/pull/538) (ray-andrew) +- Enable mimalloc local_dynamic_tls feature [#540](https://github.com/apache/datafusion-python/pull/540) (jdye64) +- Enable substrait feature to be built by default in CI, for nightlies … [#544](https://github.com/apache/datafusion-python/pull/544) (jdye64) -## [33.0.0](https://github.com/apache/arrow-datafusion-python/tree/33.0.0) (2023-11-16) +## [33.0.0](https://github.com/apache/datafusion-python/tree/33.0.0) (2023-11-16) **Merged pull requests:** -- First pass at getting architectured builds working [#350](https://github.com/apache/arrow-datafusion-python/pull/350) (charlesbluca) -- Remove libprotobuf dep [#527](https://github.com/apache/arrow-datafusion-python/pull/527) (jdye64) +- First pass at getting architectured builds working [#350](https://github.com/apache/datafusion-python/pull/350) (charlesbluca) +- Remove libprotobuf dep [#527](https://github.com/apache/datafusion-python/pull/527) (jdye64) -## [32.0.0](https://github.com/apache/arrow-datafusion-python/tree/32.0.0) (2023-10-21) +## [32.0.0](https://github.com/apache/datafusion-python/tree/32.0.0) (2023-10-21) **Implemented enhancements:** -- feat: expose PyWindowFrame [#509](https://github.com/apache/arrow-datafusion-python/pull/509) (dlovell) -- add Binary String Functions;encode,decode [#494](https://github.com/apache/arrow-datafusion-python/pull/494) (jiangzhx) -- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/arrow-datafusion-python/pull/496) (jiangzhx) -- add first_value last_value [#498](https://github.com/apache/arrow-datafusion-python/pull/498) (jiangzhx) -- add regr\_\* functions [#499](https://github.com/apache/arrow-datafusion-python/pull/499) (jiangzhx) -- Add random missing bindings [#522](https://github.com/apache/arrow-datafusion-python/pull/522) (jdye64) -- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/arrow-datafusion-python/pull/519) (jdye64) -- Add support for window function bindings [#521](https://github.com/apache/arrow-datafusion-python/pull/521) (jdye64) +- feat: expose PyWindowFrame [#509](https://github.com/apache/datafusion-python/pull/509) (dlovell) +- add Binary String Functions;encode,decode [#494](https://github.com/apache/datafusion-python/pull/494) (jiangzhx) +- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/datafusion-python/pull/496) (jiangzhx) +- add first_value last_value [#498](https://github.com/apache/datafusion-python/pull/498) (jiangzhx) +- add regr\_\* functions [#499](https://github.com/apache/datafusion-python/pull/499) (jiangzhx) +- Add random missing bindings [#522](https://github.com/apache/datafusion-python/pull/522) (jdye64) +- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/datafusion-python/pull/519) (jdye64) +- Add support for window function bindings [#521](https://github.com/apache/datafusion-python/pull/521) (jdye64) **Merged pull requests:** -- Prepare 31.0.0 release [#500](https://github.com/apache/arrow-datafusion-python/pull/500) (andygrove) -- Improve release process documentation [#505](https://github.com/apache/arrow-datafusion-python/pull/505) (andygrove) -- add Binary String Functions;encode,decode [#494](https://github.com/apache/arrow-datafusion-python/pull/494) (jiangzhx) -- build(deps): bump mimalloc from 0.1.38 to 0.1.39 [#502](https://github.com/apache/arrow-datafusion-python/pull/502) (dependabot[bot]) -- build(deps): bump syn from 2.0.32 to 2.0.35 [#503](https://github.com/apache/arrow-datafusion-python/pull/503) (dependabot[bot]) -- build(deps): bump syn from 2.0.35 to 2.0.37 [#506](https://github.com/apache/arrow-datafusion-python/pull/506) (dependabot[bot]) -- Use latest DataFusion [#511](https://github.com/apache/arrow-datafusion-python/pull/511) (andygrove) -- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/arrow-datafusion-python/pull/496) (jiangzhx) -- use DataFusion 32 [#515](https://github.com/apache/arrow-datafusion-python/pull/515) (andygrove) -- add first_value last_value [#498](https://github.com/apache/arrow-datafusion-python/pull/498) (jiangzhx) -- build(deps): bump regex-syntax from 0.7.5 to 0.8.1 [#517](https://github.com/apache/arrow-datafusion-python/pull/517) (dependabot[bot]) -- build(deps): bump pyo3-build-config from 0.19.2 to 0.20.0 [#516](https://github.com/apache/arrow-datafusion-python/pull/516) (dependabot[bot]) -- add regr\_\* functions [#499](https://github.com/apache/arrow-datafusion-python/pull/499) (jiangzhx) -- Add random missing bindings [#522](https://github.com/apache/arrow-datafusion-python/pull/522) (jdye64) -- build(deps): bump rustix from 0.38.18 to 0.38.19 [#523](https://github.com/apache/arrow-datafusion-python/pull/523) (dependabot[bot]) -- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/arrow-datafusion-python/pull/519) (jdye64) -- Add support for window function bindings [#521](https://github.com/apache/arrow-datafusion-python/pull/521) (jdye64) -- Small clippy fix [#524](https://github.com/apache/arrow-datafusion-python/pull/524) (andygrove) - -## [31.0.0](https://github.com/apache/arrow-datafusion-python/tree/31.0.0) (2023-09-12) - -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/28.0.0...31.0.0) +- Prepare 31.0.0 release [#500](https://github.com/apache/datafusion-python/pull/500) (andygrove) +- Improve release process documentation [#505](https://github.com/apache/datafusion-python/pull/505) (andygrove) +- add Binary String Functions;encode,decode [#494](https://github.com/apache/datafusion-python/pull/494) (jiangzhx) +- build(deps): bump mimalloc from 0.1.38 to 0.1.39 [#502](https://github.com/apache/datafusion-python/pull/502) (dependabot[bot]) +- build(deps): bump syn from 2.0.32 to 2.0.35 [#503](https://github.com/apache/datafusion-python/pull/503) (dependabot[bot]) +- build(deps): bump syn from 2.0.35 to 2.0.37 [#506](https://github.com/apache/datafusion-python/pull/506) (dependabot[bot]) +- Use latest DataFusion [#511](https://github.com/apache/datafusion-python/pull/511) (andygrove) +- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/datafusion-python/pull/496) (jiangzhx) +- use DataFusion 32 [#515](https://github.com/apache/datafusion-python/pull/515) (andygrove) +- add first_value last_value [#498](https://github.com/apache/datafusion-python/pull/498) (jiangzhx) +- build(deps): bump regex-syntax from 0.7.5 to 0.8.1 [#517](https://github.com/apache/datafusion-python/pull/517) (dependabot[bot]) +- build(deps): bump pyo3-build-config from 0.19.2 to 0.20.0 [#516](https://github.com/apache/datafusion-python/pull/516) (dependabot[bot]) +- add regr\_\* functions [#499](https://github.com/apache/datafusion-python/pull/499) (jiangzhx) +- Add random missing bindings [#522](https://github.com/apache/datafusion-python/pull/522) (jdye64) +- build(deps): bump rustix from 0.38.18 to 0.38.19 [#523](https://github.com/apache/datafusion-python/pull/523) (dependabot[bot]) +- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/datafusion-python/pull/519) (jdye64) +- Add support for window function bindings [#521](https://github.com/apache/datafusion-python/pull/521) (jdye64) +- Small clippy fix [#524](https://github.com/apache/datafusion-python/pull/524) (andygrove) + +## [31.0.0](https://github.com/apache/datafusion-python/tree/31.0.0) (2023-09-12) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/28.0.0...31.0.0) **Implemented enhancements:** -- feat: add case function (#447) [#448](https://github.com/apache/arrow-datafusion-python/pull/448) (mesejo) -- feat: add compression options [#456](https://github.com/apache/arrow-datafusion-python/pull/456) (mesejo) -- feat: add register_json [#458](https://github.com/apache/arrow-datafusion-python/pull/458) (mesejo) -- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/arrow-datafusion-python/pull/459) (mesejo) -- feat: add example of reading parquet from s3 [#460](https://github.com/apache/arrow-datafusion-python/pull/460) (mesejo) -- feat: add register_avro and read_table [#461](https://github.com/apache/arrow-datafusion-python/pull/461) (mesejo) -- feat: add missing scalar math functions [#465](https://github.com/apache/arrow-datafusion-python/pull/465) (mesejo) +- feat: add case function (#447) [#448](https://github.com/apache/datafusion-python/pull/448) (mesejo) +- feat: add compression options [#456](https://github.com/apache/datafusion-python/pull/456) (mesejo) +- feat: add register_json [#458](https://github.com/apache/datafusion-python/pull/458) (mesejo) +- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/datafusion-python/pull/459) (mesejo) +- feat: add example of reading parquet from s3 [#460](https://github.com/apache/datafusion-python/pull/460) (mesejo) +- feat: add register_avro and read_table [#461](https://github.com/apache/datafusion-python/pull/461) (mesejo) +- feat: add missing scalar math functions [#465](https://github.com/apache/datafusion-python/pull/465) (mesejo) **Documentation updates:** -- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/arrow-datafusion-python/pull/455) (mesejo) +- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/datafusion-python/pull/455) (mesejo) **Merged pull requests:** -- Build Linux aarch64 wheel [#443](https://github.com/apache/arrow-datafusion-python/pull/443) (gokselk) -- feat: add case function (#447) [#448](https://github.com/apache/arrow-datafusion-python/pull/448) (mesejo) -- enhancement(docs): Add user guide (#432) [#445](https://github.com/apache/arrow-datafusion-python/pull/445) (mesejo) -- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/arrow-datafusion-python/pull/455) (mesejo) -- feat: add compression options [#456](https://github.com/apache/arrow-datafusion-python/pull/456) (mesejo) -- Upgrade to DF 28.0.0-rc1 [#457](https://github.com/apache/arrow-datafusion-python/pull/457) (andygrove) -- feat: add register_json [#458](https://github.com/apache/arrow-datafusion-python/pull/458) (mesejo) -- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/arrow-datafusion-python/pull/459) (mesejo) -- feat: add example of reading parquet from s3 [#460](https://github.com/apache/arrow-datafusion-python/pull/460) (mesejo) -- feat: add register_avro and read_table [#461](https://github.com/apache/arrow-datafusion-python/pull/461) (mesejo) -- feat: add missing scalar math functions [#465](https://github.com/apache/arrow-datafusion-python/pull/465) (mesejo) -- build(deps): bump arduino/setup-protoc from 1 to 2 [#452](https://github.com/apache/arrow-datafusion-python/pull/452) (dependabot[bot]) -- Revert "build(deps): bump arduino/setup-protoc from 1 to 2 (#452)" [#474](https://github.com/apache/arrow-datafusion-python/pull/474) (viirya) -- Minor: fix wrongly copied function description [#497](https://github.com/apache/arrow-datafusion-python/pull/497) (viirya) -- Upgrade to Datafusion 31.0.0 [#491](https://github.com/apache/arrow-datafusion-python/pull/491) (judahrand) -- Add `isnan` and `iszero` [#495](https://github.com/apache/arrow-datafusion-python/pull/495) (judahrand) +- Build Linux aarch64 wheel [#443](https://github.com/apache/datafusion-python/pull/443) (gokselk) +- feat: add case function (#447) [#448](https://github.com/apache/datafusion-python/pull/448) (mesejo) +- enhancement(docs): Add user guide (#432) [#445](https://github.com/apache/datafusion-python/pull/445) (mesejo) +- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/datafusion-python/pull/455) (mesejo) +- feat: add compression options [#456](https://github.com/apache/datafusion-python/pull/456) (mesejo) +- Upgrade to DF 28.0.0-rc1 [#457](https://github.com/apache/datafusion-python/pull/457) (andygrove) +- feat: add register_json [#458](https://github.com/apache/datafusion-python/pull/458) (mesejo) +- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/datafusion-python/pull/459) (mesejo) +- feat: add example of reading parquet from s3 [#460](https://github.com/apache/datafusion-python/pull/460) (mesejo) +- feat: add register_avro and read_table [#461](https://github.com/apache/datafusion-python/pull/461) (mesejo) +- feat: add missing scalar math functions [#465](https://github.com/apache/datafusion-python/pull/465) (mesejo) +- build(deps): bump arduino/setup-protoc from 1 to 2 [#452](https://github.com/apache/datafusion-python/pull/452) (dependabot[bot]) +- Revert "build(deps): bump arduino/setup-protoc from 1 to 2 (#452)" [#474](https://github.com/apache/datafusion-python/pull/474) (viirya) +- Minor: fix wrongly copied function description [#497](https://github.com/apache/datafusion-python/pull/497) (viirya) +- Upgrade to Datafusion 31.0.0 [#491](https://github.com/apache/datafusion-python/pull/491) (judahrand) +- Add `isnan` and `iszero` [#495](https://github.com/apache/datafusion-python/pull/495) (judahrand) ## 30.0.0 @@ -167,323 +188,323 @@ - Skipped -## [28.0.0](https://github.com/apache/arrow-datafusion-python/tree/28.0.0) (2023-07-25) +## [28.0.0](https://github.com/apache/datafusion-python/tree/28.0.0) (2023-07-25) **Implemented enhancements:** -- feat: expose offset in python API [#437](https://github.com/apache/arrow-datafusion-python/pull/437) (cpcloud) +- feat: expose offset in python API [#437](https://github.com/apache/datafusion-python/pull/437) (cpcloud) **Merged pull requests:** -- File based input utils [#433](https://github.com/apache/arrow-datafusion-python/pull/433) (jdye64) -- Upgrade to 28.0.0-rc1 [#434](https://github.com/apache/arrow-datafusion-python/pull/434) (andygrove) -- Introduces utility for obtaining SqlTable information from a file like location [#398](https://github.com/apache/arrow-datafusion-python/pull/398) (jdye64) -- feat: expose offset in python API [#437](https://github.com/apache/arrow-datafusion-python/pull/437) (cpcloud) -- Use DataFusion 28 [#439](https://github.com/apache/arrow-datafusion-python/pull/439) (andygrove) +- File based input utils [#433](https://github.com/apache/datafusion-python/pull/433) (jdye64) +- Upgrade to 28.0.0-rc1 [#434](https://github.com/apache/datafusion-python/pull/434) (andygrove) +- Introduces utility for obtaining SqlTable information from a file like location [#398](https://github.com/apache/datafusion-python/pull/398) (jdye64) +- feat: expose offset in python API [#437](https://github.com/apache/datafusion-python/pull/437) (cpcloud) +- Use DataFusion 28 [#439](https://github.com/apache/datafusion-python/pull/439) (andygrove) -## [27.0.0](https://github.com/apache/arrow-datafusion-python/tree/27.0.0) (2023-07-03) +## [27.0.0](https://github.com/apache/datafusion-python/tree/27.0.0) (2023-07-03) **Merged pull requests:** -- LogicalPlan.to_variant() make public [#412](https://github.com/apache/arrow-datafusion-python/pull/412) (jdye64) -- Prepare 27.0.0 release [#423](https://github.com/apache/arrow-datafusion-python/pull/423) (andygrove) +- LogicalPlan.to_variant() make public [#412](https://github.com/apache/datafusion-python/pull/412) (jdye64) +- Prepare 27.0.0 release [#423](https://github.com/apache/datafusion-python/pull/423) (andygrove) -## [26.0.0](https://github.com/apache/arrow-datafusion-python/tree/26.0.0) (2023-06-11) +## [26.0.0](https://github.com/apache/datafusion-python/tree/26.0.0) (2023-06-11) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/25.0.0...26.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/25.0.0...26.0.0) **Merged pull requests:** -- Add Expr::Case when_then_else support to rex_call_operands function [#388](https://github.com/apache/arrow-datafusion-python/pull/388) (jdye64) -- Introduce BaseSessionContext abstract class [#390](https://github.com/apache/arrow-datafusion-python/pull/390) (jdye64) -- CRUD Schema support for `BaseSessionContext` [#392](https://github.com/apache/arrow-datafusion-python/pull/392) (jdye64) -- CRUD Table support for `BaseSessionContext` [#394](https://github.com/apache/arrow-datafusion-python/pull/394) (jdye64) +- Add Expr::Case when_then_else support to rex_call_operands function [#388](https://github.com/apache/datafusion-python/pull/388) (jdye64) +- Introduce BaseSessionContext abstract class [#390](https://github.com/apache/datafusion-python/pull/390) (jdye64) +- CRUD Schema support for `BaseSessionContext` [#392](https://github.com/apache/datafusion-python/pull/392) (jdye64) +- CRUD Table support for `BaseSessionContext` [#394](https://github.com/apache/datafusion-python/pull/394) (jdye64) -## [25.0.0](https://github.com/apache/arrow-datafusion-python/tree/25.0.0) (2023-05-23) +## [25.0.0](https://github.com/apache/datafusion-python/tree/25.0.0) (2023-05-23) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/24.0.0...25.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/24.0.0...25.0.0) **Merged pull requests:** -- Prepare 24.0.0 Release [#376](https://github.com/apache/arrow-datafusion-python/pull/376) (andygrove) -- build(deps): bump uuid from 1.3.1 to 1.3.2 [#359](https://github.com/apache/arrow-datafusion-python/pull/359) (dependabot[bot]) -- build(deps): bump mimalloc from 0.1.36 to 0.1.37 [#361](https://github.com/apache/arrow-datafusion-python/pull/361) (dependabot[bot]) -- build(deps): bump regex-syntax from 0.6.29 to 0.7.1 [#334](https://github.com/apache/arrow-datafusion-python/pull/334) (dependabot[bot]) -- upgrade maturin to 0.15.1 [#379](https://github.com/apache/arrow-datafusion-python/pull/379) (Jimexist) -- Expand Expr to include RexType basic support [#378](https://github.com/apache/arrow-datafusion-python/pull/378) (jdye64) -- Add Python script for generating changelog [#383](https://github.com/apache/arrow-datafusion-python/pull/383) (andygrove) +- Prepare 24.0.0 Release [#376](https://github.com/apache/datafusion-python/pull/376) (andygrove) +- build(deps): bump uuid from 1.3.1 to 1.3.2 [#359](https://github.com/apache/datafusion-python/pull/359) (dependabot[bot]) +- build(deps): bump mimalloc from 0.1.36 to 0.1.37 [#361](https://github.com/apache/datafusion-python/pull/361) (dependabot[bot]) +- build(deps): bump regex-syntax from 0.6.29 to 0.7.1 [#334](https://github.com/apache/datafusion-python/pull/334) (dependabot[bot]) +- upgrade maturin to 0.15.1 [#379](https://github.com/apache/datafusion-python/pull/379) (Jimexist) +- Expand Expr to include RexType basic support [#378](https://github.com/apache/datafusion-python/pull/378) (jdye64) +- Add Python script for generating changelog [#383](https://github.com/apache/datafusion-python/pull/383) (andygrove) -## [24.0.0](https://github.com/apache/arrow-datafusion-python/tree/24.0.0) (2023-05-09) +## [24.0.0](https://github.com/apache/datafusion-python/tree/24.0.0) (2023-05-09) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/23.0.0...24.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/23.0.0...24.0.0) **Documentation updates:** -- Fix link to user guide [#354](https://github.com/apache/arrow-datafusion-python/pull/354) (andygrove) +- Fix link to user guide [#354](https://github.com/apache/datafusion-python/pull/354) (andygrove) **Merged pull requests:** -- Add interface to serialize Substrait plans to Python Bytes. [#344](https://github.com/apache/arrow-datafusion-python/pull/344) (kylebrooks-8451) -- Add partition_count property to ExecutionPlan. [#346](https://github.com/apache/arrow-datafusion-python/pull/346) (kylebrooks-8451) -- Remove unsendable from all Rust pyclass types. [#348](https://github.com/apache/arrow-datafusion-python/pull/348) (kylebrooks-8451) -- Fix link to user guide [#354](https://github.com/apache/arrow-datafusion-python/pull/354) (andygrove) -- Fix SessionContext execute. [#353](https://github.com/apache/arrow-datafusion-python/pull/353) (kylebrooks-8451) -- Pub mod expr in lib.rs [#357](https://github.com/apache/arrow-datafusion-python/pull/357) (jdye64) -- Add benchmark derived from TPC-H [#355](https://github.com/apache/arrow-datafusion-python/pull/355) (andygrove) -- Add db-benchmark [#365](https://github.com/apache/arrow-datafusion-python/pull/365) (andygrove) -- First pass of documentation in mdBook [#364](https://github.com/apache/arrow-datafusion-python/pull/364) (MrPowers) -- Add 'pub' and '#[pyo3(get, set)]' to DataTypeMap [#371](https://github.com/apache/arrow-datafusion-python/pull/371) (jdye64) -- Fix db-benchmark [#369](https://github.com/apache/arrow-datafusion-python/pull/369) (andygrove) -- Docs explaining how to view query plans [#373](https://github.com/apache/arrow-datafusion-python/pull/373) (andygrove) -- Improve db-benchmark [#372](https://github.com/apache/arrow-datafusion-python/pull/372) (andygrove) -- Make expr member of PyExpr public [#375](https://github.com/apache/arrow-datafusion-python/pull/375) (jdye64) - -## [23.0.0](https://github.com/apache/arrow-datafusion-python/tree/23.0.0) (2023-04-23) - -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/22.0.0...23.0.0) +- Add interface to serialize Substrait plans to Python Bytes. [#344](https://github.com/apache/datafusion-python/pull/344) (kylebrooks-8451) +- Add partition_count property to ExecutionPlan. [#346](https://github.com/apache/datafusion-python/pull/346) (kylebrooks-8451) +- Remove unsendable from all Rust pyclass types. [#348](https://github.com/apache/datafusion-python/pull/348) (kylebrooks-8451) +- Fix link to user guide [#354](https://github.com/apache/datafusion-python/pull/354) (andygrove) +- Fix SessionContext execute. [#353](https://github.com/apache/datafusion-python/pull/353) (kylebrooks-8451) +- Pub mod expr in lib.rs [#357](https://github.com/apache/datafusion-python/pull/357) (jdye64) +- Add benchmark derived from TPC-H [#355](https://github.com/apache/datafusion-python/pull/355) (andygrove) +- Add db-benchmark [#365](https://github.com/apache/datafusion-python/pull/365) (andygrove) +- First pass of documentation in mdBook [#364](https://github.com/apache/datafusion-python/pull/364) (MrPowers) +- Add 'pub' and '#[pyo3(get, set)]' to DataTypeMap [#371](https://github.com/apache/datafusion-python/pull/371) (jdye64) +- Fix db-benchmark [#369](https://github.com/apache/datafusion-python/pull/369) (andygrove) +- Docs explaining how to view query plans [#373](https://github.com/apache/datafusion-python/pull/373) (andygrove) +- Improve db-benchmark [#372](https://github.com/apache/datafusion-python/pull/372) (andygrove) +- Make expr member of PyExpr public [#375](https://github.com/apache/datafusion-python/pull/375) (jdye64) + +## [23.0.0](https://github.com/apache/datafusion-python/tree/23.0.0) (2023-04-23) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/22.0.0...23.0.0) **Merged pull requests:** -- Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/arrow-datafusion-python/pull/321) (andygrove) -- Osx build linker args [#330](https://github.com/apache/arrow-datafusion-python/pull/330) (jdye64) -- Add requirements file for python 3.11 [#332](https://github.com/apache/arrow-datafusion-python/pull/332) (r4ntix) -- mac arm64 build [#338](https://github.com/apache/arrow-datafusion-python/pull/338) (andygrove) -- Add conda.yaml baseline workflow file [#281](https://github.com/apache/arrow-datafusion-python/pull/281) (jdye64) -- Prepare for 23.0.0 release [#335](https://github.com/apache/arrow-datafusion-python/pull/335) (andygrove) -- Reuse the Tokio Runtime [#341](https://github.com/apache/arrow-datafusion-python/pull/341) (kylebrooks-8451) +- Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/datafusion-python/pull/321) (andygrove) +- Osx build linker args [#330](https://github.com/apache/datafusion-python/pull/330) (jdye64) +- Add requirements file for python 3.11 [#332](https://github.com/apache/datafusion-python/pull/332) (r4ntix) +- mac arm64 build [#338](https://github.com/apache/datafusion-python/pull/338) (andygrove) +- Add conda.yaml baseline workflow file [#281](https://github.com/apache/datafusion-python/pull/281) (jdye64) +- Prepare for 23.0.0 release [#335](https://github.com/apache/datafusion-python/pull/335) (andygrove) +- Reuse the Tokio Runtime [#341](https://github.com/apache/datafusion-python/pull/341) (kylebrooks-8451) -## [22.0.0](https://github.com/apache/arrow-datafusion-python/tree/22.0.0) (2023-04-10) +## [22.0.0](https://github.com/apache/datafusion-python/tree/22.0.0) (2023-04-10) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/21.0.0...22.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/21.0.0...22.0.0) **Merged pull requests:** -- Fix invalid build yaml [#308](https://github.com/apache/arrow-datafusion-python/pull/308) (andygrove) -- Try fix release build [#309](https://github.com/apache/arrow-datafusion-python/pull/309) (andygrove) -- Fix release build [#310](https://github.com/apache/arrow-datafusion-python/pull/310) (andygrove) -- Enable datafusion-substrait protoc feature, to remove compile-time dependency on protoc [#312](https://github.com/apache/arrow-datafusion-python/pull/312) (andygrove) -- Fix Mac/Win release builds in CI [#313](https://github.com/apache/arrow-datafusion-python/pull/313) (andygrove) -- install protoc in docs workflow [#314](https://github.com/apache/arrow-datafusion-python/pull/314) (andygrove) -- Fix documentation generation in CI [#315](https://github.com/apache/arrow-datafusion-python/pull/315) (andygrove) -- Source wheel fix [#319](https://github.com/apache/arrow-datafusion-python/pull/319) (andygrove) +- Fix invalid build yaml [#308](https://github.com/apache/datafusion-python/pull/308) (andygrove) +- Try fix release build [#309](https://github.com/apache/datafusion-python/pull/309) (andygrove) +- Fix release build [#310](https://github.com/apache/datafusion-python/pull/310) (andygrove) +- Enable datafusion-substrait protoc feature, to remove compile-time dependency on protoc [#312](https://github.com/apache/datafusion-python/pull/312) (andygrove) +- Fix Mac/Win release builds in CI [#313](https://github.com/apache/datafusion-python/pull/313) (andygrove) +- install protoc in docs workflow [#314](https://github.com/apache/datafusion-python/pull/314) (andygrove) +- Fix documentation generation in CI [#315](https://github.com/apache/datafusion-python/pull/315) (andygrove) +- Source wheel fix [#319](https://github.com/apache/datafusion-python/pull/319) (andygrove) -## [21.0.0](https://github.com/apache/arrow-datafusion-python/tree/21.0.0) (2023-03-30) +## [21.0.0](https://github.com/apache/datafusion-python/tree/21.0.0) (2023-03-30) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/20.0.0...21.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/20.0.0...21.0.0) **Merged pull requests:** -- minor: Fix minor warning on unused import [#289](https://github.com/apache/arrow-datafusion-python/pull/289) (viirya) -- feature: Implement `describe()` method [#293](https://github.com/apache/arrow-datafusion-python/pull/293) (simicd) -- fix: Printed results not visible in debugger & notebooks [#296](https://github.com/apache/arrow-datafusion-python/pull/296) (simicd) -- add package.include and remove wildcard dependency [#295](https://github.com/apache/arrow-datafusion-python/pull/295) (andygrove) -- Update main branch name in docs workflow [#303](https://github.com/apache/arrow-datafusion-python/pull/303) (andygrove) -- Upgrade to DF 21 [#301](https://github.com/apache/arrow-datafusion-python/pull/301) (andygrove) +- minor: Fix minor warning on unused import [#289](https://github.com/apache/datafusion-python/pull/289) (viirya) +- feature: Implement `describe()` method [#293](https://github.com/apache/datafusion-python/pull/293) (simicd) +- fix: Printed results not visible in debugger & notebooks [#296](https://github.com/apache/datafusion-python/pull/296) (simicd) +- add package.include and remove wildcard dependency [#295](https://github.com/apache/datafusion-python/pull/295) (andygrove) +- Update main branch name in docs workflow [#303](https://github.com/apache/datafusion-python/pull/303) (andygrove) +- Upgrade to DF 21 [#301](https://github.com/apache/datafusion-python/pull/301) (andygrove) -## [20.0.0](https://github.com/apache/arrow-datafusion-python/tree/20.0.0) (2023-03-17) +## [20.0.0](https://github.com/apache/datafusion-python/tree/20.0.0) (2023-03-17) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/0.8.0...20.0.0) +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.8.0...20.0.0) **Implemented enhancements:** -- Empty relation bindings [#208](https://github.com/apache/arrow-datafusion-python/pull/208) (jdye64) -- wrap display_name and canonical_name functions [#214](https://github.com/apache/arrow-datafusion-python/pull/214) (jdye64) -- Add PyAlias bindings [#216](https://github.com/apache/arrow-datafusion-python/pull/216) (jdye64) -- Add bindings for scalar_variable [#218](https://github.com/apache/arrow-datafusion-python/pull/218) (jdye64) -- Bindings for LIKE type expressions [#220](https://github.com/apache/arrow-datafusion-python/pull/220) (jdye64) -- Bool expr bindings [#223](https://github.com/apache/arrow-datafusion-python/pull/223) (jdye64) -- Between bindings [#229](https://github.com/apache/arrow-datafusion-python/pull/229) (jdye64) -- Add bindings for GetIndexedField [#227](https://github.com/apache/arrow-datafusion-python/pull/227) (jdye64) -- Add bindings for case, cast, and trycast [#232](https://github.com/apache/arrow-datafusion-python/pull/232) (jdye64) -- add remaining expr bindings [#233](https://github.com/apache/arrow-datafusion-python/pull/233) (jdye64) -- feature: Additional export methods [#236](https://github.com/apache/arrow-datafusion-python/pull/236) (simicd) -- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/arrow-datafusion-python/pull/240) (iajoiner) -- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/arrow-datafusion-python/pull/242) (simicd) -- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/arrow-datafusion-python/pull/246) (iajoiner) -- feature: Set table name from ctx functions [#260](https://github.com/apache/arrow-datafusion-python/pull/260) (simicd) -- Explain bindings [#264](https://github.com/apache/arrow-datafusion-python/pull/264) (jdye64) -- Extension bindings [#266](https://github.com/apache/arrow-datafusion-python/pull/266) (jdye64) -- Subquery alias bindings [#269](https://github.com/apache/arrow-datafusion-python/pull/269) (jdye64) -- Create memory table [#271](https://github.com/apache/arrow-datafusion-python/pull/271) (jdye64) -- Create view bindings [#273](https://github.com/apache/arrow-datafusion-python/pull/273) (jdye64) -- Re-export Datafusion dependencies [#277](https://github.com/apache/arrow-datafusion-python/pull/277) (jdye64) -- Distinct bindings [#275](https://github.com/apache/arrow-datafusion-python/pull/275) (jdye64) -- Drop table bindings [#283](https://github.com/apache/arrow-datafusion-python/pull/283) (jdye64) -- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/arrow-datafusion-python/pull/285) (jdye64) -- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/arrow-datafusion-python/pull/287) (jdye64) +- Empty relation bindings [#208](https://github.com/apache/datafusion-python/pull/208) (jdye64) +- wrap display_name and canonical_name functions [#214](https://github.com/apache/datafusion-python/pull/214) (jdye64) +- Add PyAlias bindings [#216](https://github.com/apache/datafusion-python/pull/216) (jdye64) +- Add bindings for scalar_variable [#218](https://github.com/apache/datafusion-python/pull/218) (jdye64) +- Bindings for LIKE type expressions [#220](https://github.com/apache/datafusion-python/pull/220) (jdye64) +- Bool expr bindings [#223](https://github.com/apache/datafusion-python/pull/223) (jdye64) +- Between bindings [#229](https://github.com/apache/datafusion-python/pull/229) (jdye64) +- Add bindings for GetIndexedField [#227](https://github.com/apache/datafusion-python/pull/227) (jdye64) +- Add bindings for case, cast, and trycast [#232](https://github.com/apache/datafusion-python/pull/232) (jdye64) +- add remaining expr bindings [#233](https://github.com/apache/datafusion-python/pull/233) (jdye64) +- feature: Additional export methods [#236](https://github.com/apache/datafusion-python/pull/236) (simicd) +- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/datafusion-python/pull/240) (iajoiner) +- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/datafusion-python/pull/242) (simicd) +- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/datafusion-python/pull/246) (iajoiner) +- feature: Set table name from ctx functions [#260](https://github.com/apache/datafusion-python/pull/260) (simicd) +- Explain bindings [#264](https://github.com/apache/datafusion-python/pull/264) (jdye64) +- Extension bindings [#266](https://github.com/apache/datafusion-python/pull/266) (jdye64) +- Subquery alias bindings [#269](https://github.com/apache/datafusion-python/pull/269) (jdye64) +- Create memory table [#271](https://github.com/apache/datafusion-python/pull/271) (jdye64) +- Create view bindings [#273](https://github.com/apache/datafusion-python/pull/273) (jdye64) +- Re-export Datafusion dependencies [#277](https://github.com/apache/datafusion-python/pull/277) (jdye64) +- Distinct bindings [#275](https://github.com/apache/datafusion-python/pull/275) (jdye64) +- Drop table bindings [#283](https://github.com/apache/datafusion-python/pull/283) (jdye64) +- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/datafusion-python/pull/285) (jdye64) +- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/datafusion-python/pull/287) (jdye64) **Documentation updates:** -- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/arrow-datafusion-python/pull/258) (simicd) +- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/datafusion-python/pull/258) (simicd) **Merged pull requests:** -- Minor docs updates [#210](https://github.com/apache/arrow-datafusion-python/pull/210) (andygrove) -- Empty relation bindings [#208](https://github.com/apache/arrow-datafusion-python/pull/208) (jdye64) -- wrap display_name and canonical_name functions [#214](https://github.com/apache/arrow-datafusion-python/pull/214) (jdye64) -- Add PyAlias bindings [#216](https://github.com/apache/arrow-datafusion-python/pull/216) (jdye64) -- Add bindings for scalar_variable [#218](https://github.com/apache/arrow-datafusion-python/pull/218) (jdye64) -- Bindings for LIKE type expressions [#220](https://github.com/apache/arrow-datafusion-python/pull/220) (jdye64) -- Bool expr bindings [#223](https://github.com/apache/arrow-datafusion-python/pull/223) (jdye64) -- Between bindings [#229](https://github.com/apache/arrow-datafusion-python/pull/229) (jdye64) -- Add bindings for GetIndexedField [#227](https://github.com/apache/arrow-datafusion-python/pull/227) (jdye64) -- Add bindings for case, cast, and trycast [#232](https://github.com/apache/arrow-datafusion-python/pull/232) (jdye64) -- add remaining expr bindings [#233](https://github.com/apache/arrow-datafusion-python/pull/233) (jdye64) -- Pre-commit hooks [#228](https://github.com/apache/arrow-datafusion-python/pull/228) (jdye64) -- Implement new release process [#149](https://github.com/apache/arrow-datafusion-python/pull/149) (andygrove) -- feature: Additional export methods [#236](https://github.com/apache/arrow-datafusion-python/pull/236) (simicd) -- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/arrow-datafusion-python/pull/240) (iajoiner) -- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/arrow-datafusion-python/pull/242) (simicd) -- Fix release instructions [#238](https://github.com/apache/arrow-datafusion-python/pull/238) (andygrove) -- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/arrow-datafusion-python/pull/246) (iajoiner) -- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/arrow-datafusion-python/pull/258) (simicd) -- feature: Set table name from ctx functions [#260](https://github.com/apache/arrow-datafusion-python/pull/260) (simicd) -- Upgrade to DataFusion 19 [#262](https://github.com/apache/arrow-datafusion-python/pull/262) (andygrove) -- Explain bindings [#264](https://github.com/apache/arrow-datafusion-python/pull/264) (jdye64) -- Extension bindings [#266](https://github.com/apache/arrow-datafusion-python/pull/266) (jdye64) -- Subquery alias bindings [#269](https://github.com/apache/arrow-datafusion-python/pull/269) (jdye64) -- Create memory table [#271](https://github.com/apache/arrow-datafusion-python/pull/271) (jdye64) -- Create view bindings [#273](https://github.com/apache/arrow-datafusion-python/pull/273) (jdye64) -- Re-export Datafusion dependencies [#277](https://github.com/apache/arrow-datafusion-python/pull/277) (jdye64) -- Distinct bindings [#275](https://github.com/apache/arrow-datafusion-python/pull/275) (jdye64) -- build(deps): bump actions/checkout from 2 to 3 [#244](https://github.com/apache/arrow-datafusion-python/pull/244) (dependabot[bot]) -- build(deps): bump actions/upload-artifact from 2 to 3 [#245](https://github.com/apache/arrow-datafusion-python/pull/245) (dependabot[bot]) -- build(deps): bump actions/download-artifact from 2 to 3 [#243](https://github.com/apache/arrow-datafusion-python/pull/243) (dependabot[bot]) -- Use DataFusion 20 [#278](https://github.com/apache/arrow-datafusion-python/pull/278) (andygrove) -- Drop table bindings [#283](https://github.com/apache/arrow-datafusion-python/pull/283) (jdye64) -- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/arrow-datafusion-python/pull/285) (jdye64) -- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/arrow-datafusion-python/pull/287) (jdye64) - -## [0.8.0](https://github.com/apache/arrow-datafusion-python/tree/0.8.0) (2023-02-22) - -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/0.8.0-rc1...0.8.0) +- Minor docs updates [#210](https://github.com/apache/datafusion-python/pull/210) (andygrove) +- Empty relation bindings [#208](https://github.com/apache/datafusion-python/pull/208) (jdye64) +- wrap display_name and canonical_name functions [#214](https://github.com/apache/datafusion-python/pull/214) (jdye64) +- Add PyAlias bindings [#216](https://github.com/apache/datafusion-python/pull/216) (jdye64) +- Add bindings for scalar_variable [#218](https://github.com/apache/datafusion-python/pull/218) (jdye64) +- Bindings for LIKE type expressions [#220](https://github.com/apache/datafusion-python/pull/220) (jdye64) +- Bool expr bindings [#223](https://github.com/apache/datafusion-python/pull/223) (jdye64) +- Between bindings [#229](https://github.com/apache/datafusion-python/pull/229) (jdye64) +- Add bindings for GetIndexedField [#227](https://github.com/apache/datafusion-python/pull/227) (jdye64) +- Add bindings for case, cast, and trycast [#232](https://github.com/apache/datafusion-python/pull/232) (jdye64) +- add remaining expr bindings [#233](https://github.com/apache/datafusion-python/pull/233) (jdye64) +- Pre-commit hooks [#228](https://github.com/apache/datafusion-python/pull/228) (jdye64) +- Implement new release process [#149](https://github.com/apache/datafusion-python/pull/149) (andygrove) +- feature: Additional export methods [#236](https://github.com/apache/datafusion-python/pull/236) (simicd) +- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/datafusion-python/pull/240) (iajoiner) +- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/datafusion-python/pull/242) (simicd) +- Fix release instructions [#238](https://github.com/apache/datafusion-python/pull/238) (andygrove) +- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/datafusion-python/pull/246) (iajoiner) +- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/datafusion-python/pull/258) (simicd) +- feature: Set table name from ctx functions [#260](https://github.com/apache/datafusion-python/pull/260) (simicd) +- Upgrade to DataFusion 19 [#262](https://github.com/apache/datafusion-python/pull/262) (andygrove) +- Explain bindings [#264](https://github.com/apache/datafusion-python/pull/264) (jdye64) +- Extension bindings [#266](https://github.com/apache/datafusion-python/pull/266) (jdye64) +- Subquery alias bindings [#269](https://github.com/apache/datafusion-python/pull/269) (jdye64) +- Create memory table [#271](https://github.com/apache/datafusion-python/pull/271) (jdye64) +- Create view bindings [#273](https://github.com/apache/datafusion-python/pull/273) (jdye64) +- Re-export Datafusion dependencies [#277](https://github.com/apache/datafusion-python/pull/277) (jdye64) +- Distinct bindings [#275](https://github.com/apache/datafusion-python/pull/275) (jdye64) +- build(deps): bump actions/checkout from 2 to 3 [#244](https://github.com/apache/datafusion-python/pull/244) (dependabot[bot]) +- build(deps): bump actions/upload-artifact from 2 to 3 [#245](https://github.com/apache/datafusion-python/pull/245) (dependabot[bot]) +- build(deps): bump actions/download-artifact from 2 to 3 [#243](https://github.com/apache/datafusion-python/pull/243) (dependabot[bot]) +- Use DataFusion 20 [#278](https://github.com/apache/datafusion-python/pull/278) (andygrove) +- Drop table bindings [#283](https://github.com/apache/datafusion-python/pull/283) (jdye64) +- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/datafusion-python/pull/285) (jdye64) +- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/datafusion-python/pull/287) (jdye64) + +## [0.8.0](https://github.com/apache/datafusion-python/tree/0.8.0) (2023-02-22) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.8.0-rc1...0.8.0) **Implemented enhancements:** -- Add support for cuDF physical execution engine [\#202](https://github.com/apache/arrow-datafusion-python/issues/202) -- Make it easier to create a Pandas dataframe from DataFusion query results [\#139](https://github.com/apache/arrow-datafusion-python/issues/139) +- Add support for cuDF physical execution engine [\#202](https://github.com/apache/datafusion-python/issues/202) +- Make it easier to create a Pandas dataframe from DataFusion query results [\#139](https://github.com/apache/datafusion-python/issues/139) **Fixed bugs:** -- Build error: could not compile `thiserror` due to 2 previous errors [\#69](https://github.com/apache/arrow-datafusion-python/issues/69) +- Build error: could not compile `thiserror` due to 2 previous errors [\#69](https://github.com/apache/datafusion-python/issues/69) **Closed issues:** -- Integrate with the new `object_store` crate [\#22](https://github.com/apache/arrow-datafusion-python/issues/22) +- Integrate with the new `object_store` crate [\#22](https://github.com/apache/datafusion-python/issues/22) **Merged pull requests:** -- Update README in preparation for 0.8 release [\#206](https://github.com/apache/arrow-datafusion-python/pull/206) ([andygrove](https://github.com/andygrove)) -- Add support for cudf as a physical execution engine [\#205](https://github.com/apache/arrow-datafusion-python/pull/205) ([jdye64](https://github.com/jdye64)) -- Run `maturin develop` instead of `cargo build` in verification script [\#200](https://github.com/apache/arrow-datafusion-python/pull/200) ([andygrove](https://github.com/andygrove)) -- Add tests for recently added functionality [\#199](https://github.com/apache/arrow-datafusion-python/pull/199) ([andygrove](https://github.com/andygrove)) -- Implement `to_pandas()` [\#197](https://github.com/apache/arrow-datafusion-python/pull/197) ([simicd](https://github.com/simicd)) -- Add Python wrapper for LogicalPlan::Sort [\#196](https://github.com/apache/arrow-datafusion-python/pull/196) ([andygrove](https://github.com/andygrove)) -- Add Python wrapper for LogicalPlan::Aggregate [\#195](https://github.com/apache/arrow-datafusion-python/pull/195) ([andygrove](https://github.com/andygrove)) -- Add Python wrapper for LogicalPlan::Limit [\#193](https://github.com/apache/arrow-datafusion-python/pull/193) ([andygrove](https://github.com/andygrove)) -- Add Python wrapper for LogicalPlan::Filter [\#192](https://github.com/apache/arrow-datafusion-python/pull/192) ([andygrove](https://github.com/andygrove)) -- Add experimental support for executing SQL with Polars and Pandas [\#190](https://github.com/apache/arrow-datafusion-python/pull/190) ([andygrove](https://github.com/andygrove)) -- Update changelog for 0.8 release [\#188](https://github.com/apache/arrow-datafusion-python/pull/188) ([andygrove](https://github.com/andygrove)) -- Add ability to execute ExecutionPlan and get a stream of RecordBatch [\#186](https://github.com/apache/arrow-datafusion-python/pull/186) ([andygrove](https://github.com/andygrove)) -- Dffield bindings [\#185](https://github.com/apache/arrow-datafusion-python/pull/185) ([jdye64](https://github.com/jdye64)) -- Add bindings for DFSchema [\#183](https://github.com/apache/arrow-datafusion-python/pull/183) ([jdye64](https://github.com/jdye64)) -- test: Window functions [\#182](https://github.com/apache/arrow-datafusion-python/pull/182) ([simicd](https://github.com/simicd)) -- Add bindings for Projection [\#180](https://github.com/apache/arrow-datafusion-python/pull/180) ([jdye64](https://github.com/jdye64)) -- Table scan bindings [\#178](https://github.com/apache/arrow-datafusion-python/pull/178) ([jdye64](https://github.com/jdye64)) -- Make session configurable [\#176](https://github.com/apache/arrow-datafusion-python/pull/176) ([andygrove](https://github.com/andygrove)) -- Upgrade to DataFusion 18.0.0 [\#175](https://github.com/apache/arrow-datafusion-python/pull/175) ([andygrove](https://github.com/andygrove)) -- Use latest DataFusion rev in preparation for DF 18 release [\#174](https://github.com/apache/arrow-datafusion-python/pull/174) ([andygrove](https://github.com/andygrove)) -- Arrow type bindings [\#173](https://github.com/apache/arrow-datafusion-python/pull/173) ([jdye64](https://github.com/jdye64)) -- Pyo3 bump [\#171](https://github.com/apache/arrow-datafusion-python/pull/171) ([jdye64](https://github.com/jdye64)) -- feature: Add additional aggregation functions [\#170](https://github.com/apache/arrow-datafusion-python/pull/170) ([simicd](https://github.com/simicd)) -- Make from_substrait_plan return DataFrame instead of LogicalPlan [\#164](https://github.com/apache/arrow-datafusion-python/pull/164) ([andygrove](https://github.com/andygrove)) -- feature: Implement count method [\#163](https://github.com/apache/arrow-datafusion-python/pull/163) ([simicd](https://github.com/simicd)) -- CI Fixes [\#162](https://github.com/apache/arrow-datafusion-python/pull/162) ([jdye64](https://github.com/jdye64)) -- Upgrade to DataFusion 17 [\#160](https://github.com/apache/arrow-datafusion-python/pull/160) ([andygrove](https://github.com/andygrove)) -- feature: Improve string representation of datafusion classes [\#159](https://github.com/apache/arrow-datafusion-python/pull/159) ([simicd](https://github.com/simicd)) -- Make PyExecutionPlan.plan public [\#156](https://github.com/apache/arrow-datafusion-python/pull/156) ([andygrove](https://github.com/andygrove)) -- Expose methods on logical and execution plans [\#155](https://github.com/apache/arrow-datafusion-python/pull/155) ([andygrove](https://github.com/andygrove)) -- Fix clippy for new Rust version [\#154](https://github.com/apache/arrow-datafusion-python/pull/154) ([andygrove](https://github.com/andygrove)) -- Add DataFrame methods for accessing plans [\#153](https://github.com/apache/arrow-datafusion-python/pull/153) ([andygrove](https://github.com/andygrove)) -- Use DataFusion rev 5238e8c97f998b4d2cb9fab85fb182f325a1a7fb [\#150](https://github.com/apache/arrow-datafusion-python/pull/150) ([andygrove](https://github.com/andygrove)) -- build\(deps\): bump async-trait from 0.1.61 to 0.1.62 [\#148](https://github.com/apache/arrow-datafusion-python/pull/148) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Rename default branch from master to main [\#147](https://github.com/apache/arrow-datafusion-python/pull/147) ([andygrove](https://github.com/andygrove)) -- Substrait bindings [\#145](https://github.com/apache/arrow-datafusion-python/pull/145) ([jdye64](https://github.com/jdye64)) -- build\(deps\): bump uuid from 0.8.2 to 1.2.2 [\#143](https://github.com/apache/arrow-datafusion-python/pull/143) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Prepare for 0.8.0 release [\#141](https://github.com/apache/arrow-datafusion-python/pull/141) ([andygrove](https://github.com/andygrove)) -- Improve README and add more examples [\#137](https://github.com/apache/arrow-datafusion-python/pull/137) ([andygrove](https://github.com/andygrove)) -- test: Expand tests for built-in functions [\#129](https://github.com/apache/arrow-datafusion-python/pull/129) ([simicd](https://github.com/simicd)) -- build\(deps\): bump object_store from 0.5.2 to 0.5.3 [\#126](https://github.com/apache/arrow-datafusion-python/pull/126) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump mimalloc from 0.1.32 to 0.1.34 [\#125](https://github.com/apache/arrow-datafusion-python/pull/125) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Introduce conda directory containing datafusion-dev.yaml conda enviro… [\#124](https://github.com/apache/arrow-datafusion-python/pull/124) ([jdye64](https://github.com/jdye64)) -- build\(deps\): bump bzip2 from 0.4.3 to 0.4.4 [\#121](https://github.com/apache/arrow-datafusion-python/pull/121) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump tokio from 1.23.0 to 1.24.1 [\#119](https://github.com/apache/arrow-datafusion-python/pull/119) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump async-trait from 0.1.60 to 0.1.61 [\#118](https://github.com/apache/arrow-datafusion-python/pull/118) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Upgrade to DataFusion 16.0.0 [\#115](https://github.com/apache/arrow-datafusion-python/pull/115) ([andygrove](https://github.com/andygrove)) -- Bump async-trait from 0.1.57 to 0.1.60 [\#114](https://github.com/apache/arrow-datafusion-python/pull/114) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump object_store from 0.5.1 to 0.5.2 [\#112](https://github.com/apache/arrow-datafusion-python/pull/112) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump tokio from 1.21.2 to 1.23.0 [\#109](https://github.com/apache/arrow-datafusion-python/pull/109) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Add entries for publishing production \(asf-site\) and staging docs [\#107](https://github.com/apache/arrow-datafusion-python/pull/107) ([martin-g](https://github.com/martin-g)) -- Add a workflow that builds the docs and deploys them at staged or production [\#104](https://github.com/apache/arrow-datafusion-python/pull/104) ([martin-g](https://github.com/martin-g)) -- Upgrade to DataFusion 15.0.0 [\#103](https://github.com/apache/arrow-datafusion-python/pull/103) ([andygrove](https://github.com/andygrove)) -- build\(deps\): bump futures from 0.3.24 to 0.3.25 [\#102](https://github.com/apache/arrow-datafusion-python/pull/102) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump pyo3 from 0.17.2 to 0.17.3 [\#101](https://github.com/apache/arrow-datafusion-python/pull/101) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump mimalloc from 0.1.30 to 0.1.32 [\#98](https://github.com/apache/arrow-datafusion-python/pull/98) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump rand from 0.7.3 to 0.8.5 [\#97](https://github.com/apache/arrow-datafusion-python/pull/97) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Fix GitHub actions warnings [\#95](https://github.com/apache/arrow-datafusion-python/pull/95) ([martin-g](https://github.com/martin-g)) -- Fixes \#81 - Add CI workflow for source distribution [\#93](https://github.com/apache/arrow-datafusion-python/pull/93) ([martin-g](https://github.com/martin-g)) -- post-release updates [\#91](https://github.com/apache/arrow-datafusion-python/pull/91) ([andygrove](https://github.com/andygrove)) -- Build for manylinux 2014 [\#88](https://github.com/apache/arrow-datafusion-python/pull/88) ([martin-g](https://github.com/martin-g)) -- update release readme tag [\#86](https://github.com/apache/arrow-datafusion-python/pull/86) ([Jimexist](https://github.com/Jimexist)) -- Upgrade Maturin to 0.14.2 [\#85](https://github.com/apache/arrow-datafusion-python/pull/85) ([martin-g](https://github.com/martin-g)) -- Update release instructions [\#83](https://github.com/apache/arrow-datafusion-python/pull/83) ([andygrove](https://github.com/andygrove)) -- \[Functions\] - Add python function binding to `functions` [\#73](https://github.com/apache/arrow-datafusion-python/pull/73) ([francis-du](https://github.com/francis-du)) - -## [0.8.0-rc1](https://github.com/apache/arrow-datafusion-python/tree/0.8.0-rc1) (2023-02-17) - -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/0.7.0-rc2...0.8.0-rc1) +- Update README in preparation for 0.8 release [\#206](https://github.com/apache/datafusion-python/pull/206) ([andygrove](https://github.com/andygrove)) +- Add support for cudf as a physical execution engine [\#205](https://github.com/apache/datafusion-python/pull/205) ([jdye64](https://github.com/jdye64)) +- Run `maturin develop` instead of `cargo build` in verification script [\#200](https://github.com/apache/datafusion-python/pull/200) ([andygrove](https://github.com/andygrove)) +- Add tests for recently added functionality [\#199](https://github.com/apache/datafusion-python/pull/199) ([andygrove](https://github.com/andygrove)) +- Implement `to_pandas()` [\#197](https://github.com/apache/datafusion-python/pull/197) ([simicd](https://github.com/simicd)) +- Add Python wrapper for LogicalPlan::Sort [\#196](https://github.com/apache/datafusion-python/pull/196) ([andygrove](https://github.com/andygrove)) +- Add Python wrapper for LogicalPlan::Aggregate [\#195](https://github.com/apache/datafusion-python/pull/195) ([andygrove](https://github.com/andygrove)) +- Add Python wrapper for LogicalPlan::Limit [\#193](https://github.com/apache/datafusion-python/pull/193) ([andygrove](https://github.com/andygrove)) +- Add Python wrapper for LogicalPlan::Filter [\#192](https://github.com/apache/datafusion-python/pull/192) ([andygrove](https://github.com/andygrove)) +- Add experimental support for executing SQL with Polars and Pandas [\#190](https://github.com/apache/datafusion-python/pull/190) ([andygrove](https://github.com/andygrove)) +- Update changelog for 0.8 release [\#188](https://github.com/apache/datafusion-python/pull/188) ([andygrove](https://github.com/andygrove)) +- Add ability to execute ExecutionPlan and get a stream of RecordBatch [\#186](https://github.com/apache/datafusion-python/pull/186) ([andygrove](https://github.com/andygrove)) +- Dffield bindings [\#185](https://github.com/apache/datafusion-python/pull/185) ([jdye64](https://github.com/jdye64)) +- Add bindings for DFSchema [\#183](https://github.com/apache/datafusion-python/pull/183) ([jdye64](https://github.com/jdye64)) +- test: Window functions [\#182](https://github.com/apache/datafusion-python/pull/182) ([simicd](https://github.com/simicd)) +- Add bindings for Projection [\#180](https://github.com/apache/datafusion-python/pull/180) ([jdye64](https://github.com/jdye64)) +- Table scan bindings [\#178](https://github.com/apache/datafusion-python/pull/178) ([jdye64](https://github.com/jdye64)) +- Make session configurable [\#176](https://github.com/apache/datafusion-python/pull/176) ([andygrove](https://github.com/andygrove)) +- Upgrade to DataFusion 18.0.0 [\#175](https://github.com/apache/datafusion-python/pull/175) ([andygrove](https://github.com/andygrove)) +- Use latest DataFusion rev in preparation for DF 18 release [\#174](https://github.com/apache/datafusion-python/pull/174) ([andygrove](https://github.com/andygrove)) +- Arrow type bindings [\#173](https://github.com/apache/datafusion-python/pull/173) ([jdye64](https://github.com/jdye64)) +- Pyo3 bump [\#171](https://github.com/apache/datafusion-python/pull/171) ([jdye64](https://github.com/jdye64)) +- feature: Add additional aggregation functions [\#170](https://github.com/apache/datafusion-python/pull/170) ([simicd](https://github.com/simicd)) +- Make from_substrait_plan return DataFrame instead of LogicalPlan [\#164](https://github.com/apache/datafusion-python/pull/164) ([andygrove](https://github.com/andygrove)) +- feature: Implement count method [\#163](https://github.com/apache/datafusion-python/pull/163) ([simicd](https://github.com/simicd)) +- CI Fixes [\#162](https://github.com/apache/datafusion-python/pull/162) ([jdye64](https://github.com/jdye64)) +- Upgrade to DataFusion 17 [\#160](https://github.com/apache/datafusion-python/pull/160) ([andygrove](https://github.com/andygrove)) +- feature: Improve string representation of datafusion classes [\#159](https://github.com/apache/datafusion-python/pull/159) ([simicd](https://github.com/simicd)) +- Make PyExecutionPlan.plan public [\#156](https://github.com/apache/datafusion-python/pull/156) ([andygrove](https://github.com/andygrove)) +- Expose methods on logical and execution plans [\#155](https://github.com/apache/datafusion-python/pull/155) ([andygrove](https://github.com/andygrove)) +- Fix clippy for new Rust version [\#154](https://github.com/apache/datafusion-python/pull/154) ([andygrove](https://github.com/andygrove)) +- Add DataFrame methods for accessing plans [\#153](https://github.com/apache/datafusion-python/pull/153) ([andygrove](https://github.com/andygrove)) +- Use DataFusion rev 5238e8c97f998b4d2cb9fab85fb182f325a1a7fb [\#150](https://github.com/apache/datafusion-python/pull/150) ([andygrove](https://github.com/andygrove)) +- build\(deps\): bump async-trait from 0.1.61 to 0.1.62 [\#148](https://github.com/apache/datafusion-python/pull/148) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Rename default branch from master to main [\#147](https://github.com/apache/datafusion-python/pull/147) ([andygrove](https://github.com/andygrove)) +- Substrait bindings [\#145](https://github.com/apache/datafusion-python/pull/145) ([jdye64](https://github.com/jdye64)) +- build\(deps\): bump uuid from 0.8.2 to 1.2.2 [\#143](https://github.com/apache/datafusion-python/pull/143) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Prepare for 0.8.0 release [\#141](https://github.com/apache/datafusion-python/pull/141) ([andygrove](https://github.com/andygrove)) +- Improve README and add more examples [\#137](https://github.com/apache/datafusion-python/pull/137) ([andygrove](https://github.com/andygrove)) +- test: Expand tests for built-in functions [\#129](https://github.com/apache/datafusion-python/pull/129) ([simicd](https://github.com/simicd)) +- build\(deps\): bump object_store from 0.5.2 to 0.5.3 [\#126](https://github.com/apache/datafusion-python/pull/126) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump mimalloc from 0.1.32 to 0.1.34 [\#125](https://github.com/apache/datafusion-python/pull/125) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Introduce conda directory containing datafusion-dev.yaml conda enviro… [\#124](https://github.com/apache/datafusion-python/pull/124) ([jdye64](https://github.com/jdye64)) +- build\(deps\): bump bzip2 from 0.4.3 to 0.4.4 [\#121](https://github.com/apache/datafusion-python/pull/121) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump tokio from 1.23.0 to 1.24.1 [\#119](https://github.com/apache/datafusion-python/pull/119) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump async-trait from 0.1.60 to 0.1.61 [\#118](https://github.com/apache/datafusion-python/pull/118) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Upgrade to DataFusion 16.0.0 [\#115](https://github.com/apache/datafusion-python/pull/115) ([andygrove](https://github.com/andygrove)) +- Bump async-trait from 0.1.57 to 0.1.60 [\#114](https://github.com/apache/datafusion-python/pull/114) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump object_store from 0.5.1 to 0.5.2 [\#112](https://github.com/apache/datafusion-python/pull/112) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tokio from 1.21.2 to 1.23.0 [\#109](https://github.com/apache/datafusion-python/pull/109) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add entries for publishing production \(asf-site\) and staging docs [\#107](https://github.com/apache/datafusion-python/pull/107) ([martin-g](https://github.com/martin-g)) +- Add a workflow that builds the docs and deploys them at staged or production [\#104](https://github.com/apache/datafusion-python/pull/104) ([martin-g](https://github.com/martin-g)) +- Upgrade to DataFusion 15.0.0 [\#103](https://github.com/apache/datafusion-python/pull/103) ([andygrove](https://github.com/andygrove)) +- build\(deps\): bump futures from 0.3.24 to 0.3.25 [\#102](https://github.com/apache/datafusion-python/pull/102) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump pyo3 from 0.17.2 to 0.17.3 [\#101](https://github.com/apache/datafusion-python/pull/101) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump mimalloc from 0.1.30 to 0.1.32 [\#98](https://github.com/apache/datafusion-python/pull/98) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump rand from 0.7.3 to 0.8.5 [\#97](https://github.com/apache/datafusion-python/pull/97) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Fix GitHub actions warnings [\#95](https://github.com/apache/datafusion-python/pull/95) ([martin-g](https://github.com/martin-g)) +- Fixes \#81 - Add CI workflow for source distribution [\#93](https://github.com/apache/datafusion-python/pull/93) ([martin-g](https://github.com/martin-g)) +- post-release updates [\#91](https://github.com/apache/datafusion-python/pull/91) ([andygrove](https://github.com/andygrove)) +- Build for manylinux 2014 [\#88](https://github.com/apache/datafusion-python/pull/88) ([martin-g](https://github.com/martin-g)) +- update release readme tag [\#86](https://github.com/apache/datafusion-python/pull/86) ([Jimexist](https://github.com/Jimexist)) +- Upgrade Maturin to 0.14.2 [\#85](https://github.com/apache/datafusion-python/pull/85) ([martin-g](https://github.com/martin-g)) +- Update release instructions [\#83](https://github.com/apache/datafusion-python/pull/83) ([andygrove](https://github.com/andygrove)) +- \[Functions\] - Add python function binding to `functions` [\#73](https://github.com/apache/datafusion-python/pull/73) ([francis-du](https://github.com/francis-du)) + +## [0.8.0-rc1](https://github.com/apache/datafusion-python/tree/0.8.0-rc1) (2023-02-17) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.7.0-rc2...0.8.0-rc1) **Implemented enhancements:** -- Add bindings for datafusion_common::DFField [\#184](https://github.com/apache/arrow-datafusion-python/issues/184) -- Add bindings for DFSchema/DFSchemaRef [\#181](https://github.com/apache/arrow-datafusion-python/issues/181) -- Add bindings for datafusion_expr Projection [\#179](https://github.com/apache/arrow-datafusion-python/issues/179) -- Add bindings for `TableScan` struct from `datafusion_expr::TableScan` [\#177](https://github.com/apache/arrow-datafusion-python/issues/177) -- Add a "mapping" struct for types [\#172](https://github.com/apache/arrow-datafusion-python/issues/172) -- Improve string representation of datafusion classes \(dataframe, context, expression, ...\) [\#158](https://github.com/apache/arrow-datafusion-python/issues/158) -- Add DataFrame count method [\#151](https://github.com/apache/arrow-datafusion-python/issues/151) -- \[REQUEST\] Github Actions Improvements [\#146](https://github.com/apache/arrow-datafusion-python/issues/146) -- Change default branch name from master to main [\#144](https://github.com/apache/arrow-datafusion-python/issues/144) -- Bump pyo3 to 0.18.0 [\#140](https://github.com/apache/arrow-datafusion-python/issues/140) -- Add script for Python linting [\#134](https://github.com/apache/arrow-datafusion-python/issues/134) -- Add Python bindings for substrait module [\#132](https://github.com/apache/arrow-datafusion-python/issues/132) -- Expand unit tests for built-in functions [\#128](https://github.com/apache/arrow-datafusion-python/issues/128) -- support creating arrow-datafusion-python conda environment [\#122](https://github.com/apache/arrow-datafusion-python/issues/122) -- Build Python source distribution in GitHub workflow [\#81](https://github.com/apache/arrow-datafusion-python/issues/81) -- EPIC: Add all functions to python binding `functions` [\#72](https://github.com/apache/arrow-datafusion-python/issues/72) +- Add bindings for datafusion_common::DFField [\#184](https://github.com/apache/datafusion-python/issues/184) +- Add bindings for DFSchema/DFSchemaRef [\#181](https://github.com/apache/datafusion-python/issues/181) +- Add bindings for datafusion_expr Projection [\#179](https://github.com/apache/datafusion-python/issues/179) +- Add bindings for `TableScan` struct from `datafusion_expr::TableScan` [\#177](https://github.com/apache/datafusion-python/issues/177) +- Add a "mapping" struct for types [\#172](https://github.com/apache/datafusion-python/issues/172) +- Improve string representation of datafusion classes \(dataframe, context, expression, ...\) [\#158](https://github.com/apache/datafusion-python/issues/158) +- Add DataFrame count method [\#151](https://github.com/apache/datafusion-python/issues/151) +- \[REQUEST\] Github Actions Improvements [\#146](https://github.com/apache/datafusion-python/issues/146) +- Change default branch name from master to main [\#144](https://github.com/apache/datafusion-python/issues/144) +- Bump pyo3 to 0.18.0 [\#140](https://github.com/apache/datafusion-python/issues/140) +- Add script for Python linting [\#134](https://github.com/apache/datafusion-python/issues/134) +- Add Python bindings for substrait module [\#132](https://github.com/apache/datafusion-python/issues/132) +- Expand unit tests for built-in functions [\#128](https://github.com/apache/datafusion-python/issues/128) +- support creating arrow-datafusion-python conda environment [\#122](https://github.com/apache/datafusion-python/issues/122) +- Build Python source distribution in GitHub workflow [\#81](https://github.com/apache/datafusion-python/issues/81) +- EPIC: Add all functions to python binding `functions` [\#72](https://github.com/apache/datafusion-python/issues/72) **Fixed bugs:** -- Build is broken [\#161](https://github.com/apache/arrow-datafusion-python/issues/161) -- Out of memory when sorting [\#157](https://github.com/apache/arrow-datafusion-python/issues/157) -- window_lead test appears to be non-deterministic [\#135](https://github.com/apache/arrow-datafusion-python/issues/135) -- Reading csv does not work [\#130](https://github.com/apache/arrow-datafusion-python/issues/130) -- Github actions produce a lot of warnings [\#94](https://github.com/apache/arrow-datafusion-python/issues/94) -- ASF source release tarball has wrong directory name [\#90](https://github.com/apache/arrow-datafusion-python/issues/90) -- Python Release Build failing after upgrading to maturin 14.2 [\#87](https://github.com/apache/arrow-datafusion-python/issues/87) -- Maturin build hangs on Linux ARM64 [\#84](https://github.com/apache/arrow-datafusion-python/issues/84) -- Cannot install on Mac M1 from source tarball from testpypi [\#82](https://github.com/apache/arrow-datafusion-python/issues/82) -- ImportPathMismatchError when running pytest locally [\#77](https://github.com/apache/arrow-datafusion-python/issues/77) +- Build is broken [\#161](https://github.com/apache/datafusion-python/issues/161) +- Out of memory when sorting [\#157](https://github.com/apache/datafusion-python/issues/157) +- window_lead test appears to be non-deterministic [\#135](https://github.com/apache/datafusion-python/issues/135) +- Reading csv does not work [\#130](https://github.com/apache/datafusion-python/issues/130) +- Github actions produce a lot of warnings [\#94](https://github.com/apache/datafusion-python/issues/94) +- ASF source release tarball has wrong directory name [\#90](https://github.com/apache/datafusion-python/issues/90) +- Python Release Build failing after upgrading to maturin 14.2 [\#87](https://github.com/apache/datafusion-python/issues/87) +- Maturin build hangs on Linux ARM64 [\#84](https://github.com/apache/datafusion-python/issues/84) +- Cannot install on Mac M1 from source tarball from testpypi [\#82](https://github.com/apache/datafusion-python/issues/82) +- ImportPathMismatchError when running pytest locally [\#77](https://github.com/apache/datafusion-python/issues/77) **Closed issues:** -- Publish documentation for Python bindings [\#39](https://github.com/apache/arrow-datafusion-python/issues/39) -- Add Python binding for `approx_median` [\#32](https://github.com/apache/arrow-datafusion-python/issues/32) -- Release version 0.7.0 [\#7](https://github.com/apache/arrow-datafusion-python/issues/7) +- Publish documentation for Python bindings [\#39](https://github.com/apache/datafusion-python/issues/39) +- Add Python binding for `approx_median` [\#32](https://github.com/apache/datafusion-python/issues/32) +- Release version 0.7.0 [\#7](https://github.com/apache/datafusion-python/issues/7) -## [0.7.0-rc2](https://github.com/apache/arrow-datafusion-python/tree/0.7.0-rc2) (2022-11-26) +## [0.7.0-rc2](https://github.com/apache/datafusion-python/tree/0.7.0-rc2) (2022-11-26) -[Full Changelog](https://github.com/apache/arrow-datafusion-python/compare/0.7.0...0.7.0-rc2) +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.7.0...0.7.0-rc2) ## [Unreleased](https://github.com/datafusion-contrib/datafusion-python/tree/HEAD) From dfbb3ca1e7080c26a23ec774618bbdd4b6b1d34e Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 21 May 2024 12:48:22 -0400 Subject: [PATCH 145/413] tsaucer/run TPC-H examples in CI (#711) * Mostly small updates to tpc-h examples to make their results consistent with spec * Mostly prepares tpch examples for CI by allowing us to check the path for the files regardless of where it is run from. There are a few small updates to make the tests match the expected answer file provided by dbgen. * Expose the substring command * Add the script to run all tpch examples in pytest * Update tpch generator script to allow for non-interactive terminals, such as when running in CI * Add tpch examples to github workflow for testing --- .github/workflows/test.yaml | 20 ++++ benchmarks/tpch/tpch-gen.sh | 13 +- examples/tpch/_tests.py | 111 ++++++++++++++++++ examples/tpch/convert_data_to_parquet.py | 18 +-- examples/tpch/q01_pricing_summary_report.py | 11 +- examples/tpch/q02_minimum_cost_supplier.py | 14 ++- examples/tpch/q03_shipping_priority.py | 11 +- examples/tpch/q04_order_priority_checking.py | 5 +- examples/tpch/q05_local_supplier_volume.py | 13 +- .../tpch/q06_forecasting_revenue_change.py | 3 +- examples/tpch/q07_volume_shipping.py | 11 +- examples/tpch/q08_market_share.py | 15 +-- .../tpch/q09_product_type_profit_measure.py | 15 +-- examples/tpch/q10_returned_item_reporting.py | 13 +- .../q11_important_stock_identification.py | 9 +- examples/tpch/q12_ship_mode_order_priority.py | 5 +- examples/tpch/q13_customer_distribution.py | 7 +- examples/tpch/q14_promotion_effect.py | 7 +- examples/tpch/q15_top_supplier.py | 9 +- .../tpch/q16_part_supplier_relationship.py | 11 +- examples/tpch/q17_small_quantity_order.py | 9 +- examples/tpch/q18_large_volume_customer.py | 7 +- examples/tpch/q19_discounted_revenue.py | 12 +- examples/tpch/q20_potential_part_promotion.py | 15 +-- .../tpch/q21_suppliers_kept_orders_waiting.py | 11 +- examples/tpch/q22_global_sales_opportunity.py | 14 ++- examples/tpch/util.py | 33 ++++++ src/functions.rs | 2 + 28 files changed, 311 insertions(+), 113 deletions(-) create mode 100644 examples/tpch/_tests.py create mode 100644 examples/tpch/util.py diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a486f2778..d8909b6b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -111,3 +111,23 @@ jobs: source venv/bin/activate pip install -e . -vv pytest -v . + + - name: Cache the generated dataset + id: cache-tpch-dataset + uses: actions/cache@v3 + with: + path: benchmarks/tpch/data + key: tpch-data-2.18.0 + + - name: Run dbgen to create 1 Gb dataset + if: ${{ steps.cache-tpch-dataset.outputs.cache-hit != 'true' }} + run: | + cd benchmarks/tpch + RUN_IN_CI=TRUE ./tpch-gen.sh 1 + + - name: Run TPC-H examples + run: | + source venv/bin/activate + cd examples/tpch + python convert_data_to_parquet.py + pytest _tests.py diff --git a/benchmarks/tpch/tpch-gen.sh b/benchmarks/tpch/tpch-gen.sh index 15cab12a5..139c300a2 100755 --- a/benchmarks/tpch/tpch-gen.sh +++ b/benchmarks/tpch/tpch-gen.sh @@ -20,6 +20,15 @@ mkdir -p data/answers 2>/dev/null set -e +# If RUN_IN_CI is set, then do not produce verbose output or use an interactive terminal +if [[ -z "${RUN_IN_CI}" ]]; then + TERMINAL_FLAG="-it" + VERBOSE_OUTPUT="-vf" +else + TERMINAL_FLAG="" + VERBOSE_OUTPUT="-f" +fi + #pushd .. #. ./dev/build-set-env.sh #popd @@ -29,7 +38,7 @@ FILE=./data/supplier.tbl if test -f "$FILE"; then echo "$FILE exists." else - docker run -v `pwd`/data:/data -it --rm ghcr.io/scalytics/tpch-docker:main -vf -s $1 + docker run -v `pwd`/data:/data $TERMINAL_FLAG --rm ghcr.io/scalytics/tpch-docker:main $VERBOSE_OUTPUT -s $1 # workaround for https://github.com/apache/arrow-datafusion/issues/6147 mv data/customer.tbl data/customer.csv @@ -49,5 +58,5 @@ FILE=./data/answers/q1.out if test -f "$FILE"; then echo "$FILE exists." else - docker run -v `pwd`/data:/data -it --entrypoint /bin/bash --rm ghcr.io/scalytics/tpch-docker:main -c "cp /opt/tpch/2.18.0_rc2/dbgen/answers/* /data/answers/" + docker run -v `pwd`/data:/data $TERMINAL_FLAG --entrypoint /bin/bash --rm ghcr.io/scalytics/tpch-docker:main -c "cp /opt/tpch/2.18.0_rc2/dbgen/answers/* /data/answers/" fi diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py new file mode 100644 index 000000000..049b43dff --- /dev/null +++ b/examples/tpch/_tests.py @@ -0,0 +1,111 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest +from importlib import import_module +import pyarrow as pa +from datafusion import col, lit, functions as F +from util import get_answer_file + +def df_selection(col_name, col_type): + if col_type == pa.float64() or isinstance(col_type, pa.Decimal128Type): + return F.round(col(col_name), lit(2)).alias(col_name) + elif col_type == pa.string(): + return F.trim(col(col_name)).alias(col_name) + else: + return col(col_name) + +def load_schema(col_name, col_type): + if col_type == pa.int64() or col_type == pa.int32(): + return col_name, pa.string() + elif isinstance(col_type, pa.Decimal128Type): + return col_name, pa.float64() + else: + return col_name, col_type + +def expected_selection(col_name, col_type): + if col_type == pa.int64() or col_type == pa.int32(): + return F.trim(col(col_name)).cast(col_type).alias(col_name) + elif col_type == pa.string(): + return F.trim(col(col_name)).alias(col_name) + else: + return col(col_name) + +def selections_and_schema(original_schema): + columns = [ (c, original_schema.field(c).type) for c in original_schema.names ] + + df_selections = [ df_selection(c, t) for (c, t) in columns] + expected_schema = [ load_schema(c, t) for (c, t) in columns] + expected_selections = [ expected_selection(c, t) for (c, t) in columns] + + return (df_selections, expected_schema, expected_selections) + +def check_q17(df): + raw_value = float(df.collect()[0]["avg_yearly"][0].as_py()) + value = round(raw_value, 2) + assert abs(value - 348406.05) < 0.001 + +@pytest.mark.parametrize( + ("query_code", "answer_file"), + [ + ("q01_pricing_summary_report", "q1"), + ("q02_minimum_cost_supplier", "q2"), + ("q03_shipping_priority", "q3"), + ("q04_order_priority_checking", "q4"), + ("q05_local_supplier_volume", "q5"), + ("q06_forecasting_revenue_change", "q6"), + ("q07_volume_shipping", "q7"), + ("q08_market_share", "q8"), + ("q09_product_type_profit_measure", "q9"), + ("q10_returned_item_reporting", "q10"), + ("q11_important_stock_identification", "q11"), + ("q12_ship_mode_order_priority", "q12"), + ("q13_customer_distribution", "q13"), + ("q14_promotion_effect", "q14"), + ("q15_top_supplier", "q15"), + ("q16_part_supplier_relationship", "q16"), + ("q17_small_quantity_order", "q17"), + ("q18_large_volume_customer", "q18"), + ("q19_discounted_revenue", "q19"), + ("q20_potential_part_promotion", "q20"), + ("q21_suppliers_kept_orders_waiting", "q21"), + ("q22_global_sales_opportunity", "q22"), + ], +) +def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): + module = import_module(query_code) + df = module.df + + # Treat q17 as a special case. The answer file does not match the spec. Running at + # scale factor 1, we have manually verified this result does match the expected value. + if answer_file == "q17": + return check_q17(df) + + (df_selections, expected_schema, expected_selections) = selections_and_schema(df.schema()) + + df = df.select(*df_selections) + + read_schema = pa.schema(expected_schema) + + df_expected = module.ctx.read_csv(get_answer_file(answer_file), schema=read_schema, delimiter="|", file_extension=".out") + + df_expected = df_expected.select(*expected_selections) + + cols = list(read_schema.names) + + assert df.join(df_expected, (cols, cols), "anti").count() == 0 + assert df.count() == df_expected.count() diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py index 178b7fb39..5da60bc28 100644 --- a/examples/tpch/convert_data_to_parquet.py +++ b/examples/tpch/convert_data_to_parquet.py @@ -36,7 +36,7 @@ ("C_ADDRESS", pyarrow.string()), ("C_NATIONKEY", pyarrow.int32()), ("C_PHONE", pyarrow.string()), - ("C_ACCTBAL", pyarrow.float32()), + ("C_ACCTBAL", pyarrow.decimal128(15, 2)), ("C_MKTSEGMENT", pyarrow.string()), ("C_COMMENT", pyarrow.string()), ] @@ -46,10 +46,10 @@ ("L_PARTKEY", pyarrow.int32()), ("L_SUPPKEY", pyarrow.int32()), ("L_LINENUMBER", pyarrow.int32()), - ("L_QUANTITY", pyarrow.float32()), - ("L_EXTENDEDPRICE", pyarrow.float32()), - ("L_DISCOUNT", pyarrow.float32()), - ("L_TAX", pyarrow.float32()), + ("L_QUANTITY", pyarrow.decimal128(15, 2)), + ("L_EXTENDEDPRICE", pyarrow.decimal128(15, 2)), + ("L_DISCOUNT", pyarrow.decimal128(15, 2)), + ("L_TAX", pyarrow.decimal128(15, 2)), ("L_RETURNFLAG", pyarrow.string()), ("L_LINESTATUS", pyarrow.string()), ("L_SHIPDATE", pyarrow.date32()), @@ -71,7 +71,7 @@ ("O_ORDERKEY", pyarrow.int32()), ("O_CUSTKEY", pyarrow.int32()), ("O_ORDERSTATUS", pyarrow.string()), - ("O_TOTALPRICE", pyarrow.float32()), + ("O_TOTALPRICE", pyarrow.decimal128(15, 2)), ("O_ORDERDATE", pyarrow.date32()), ("O_ORDERPRIORITY", pyarrow.string()), ("O_CLERK", pyarrow.string()), @@ -87,7 +87,7 @@ ("P_TYPE", pyarrow.string()), ("P_SIZE", pyarrow.int32()), ("P_CONTAINER", pyarrow.string()), - ("P_RETAILPRICE", pyarrow.float32()), + ("P_RETAILPRICE", pyarrow.decimal128(15, 2)), ("P_COMMENT", pyarrow.string()), ] @@ -95,7 +95,7 @@ ("PS_PARTKEY", pyarrow.int32()), ("PS_SUPPKEY", pyarrow.int32()), ("PS_AVAILQTY", pyarrow.int32()), - ("PS_SUPPLYCOST", pyarrow.float32()), + ("PS_SUPPLYCOST", pyarrow.decimal128(15, 2)), ("PS_COMMENT", pyarrow.string()), ] @@ -111,7 +111,7 @@ ("S_ADDRESS", pyarrow.string()), ("S_NATIONKEY", pyarrow.int32()), ("S_PHONE", pyarrow.string()), - ("S_ACCTBAL", pyarrow.float32()), + ("S_ACCTBAL", pyarrow.decimal128(15, 2)), ("S_COMMENT", pyarrow.string()), ] diff --git a/examples/tpch/q01_pricing_summary_report.py b/examples/tpch/q01_pricing_summary_report.py index 1aafccab0..7e86055d9 100644 --- a/examples/tpch/q01_pricing_summary_report.py +++ b/examples/tpch/q01_pricing_summary_report.py @@ -31,10 +31,11 @@ import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path ctx = SessionContext() -df = ctx.read_parquet("data/lineitem.parquet") +df = ctx.read_parquet(get_data_path("lineitem.parquet")) # It may be that the date can be hard coded, based on examples shown. # This approach will work with any date range in the provided data set. @@ -45,7 +46,7 @@ # From the given problem, this is how close to the last date in the database we # want to report results for. It should be between 60-120 days before the end. -DAYS_BEFORE_FINAL = 68 +DAYS_BEFORE_FINAL = 90 # Note: this is a hack on setting the values. It should be set differently once # https://github.com/apache/datafusion-python/issues/665 is resolved. @@ -63,13 +64,13 @@ [ F.sum(col("l_quantity")).alias("sum_qty"), F.sum(col("l_extendedprice")).alias("sum_base_price"), - F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias( + F.sum(col("l_extendedprice") * (lit(1) - col("l_discount"))).alias( "sum_disc_price" ), F.sum( col("l_extendedprice") - * (lit(1.0) - col("l_discount")) - * (lit(1.0) + col("l_tax")) + * (lit(1) - col("l_discount")) + * (lit(1) + col("l_tax")) ).alias("sum_charge"), F.avg(col("l_quantity")).alias("avg_qty"), F.avg(col("l_extendedprice")).alias("avg_price"), diff --git a/examples/tpch/q02_minimum_cost_supplier.py b/examples/tpch/q02_minimum_cost_supplier.py index 262e2cf46..f4020d7bb 100644 --- a/examples/tpch/q02_minimum_cost_supplier.py +++ b/examples/tpch/q02_minimum_cost_supplier.py @@ -31,8 +31,10 @@ import datafusion from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path -# This is the part we're looking for +# This is the part we're looking for. Values selected here differ from the spec in order to run +# unit tests on a small data set. SIZE_OF_INTEREST = 15 TYPE_OF_INTEREST = "BRASS" REGION_OF_INTEREST = "EUROPE" @@ -41,10 +43,10 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( "p_partkey", "p_mfgr", "p_type", "p_size" ) -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_acctbal", "s_name", "s_address", @@ -53,13 +55,13 @@ "s_nationkey", "s_suppkey", ) -df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( "ps_partkey", "ps_suppkey", "ps_supplycost" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_regionkey", "n_name" ) -df_region = ctx.read_parquet("data/region.parquet").select_columns( +df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns( "r_regionkey", "r_name" ) diff --git a/examples/tpch/q03_shipping_priority.py b/examples/tpch/q03_shipping_priority.py index 78993e9e4..6a4886d83 100644 --- a/examples/tpch/q03_shipping_priority.py +++ b/examples/tpch/q03_shipping_priority.py @@ -28,6 +28,7 @@ """ from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path SEGMENT_OF_INTEREST = "BUILDING" DATE_OF_INTEREST = "1995-03-15" @@ -36,13 +37,13 @@ ctx = SessionContext() -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_mktsegment", "c_custkey" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderdate", "o_shippriority", "o_custkey", "o_orderkey" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_orderkey", "l_extendedprice", "l_discount", "l_shipdate" ) @@ -73,9 +74,9 @@ df = df.sort(col("revenue").sort(ascending=False), col("o_orderdate").sort()) -# Only return 100 results +# Only return 10 results -df = df.limit(100) +df = df.limit(10) # Change the order that the columns are reported in just to match the spec diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py index b691d5b19..40eab6970 100644 --- a/examples/tpch/q04_order_priority_checking.py +++ b/examples/tpch/q04_order_priority_checking.py @@ -29,6 +29,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path # Ideally we could put 3 months into the interval. See note below. INTERVAL_DAYS = 92 @@ -38,10 +39,10 @@ ctx = SessionContext() -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderdate", "o_orderpriority", "o_orderkey" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_orderkey", "l_commitdate", "l_receiptdate" ) diff --git a/examples/tpch/q05_local_supplier_volume.py b/examples/tpch/q05_local_supplier_volume.py index 7cb6e6324..27b4b84c7 100644 --- a/examples/tpch/q05_local_supplier_volume.py +++ b/examples/tpch/q05_local_supplier_volume.py @@ -32,6 +32,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path DATE_OF_INTEREST = "1994-01-01" @@ -48,22 +49,22 @@ ctx = SessionContext() -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_custkey", "c_nationkey" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_custkey", "o_orderkey", "o_orderdate" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_orderkey", "l_suppkey", "l_extendedprice", "l_discount" ) -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_regionkey", "n_name" ) -df_region = ctx.read_parquet("data/region.parquet").select_columns( +df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns( "r_regionkey", "r_name" ) diff --git a/examples/tpch/q06_forecasting_revenue_change.py b/examples/tpch/q06_forecasting_revenue_change.py index 5fbb91778..3f58c5ec0 100644 --- a/examples/tpch/q06_forecasting_revenue_change.py +++ b/examples/tpch/q06_forecasting_revenue_change.py @@ -32,6 +32,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path # Variables from the example query @@ -52,7 +53,7 @@ ctx = SessionContext() -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_shipdate", "l_quantity", "l_extendedprice", "l_discount" ) diff --git a/examples/tpch/q07_volume_shipping.py b/examples/tpch/q07_volume_shipping.py index 3c87f9375..fd7323b79 100644 --- a/examples/tpch/q07_volume_shipping.py +++ b/examples/tpch/q07_volume_shipping.py @@ -31,6 +31,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path # Variables of interest to query over @@ -48,19 +49,19 @@ ctx = SessionContext() -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_shipdate", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_custkey" ) -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_custkey", "c_nationkey" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name" ) diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py index a415156ec..d13a71dff 100644 --- a/examples/tpch/q08_market_share.py +++ b/examples/tpch/q08_market_share.py @@ -30,6 +30,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path supplier_nation = lit("BRAZIL") customer_region = lit("AMERICA") @@ -46,23 +47,23 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_type") -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_type") +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_partkey", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_custkey", "o_orderdate" ) -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_custkey", "c_nationkey" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name", "n_regionkey" ) -df_region = ctx.read_parquet("data/region.parquet").select_columns( +df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns( "r_regionkey", "r_name" ) diff --git a/examples/tpch/q09_product_type_profit_measure.py b/examples/tpch/q09_product_type_profit_measure.py index 4fdfc1cba..29ffceed4 100644 --- a/examples/tpch/q09_product_type_profit_measure.py +++ b/examples/tpch/q09_product_type_profit_measure.py @@ -31,6 +31,7 @@ import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path part_color = lit("green") @@ -38,14 +39,14 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_name") -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_name") +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) -df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( "ps_suppkey", "ps_partkey", "ps_supplycost" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_partkey", "l_extendedprice", "l_discount", @@ -53,10 +54,10 @@ "l_orderkey", "l_quantity", ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_custkey", "o_orderdate" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name", "n_regionkey" ) @@ -77,7 +78,7 @@ col("n_name").alias("nation"), F.datepart(lit("year"), col("o_orderdate")).cast(pa.int32()).alias("o_year"), ( - col("l_extendedprice") * (lit(1.0) - col("l_discount")) + (col("l_extendedprice") * (lit(1) - col("l_discount"))) - (col("ps_supplycost") * col("l_quantity")) ).alias("amount"), ) diff --git a/examples/tpch/q10_returned_item_reporting.py b/examples/tpch/q10_returned_item_reporting.py index 1879027c1..ed88c2995 100644 --- a/examples/tpch/q10_returned_item_reporting.py +++ b/examples/tpch/q10_returned_item_reporting.py @@ -32,6 +32,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path DATE_START_OF_QUARTER = "1993-10-01" @@ -39,13 +40,13 @@ # Note: this is a hack on setting the values. It should be set differently once # https://github.com/apache/datafusion-python/issues/665 is resolved. -interval_one_quarter = lit(pa.scalar((0, 0, 120), type=pa.month_day_nano_interval())) +interval_one_quarter = lit(pa.scalar((0, 0, 92), type=pa.month_day_nano_interval())) # Load the dataframes we need ctx = SessionContext() -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_custkey", "c_nationkey", "c_name", @@ -54,13 +55,13 @@ "c_phone", "c_comment", ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_extendedprice", "l_discount", "l_orderkey", "l_returnflag" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_custkey", "o_orderdate" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name", "n_regionkey" ) @@ -80,7 +81,7 @@ # Compute the revenue df = df.aggregate( [col("o_custkey")], - [F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias("revenue")], + [F.sum(col("l_extendedprice") * (lit(1) - col("l_discount"))).alias("revenue")], ) # Now join in the customer data diff --git a/examples/tpch/q11_important_stock_identification.py b/examples/tpch/q11_important_stock_identification.py index 78fe26dbf..267248707 100644 --- a/examples/tpch/q11_important_stock_identification.py +++ b/examples/tpch/q11_important_stock_identification.py @@ -28,6 +28,7 @@ """ from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from util import get_data_path NATION = "GERMANY" FRACTION = 0.0001 @@ -36,13 +37,13 @@ ctx = SessionContext() -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) -df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( "ps_supplycost", "ps_availqty", "ps_suppkey", "ps_partkey" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name" ) @@ -71,7 +72,7 @@ ) # Limit to the parts for which there is a significant value based on the fraction of the total -df = df.filter(col("value") / col("total_value") > lit(FRACTION)) +df = df.filter(col("value") / col("total_value") >= lit(FRACTION)) # We only need to report on these two columns df = df.select_columns("ps_partkey", "value") diff --git a/examples/tpch/q12_ship_mode_order_priority.py b/examples/tpch/q12_ship_mode_order_priority.py index e76efa54e..d3dd7d283 100644 --- a/examples/tpch/q12_ship_mode_order_priority.py +++ b/examples/tpch/q12_ship_mode_order_priority.py @@ -32,6 +32,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path SHIP_MODE_1 = "MAIL" SHIP_MODE_2 = "SHIP" @@ -41,10 +42,10 @@ ctx = SessionContext() -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_orderpriority" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_orderkey", "l_shipmode", "l_commitdate", "l_shipdate", "l_receiptdate" ) diff --git a/examples/tpch/q13_customer_distribution.py b/examples/tpch/q13_customer_distribution.py index 1eb9ca303..2b6e7e208 100644 --- a/examples/tpch/q13_customer_distribution.py +++ b/examples/tpch/q13_customer_distribution.py @@ -29,6 +29,7 @@ """ from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path WORD_1 = "special" WORD_2 = "requests" @@ -37,10 +38,10 @@ ctx = SessionContext() -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_custkey", "o_comment" ) -df_customer = ctx.read_parquet("data/customer.parquet").select_columns("c_custkey") +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns("c_custkey") # Use a regex to remove special cases df_orders = df_orders.filter( @@ -51,7 +52,7 @@ df = df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="left") # Find the number of orders for each customer -df = df.aggregate([col("c_custkey")], [F.count(col("c_custkey")).alias("c_count")]) +df = df.aggregate([col("c_custkey")], [F.count(col("o_custkey")).alias("c_count")]) # Ultimately we want to know the number of customers that have that customer count df = df.aggregate([col("c_count")], [F.count(col("c_count")).alias("custdist")]) diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py index 9ec38366b..333398c17 100644 --- a/examples/tpch/q14_promotion_effect.py +++ b/examples/tpch/q14_promotion_effect.py @@ -29,6 +29,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path DATE = "1995-09-01" @@ -41,15 +42,15 @@ ctx = SessionContext() -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_partkey", "l_shipdate", "l_extendedprice", "l_discount" ) -df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_type") +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_type") # Check part type begins with PROMO df_part = df_part.filter( - F.substr(col("p_type"), lit(0), lit(6)) == lit("PROMO") + F.substring(col("p_type"), lit(0), lit(6)) == lit("PROMO") ).with_column("promo_factor", lit(1.0)) df_lineitem = df_lineitem.filter(col("l_shipdate") >= date_of_interest).filter( diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py index 7113e04fe..91af34a9f 100644 --- a/examples/tpch/q15_top_supplier.py +++ b/examples/tpch/q15_top_supplier.py @@ -29,22 +29,23 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from util import get_data_path DATE = "1996-01-01" date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date()) # Note: this is a hack on setting the values. It should be set differently once # https://github.com/apache/datafusion-python/issues/665 is resolved. -interval_3_months = lit(pa.scalar((0, 0, 90), type=pa.month_day_nano_interval())) +interval_3_months = lit(pa.scalar((0, 0, 91), type=pa.month_day_nano_interval())) # Load the dataframes we need ctx = SessionContext() -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_suppkey", "l_shipdate", "l_extendedprice", "l_discount" ) -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_name", "s_address", @@ -59,7 +60,7 @@ df = df_lineitem.aggregate( [col("l_suppkey")], [ - F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias( + F.sum(col("l_extendedprice") * (lit(1) - col("l_discount"))).alias( "total_revenue" ) ], diff --git a/examples/tpch/q16_part_supplier_relationship.py b/examples/tpch/q16_part_supplier_relationship.py index 5f941d5ad..0db2d1b84 100644 --- a/examples/tpch/q16_part_supplier_relationship.py +++ b/examples/tpch/q16_part_supplier_relationship.py @@ -30,6 +30,7 @@ import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path BRAND = "Brand#45" TYPE_TO_IGNORE = "MEDIUM POLISHED" @@ -39,13 +40,13 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( "p_partkey", "p_brand", "p_type", "p_size" ) -df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( "ps_suppkey", "ps_partkey" ) -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_comment" ) @@ -59,9 +60,9 @@ ) # Select the parts we are interested in -df_part = df_part.filter(col("p_brand") == lit(BRAND)) +df_part = df_part.filter(col("p_brand") != lit(BRAND)) df_part = df_part.filter( - F.substr(col("p_type"), lit(0), lit(len(TYPE_TO_IGNORE) + 1)) != lit(TYPE_TO_IGNORE) + F.substring(col("p_type"), lit(0), lit(len(TYPE_TO_IGNORE) + 1)) != lit(TYPE_TO_IGNORE) ) # Python conversion of integer to literal casts it to int64 but the data for diff --git a/examples/tpch/q17_small_quantity_order.py b/examples/tpch/q17_small_quantity_order.py index aae238b2f..5880e7ed4 100644 --- a/examples/tpch/q17_small_quantity_order.py +++ b/examples/tpch/q17_small_quantity_order.py @@ -29,6 +29,7 @@ """ from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from util import get_data_path BRAND = "Brand#23" CONTAINER = "MED BOX" @@ -37,10 +38,10 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( "p_partkey", "p_brand", "p_container" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_partkey", "l_quantity", "l_extendedprice" ) @@ -55,7 +56,7 @@ # Find the average quantity window_frame = WindowFrame("rows", None, None) df = df.with_column( - "avg_quantity", F.window("avg", [col("l_quantity")], window_frame=window_frame) + "avg_quantity", F.window("avg", [col("l_quantity")], window_frame=window_frame, partition_by=[col("l_partkey")]) ) df = df.filter(col("l_quantity") < lit(0.2) * col("avg_quantity")) @@ -64,6 +65,6 @@ df = df.aggregate([], [F.sum(col("l_extendedprice")).alias("total")]) # Divide by number of years in the problem statement to get average -df = df.select((col("total") / lit(7.0)).alias("avg_yearly")) +df = df.select((col("total") / lit(7)).alias("avg_yearly")) df.show() diff --git a/examples/tpch/q18_large_volume_customer.py b/examples/tpch/q18_large_volume_customer.py index 96ca08ff7..10c5f6e6a 100644 --- a/examples/tpch/q18_large_volume_customer.py +++ b/examples/tpch/q18_large_volume_customer.py @@ -27,6 +27,7 @@ """ from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path QUANTITY = 300 @@ -34,13 +35,13 @@ ctx = SessionContext() -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_custkey", "c_name" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_custkey", "o_orderdate", "o_totalprice" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_orderkey", "l_quantity", "l_extendedprice" ) diff --git a/examples/tpch/q19_discounted_revenue.py b/examples/tpch/q19_discounted_revenue.py index 20ad48a77..b15cd98bf 100644 --- a/examples/tpch/q19_discounted_revenue.py +++ b/examples/tpch/q19_discounted_revenue.py @@ -28,6 +28,7 @@ import pyarrow as pa from datafusion import SessionContext, col, lit, udf, functions as F +from util import get_data_path items_of_interest = { "Brand#12": { @@ -51,10 +52,10 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( "p_partkey", "p_brand", "p_container", "p_size" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_partkey", "l_quantity", "l_shipmode", @@ -67,9 +68,8 @@ df = df_lineitem.filter(col("l_shipinstruct") == lit("DELIVER IN PERSON")) -# Small note: The data generated uses "REG AIR" but the spec says "AIR REG" df = df.filter( - (col("l_shipmode") == lit("AIR")) | (col("l_shipmode") == lit("REG AIR")) + (col("l_shipmode") == lit("AIR")) | (col("l_shipmode") == lit("AIR REG")) ) df = df.join(df_part, (["l_partkey"], ["p_partkey"]), "inner") @@ -117,7 +117,7 @@ def is_of_interest( # Turn the above function into a UDF that DataFusion can understand is_of_interest_udf = udf( is_of_interest, - [pa.utf8(), pa.utf8(), pa.float32(), pa.int32()], + [pa.utf8(), pa.utf8(), pa.decimal128(15, 2), pa.int32()], pa.bool_(), "stable", ) @@ -131,7 +131,7 @@ def is_of_interest( df = df.aggregate( [], - [F.sum(col("l_extendedprice") * (lit(1.0) - col("l_discount"))).alias("revenue")], + [F.sum(col("l_extendedprice") * (lit(1) - col("l_discount"))).alias("revenue")], ) df.show() diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index 09686db05..4a602846f 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -30,6 +30,7 @@ from datetime import datetime import pyarrow as pa from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path COLOR_OF_INTEREST = "forest" DATE_OF_INTEREST = "1994-01-01" @@ -39,17 +40,17 @@ ctx = SessionContext() -df_part = ctx.read_parquet("data/part.parquet").select_columns("p_partkey", "p_name") -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_name") +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_shipdate", "l_partkey", "l_suppkey", "l_quantity" ) -df_partsupp = ctx.read_parquet("data/partsupp.parquet").select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( "ps_partkey", "ps_suppkey", "ps_availqty" ) -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_address", "s_name", "s_nationkey" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name" ) @@ -62,7 +63,7 @@ # Filter down dataframes df_nation = df_nation.filter(col("n_name") == lit(NATION_OF_INTEREST)) df_part = df_part.filter( - F.substr(col("p_name"), lit(0), lit(len(COLOR_OF_INTEREST) + 1)) + F.substring(col("p_name"), lit(0), lit(len(COLOR_OF_INTEREST) + 1)) == lit(COLOR_OF_INTEREST) ) @@ -90,7 +91,7 @@ df = df.join(df_nation, (["s_nationkey"], ["n_nationkey"]), "inner") # Restrict to the requested data per the problem statement -df = df.select_columns("s_name", "s_address") +df = df.select_columns("s_name", "s_address").distinct() df = df.sort(col("s_name").sort()) diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py index 2f58d6e79..9f59804e5 100644 --- a/examples/tpch/q21_suppliers_kept_orders_waiting.py +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -27,6 +27,7 @@ """ from datafusion import SessionContext, col, lit, functions as F +from util import get_data_path NATION_OF_INTEREST = "SAUDI ARABIA" @@ -34,16 +35,16 @@ ctx = SessionContext() -df_orders = ctx.read_parquet("data/orders.parquet").select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_orderkey", "o_orderstatus" ) -df_lineitem = ctx.read_parquet("data/lineitem.parquet").select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_orderkey", "l_receiptdate", "l_commitdate", "l_suppkey" ) -df_supplier = ctx.read_parquet("data/supplier.parquet").select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_name", "s_nationkey" ) -df_nation = ctx.read_parquet("data/nation.parquet").select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( "n_nationkey", "n_name" ) @@ -107,7 +108,7 @@ df = df.aggregate([col("s_name")], [F.count(col("o_orderkey")).alias("numwait")]) # Return in descending order -df = df.sort(col("numwait").sort(ascending=False)) +df = df.sort(col("numwait").sort(ascending=False), col("s_name").sort()) df = df.limit(100) diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py index d2d0c5a0d..dfde19cb4 100644 --- a/examples/tpch/q22_global_sales_opportunity.py +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -27,27 +27,28 @@ """ from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from util import get_data_path -NATION_CODE = 13 +NATION_CODES = [13, 31, 23, 29, 30, 18, 17] # Load the dataframes we need ctx = SessionContext() -df_customer = ctx.read_parquet("data/customer.parquet").select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_phone", "c_acctbal", "c_custkey" ) -df_orders = ctx.read_parquet("data/orders.parquet").select_columns("o_custkey") +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns("o_custkey") # The nation code is a two digit number, but we need to convert it to a string literal -nation_code = lit(str(NATION_CODE)) +nation_codes = F.make_array(*[lit(str(n)) for n in NATION_CODES]) # Use the substring operation to extract the first two charaters of the phone number -df = df_customer.with_column("cntrycode", F.substr(col("c_phone"), lit(0), lit(3))) +df = df_customer.with_column("cntrycode", F.substring(col("c_phone"), lit(0), lit(3))) # Limit our search to customers with some balance and in the country code above df = df.filter(col("c_acctbal") > lit(0.0)) -df = df.filter(nation_code == col("cntrycode")) +df = df.filter(~F.array_position(nation_codes, col("cntrycode")).is_null()) # Compute the average balance. By default, the window frame is from unbounded preceeding to the # current row. We want our frame to cover the entire data frame. @@ -56,6 +57,7 @@ "avg_balance", F.window("avg", [col("c_acctbal")], window_frame=window_frame) ) +df.show() # Limit results to customers with above average balance df = df.filter(col("c_acctbal") > col("avg_balance")) diff --git a/examples/tpch/util.py b/examples/tpch/util.py new file mode 100644 index 000000000..191fa6092 --- /dev/null +++ b/examples/tpch/util.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Common utilities for running TPC-H examples. +""" + +import os +from pathlib import Path + +def get_data_path(filename: str) -> str: + path = os.path.dirname(os.path.abspath(__file__)) + + return os.path.join(path, "data", filename) + +def get_answer_file(answer_file: str) -> str: + path = os.path.dirname(os.path.abspath(__file__)) + + return os.path.join(path, "../../benchmarks/tpch/data/answers", f"{answer_file}.out") diff --git a/src/functions.rs b/src/functions.rs index 975025b6b..a4bd9860f 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -477,6 +477,7 @@ expr_fn!(sqrt, num); expr_fn!(starts_with, string prefix, "Returns true if string starts with prefix."); expr_fn!(strpos, string substring, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); expr_fn!(substr, string position); +expr_fn!(substring, string position length); expr_fn!(tan, num); expr_fn!(tanh, num); expr_fn!( @@ -717,6 +718,7 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(strpos))?; m.add_wrapped(wrap_pyfunction!(r#struct))?; // Use raw identifier since struct is a keyword m.add_wrapped(wrap_pyfunction!(substr))?; + m.add_wrapped(wrap_pyfunction!(substring))?; m.add_wrapped(wrap_pyfunction!(sum))?; m.add_wrapped(wrap_pyfunction!(tan))?; m.add_wrapped(wrap_pyfunction!(tanh))?; From e029559f314c71d1050df985e0418307b759af4a Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Thu, 23 May 2024 17:08:53 -0500 Subject: [PATCH 146/413] test: parametrize test_array_functions (#678) test_array_functions now has 56 passing test cases and 1 expected failure (`array_slice` being the expected failure Ref #670). test_array_function_flatten was broken out as a single test because it was an outlier in terms of test-input. test_array_function_obj_tests had a different set of asserts, so was broken out for 5 test cases. Ref #671 --- python/datafusion/tests/test_functions.py | 414 ++++++++++++---------- 1 file changed, 223 insertions(+), 191 deletions(-) diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index d34e46b33..493b6b6ba 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -209,324 +209,356 @@ def test_math_functions(): ) -def test_array_functions(): - data = [[1.0, 2.0, 3.0, 3.0], [4.0, 5.0, 3.0], [6.0]] - ctx = SessionContext() - batch = pa.RecordBatch.from_arrays([np.array(data, dtype=object)], names=["arr"]) - df = ctx.create_dataframe([[batch]]) +def py_indexof(arr, v): + try: + return arr.index(v) + 1 + except ValueError: + return np.nan + - def py_indexof(arr, v): +def py_arr_remove(arr, v, n=None): + new_arr = arr[:] + found = 0 + while found != n: try: - return arr.index(v) + 1 + new_arr.remove(v) + found += 1 except ValueError: - return np.nan - - def py_arr_remove(arr, v, n=None): - new_arr = arr[:] - found = 0 - while found != n: - try: - new_arr.remove(v) - found += 1 - except ValueError: - break - - return new_arr - - def py_arr_replace(arr, from_, to, n=None): - new_arr = arr[:] - found = 0 - while found != n: - try: - idx = new_arr.index(from_) - new_arr[idx] = to - found += 1 - except ValueError: - break - - return new_arr - - def py_arr_resize(arr, size, value): - arr = np.asarray(arr) - return np.pad( - arr, - [(0, size - arr.shape[0])], - "constant", - constant_values=value, - ) + break - def py_flatten(arr): - result = [] - for elem in arr: - if isinstance(elem, list): - result.extend(py_flatten(elem)) - else: - result.append(elem) - return result + return new_arr - col = column("arr") - test_items = [ + +def py_arr_replace(arr, from_, to, n=None): + new_arr = arr[:] + found = 0 + while found != n: + try: + idx = new_arr.index(from_) + new_arr[idx] = to + found += 1 + except ValueError: + break + + return new_arr + + +def py_arr_resize(arr, size, value): + arr = np.asarray(arr) + return np.pad( + arr, + [(0, size - arr.shape[0])], + "constant", + constant_values=value, + ) + + +def py_flatten(arr): + result = [] + for elem in arr: + if isinstance(elem, list): + result.extend(py_flatten(elem)) + else: + result.append(elem) + return result + + +@pytest.mark.parametrize( + ("stmt", "py_expr"), + [ [ - f.array_append(col, literal(99.0)), - lambda: [np.append(arr, 99.0) for arr in data], + lambda col: f.array_append(col, literal(99.0)), + lambda data: [np.append(arr, 99.0) for arr in data], ], [ - f.array_push_back(col, literal(99.0)), - lambda: [np.append(arr, 99.0) for arr in data], + lambda col: f.array_push_back(col, literal(99.0)), + lambda data: [np.append(arr, 99.0) for arr in data], ], [ - f.list_append(col, literal(99.0)), - lambda: [np.append(arr, 99.0) for arr in data], + lambda col: f.list_append(col, literal(99.0)), + lambda data: [np.append(arr, 99.0) for arr in data], ], [ - f.list_push_back(col, literal(99.0)), - lambda: [np.append(arr, 99.0) for arr in data], + lambda col: f.list_push_back(col, literal(99.0)), + lambda data: [np.append(arr, 99.0) for arr in data], ], [ - f.array_concat(col, col), - lambda: [np.concatenate([arr, arr]) for arr in data], + lambda col: f.array_concat(col, col), + lambda data: [np.concatenate([arr, arr]) for arr in data], ], [ - f.array_cat(col, col), - lambda: [np.concatenate([arr, arr]) for arr in data], + lambda col: f.array_cat(col, col), + lambda data: [np.concatenate([arr, arr]) for arr in data], ], [ - f.array_dims(col), - lambda: [[len(r)] for r in data], + lambda col: f.array_dims(col), + lambda data: [[len(r)] for r in data], ], [ - f.array_distinct(col), - lambda: [list(set(r)) for r in data], + lambda col: f.array_distinct(col), + lambda data: [list(set(r)) for r in data], ], [ - f.list_distinct(col), - lambda: [list(set(r)) for r in data], + lambda col: f.list_distinct(col), + lambda data: [list(set(r)) for r in data], ], [ - f.list_dims(col), - lambda: [[len(r)] for r in data], + lambda col: f.list_dims(col), + lambda data: [[len(r)] for r in data], ], [ - f.array_element(col, literal(1)), - lambda: [r[0] for r in data], + lambda col: f.array_element(col, literal(1)), + lambda data: [r[0] for r in data], ], [ - f.array_extract(col, literal(1)), - lambda: [r[0] for r in data], + lambda col: f.array_extract(col, literal(1)), + lambda data: [r[0] for r in data], ], [ - f.list_element(col, literal(1)), - lambda: [r[0] for r in data], + lambda col: f.list_element(col, literal(1)), + lambda data: [r[0] for r in data], ], [ - f.list_extract(col, literal(1)), - lambda: [r[0] for r in data], + lambda col: f.list_extract(col, literal(1)), + lambda data: [r[0] for r in data], ], [ - f.array_length(col), - lambda: [len(r) for r in data], + lambda col: f.array_length(col), + lambda data: [len(r) for r in data], ], [ - f.list_length(col), - lambda: [len(r) for r in data], + lambda col: f.list_length(col), + lambda data: [len(r) for r in data], ], [ - f.array_has(col, literal(1.0)), - lambda: [1.0 in r for r in data], + lambda col: f.array_has(col, literal(1.0)), + lambda data: [1.0 in r for r in data], ], [ - f.array_has_all(col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]])), - lambda: [np.all([v in r for v in [1.0, 3.0, 5.0]]) for r in data], + lambda col: f.array_has_all( + col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) + ), + lambda data: [np.all([v in r for v in [1.0, 3.0, 5.0]]) for r in data], ], [ - f.array_has_any(col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]])), - lambda: [np.any([v in r for v in [1.0, 3.0, 5.0]]) for r in data], + lambda col: f.array_has_any( + col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) + ), + lambda data: [np.any([v in r for v in [1.0, 3.0, 5.0]]) for r in data], ], [ - f.array_position(col, literal(1.0)), - lambda: [py_indexof(r, 1.0) for r in data], + lambda col: f.array_position(col, literal(1.0)), + lambda data: [py_indexof(r, 1.0) for r in data], ], [ - f.array_indexof(col, literal(1.0)), - lambda: [py_indexof(r, 1.0) for r in data], + lambda col: f.array_indexof(col, literal(1.0)), + lambda data: [py_indexof(r, 1.0) for r in data], ], [ - f.list_position(col, literal(1.0)), - lambda: [py_indexof(r, 1.0) for r in data], + lambda col: f.list_position(col, literal(1.0)), + lambda data: [py_indexof(r, 1.0) for r in data], ], [ - f.list_indexof(col, literal(1.0)), - lambda: [py_indexof(r, 1.0) for r in data], + lambda col: f.list_indexof(col, literal(1.0)), + lambda data: [py_indexof(r, 1.0) for r in data], ], [ - f.array_positions(col, literal(1.0)), - lambda: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], + lambda col: f.array_positions(col, literal(1.0)), + lambda data: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], ], [ - f.list_positions(col, literal(1.0)), - lambda: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], + lambda col: f.list_positions(col, literal(1.0)), + lambda data: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], ], [ - f.array_ndims(col), - lambda: [np.array(r).ndim for r in data], + lambda col: f.array_ndims(col), + lambda data: [np.array(r).ndim for r in data], ], [ - f.list_ndims(col), - lambda: [np.array(r).ndim for r in data], + lambda col: f.list_ndims(col), + lambda data: [np.array(r).ndim for r in data], ], [ - f.array_prepend(literal(99.0), col), - lambda: [np.insert(arr, 0, 99.0) for arr in data], + lambda col: f.array_prepend(literal(99.0), col), + lambda data: [np.insert(arr, 0, 99.0) for arr in data], ], [ - f.array_push_front(literal(99.0), col), - lambda: [np.insert(arr, 0, 99.0) for arr in data], + lambda col: f.array_push_front(literal(99.0), col), + lambda data: [np.insert(arr, 0, 99.0) for arr in data], ], [ - f.list_prepend(literal(99.0), col), - lambda: [np.insert(arr, 0, 99.0) for arr in data], + lambda col: f.list_prepend(literal(99.0), col), + lambda data: [np.insert(arr, 0, 99.0) for arr in data], ], [ - f.list_push_front(literal(99.0), col), - lambda: [np.insert(arr, 0, 99.0) for arr in data], + lambda col: f.list_push_front(literal(99.0), col), + lambda data: [np.insert(arr, 0, 99.0) for arr in data], ], [ - f.array_pop_back(col), - lambda: [arr[:-1] for arr in data], + lambda col: f.array_pop_back(col), + lambda data: [arr[:-1] for arr in data], ], [ - f.array_pop_front(col), - lambda: [arr[1:] for arr in data], + lambda col: f.array_pop_front(col), + lambda data: [arr[1:] for arr in data], ], [ - f.array_remove(col, literal(3.0)), - lambda: [py_arr_remove(arr, 3.0, 1) for arr in data], + lambda col: f.array_remove(col, literal(3.0)), + lambda data: [py_arr_remove(arr, 3.0, 1) for arr in data], ], [ - f.list_remove(col, literal(3.0)), - lambda: [py_arr_remove(arr, 3.0, 1) for arr in data], + lambda col: f.list_remove(col, literal(3.0)), + lambda data: [py_arr_remove(arr, 3.0, 1) for arr in data], ], [ - f.array_remove_n(col, literal(3.0), literal(2)), - lambda: [py_arr_remove(arr, 3.0, 2) for arr in data], + lambda col: f.array_remove_n(col, literal(3.0), literal(2)), + lambda data: [py_arr_remove(arr, 3.0, 2) for arr in data], ], [ - f.list_remove_n(col, literal(3.0), literal(2)), - lambda: [py_arr_remove(arr, 3.0, 2) for arr in data], + lambda col: f.list_remove_n(col, literal(3.0), literal(2)), + lambda data: [py_arr_remove(arr, 3.0, 2) for arr in data], ], [ - f.array_remove_all(col, literal(3.0)), - lambda: [py_arr_remove(arr, 3.0) for arr in data], + lambda col: f.array_remove_all(col, literal(3.0)), + lambda data: [py_arr_remove(arr, 3.0) for arr in data], ], [ - f.list_remove_all(col, literal(3.0)), - lambda: [py_arr_remove(arr, 3.0) for arr in data], + lambda col: f.list_remove_all(col, literal(3.0)), + lambda data: [py_arr_remove(arr, 3.0) for arr in data], ], [ - f.array_repeat(col, literal(2)), - lambda: [[arr] * 2 for arr in data], + lambda col: f.array_repeat(col, literal(2)), + lambda data: [[arr] * 2 for arr in data], ], [ - f.array_replace(col, literal(3.0), literal(4.0)), - lambda: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], + lambda col: f.array_replace(col, literal(3.0), literal(4.0)), + lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], ], [ - f.list_replace(col, literal(3.0), literal(4.0)), - lambda: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], + lambda col: f.list_replace(col, literal(3.0), literal(4.0)), + lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], ], [ - f.array_replace_n(col, literal(3.0), literal(4.0), literal(1)), - lambda: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], + lambda col: f.array_replace_n(col, literal(3.0), literal(4.0), literal(1)), + lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], ], [ - f.list_replace_n(col, literal(3.0), literal(4.0), literal(2)), - lambda: [py_arr_replace(arr, 3.0, 4.0, 2) for arr in data], + lambda col: f.list_replace_n(col, literal(3.0), literal(4.0), literal(2)), + lambda data: [py_arr_replace(arr, 3.0, 4.0, 2) for arr in data], ], [ - f.array_replace_all(col, literal(3.0), literal(4.0)), - lambda: [py_arr_replace(arr, 3.0, 4.0) for arr in data], + lambda col: f.array_replace_all(col, literal(3.0), literal(4.0)), + lambda data: [py_arr_replace(arr, 3.0, 4.0) for arr in data], ], [ - f.list_replace_all(col, literal(3.0), literal(4.0)), - lambda: [py_arr_replace(arr, 3.0, 4.0) for arr in data], + lambda col: f.list_replace_all(col, literal(3.0), literal(4.0)), + lambda data: [py_arr_replace(arr, 3.0, 4.0) for arr in data], ], [ - f.array_slice(col, literal(2), literal(4)), - lambda: [arr[1:4] for arr in data], + lambda col: f.array_slice(col, literal(2), literal(4)), + lambda data: [arr[1:4] for arr in data], ], - # [ - # f.list_slice(col, literal(-1), literal(2)), - # lambda: [arr[-1:2] for arr in data], - # ], + pytest.param( + lambda col: f.list_slice(col, literal(-1), literal(2)), + lambda data: [arr[-1:2] for arr in data], + marks=pytest.mark.xfail, + ), [ - f.array_intersect(col, literal([3.0, 4.0])), - lambda: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], + lambda col: f.array_intersect(col, literal([3.0, 4.0])), + lambda data: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], ], [ - f.list_intersect(col, literal([3.0, 4.0])), - lambda: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], + lambda col: f.list_intersect(col, literal([3.0, 4.0])), + lambda data: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], ], [ - f.array_union(col, literal([12.0, 999.0])), - lambda: [np.union1d(arr, [12.0, 999.0]) for arr in data], + lambda col: f.array_union(col, literal([12.0, 999.0])), + lambda data: [np.union1d(arr, [12.0, 999.0]) for arr in data], ], [ - f.list_union(col, literal([12.0, 999.0])), - lambda: [np.union1d(arr, [12.0, 999.0]) for arr in data], + lambda col: f.list_union(col, literal([12.0, 999.0])), + lambda data: [np.union1d(arr, [12.0, 999.0]) for arr in data], ], [ - f.array_except(col, literal([3.0])), - lambda: [np.setdiff1d(arr, [3.0]) for arr in data], + lambda col: f.array_except(col, literal([3.0])), + lambda data: [np.setdiff1d(arr, [3.0]) for arr in data], ], [ - f.list_except(col, literal([3.0])), - lambda: [np.setdiff1d(arr, [3.0]) for arr in data], + lambda col: f.list_except(col, literal([3.0])), + lambda data: [np.setdiff1d(arr, [3.0]) for arr in data], ], [ - f.array_resize(col, literal(10), literal(0.0)), - lambda: [py_arr_resize(arr, 10, 0.0) for arr in data], + lambda col: f.array_resize(col, literal(10), literal(0.0)), + lambda data: [py_arr_resize(arr, 10, 0.0) for arr in data], ], [ - f.list_resize(col, literal(10), literal(0.0)), - lambda: [py_arr_resize(arr, 10, 0.0) for arr in data], + lambda col: f.list_resize(col, literal(10), literal(0.0)), + lambda data: [py_arr_resize(arr, 10, 0.0) for arr in data], ], - [f.flatten(literal(data)), lambda: [py_flatten(data)]], [ - f.range(literal(1), literal(5), literal(2)), - lambda: [np.arange(1, 5, 2)], + lambda col: f.range(literal(1), literal(5), literal(2)), + lambda data: [np.arange(1, 5, 2)], ], - ] + ], +) +def test_array_functions(stmt, py_expr): + data = [[1.0, 2.0, 3.0, 3.0], [4.0, 5.0, 3.0], [6.0]] + ctx = SessionContext() + batch = pa.RecordBatch.from_arrays([np.array(data, dtype=object)], names=["arr"]) + df = ctx.create_dataframe([[batch]]) - for stmt, py_expr in test_items: - query_result = df.select(stmt).collect()[0].column(0) - for a, b in zip(query_result, py_expr()): - np.testing.assert_array_almost_equal( - np.array(a.as_py(), dtype=float), np.array(b, dtype=float) - ) + col = column("arr") + query_result = df.select(stmt(col)).collect()[0].column(0) + for a, b in zip(query_result, py_expr(data)): + np.testing.assert_array_almost_equal( + np.array(a.as_py(), dtype=float), np.array(b, dtype=float) + ) - obj_test_items = [ + +def test_array_function_flatten(): + data = [[1.0, 2.0, 3.0, 3.0], [4.0, 5.0, 3.0], [6.0]] + ctx = SessionContext() + batch = pa.RecordBatch.from_arrays([np.array(data, dtype=object)], names=["arr"]) + df = ctx.create_dataframe([[batch]]) + + stmt = f.flatten(literal(data)) + py_expr = [py_flatten(data)] + query_result = df.select(stmt).collect()[0].column(0) + for a, b in zip(query_result, py_expr): + np.testing.assert_array_almost_equal( + np.array(a.as_py(), dtype=float), np.array(b, dtype=float) + ) + + +@pytest.mark.parametrize( + ("stmt", "py_expr"), + [ [ - f.array_to_string(col, literal(",")), - lambda: [",".join([str(int(v)) for v in r]) for r in data], + f.array_to_string(column("arr"), literal(",")), + lambda data: [",".join([str(int(v)) for v in r]) for r in data], ], [ - f.array_join(col, literal(",")), - lambda: [",".join([str(int(v)) for v in r]) for r in data], + f.array_join(column("arr"), literal(",")), + lambda data: [",".join([str(int(v)) for v in r]) for r in data], ], [ - f.list_to_string(col, literal(",")), - lambda: [",".join([str(int(v)) for v in r]) for r in data], + f.list_to_string(column("arr"), literal(",")), + lambda data: [",".join([str(int(v)) for v in r]) for r in data], ], [ - f.list_join(col, literal(",")), - lambda: [",".join([str(int(v)) for v in r]) for r in data], + f.list_join(column("arr"), literal(",")), + lambda data: [",".join([str(int(v)) for v in r]) for r in data], ], - ] - - for stmt, py_expr in obj_test_items: - query_result = np.array(df.select(stmt).collect()[0].column(0)) - for a, b in zip(query_result, py_expr()): - assert a == b + ], +) +def test_array_function_obj_tests(stmt, py_expr): + data = [[1.0, 2.0, 3.0, 3.0], [4.0, 5.0, 3.0], [6.0]] + ctx = SessionContext() + batch = pa.RecordBatch.from_arrays([np.array(data, dtype=object)], names=["arr"]) + df = ctx.create_dataframe([[batch]]) + query_result = np.array(df.select(stmt).collect()[0].column(0)) + for a, b in zip(query_result, py_expr(data)): + assert a == b def test_string_functions(df): From 0d42e77b469708f10ec69ff51366a529d3454efb Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 23 May 2024 18:09:39 -0400 Subject: [PATCH 147/413] Use int64 for TPC-H keys and set input schema to not nullable (#714) --- examples/tpch/convert_data_to_parquet.py | 30 +++++++++++++----------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py index 5da60bc28..d81ec290d 100644 --- a/examples/tpch/convert_data_to_parquet.py +++ b/examples/tpch/convert_data_to_parquet.py @@ -31,10 +31,10 @@ all_schemas = {} all_schemas["customer"] = [ - ("C_CUSTKEY", pyarrow.int32()), + ("C_CUSTKEY", pyarrow.int64()), ("C_NAME", pyarrow.string()), ("C_ADDRESS", pyarrow.string()), - ("C_NATIONKEY", pyarrow.int32()), + ("C_NATIONKEY", pyarrow.int64()), ("C_PHONE", pyarrow.string()), ("C_ACCTBAL", pyarrow.decimal128(15, 2)), ("C_MKTSEGMENT", pyarrow.string()), @@ -42,9 +42,9 @@ ] all_schemas["lineitem"] = [ - ("L_ORDERKEY", pyarrow.int32()), - ("L_PARTKEY", pyarrow.int32()), - ("L_SUPPKEY", pyarrow.int32()), + ("L_ORDERKEY", pyarrow.int64()), + ("L_PARTKEY", pyarrow.int64()), + ("L_SUPPKEY", pyarrow.int64()), ("L_LINENUMBER", pyarrow.int32()), ("L_QUANTITY", pyarrow.decimal128(15, 2)), ("L_EXTENDEDPRICE", pyarrow.decimal128(15, 2)), @@ -61,15 +61,15 @@ ] all_schemas["nation"] = [ - ("N_NATIONKEY", pyarrow.int32()), + ("N_NATIONKEY", pyarrow.int64()), ("N_NAME", pyarrow.string()), - ("N_REGIONKEY", pyarrow.int32()), + ("N_REGIONKEY", pyarrow.int64()), ("N_COMMENT", pyarrow.string()), ] all_schemas["orders"] = [ - ("O_ORDERKEY", pyarrow.int32()), - ("O_CUSTKEY", pyarrow.int32()), + ("O_ORDERKEY", pyarrow.int64()), + ("O_CUSTKEY", pyarrow.int64()), ("O_ORDERSTATUS", pyarrow.string()), ("O_TOTALPRICE", pyarrow.decimal128(15, 2)), ("O_ORDERDATE", pyarrow.date32()), @@ -80,7 +80,7 @@ ] all_schemas["part"] = [ - ("P_PARTKEY", pyarrow.int32()), + ("P_PARTKEY", pyarrow.int64()), ("P_NAME", pyarrow.string()), ("P_MFGR", pyarrow.string()), ("P_BRAND", pyarrow.string()), @@ -92,21 +92,21 @@ ] all_schemas["partsupp"] = [ - ("PS_PARTKEY", pyarrow.int32()), - ("PS_SUPPKEY", pyarrow.int32()), + ("PS_PARTKEY", pyarrow.int64()), + ("PS_SUPPKEY", pyarrow.int64()), ("PS_AVAILQTY", pyarrow.int32()), ("PS_SUPPLYCOST", pyarrow.decimal128(15, 2)), ("PS_COMMENT", pyarrow.string()), ] all_schemas["region"] = [ - ("r_REGIONKEY", pyarrow.int32()), + ("r_REGIONKEY", pyarrow.int64()), ("r_NAME", pyarrow.string()), ("r_COMMENT", pyarrow.string()), ] all_schemas["supplier"] = [ - ("S_SUPPKEY", pyarrow.int32()), + ("S_SUPPKEY", pyarrow.int64()), ("S_NAME", pyarrow.string()), ("S_ADDRESS", pyarrow.string()), ("S_NATIONKEY", pyarrow.int32()), @@ -125,6 +125,8 @@ # in to handle the trailing | in the file output_cols = [r[0] for r in curr_schema] + curr_schema = [ pyarrow.field(r[0], r[1], nullable=False) for r in curr_schema] + # Trailing | requires extra field for in processing curr_schema.append(("some_null", pyarrow.null())) From ea4a4dc7990c72cb65b999137da369a4e814cdbb Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 25 May 2024 06:45:09 -0600 Subject: [PATCH 148/413] change version to 38.0.1 (#716) --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b7ed560f..c9e0e6d81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1016,7 +1016,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "38.0.0" +version = "38.0.1" dependencies = [ "async-trait", "datafusion", diff --git a/Cargo.toml b/Cargo.toml index cde3be222..fd6f4bb31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "38.0.0" +version = "38.0.1" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] From ef889211a0365f208a39221a93e6fee54f3ad4ed Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 31 May 2024 09:58:36 -0500 Subject: [PATCH 149/413] ci: add substrait feature to linux builds (#720) Fixes #646 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 239b1718b..a4c31c09b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -189,7 +189,7 @@ jobs: target: x86_64 manylinux: auto rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 - args: --release --manylinux 2014 --features protoc + args: --release --manylinux 2014 --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v3 with: @@ -219,7 +219,7 @@ jobs: # Use manylinux_2_28-cross because the manylinux2014-cross has GCC 4.8.5, which causes the build to fail manylinux: 2_28 rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 - args: --release --features protoc + args: --release --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v3 with: @@ -245,7 +245,7 @@ jobs: rust-toolchain: stable manylinux: auto rustup-components: rust-std rustfmt - args: --release --sdist --out dist --features protoc + args: --release --sdist --out dist --features protoc,substrait - name: Archive wheels uses: actions/upload-artifact@v3 with: From 84c1195d9c99e08ba57366eb07e89d1ec2654bd1 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 31 May 2024 10:39:01 -0500 Subject: [PATCH 150/413] Docs deploy action (#721) * ci: add debug print for githuc docs actions * ci: update docs job condition --- .github/workflows/docs.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 855164852..db23462a6 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,12 +11,21 @@ on: name: Deploy DataFusion Python site jobs: + debug-github-context: + name: Print github context + runs-on: ubuntu-latest + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: | + echo "$GITHUB_CONTEXT" build-docs: name: Build docs runs-on: ubuntu-latest steps: - name: Set target branch - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag') id: target-branch run: | set -x @@ -31,7 +40,7 @@ jobs: - name: Checkout docs sources uses: actions/checkout@v3 - name: Checkout docs target branch - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag') uses: actions/checkout@v3 with: fetch-depth: 0 @@ -69,7 +78,7 @@ jobs: make html - name: Copy & push the generated HTML - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag') run: | set -x cd docs-target From 1d4743fefd9a6992039c3162c49f2006fc8f1560 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Thu, 6 Jun 2024 15:09:51 -0500 Subject: [PATCH 151/413] update deps (#723) * chore: update rust deps * ci: bump actions/checkout from 3 to 4 * ci: bump actions/setup-python from 4 to 5 * ci: bump actions/cache from 3 to 4 * ci: bump conda-incubator/setup-miniconda from 2.2.0 to 3.0.4 --- .github/workflows/build.yml | 20 +- .github/workflows/conda.yml | 4 +- .github/workflows/dev.yml | 4 +- .github/workflows/docs.yaml | 6 +- .github/workflows/test.yaml | 8 +- Cargo.lock | 781 +++++++++++++++++------------------- 6 files changed, 386 insertions(+), 437 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4c31c09b..7df129ec8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,9 +27,9 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install dependencies @@ -43,7 +43,7 @@ jobs: generate-license: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions-rs/toolchain@v1 with: profile: minimal @@ -66,9 +66,9 @@ jobs: python-version: ["3.10"] os: [macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -124,9 +124,9 @@ jobs: matrix: python-version: ["3.10"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -172,7 +172,7 @@ jobs: name: Manylinux runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: rm LICENSE.txt - name: Download LICENSE.txt uses: actions/download-artifact@v3 @@ -201,7 +201,7 @@ jobs: name: Manylinux arm64 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: rm LICENSE.txt - name: Download LICENSE.txt uses: actions/download-artifact@v3 @@ -231,7 +231,7 @@ jobs: name: Source distribution runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: rm LICENSE.txt - name: Download LICENSE.txt uses: actions/download-artifact@v3 diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index c33ae551f..f25a431c4 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -64,11 +64,11 @@ jobs: sudo chmod 600 /swapfile || true sudo mkswap /swapfile || true sudo swapon /swapfile || true - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Set up Python - uses: conda-incubator/setup-miniconda@v2.2.0 + uses: conda-incubator/setup-miniconda@v3.0.4 with: miniforge-variant: Mambaforge use-mamba: true diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index 05cf8ce68..44481818e 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -25,9 +25,9 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Audit licenses diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index db23462a6..60dc15927 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -38,16 +38,16 @@ jobs: exit 1 fi - name: Checkout docs sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Checkout docs target branch if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag') - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 ref: ${{ steps.target-branch.outputs.value }} path: docs-target - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d8909b6b0..897c384c6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -42,7 +42,7 @@ jobs: - python-version: "3.7" toolchain: "stable" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Setup Rust Toolchain uses: actions-rs/toolchain@v1 @@ -58,12 +58,12 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Cache Cargo - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cargo key: cargo-cache-${{ steps.rust-toolchain.outputs.rustc_hash }}-${{ hashFiles('Cargo.lock') }} @@ -114,7 +114,7 @@ jobs: - name: Cache the generated dataset id: cache-tpch-dataset - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: benchmarks/tpch/data key: tpch-data-2.18.0 diff --git a/Cargo.lock b/Cargo.lock index c9e0e6d81..ed1ee3b80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -25,9 +25,9 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", @@ -39,9 +39,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "android-tzdata" @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "apache-avro" @@ -178,7 +178,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.3", + "hashbrown", "num", ] @@ -307,7 +307,7 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown 0.14.3", + "hashbrown", ] [[package]] @@ -316,7 +316,7 @@ version = "51.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", ] [[package]] @@ -352,9 +352,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.6" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a116f46a969224200a0a97f29cfd4c50e7534e4b4826bd23ea2c3c533039c82c" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "bzip2", "flate2", @@ -364,19 +364,19 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.13.0", - "zstd-safe 7.0.0", + "zstd 0.13.1", + "zstd-safe 7.1.0", ] [[package]] name = "async-recursion" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -387,7 +387,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -401,24 +401,24 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "autotools" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef8da1805e028a172334c3b680f93e71126f2327622faef2ec3d893c0a4ad77" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" dependencies = [ "cc", ] [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -449,9 +449,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "blake2" @@ -464,9 +464,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" +checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" dependencies = [ "arrayref", "arrayvec", @@ -486,9 +486,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.4.0" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" +checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -507,9 +507,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" @@ -519,9 +519,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "bzip2" @@ -546,12 +546,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -562,22 +563,22 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] name = "chrono-tz" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d7b79e99bfaa0d47da0687c43aa3b7381938a62ad3a6498599039321f660b7" +checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", "chrono-tz-build", @@ -597,20 +598,20 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.1.0" +version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" +checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ - "strum 0.25.0", - "strum_macros 0.25.3", + "strum 0.26.2", + "strum_macros 0.26.4", "unicode-width", ] [[package]] name = "const-random" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaf16c9c2c612020bcfd042e170f6e32de9b9d75adb5277cdbbd2e2c8c8299a" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" dependencies = [ "const-random-macro", ] @@ -668,9 +669,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] @@ -725,7 +726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ "cfg-if", - "hashbrown 0.14.3", + "hashbrown", "lock_api", "once_cell", "parking_lot_core", @@ -764,9 +765,9 @@ dependencies = [ "futures", "glob", "half", - "hashbrown 0.14.3", + "hashbrown", "indexmap", - "itertools 0.12.0", + "itertools", "log", "num-traits", "num_cpus", @@ -782,7 +783,7 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.0", + "zstd 0.13.1", ] [[package]] @@ -829,7 +830,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.3", + "hashbrown", "log", "object_store", "parking_lot", @@ -852,8 +853,8 @@ dependencies = [ "paste", "serde_json", "sqlparser", - "strum 0.26.1", - "strum_macros 0.26.1", + "strum 0.26.2", + "strum_macros 0.26.4", ] [[package]] @@ -871,9 +872,9 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.3", + "hashbrown", "hex", - "itertools 0.12.0", + "itertools", "log", "md-5", "rand", @@ -914,7 +915,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", - "itertools 0.12.0", + "itertools", "log", "paste", ] @@ -931,9 +932,9 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.3", + "hashbrown", "indexmap", - "itertools 0.12.0", + "itertools", "log", "regex-syntax", ] @@ -959,10 +960,10 @@ dependencies = [ "datafusion-functions-aggregate", "datafusion-physical-expr-common", "half", - "hashbrown 0.14.3", + "hashbrown", "hex", "indexmap", - "itertools 0.12.0", + "itertools", "log", "paste", "petgraph", @@ -1003,9 +1004,9 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.14.3", + "hashbrown", "indexmap", - "itertools 0.12.0", + "itertools", "log", "once_cell", "parking_lot", @@ -1036,7 +1037,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.63", + "syn 2.0.66", "tokio", "url", "uuid", @@ -1055,7 +1056,7 @@ dependencies = [ "datafusion-expr", "log", "sqlparser", - "strum 0.26.1", + "strum 0.26.2", ] [[package]] @@ -1067,7 +1068,7 @@ dependencies = [ "async-recursion", "chrono", "datafusion", - "itertools 0.12.0", + "itertools", "object_store", "prost", "prost-types", @@ -1093,21 +1094,21 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "dyn-clone" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" -version = "1.9.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -1120,9 +1121,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", @@ -1130,9 +1131,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "fixedbitset" @@ -1152,9 +1153,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "miniz_oxide", @@ -1231,7 +1232,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -1276,9 +1277,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -1287,9 +1288,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -1318,9 +1319,9 @@ dependencies = [ [[package]] name = "half" -version = "2.3.1" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "cfg-if", "crunchy", @@ -1329,18 +1330,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", @@ -1360,9 +1352,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -1370,20 +1362,11 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "home" -version = "0.5.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" -dependencies = [ - "windows-sys 0.52.0", -] - [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -1421,9 +1404,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -1459,9 +1442,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1497,20 +1480,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", - "hashbrown 0.14.3", + "hashbrown", ] [[package]] name = "indoc" -version = "2.0.4" +version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" [[package]] name = "instant" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if", "js-sys", @@ -1532,42 +1515,33 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25db6b064527c5d482d0423354fcd07a89a2dfe07b67892e62411946db7f07b0" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -1644,15 +1618,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.152" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libflate" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7d5654ae1795afc7ff76f4365c2c8791b0feb18e8996a96adad8ffd7c3b2bf" +checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" dependencies = [ "adler32", "core2", @@ -1663,12 +1637,12 @@ dependencies = [ [[package]] name = "libflate_lz77" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5f52fb8c451576ec6b79d3f4deb327398bc05bbdbd99021a6e77a4c855d524" +checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" dependencies = [ "core2", - "hashbrown 0.13.2", + "hashbrown", "rle-decode-fast", ] @@ -1680,9 +1654,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.35" +version = "0.1.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" +checksum = "0e7bb23d733dfcc8af652a78b7bf232f0e967710d044732185e561e47c0336b6" dependencies = [ "cc", "libc", @@ -1690,15 +1664,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1706,15 +1680,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lz4_flex" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "912b45c753ff5f7f5208307e8ace7d2a2e30d024e26d3509f3dce546c044ce15" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ "twox-hash", ] @@ -1742,24 +1716,24 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] [[package]] name = "mimalloc" -version = "0.1.39" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa01922b5ea280a911e323e4d2fd24b7fe5cc4042e0d2cda3c40775cdc4bdc9c" +checksum = "e9186d86b79b52f4a77af65604b51225e8db1d6ee7e3f41aec1e40829c71a176" dependencies = [ "libmimalloc-sys", ] @@ -1772,9 +1746,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] @@ -1792,15 +1766,15 @@ dependencies = [ [[package]] name = "multimap" -version = "0.8.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" [[package]] name = "num" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" dependencies = [ "num-bigint", "num-complex", @@ -1812,39 +1786,37 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608e7659b5c3d7cba262d894801b9ec9d00de989e8a82bd4bef91d08da45cdc0" +checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" dependencies = [ - "autocfg", "num-integer", "num-traits", ] [[package]] name = "num-complex" -version = "0.4.4" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ba157ca0885411de85d6ca030ba7e2a83a28636056c7c699b07c8b6f7383214" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "num-traits", ] [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -1853,11 +1825,10 @@ dependencies = [ [[package]] name = "num-rational" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" dependencies = [ - "autocfg", "num-bigint", "num-integer", "num-traits", @@ -1865,9 +1836,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -1885,9 +1856,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -1905,7 +1876,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.12.0", + "itertools", "md-5", "parking_lot", "percent-encoding", @@ -1913,7 +1884,7 @@ dependencies = [ "rand", "reqwest", "ring", - "rustls-pemfile 2.0.0", + "rustls-pemfile 2.1.2", "serde", "serde_json", "snafu", @@ -1946,9 +1917,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1956,15 +1927,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.5", ] [[package]] @@ -1988,7 +1959,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.14.3", + "hashbrown", "lz4_flex", "num", "num-bigint", @@ -1999,23 +1970,23 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.0", + "zstd 0.13.1", ] [[package]] name = "parse-zoneinfo" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" dependencies = [ "regex", ] [[package]] name = "paste" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" @@ -2025,9 +1996,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", "indexmap", @@ -2073,9 +2044,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -2085,9 +2056,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "portable-atomic" @@ -2103,28 +2074,28 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "prettyplease" -version = "0.2.16" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.12.4" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f5d036824e4761737860779c906171497f6d55681139d8312388f8fe398922" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", "prost-derive", @@ -2132,13 +2103,13 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.3" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55e02e35260070b6f716a2423c2ff1c3bb1642ddca6f99e1f26d06268a0e2d2" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", - "heck 0.4.1", - "itertools 0.11.0", + "heck 0.5.0", + "itertools", "log", "multimap", "once_cell", @@ -2147,27 +2118,26 @@ dependencies = [ "prost-types", "regex", "tempfile", - "which", ] [[package]] name = "prost-derive" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9554e3ab233f0a932403704f1a1d08c30d5ccd931adfdfa1e8b5a19b52c1d55a" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools 0.12.0", + "itertools", "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] name = "prost-types" -version = "0.12.3" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ "prost", ] @@ -2228,7 +2198,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2241,7 +2211,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2262,9 +2232,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -2301,18 +2271,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", ] [[package]] name = "regex" -version = "1.10.2" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", @@ -2322,9 +2292,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", @@ -2339,9 +2309,9 @@ checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "regress" @@ -2349,15 +2319,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f5f39ba4513916c1b2657b72af6ec671f091cd637992f58d0ede5cae4e5dea0" dependencies = [ - "hashbrown 0.14.3", + "hashbrown", "memchr", ] [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", "bytes", @@ -2382,6 +2352,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-rustls", @@ -2397,16 +2368,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin", "untrusted", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2417,9 +2389,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc_version" @@ -2432,11 +2404,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys", @@ -2478,19 +2450,19 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.0.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.1.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" @@ -2504,15 +2476,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -2534,9 +2506,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.16" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" +checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" dependencies = [ "dyn-clone", "schemars_derive", @@ -2546,14 +2518,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.16" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" +checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 1.0.109", + "syn 2.0.66", ] [[package]] @@ -2574,11 +2546,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -2587,9 +2559,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -2597,9 +2569,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "seq-macro" @@ -2609,40 +2581,40 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.199" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9f6e76df036c77cd94996771fb40db98187f096dd0b9af39c6c6e452ba966a" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.199" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11bd257a6541e141e42ca6d24ae26f7714887b47e89aa739099104c7e4d3b7fc" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.66", ] [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -2658,7 +2630,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2714,9 +2686,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "snafu" @@ -2748,12 +2720,12 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2780,7 +2752,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2797,11 +2769,11 @@ checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" [[package]] name = "strum" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723b93e8addf9aa965ebe2d11da6d7540fa2283fcea14b3371ff055f7ba13f5f" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" dependencies = [ - "strum_macros 0.26.1", + "strum_macros 0.26.4", ] [[package]] @@ -2814,20 +2786,20 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] name = "strum_macros" -version = "0.26.1" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2847,7 +2819,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.63", + "syn 2.0.66", "typify", "walkdir", ] @@ -2871,15 +2843,21 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.63" +version = "2.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf5be731623ca1a1fb7d8be6f261a3be6d3e2337b8a1f97be944d020c8fcb704" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "system-configuration" version = "0.5.1" @@ -2903,41 +2881,40 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.13" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" +checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "redox_syscall", "rustix", "windows-sys 0.52.0", ] [[package]] name = "thiserror" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.59" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -2977,9 +2954,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -2994,13 +2971,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3015,16 +2992,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -3052,7 +3028,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3097,7 +3073,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3129,7 +3105,7 @@ dependencies = [ "regress", "schemars", "serde_json", - "syn 2.0.63", + "syn 2.0.66", "thiserror", "unicode-ident", ] @@ -3146,7 +3122,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.63", + "syn 2.0.66", "typify-impl", ] @@ -3164,24 +3140,24 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" [[package]] name = "unicode-width" -version = "0.1.11" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51733f11c9c4f72aa0c160008246859e340b00807569a0da0e7a1079b27ba85" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" [[package]] name = "unindent" @@ -3255,9 +3231,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -3265,24 +3241,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -3292,9 +3268,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3302,28 +3278,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -3334,64 +3310,30 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", ] -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -3409,7 +3351,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -3429,17 +3371,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -3450,9 +3393,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -3462,9 +3405,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -3474,9 +3417,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -3486,9 +3435,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -3498,9 +3447,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -3510,9 +3459,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -3522,9 +3471,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" @@ -3547,22 +3496,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.63", + "syn 2.0.66", ] [[package]] @@ -3576,11 +3525,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" dependencies = [ - "zstd-safe 7.0.0", + "zstd-safe 7.1.0", ] [[package]] @@ -3595,18 +3544,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", "pkg-config", From 860283a466ddefb47c3fce9299f38cd323bc6933 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sat, 8 Jun 2024 16:22:38 -0500 Subject: [PATCH 152/413] Upgrade maturin (#725) * chore: update maturin dep to 1.5.1 * chore: pip-compile for requirements-310.txt and requirements-311.txt * chore: update maturin version in conda recipe --- .github/workflows/build.yml | 4 +- conda/environments/datafusion-dev.yaml | 2 +- conda/recipes/meta.yaml | 4 +- pyproject.toml | 2 +- requirements-310.txt | 322 ++++++++++++------------- requirements-311.txt | 218 ++++++++--------- requirements.in | 2 +- 7 files changed, 276 insertions(+), 278 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7df129ec8..350be46d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -80,7 +80,7 @@ jobs: run: python -m pip install --upgrade pip - name: Install maturin - run: pip install maturin==0.15.1 + run: pip install maturin==1.5.1 - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -141,7 +141,7 @@ jobs: run: python -m pip install --upgrade pip - name: Install maturin - run: pip install maturin==0.15.1 + run: pip install maturin==1.5.1 - run: rm LICENSE.txt - name: Download LICENSE.txt diff --git a/conda/environments/datafusion-dev.yaml b/conda/environments/datafusion-dev.yaml index ceab504c8..1f6f23942 100644 --- a/conda/environments/datafusion-dev.yaml +++ b/conda/environments/datafusion-dev.yaml @@ -21,7 +21,7 @@ dependencies: - black - flake8 - isort - - maturin>=0.15 + - maturin>=1.5.1 - mypy - numpy - pyarrow>=11.0.0 diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index 113e7a441..72ac7f501 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -40,11 +40,11 @@ requirements: - {{ compiler('c') }} - {{ compiler('rust') }} - cargo-bundle-licenses - - maturin >=0.15,<0.16 + - maturin >=1.5.1,<1.6.0 - libprotobuf =3 host: - python - - maturin >=0.15,<0.16 + - maturin >=1.5.1,<1.6.0 - pip - zlib - xz # [linux64] diff --git a/pyproject.toml b/pyproject.toml index 444aa8ee7..b706065a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [build-system] -requires = ["maturin>=0.15,<0.16"] +requires = ["maturin>=1.5.1,<1.6.0"] build-backend = "maturin" [project] diff --git a/requirements-310.txt b/requirements-310.txt index 915f74d54..9c3b6e141 100644 --- a/requirements-310.txt +++ b/requirements-310.txt @@ -4,185 +4,183 @@ # # pip-compile --generate-hashes --output-file=requirements-310.txt # -attrs==21.2.0 \ - --hash=sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1 \ - --hash=sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb +exceptiongroup==1.2.1 \ + --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \ + --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16 # via pytest -iniconfig==1.1.1 \ - --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ - --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 +iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via pytest -maturin==0.15.1 \ - --hash=sha256:0e89a87549d671056f9358832c8aa1bb522abcf13eeca83327a58b091d4f5a98 \ - --hash=sha256:229d7eb6e14455a3c69a10a4546f082c7bd5490b8ec7f50d5d10edcea600dc64 \ - --hash=sha256:247bec13d82021972e5cb4eb38e7a7aea0e7a034beab60f0e0464ffe7423f24b \ - --hash=sha256:37ddbc261db778c73829173f43624b6178dd4244fae8e4b278323e050e899a14 \ - --hash=sha256:3b4c296b12736756d1d084c04b5a60281011e77b763eb9c69c39444d2421ad31 \ - --hash=sha256:616220c8c875526b22605bbb0be189ff859956abad8fe6e49c9ad2caa761ebca \ - --hash=sha256:61eda19c37394660c3b0b9c6dbfa7d411a5dca1a93c9dcf7baa718f3a03fda04 \ - --hash=sha256:7a7e02f265c30bac9f16e65c9d14184b3228c26b03dbe83a15220ab09fceabd7 \ - --hash=sha256:a40c96dc135e1571159155bdbd8c06509dea14a9317ca0e5b7cae4c8d2016b93 \ - --hash=sha256:ab5f8d8858440f35d833bce8bbf8233e595afa23fe00c513f7b671183946ea02 \ - --hash=sha256:b54421ae71e1e3dde92105d300187b9ac3b257a5b33c1d670ff9ee5266a1bd81 \ - --hash=sha256:dabb8ff46461c6fb1d68e8972a172cf1dede3c9825a41e4a6caecc95c26ca3b4 \ - --hash=sha256:f604b65fd9f0b94856e88cf8b345e21a27276297e6df4ad9305937887feda13b +maturin==1.6.0 \ + --hash=sha256:16ef860df20028618b5a064da06b02c1c47acba064a4d25aaf84662a459ec599 \ + --hash=sha256:337899784955934dd67b30497d1dd5fab22da89f60bb079dbaf2eaa446b97a10 \ + --hash=sha256:4e931c92037128ade49cd26dd040d9c46ad8092d8170cc44f5c3a0b4a052d576 \ + --hash=sha256:50133965e52d8b5b969381fee3fde111ae2383905cdaba7650f256e08ccddcd4 \ + --hash=sha256:a2a2436628c36d98dabd79b52256df7e12fc4fd1b122984d9373fdf918fd4609 \ + --hash=sha256:aa4eb7dca7d246b466392f21016f67ff09a9aff2305fa714ca25a2344e4639e7 \ + --hash=sha256:b955025c24c8babc808db49e0ff90db8b4b1320dcc16b14eb26132841737230d \ + --hash=sha256:bd85edcb1b8e2bcddc1b7d16ce58ce00a66aa80c422745c8ad9e132ac40d4b48 \ + --hash=sha256:c87d1a7596c42b589099adb831343a56e02373588366e4cede96cbdf8bd68f9d \ + --hash=sha256:d67ca8dc7f3b2314bd3bf83c4de52645e220ee312fd526e53acc6a735f233fad \ + --hash=sha256:d8620970bd0b6a0acb99dbd0b1c2ebb7a69909d25f6023bdff9635a39001aa51 \ + --hash=sha256:d92b045e90ed919a8a2520dda64e3f384e5e746ea51e1498cc6ac3e9e5c76054 \ + --hash=sha256:dbbbf25dc3c207b0a7bd4f3aea1df33d4f22b8508592796a6f36f4d8ed216db0 # via -r requirements.in -mypy==0.910 \ - --hash=sha256:088cd9c7904b4ad80bec811053272986611b84221835e079be5bcad029e79dd9 \ - --hash=sha256:0aadfb2d3935988ec3815952e44058a3100499f5be5b28c34ac9d79f002a4a9a \ - --hash=sha256:119bed3832d961f3a880787bf621634ba042cb8dc850a7429f643508eeac97b9 \ - --hash=sha256:1a85e280d4d217150ce8cb1a6dddffd14e753a4e0c3cf90baabb32cefa41b59e \ - --hash=sha256:3c4b8ca36877fc75339253721f69603a9c7fdb5d4d5a95a1a1b899d8b86a4de2 \ - --hash=sha256:3e382b29f8e0ccf19a2df2b29a167591245df90c0b5a2542249873b5c1d78212 \ - --hash=sha256:42c266ced41b65ed40a282c575705325fa7991af370036d3f134518336636f5b \ - --hash=sha256:53fd2eb27a8ee2892614370896956af2ff61254c275aaee4c230ae771cadd885 \ - --hash=sha256:704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150 \ - --hash=sha256:7df1ead20c81371ccd6091fa3e2878559b5c4d4caadaf1a484cf88d93ca06703 \ - --hash=sha256:866c41f28cee548475f146aa4d39a51cf3b6a84246969f3759cb3e9c742fc072 \ - --hash=sha256:a155d80ea6cee511a3694b108c4494a39f42de11ee4e61e72bc424c490e46457 \ - --hash=sha256:adaeee09bfde366d2c13fe6093a7df5df83c9a2ba98638c7d76b010694db760e \ - --hash=sha256:b6fb13123aeef4a3abbcfd7e71773ff3ff1526a7d3dc538f3929a49b42be03f0 \ - --hash=sha256:b94e4b785e304a04ea0828759172a15add27088520dc7e49ceade7834275bedb \ - --hash=sha256:c0df2d30ed496a08de5daed2a9ea807d07c21ae0ab23acf541ab88c24b26ab97 \ - --hash=sha256:c6c2602dffb74867498f86e6129fd52a2770c48b7cd3ece77ada4fa38f94eba8 \ - --hash=sha256:ceb6e0a6e27fb364fb3853389607cf7eb3a126ad335790fa1e14ed02fba50811 \ - --hash=sha256:d9dd839eb0dc1bbe866a288ba3c1afc33a202015d2ad83b31e875b5905a079b6 \ - --hash=sha256:e4dab234478e3bd3ce83bac4193b2ecd9cf94e720ddd95ce69840273bf44f6de \ - --hash=sha256:ec4e0cd079db280b6bdabdc807047ff3e199f334050db5cbb91ba3e959a67504 \ - --hash=sha256:ecd2c3fe726758037234c93df7e98deb257fd15c24c9180dacf1ef829da5f921 \ - --hash=sha256:ef565033fa5a958e62796867b1df10c40263ea9ded87164d67572834e57a174d +mypy==1.10.0 \ + --hash=sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061 \ + --hash=sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99 \ + --hash=sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de \ + --hash=sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a \ + --hash=sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9 \ + --hash=sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec \ + --hash=sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1 \ + --hash=sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131 \ + --hash=sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f \ + --hash=sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821 \ + --hash=sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5 \ + --hash=sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee \ + --hash=sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e \ + --hash=sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746 \ + --hash=sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2 \ + --hash=sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0 \ + --hash=sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b \ + --hash=sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53 \ + --hash=sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30 \ + --hash=sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda \ + --hash=sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051 \ + --hash=sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2 \ + --hash=sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7 \ + --hash=sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee \ + --hash=sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727 \ + --hash=sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976 \ + --hash=sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4 # via -r requirements.in -mypy-extensions==0.4.3 \ - --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ - --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 +mypy-extensions==1.0.0 \ + --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ + --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 # via mypy -numpy==1.21.3 \ - --hash=sha256:043e83bfc274649c82a6f09836943e4a4aebe5e33656271c7dbf9621dd58b8ec \ - --hash=sha256:160ccc1bed3a8371bf0d760971f09bfe80a3e18646620e9ded0ad159d9749baa \ - --hash=sha256:188031f833bbb623637e66006cf75e933e00e7231f67e2b45cf8189612bb5dc3 \ - --hash=sha256:28f15209fb535dd4c504a7762d3bc440779b0e37d50ed810ced209e5cea60d96 \ - --hash=sha256:29fb3dcd0468b7715f8ce2c0c2d9bbbaf5ae686334951343a41bd8d155c6ea27 \ - --hash=sha256:2a6ee9620061b2a722749b391c0d80a0e2ae97290f1b32e28d5a362e21941ee4 \ - --hash=sha256:300321e3985c968e3ae7fbda187237b225f3ffe6528395a5b7a5407f73cf093e \ - --hash=sha256:32437f0b275c1d09d9c3add782516413e98cd7c09e6baf4715cbce781fc29912 \ - --hash=sha256:3c09418a14471c7ae69ba682e2428cae5b4420a766659605566c0fa6987f6b7e \ - --hash=sha256:49c6249260890e05b8111ebfc391ed58b3cb4b33e63197b2ec7f776e45330721 \ - --hash=sha256:4cc9b512e9fb590797474f58b7f6d1f1b654b3a94f4fa8558b48ca8b3cfc97cf \ - --hash=sha256:508b0b513fa1266875524ba8a9ecc27b02ad771fe1704a16314dc1a816a68737 \ - --hash=sha256:50cd26b0cf6664cb3b3dd161ba0a09c9c1343db064e7c69f9f8b551f5104d654 \ - --hash=sha256:5c4193f70f8069550a1788bd0cd3268ab7d3a2b70583dfe3b2e7f421e9aace06 \ - --hash=sha256:5dfe9d6a4c39b8b6edd7990091fea4f852888e41919d0e6722fe78dd421db0eb \ - --hash=sha256:63571bb7897a584ca3249c86dd01c10bcb5fe4296e3568b2e9c1a55356b6410e \ - --hash=sha256:75621882d2230ab77fb6a03d4cbccd2038511491076e7964ef87306623aa5272 \ - --hash=sha256:75eb7cadc8da49302f5b659d40ba4f6d94d5045fbd9569c9d058e77b0514c9e4 \ - --hash=sha256:88a5d6b268e9ad18f3533e184744acdaa2e913b13148160b1152300c949bbb5f \ - --hash=sha256:8a10968963640e75cc0193e1847616ab4c718e83b6938ae74dea44953950f6b7 \ - --hash=sha256:90bec6a86b348b4559b6482e2b684db4a9a7eed1fa054b86115a48d58fbbf62a \ - --hash=sha256:98339aa9911853f131de11010f6dd94c8cec254d3d1f7261528c3b3e3219f139 \ - --hash=sha256:a99a6b067e5190ac6d12005a4d85aa6227c5606fa93211f86b1dafb16233e57d \ - --hash=sha256:bffa2eee3b87376cc6b31eee36d05349571c236d1de1175b804b348dc0941e3f \ - --hash=sha256:c6c2d535a7beb1f8790aaa98fd089ceab2e3dd7ca48aca0af7dc60e6ef93ffe1 \ - --hash=sha256:cc14e7519fab2a4ed87d31f99c31a3796e4e1fe63a86ebdd1c5a1ea78ebd5896 \ - --hash=sha256:dd0482f3fc547f1b1b5d6a8b8e08f63fdc250c58ce688dedd8851e6e26cff0f3 \ - --hash=sha256:dde972a1e11bb7b702ed0e447953e7617723760f420decb97305e66fb4afc54f \ - --hash=sha256:e54af82d68ef8255535a6cdb353f55d6b8cf418a83e2be3569243787a4f4866f \ - --hash=sha256:e606e6316911471c8d9b4618e082635cfe98876007556e89ce03d52ff5e8fcf0 \ - --hash=sha256:f41b018f126aac18583956c54544db437f25c7ee4794bcb23eb38bef8e5e192a \ - --hash=sha256:f8f4625536926a155b80ad2bbff44f8cc59e9f2ad14cdda7acf4c135b4dc8ff2 \ - --hash=sha256:fe52dbe47d9deb69b05084abd4b0df7abb39a3c51957c09f635520abd49b29dd +numpy==1.26.4 \ + --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ + --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ + --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ + --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ + --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ + --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ + --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ + --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ + --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ + --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ + --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ + --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ + --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ + --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ + --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ + --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ + --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ + --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ + --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ + --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ + --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ + --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ + --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ + --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ + --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ + --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ + --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ + --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ + --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ + --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ + --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ + --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ + --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ + --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ + --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ + --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f # via # -r requirements.in # pyarrow -packaging==21.0 \ - --hash=sha256:7dc96269f53a4ccec5c0670940a4281106dd0bb343f47b7471f779df49c2fbe7 \ - --hash=sha256:c86254f9220d55e31cc94d69bade760f0847da8000def4dfe1c6b872fd14ff14 +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 # via pytest -pluggy==1.0.0 \ - --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ - --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 +pluggy==1.5.0 \ + --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ + --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 # via pytest -py==1.10.0 \ - --hash=sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3 \ - --hash=sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a - # via pytest -pyarrow==15.0.0 \ - --hash=sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4 \ - --hash=sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970 \ - --hash=sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555 \ - --hash=sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b \ - --hash=sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d \ - --hash=sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2 \ - --hash=sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99 \ - --hash=sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac \ - --hash=sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6 \ - --hash=sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5 \ - --hash=sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f \ - --hash=sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3 \ - --hash=sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb \ - --hash=sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642 \ - --hash=sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2 \ - --hash=sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f \ - --hash=sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929 \ - --hash=sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e \ - --hash=sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83 \ - --hash=sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340 \ - --hash=sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540 \ - --hash=sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed \ - --hash=sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351 \ - --hash=sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177 \ - --hash=sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4 \ - --hash=sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7 \ - --hash=sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593 \ - --hash=sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7 \ - --hash=sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31 \ - --hash=sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b \ - --hash=sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e \ - --hash=sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6 \ - --hash=sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4 \ - --hash=sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40 \ - --hash=sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5 \ - --hash=sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19 +pyarrow==16.1.0 \ + --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ + --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ + --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ + --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ + --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ + --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ + --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ + --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ + --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ + --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ + --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ + --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ + --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ + --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ + --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ + --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ + --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ + --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ + --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ + --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ + --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ + --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ + --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ + --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ + --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ + --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ + --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ + --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ + --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ + --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ + --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ + --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ + --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ + --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ + --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ + --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 # via -r requirements.in -pyparsing==3.0.3 \ - --hash=sha256:9e3511118010f112a4b4b435ae50e1eaa610cda191acb9e421d60cf5fde83455 \ - --hash=sha256:f8d3fe9fc404576c5164f0f0c4e382c96b85265e023c409c43d48f65da9d60d0 - # via packaging -pytest==6.2.5 \ - --hash=sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89 \ - --hash=sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134 +pytest==8.2.2 \ + --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ + --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 # via -r requirements.in -ruff==0.3.0 \ - --hash=sha256:0886184ba2618d815067cf43e005388967b67ab9c80df52b32ec1152ab49f53a \ - --hash=sha256:128265876c1d703e5f5e5a4543bd8be47c73a9ba223fd3989d4aa87dd06f312f \ - --hash=sha256:19eacceb4c9406f6c41af806418a26fdb23120dfe53583df76d1401c92b7c14b \ - --hash=sha256:23dbb808e2f1d68eeadd5f655485e235c102ac6f12ad31505804edced2a5ae77 \ - --hash=sha256:2f7dbba46e2827dfcb0f0cc55fba8e96ba7c8700e0a866eb8cef7d1d66c25dcb \ - --hash=sha256:3ef655c51f41d5fa879f98e40c90072b567c666a7114fa2d9fe004dffba00932 \ - --hash=sha256:5da894a29ec018a8293d3d17c797e73b374773943e8369cfc50495573d396933 \ - --hash=sha256:755c22536d7f1889be25f2baf6fedd019d0c51d079e8417d4441159f3bcd30c2 \ - --hash=sha256:7deb528029bacf845bdbb3dbb2927d8ef9b4356a5e731b10eef171e3f0a85944 \ - --hash=sha256:9343690f95710f8cf251bee1013bf43030072b9f8d012fbed6ad702ef70d360a \ - --hash=sha256:a1f3ed501a42f60f4dedb7805fa8d4534e78b4e196f536bac926f805f0743d49 \ - --hash=sha256:b08b356d06a792e49a12074b62222f9d4ea2a11dca9da9f68163b28c71bf1dd4 \ - --hash=sha256:cc30a9053ff2f1ffb505a585797c23434d5f6c838bacfe206c0e6cf38c921a1e \ - --hash=sha256:d0d3d7ef3d4f06433d592e5f7d813314a34601e6c5be8481cccb7fa760aa243e \ - --hash=sha256:dd73fe7f4c28d317855da6a7bc4aa29a1500320818dd8f27df95f70a01b8171f \ - --hash=sha256:e1e0d4381ca88fb2b73ea0766008e703f33f460295de658f5467f6f229658c19 \ - --hash=sha256:e3a4a6d46aef0a84b74fcd201a4401ea9a6cd85614f6a9435f2d33dd8cefbf83 +ruff==0.4.8 \ + --hash=sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780 \ + --hash=sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268 \ + --hash=sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb \ + --hash=sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed \ + --hash=sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9 \ + --hash=sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a \ + --hash=sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b \ + --hash=sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883 \ + --hash=sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066 \ + --hash=sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199 \ + --hash=sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764 \ + --hash=sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3 \ + --hash=sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45 \ + --hash=sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d \ + --hash=sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa \ + --hash=sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913 \ + --hash=sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc # via -r requirements.in toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f + # via -r requirements.in +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f # via - # -r requirements.in + # maturin # mypy # pytest -tomli==1.2.2 \ - --hash=sha256:c6ce0015eb38820eaf32b5db832dbc26deb3dd427bd5f6556cf0acac2c214fee \ - --hash=sha256:f04066f68f5554911363063a30b108d2b5a5b1a010aa8b6132af78489fe3aade - # via maturin -typing-extensions==3.10.0.2 \ - --hash=sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e \ - --hash=sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7 \ - --hash=sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34 +typing-extensions==4.12.1 \ + --hash=sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a \ + --hash=sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1 # via mypy diff --git a/requirements-311.txt b/requirements-311.txt index 876311e6c..35b91133c 100644 --- a/requirements-311.txt +++ b/requirements-311.txt @@ -8,49 +8,49 @@ iniconfig==2.0.0 \ --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 # via pytest -maturin==1.4.0 \ - --hash=sha256:01473dc30aed8f2cee3572b3e99e3ea75bf09c84b028bf6077f7643a189699c8 \ - --hash=sha256:076970a73da7fa3648204a584cd347b899c1ea67f8124b212bccd06728e63ed9 \ - --hash=sha256:16239a7648ef17976585353e381840c18e650d352576ed9545abca407d65e534 \ - --hash=sha256:2979175a7eee837dc3a6931980b37ddc86b9ced54d600856668fc074ca2530ef \ - --hash=sha256:77428c043d585f038f4b056c4d617e00a8027b49598ab6d065b8f6b9b9b8d144 \ - --hash=sha256:b4b2f006db1e92687c814576029157dcc2d97b5750fd35fd4f3aabb97e36444f \ - --hash=sha256:b84bee85620e1b7b662a7af71289f7f6c23df8269e42c0f76882676dfc9c733f \ - --hash=sha256:e2c1b157397ef3721b9c2f3f24d9a5a60bd84322aac13b4dd0704a80448741b0 \ - --hash=sha256:e669ba5984c15e29b8545b295ba6738974180b44f47f5d9e75569a5ce6b8add5 \ - --hash=sha256:ed12e1768094a7adeafc3a74ebdb8dc2201fa64c4e7e31f14cfc70378bf93790 \ - --hash=sha256:f8eded83abdb30b2b6ae6d32c80b8192bdd8bcfec0ebfacee6ac02434aa499d6 \ - --hash=sha256:ff95a4494d9e57b6e74d4d7f8a9a2ee8ed29bd7f0e61855656ad959a432c0efc \ - --hash=sha256:ffe4e967080ceb83c156e73a37d3974b30cad01c376a86dc39a76a0c6bccf9b0 +maturin==1.6.0 \ + --hash=sha256:16ef860df20028618b5a064da06b02c1c47acba064a4d25aaf84662a459ec599 \ + --hash=sha256:337899784955934dd67b30497d1dd5fab22da89f60bb079dbaf2eaa446b97a10 \ + --hash=sha256:4e931c92037128ade49cd26dd040d9c46ad8092d8170cc44f5c3a0b4a052d576 \ + --hash=sha256:50133965e52d8b5b969381fee3fde111ae2383905cdaba7650f256e08ccddcd4 \ + --hash=sha256:a2a2436628c36d98dabd79b52256df7e12fc4fd1b122984d9373fdf918fd4609 \ + --hash=sha256:aa4eb7dca7d246b466392f21016f67ff09a9aff2305fa714ca25a2344e4639e7 \ + --hash=sha256:b955025c24c8babc808db49e0ff90db8b4b1320dcc16b14eb26132841737230d \ + --hash=sha256:bd85edcb1b8e2bcddc1b7d16ce58ce00a66aa80c422745c8ad9e132ac40d4b48 \ + --hash=sha256:c87d1a7596c42b589099adb831343a56e02373588366e4cede96cbdf8bd68f9d \ + --hash=sha256:d67ca8dc7f3b2314bd3bf83c4de52645e220ee312fd526e53acc6a735f233fad \ + --hash=sha256:d8620970bd0b6a0acb99dbd0b1c2ebb7a69909d25f6023bdff9635a39001aa51 \ + --hash=sha256:d92b045e90ed919a8a2520dda64e3f384e5e746ea51e1498cc6ac3e9e5c76054 \ + --hash=sha256:dbbbf25dc3c207b0a7bd4f3aea1df33d4f22b8508592796a6f36f4d8ed216db0 # via -r requirements.in -mypy==1.8.0 \ - --hash=sha256:028cf9f2cae89e202d7b6593cd98db6759379f17a319b5faf4f9978d7084cdc6 \ - --hash=sha256:2afecd6354bbfb6e0160f4e4ad9ba6e4e003b767dd80d85516e71f2e955ab50d \ - --hash=sha256:2b5b6c721bd4aabaadead3a5e6fa85c11c6c795e0c81a7215776ef8afc66de02 \ - --hash=sha256:42419861b43e6962a649068a61f4a4839205a3ef525b858377a960b9e2de6e0d \ - --hash=sha256:42c6680d256ab35637ef88891c6bd02514ccb7e1122133ac96055ff458f93fc3 \ - --hash=sha256:485a8942f671120f76afffff70f259e1cd0f0cfe08f81c05d8816d958d4577d3 \ - --hash=sha256:4c886c6cce2d070bd7df4ec4a05a13ee20c0aa60cb587e8d1265b6c03cf91da3 \ - --hash=sha256:4e6d97288757e1ddba10dd9549ac27982e3e74a49d8d0179fc14d4365c7add66 \ - --hash=sha256:4ef4be7baf08a203170f29e89d79064463b7fc7a0908b9d0d5114e8009c3a259 \ - --hash=sha256:51720c776d148bad2372ca21ca29256ed483aa9a4cdefefcef49006dff2a6835 \ - --hash=sha256:52825b01f5c4c1c4eb0db253ec09c7aa17e1a7304d247c48b6f3599ef40db8bd \ - --hash=sha256:538fd81bb5e430cc1381a443971c0475582ff9f434c16cd46d2c66763ce85d9d \ - --hash=sha256:5c1538c38584029352878a0466f03a8ee7547d7bd9f641f57a0f3017a7c905b8 \ - --hash=sha256:6ff8b244d7085a0b425b56d327b480c3b29cafbd2eff27316a004f9a7391ae07 \ - --hash=sha256:7178def594014aa6c35a8ff411cf37d682f428b3b5617ca79029d8ae72f5402b \ - --hash=sha256:720a5ca70e136b675af3af63db533c1c8c9181314d207568bbe79051f122669e \ - --hash=sha256:7f1478736fcebb90f97e40aff11a5f253af890c845ee0c850fe80aa060a267c6 \ - --hash=sha256:855fe27b80375e5c5878492f0729540db47b186509c98dae341254c8f45f42ae \ - --hash=sha256:8963b83d53ee733a6e4196954502b33567ad07dfd74851f32be18eb932fb1cb9 \ - --hash=sha256:9261ed810972061388918c83c3f5cd46079d875026ba97380f3e3978a72f503d \ - --hash=sha256:99b00bc72855812a60d253420d8a2eae839b0afa4938f09f4d2aa9bb4654263a \ - --hash=sha256:ab3c84fa13c04aeeeabb2a7f67a25ef5d77ac9d6486ff33ded762ef353aa5592 \ - --hash=sha256:afe3fe972c645b4632c563d3f3eff1cdca2fa058f730df2b93a35e3b0c538218 \ - --hash=sha256:d19c413b3c07cbecf1f991e2221746b0d2a9410b59cb3f4fb9557f0365a1a817 \ - --hash=sha256:df9824ac11deaf007443e7ed2a4a26bebff98d2bc43c6da21b2b64185da011c4 \ - --hash=sha256:e46f44b54ebddbeedbd3d5b289a893219065ef805d95094d16a0af6630f5d410 \ - --hash=sha256:f5ac9a4eeb1ec0f1ccdc6f326bcdb464de5f80eb07fb38b5ddd7b0de6bc61e55 +mypy==1.10.0 \ + --hash=sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061 \ + --hash=sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99 \ + --hash=sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de \ + --hash=sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a \ + --hash=sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9 \ + --hash=sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec \ + --hash=sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1 \ + --hash=sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131 \ + --hash=sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f \ + --hash=sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821 \ + --hash=sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5 \ + --hash=sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee \ + --hash=sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e \ + --hash=sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746 \ + --hash=sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2 \ + --hash=sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0 \ + --hash=sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b \ + --hash=sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53 \ + --hash=sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30 \ + --hash=sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda \ + --hash=sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051 \ + --hash=sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2 \ + --hash=sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7 \ + --hash=sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee \ + --hash=sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727 \ + --hash=sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976 \ + --hash=sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4 # via -r requirements.in mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ @@ -96,80 +96,80 @@ numpy==1.26.4 \ # via # -r requirements.in # pyarrow -packaging==23.2 \ - --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ - --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 +packaging==24.0 \ + --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ + --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 # via pytest -pluggy==1.4.0 \ - --hash=sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981 \ - --hash=sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be +pluggy==1.5.0 \ + --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ + --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 # via pytest -pyarrow==15.0.0 \ - --hash=sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4 \ - --hash=sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970 \ - --hash=sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555 \ - --hash=sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b \ - --hash=sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d \ - --hash=sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2 \ - --hash=sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99 \ - --hash=sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac \ - --hash=sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6 \ - --hash=sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5 \ - --hash=sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f \ - --hash=sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3 \ - --hash=sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb \ - --hash=sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642 \ - --hash=sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2 \ - --hash=sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f \ - --hash=sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929 \ - --hash=sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e \ - --hash=sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83 \ - --hash=sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340 \ - --hash=sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540 \ - --hash=sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed \ - --hash=sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351 \ - --hash=sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177 \ - --hash=sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4 \ - --hash=sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7 \ - --hash=sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593 \ - --hash=sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7 \ - --hash=sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31 \ - --hash=sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b \ - --hash=sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e \ - --hash=sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6 \ - --hash=sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4 \ - --hash=sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40 \ - --hash=sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5 \ - --hash=sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19 +pyarrow==16.1.0 \ + --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ + --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ + --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ + --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ + --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ + --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ + --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ + --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ + --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ + --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ + --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ + --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ + --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ + --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ + --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ + --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ + --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ + --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ + --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ + --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ + --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ + --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ + --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ + --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ + --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ + --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ + --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ + --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ + --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ + --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ + --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ + --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ + --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ + --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ + --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ + --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 # via -r requirements.in -pytest==8.0.2 \ - --hash=sha256:d4051d623a2e0b7e51960ba963193b09ce6daeb9759a451844a21e4ddedfc1bd \ - --hash=sha256:edfaaef32ce5172d5466b5127b42e0d6d35ebbe4453f0e3505d96afd93f6b096 +pytest==8.2.2 \ + --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ + --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 # via -r requirements.in -ruff==0.3.0 \ - --hash=sha256:0886184ba2618d815067cf43e005388967b67ab9c80df52b32ec1152ab49f53a \ - --hash=sha256:128265876c1d703e5f5e5a4543bd8be47c73a9ba223fd3989d4aa87dd06f312f \ - --hash=sha256:19eacceb4c9406f6c41af806418a26fdb23120dfe53583df76d1401c92b7c14b \ - --hash=sha256:23dbb808e2f1d68eeadd5f655485e235c102ac6f12ad31505804edced2a5ae77 \ - --hash=sha256:2f7dbba46e2827dfcb0f0cc55fba8e96ba7c8700e0a866eb8cef7d1d66c25dcb \ - --hash=sha256:3ef655c51f41d5fa879f98e40c90072b567c666a7114fa2d9fe004dffba00932 \ - --hash=sha256:5da894a29ec018a8293d3d17c797e73b374773943e8369cfc50495573d396933 \ - --hash=sha256:755c22536d7f1889be25f2baf6fedd019d0c51d079e8417d4441159f3bcd30c2 \ - --hash=sha256:7deb528029bacf845bdbb3dbb2927d8ef9b4356a5e731b10eef171e3f0a85944 \ - --hash=sha256:9343690f95710f8cf251bee1013bf43030072b9f8d012fbed6ad702ef70d360a \ - --hash=sha256:a1f3ed501a42f60f4dedb7805fa8d4534e78b4e196f536bac926f805f0743d49 \ - --hash=sha256:b08b356d06a792e49a12074b62222f9d4ea2a11dca9da9f68163b28c71bf1dd4 \ - --hash=sha256:cc30a9053ff2f1ffb505a585797c23434d5f6c838bacfe206c0e6cf38c921a1e \ - --hash=sha256:d0d3d7ef3d4f06433d592e5f7d813314a34601e6c5be8481cccb7fa760aa243e \ - --hash=sha256:dd73fe7f4c28d317855da6a7bc4aa29a1500320818dd8f27df95f70a01b8171f \ - --hash=sha256:e1e0d4381ca88fb2b73ea0766008e703f33f460295de658f5467f6f229658c19 \ - --hash=sha256:e3a4a6d46aef0a84b74fcd201a4401ea9a6cd85614f6a9435f2d33dd8cefbf83 +ruff==0.4.8 \ + --hash=sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780 \ + --hash=sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268 \ + --hash=sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb \ + --hash=sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed \ + --hash=sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9 \ + --hash=sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a \ + --hash=sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b \ + --hash=sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883 \ + --hash=sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066 \ + --hash=sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199 \ + --hash=sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764 \ + --hash=sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3 \ + --hash=sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45 \ + --hash=sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d \ + --hash=sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa \ + --hash=sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913 \ + --hash=sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc # via -r requirements.in toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f # via -r requirements.in -typing-extensions==4.10.0 \ - --hash=sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475 \ - --hash=sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb +typing-extensions==4.12.1 \ + --hash=sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a \ + --hash=sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1 # via mypy diff --git a/requirements.in b/requirements.in index 2365d23e0..b2a1a48df 100644 --- a/requirements.in +++ b/requirements.in @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -maturin>=0.15 +maturin>=1.5.1 mypy numpy pyarrow>=11.0.0 From b5446efbc2e50fb017454173f6b11e4fad53fe5a Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 14 Jun 2024 13:19:03 -0500 Subject: [PATCH 153/413] Upgrade datafusion 39 (#728) * deps: update datafusion to 39.0.0, pyo3 to 0.21, and object_store to 0.10.1 `datafusion-common` also depends on `pyo3`, so they need to be upgraded together. * feat: remove GetIndexField datafusion replaced Expr::GetIndexField with a FieldAccessor trait. Ref https://github.com/apache/datafusion/pull/10568 Ref https://github.com/apache/datafusion/pull/10769 * feat: update ScalarFunction The field `func_name` was changed to `func` as part of removing `ScalarFunctionDefinition` upstream. Ref https://github.com/apache/datafusion/pull/10325 * feat: incorporate upstream array_slice fixes Fixes #670 * update ExectionPlan::children impl for DatasetExec Ref https://github.com/apache/datafusion/pull/10543 * update value_interval_daytime Ref https://github.com/apache/arrow-rs/pull/5769 * update regexp_replace and regexp_match Fixes #677 * add gil-refs feature to pyo3 This silences pyo3's deprecation warnings for its new Bounds api. It's the 1st step of the migration, and should be removed before merge. Ref https://pyo3.rs/v0.21.0/migration#from-020-to-021 * fix signature for octet_length Ref https://github.com/apache/datafusion/pull/10726 * update signature for covar_samp AggregateUDF expressions now have a builder API design, which removes arguments like filter and order_by Ref https://github.com/apache/datafusion/issues/10545 Ref https://github.com/apache/datafusion/pull/10492 * convert covar_pop to expr_fn api Ref: https://github.com/apache/datafusion/pull/10418/files * convert median to expr_fn api Ref https://github.com/apache/datafusion/pull/10644 * convert variance sample to UDF Ref https://github.com/apache/datafusion/issues/10667 * convert first_value and last_value to UDFs Ref https://github.com/apache/datafusion/pull/10648 * checkpointing with a few todos to fix remaining compile errors * impl PyExpr::python_value for IntervalDayTime and IntervalMonthDayNano * convert sum aggregate function to UDF * remove unnecessary clone on double reference * apply cargo fmt * remove duplicate allow-dead-code annotation * update tpch examples for new pyarrow interval Fixes #665 * marked q11 tpch example as expected fail Ref #730 * add default stride of None back to array_slice --- Cargo.lock | 540 +++++++++++------- Cargo.toml | 22 +- .../common-operations/functions.rst | 3 +- examples/tpch/_tests.py | 5 +- examples/tpch/q01_pricing_summary_report.py | 4 +- examples/tpch/q04_order_priority_checking.py | 4 +- examples/tpch/q05_local_supplier_volume.py | 4 +- .../tpch/q06_forecasting_revenue_change.py | 4 +- examples/tpch/q10_returned_item_reporting.py | 4 +- examples/tpch/q12_ship_mode_order_priority.py | 4 +- examples/tpch/q14_promotion_effect.py | 5 +- examples/tpch/q15_top_supplier.py | 5 +- examples/tpch/q20_potential_part_promotion.py | 4 +- python/datafusion/__init__.py | 2 - python/datafusion/tests/test_functions.py | 1 - python/datafusion/tests/test_imports.py | 2 - src/common/data_type.rs | 1 + src/dataset_exec.rs | 2 +- src/expr.rs | 32 +- src/expr/literal.rs | 7 +- src/expr/signature.rs | 1 - src/functions.rs | 136 +++-- src/physical_plan.rs | 2 +- 23 files changed, 467 insertions(+), 327 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed1ee3b80..eb674d0f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" dependencies = [ "bytes", "half", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ "arrow-buffer", "arrow-schema", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash", "arrow-array", @@ -312,18 +312,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" dependencies = [ "bitflags 2.5.0", ] [[package]] name = "arrow-select" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash", "arrow-array", @@ -335,9 +335,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ "arrow-array", "arrow-buffer", @@ -364,8 +364,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.13.1", - "zstd-safe 7.1.0", + "zstd 0.13.0", + "zstd-safe 7.0.0", ] [[package]] @@ -400,19 +400,16 @@ dependencies = [ ] [[package]] -name = "autocfg" -version = "1.3.0" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "autotools" -version = "0.2.7" +name = "autocfg" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" -dependencies = [ - "cc", -] +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" @@ -486,9 +483,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.5.0" +version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -497,9 +494,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -576,9 +573,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.8.6" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" dependencies = [ "chrono", "chrono-tz-build", @@ -587,15 +584,24 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433e39f13c9a060046954e0592a8d0a4bcb1040125cbf91cb8ee58964cfb350f" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" dependencies = [ "parse-zoneinfo", "phf", "phf_codegen", ] +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "comfy-table" version = "7.1.1" @@ -734,9 +740,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05fb4eeeb7109393a0739ac5b8fd892f95ccef691421491c85544f7997366f68" +checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" dependencies = [ "ahash", "apache-avro", @@ -759,6 +765,7 @@ dependencies = [ "datafusion-functions-array", "datafusion-optimizer", "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-sql", "flate2", @@ -767,13 +774,14 @@ dependencies = [ "half", "hashbrown", "indexmap", - "itertools", + "itertools 0.12.1", "log", "num-traits", "num_cpus", "object_store", "parking_lot", "parquet", + "paste", "pin-project-lite", "rand", "sqlparser", @@ -783,14 +791,14 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.1", + "zstd 0.13.0", ] [[package]] name = "datafusion-common" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "741aeac15c82f239f2fc17deccaab19873abbd62987be20023689b15fa72fa09" +checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" dependencies = [ "ahash", "apache-avro", @@ -800,6 +808,7 @@ dependencies = [ "arrow-schema", "chrono", "half", + "hashbrown", "instant", "libc", "num_cpus", @@ -811,18 +820,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8ddfb8d8cb51646a30da0122ecfffb81ca16919ae9a3495a9e7468bdcd52b8" +checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282122f90b20e8f98ebfa101e4bf20e718fd2684cf81bef4e8c6366571c64404" +checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" dependencies = [ "arrow", "chrono", @@ -841,13 +850,14 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5478588f733df0dfd87a62671c7478f590952c95fa2fa5c137e3ff2929491e22" +checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", @@ -859,9 +869,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4afd261cea6ac9c3ca1192fd5e9f940596d8e9208c5b1333f4961405db53185" +checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" dependencies = [ "arrow", "base64 0.22.1", @@ -874,7 +884,7 @@ dependencies = [ "datafusion-physical-expr", "hashbrown", "hex", - "itertools", + "itertools 0.12.1", "log", "md-5", "rand", @@ -886,11 +896,13 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b36a6c4838ab94b5bf8f7a96ce6ce059d805c5d1dcaa6ace49e034eb65cd999" +checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" dependencies = [ + "ahash", "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -902,9 +914,9 @@ dependencies = [ [[package]] name = "datafusion-functions-array" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fdd200a6233f48d3362e7ccb784f926f759100e44ae2137a5e2dcb986a59c4" +checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" dependencies = [ "arrow", "arrow-array", @@ -915,16 +927,16 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", - "itertools", + "itertools 0.12.1", "log", "paste", ] [[package]] name = "datafusion-optimizer" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54f2820938810e8a2d71228fd6f59f33396aebc5f5f687fcbf14de5aab6a7e1a" +checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" dependencies = [ "arrow", "async-trait", @@ -934,16 +946,16 @@ dependencies = [ "datafusion-physical-expr", "hashbrown", "indexmap", - "itertools", + "itertools 0.12.1", "log", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9adf8eb12716f52ddf01e09eb6c94d3c9b291e062c05c91b839a448bddba2ff8" +checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" dependencies = [ "ahash", "arrow", @@ -963,7 +975,7 @@ dependencies = [ "hashbrown", "hex", "indexmap", - "itertools", + "itertools 0.12.1", "log", "paste", "petgraph", @@ -972,20 +984,21 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d5472c3230584c150197b3f2c23f2392b9dc54dbfb62ad41e7e36447cfce4be" +checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" dependencies = [ "arrow", "datafusion-common", "datafusion-expr", + "rand", ] [[package]] name = "datafusion-physical-plan" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18ae750c38389685a8b62e5b899bbbec488950755ad6d218f3662d35b800c4fe" +checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" dependencies = [ "ahash", "arrow", @@ -1006,7 +1019,7 @@ dependencies = [ "half", "hashbrown", "indexmap", - "itertools", + "itertools 0.12.1", "log", "once_cell", "parking_lot", @@ -1017,7 +1030,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "38.0.1" +version = "39.0.0" dependencies = [ "async-trait", "datafusion", @@ -1045,9 +1058,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "befc67a3cdfbfa76853f43b10ac27337821bb98e519ab6baf431fcc0bcfcafdb" +checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" dependencies = [ "arrow", "arrow-array", @@ -1055,23 +1068,25 @@ dependencies = [ "datafusion-common", "datafusion-expr", "log", + "regex", "sqlparser", "strum 0.26.2", ] [[package]] name = "datafusion-substrait" -version = "38.0.0" +version = "39.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f62542caa77df003e23a8bc2f1b8a1ffc682fe447c7fcb4905d109e3d7a5b9d" +checksum = "8249d69665c1cd32e07789ed6dd1da6528a23019ef16d3483db52952b6f9f68a" dependencies = [ + "arrow-buffer", "async-recursion", "chrono", "datafusion", - "itertools", + "itertools 0.12.1", "object_store", + "pbjson-types", "prost", - "prost-types", "substrait", ] @@ -1104,15 +1119,6 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" -[[package]] -name = "encoding_rs" -version = "0.8.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -1143,9 +1149,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1300,15 +1306,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.3.26" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" dependencies = [ + "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "futures-util", "http", "indexmap", "slab", @@ -1364,9 +1370,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "0.2.12" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -1375,12 +1381,24 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.6" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", + "futures-core", "http", + "http-body", "pin-project-lite", ] @@ -1390,12 +1408,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - [[package]] name = "humantime" version = "2.1.0" @@ -1404,40 +1416,59 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", - "futures-core", "futures-util", "h2", "http", "http-body", "httparse", - "httpdate", "itoa", "pin-project-lite", - "socket2", + "smallvec", "tokio", - "tower-service", - "tracing", "want", ] [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" dependencies = [ "futures-util", "http", "hyper", + "hyper-util", "rustls", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1513,6 +1544,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1865,18 +1905,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" +checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" dependencies = [ "async-trait", - "base64 0.21.7", + "base64 0.22.1", "bytes", "chrono", "futures", "humantime", "hyper", - "itertools", + "itertools 0.12.1", "md-5", "parking_lot", "percent-encoding", @@ -1884,7 +1924,7 @@ dependencies = [ "rand", "reqwest", "ring", - "rustls-pemfile 2.1.2", + "rustls-pemfile", "serde", "serde_json", "snafu", @@ -1940,9 +1980,9 @@ dependencies = [ [[package]] name = "parquet" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" +checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash", "arrow-array", @@ -1970,7 +2010,8 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.1", + "zstd 0.13.0", + "zstd-sys", ] [[package]] @@ -1988,6 +2029,43 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbjson" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" +dependencies = [ + "heck 0.4.1", + "itertools 0.11.0", + "prost", + "prost-types", +] + +[[package]] +name = "pbjson-types" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +dependencies = [ + "bytes", + "chrono", + "pbjson", + "pbjson-build", + "prost", + "prost-build", + "serde", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2042,6 +2120,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2109,14 +2207,16 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", "heck 0.5.0", - "itertools", + "itertools 0.12.1", "log", "multimap", "once_cell", "petgraph", + "prettyplease", "prost", "prost-types", "regex", + "syn 2.0.66", "tempfile", ] @@ -2127,7 +2227,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.66", @@ -2144,18 +2244,18 @@ dependencies = [ [[package]] name = "protobuf-src" -version = "1.1.0+21.5" +version = "2.0.1+26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ac8852baeb3cc6fb83b93646fb93c0ffe5d14bf138c945ceb4b9948ee0e3c1" +checksum = "f8ba1cfa4b9dc098926b8cce388bf434b93516db3ecf6e8b1a37eb643d733ee7" dependencies = [ - "autotools", + "cmake", ] [[package]] name = "pyo3" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" dependencies = [ "cfg-if", "indoc", @@ -2171,9 +2271,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" dependencies = [ "once_cell", "target-lexicon", @@ -2181,9 +2281,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" dependencies = [ "libc", "pyo3-build-config", @@ -2191,9 +2291,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2203,9 +2303,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -2315,9 +2415,9 @@ checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "regress" -version = "0.8.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5f39ba4513916c1b2657b72af6ec671f091cd637992f58d0ede5cae4e5dea0" +checksum = "0eae2a1ebfecc58aff952ef8ccd364329abe627762f5bf09ff42eb9d98522479" dependencies = [ "hashbrown", "memchr", @@ -2325,20 +2425,21 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.27" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", - "encoding_rs", "futures-core", "futures-util", "h2", "http", "http-body", + "http-body-util", "hyper", "hyper-rustls", + "hyper-util", "ipnet", "js-sys", "log", @@ -2348,12 +2449,12 @@ dependencies = [ "pin-project-lite", "rustls", "rustls-native-certs", - "rustls-pemfile 1.0.4", + "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-rustls", "tokio-util", @@ -2417,37 +2518,31 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.12" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" dependencies = [ "log", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.4", + "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", -] - [[package]] name = "rustls-pemfile" version = "2.1.2" @@ -2466,11 +2561,12 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] @@ -2534,16 +2630,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" version = "2.11.0" @@ -2572,6 +2658,9 @@ name = "semver" version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +dependencies = [ + "serde", +] [[package]] name = "seq-macro" @@ -2736,9 +2825,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.45.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", "sqlparser_derive", @@ -2804,11 +2893,14 @@ dependencies = [ [[package]] name = "substrait" -version = "0.32.0" +version = "0.34.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f01344023c2614171a9ffd6e387eea14e12f7387c5b6adb33f1563187d65e376" +checksum = "04c77dec9b6c4e48ac828937bbe7cf473b0933168c5d76d51a5816ace7046be9" dependencies = [ "heck 0.5.0", + "pbjson", + "pbjson-build", + "pbjson-types", "prettyplease", "prost", "prost-build", @@ -2858,27 +2950,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "target-lexicon" version = "0.12.14" @@ -2982,11 +3053,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] @@ -3003,6 +3075,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -3084,9 +3177,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" -version = "0.0.16" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c61e9db210bbff218e6535c664b37ec47da449169b98e7866d0580d0db75529" +checksum = "adb6beec125971dda80a086f90b4a70f60f222990ce4d63ad0fc140492f53444" dependencies = [ "typify-impl", "typify-macro", @@ -3094,16 +3187,18 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.0.16" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e32f38493804f88e2dc7a5412eccd872ea5452b4db9b0a77de4df180f2a87e" +checksum = "93bbb24e990654aff858d80fee8114f4322f7d7a1b1ecb45129e2fcb0d0ad5ae" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "log", "proc-macro2", "quote", "regress", "schemars", + "semver", + "serde", "serde_json", "syn 2.0.66", "thiserror", @@ -3112,13 +3207,14 @@ dependencies = [ [[package]] name = "typify-macro" -version = "0.0.16" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc09508b72f63d521d68e42c7f172c7416d67986df44b3c7d1f7f9963948ed32" +checksum = "f8e6491896e955692d68361c68db2b263e3bec317ec0b684e0e2fa882fb6e31e" dependencies = [ "proc-macro2", "quote", "schemars", + "semver", "serde", "serde_json", "serde_tokenstream", @@ -3477,9 +3573,9 @@ checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winreg" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" dependencies = [ "cfg-if", "windows-sys 0.48.0", @@ -3514,6 +3610,12 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + [[package]] name = "zstd" version = "0.12.4" @@ -3525,11 +3627,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" dependencies = [ - "zstd-safe 7.1.0", + "zstd-safe 7.0.0", ] [[package]] @@ -3544,18 +3646,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.9+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index fd6f4bb31..85a19d1e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "38.0.1" +version = "39.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -36,28 +36,28 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" -pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] } -datafusion = { version = "38.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-common = { version = "38.0.0", features = ["pyarrow"] } -datafusion-expr = "38.0.0" -datafusion-functions-array = "38.0.0" -datafusion-optimizer = "38.0.0" -datafusion-sql = "38.0.0" -datafusion-substrait = { version = "38.0.0", optional = true } +pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38", "gil-refs"] } +datafusion = { version = "39.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-common = { version = "39.0.0", features = ["pyarrow"] } +datafusion-expr = "39.0.0" +datafusion-functions-array = "39.0.0" +datafusion-optimizer = "39.0.0" +datafusion-sql = "39.0.0" +datafusion-substrait = { version = "39.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.8", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.9.1", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" syn = "2.0.43" url = "2.2" [build-dependencies] -pyo3-build-config = "0.20.0" +pyo3-build-config = "0.21" [lib] name = "datafusion_python" diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index 50b493098..d793314f7 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -92,12 +92,13 @@ DataFusion offers a range of helpful options. f.left(col('"Name"'), literal(4)).alias("code") ) -This also includes the functions for regular expressions like :func:`.regexp_match` +This also includes the functions for regular expressions like :func:`.regexp_replace` and :func:`.regexp_match` .. ipython:: python df.select( f.regexp_match(col('"Name"'), literal("Char")).alias("dragons"), + f.regexp_replace(col('"Name"'), literal("saur"), literal("fleur")).alias("flowers") ) diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 049b43dff..aa9491bfd 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -72,7 +72,10 @@ def check_q17(df): ("q08_market_share", "q8"), ("q09_product_type_profit_measure", "q9"), ("q10_returned_item_reporting", "q10"), - ("q11_important_stock_identification", "q11"), + pytest.param( + "q11_important_stock_identification", "q11", + marks=pytest.mark.xfail # https://github.com/apache/datafusion-python/issues/730 + ), ("q12_ship_mode_order_priority", "q12"), ("q13_customer_distribution", "q13"), ("q14_promotion_effect", "q14"), diff --git a/examples/tpch/q01_pricing_summary_report.py b/examples/tpch/q01_pricing_summary_report.py index 7e86055d9..cb9485a7a 100644 --- a/examples/tpch/q01_pricing_summary_report.py +++ b/examples/tpch/q01_pricing_summary_report.py @@ -48,9 +48,7 @@ # want to report results for. It should be between 60-120 days before the end. DAYS_BEFORE_FINAL = 90 -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval = pa.scalar((0, 0, DAYS_BEFORE_FINAL), type=pa.month_day_nano_interval()) +interval = pa.scalar((0, DAYS_BEFORE_FINAL, 0), type=pa.month_day_nano_interval()) print("Final date in database:", greatest_ship_date) diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py index 40eab6970..9dbd81674 100644 --- a/examples/tpch/q04_order_priority_checking.py +++ b/examples/tpch/q04_order_priority_checking.py @@ -49,9 +49,7 @@ # Create a date object from the string date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval()) +interval = pa.scalar((0, INTERVAL_DAYS, 0), type=pa.month_day_nano_interval()) # Limit results to cases where commitment date before receipt date # Aggregate the results so we only get one row to join with the order table. diff --git a/examples/tpch/q05_local_supplier_volume.py b/examples/tpch/q05_local_supplier_volume.py index 27b4b84c7..f17f600a4 100644 --- a/examples/tpch/q05_local_supplier_volume.py +++ b/examples/tpch/q05_local_supplier_volume.py @@ -41,9 +41,7 @@ date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval()) +interval = pa.scalar((0, INTERVAL_DAYS, 0), type=pa.month_day_nano_interval()) # Load the dataframes we need diff --git a/examples/tpch/q06_forecasting_revenue_change.py b/examples/tpch/q06_forecasting_revenue_change.py index 3f58c5ec0..ec98aaf5e 100644 --- a/examples/tpch/q06_forecasting_revenue_change.py +++ b/examples/tpch/q06_forecasting_revenue_change.py @@ -45,9 +45,7 @@ date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval = pa.scalar((0, 0, INTERVAL_DAYS), type=pa.month_day_nano_interval()) +interval = pa.scalar((0, INTERVAL_DAYS, 0), type=pa.month_day_nano_interval()) # Load the dataframes we need diff --git a/examples/tpch/q10_returned_item_reporting.py b/examples/tpch/q10_returned_item_reporting.py index ed88c2995..78327c3ad 100644 --- a/examples/tpch/q10_returned_item_reporting.py +++ b/examples/tpch/q10_returned_item_reporting.py @@ -38,9 +38,7 @@ date_start_of_quarter = lit(datetime.strptime(DATE_START_OF_QUARTER, "%Y-%m-%d").date()) -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval_one_quarter = lit(pa.scalar((0, 0, 92), type=pa.month_day_nano_interval())) +interval_one_quarter = lit(pa.scalar((0, 92, 0), type=pa.month_day_nano_interval())) # Load the dataframes we need diff --git a/examples/tpch/q12_ship_mode_order_priority.py b/examples/tpch/q12_ship_mode_order_priority.py index d3dd7d283..150870c64 100644 --- a/examples/tpch/q12_ship_mode_order_priority.py +++ b/examples/tpch/q12_ship_mode_order_priority.py @@ -51,9 +51,7 @@ date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval = pa.scalar((0, 0, 365), type=pa.month_day_nano_interval()) +interval = pa.scalar((0, 365, 0), type=pa.month_day_nano_interval()) df = df_lineitem.filter(col("l_receiptdate") >= lit(date)).filter( diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py index 333398c17..75fa363ad 100644 --- a/examples/tpch/q14_promotion_effect.py +++ b/examples/tpch/q14_promotion_effect.py @@ -34,9 +34,8 @@ DATE = "1995-09-01" date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date()) -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval_one_month = lit(pa.scalar((0, 0, 30), type=pa.month_day_nano_interval())) + +interval_one_month = lit(pa.scalar((0, 30, 0), type=pa.month_day_nano_interval())) # Load the dataframes we need diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py index 91af34a9f..4b9e4c1dd 100644 --- a/examples/tpch/q15_top_supplier.py +++ b/examples/tpch/q15_top_supplier.py @@ -34,9 +34,8 @@ DATE = "1996-01-01" date_of_interest = lit(datetime.strptime(DATE, "%Y-%m-%d").date()) -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval_3_months = lit(pa.scalar((0, 0, 91), type=pa.month_day_nano_interval())) + +interval_3_months = lit(pa.scalar((0, 91, 0), type=pa.month_day_nano_interval())) # Load the dataframes we need diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index 4a602846f..85e7226f7 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -56,9 +56,7 @@ date = datetime.strptime(DATE_OF_INTEREST, "%Y-%m-%d").date() -# Note: this is a hack on setting the values. It should be set differently once -# https://github.com/apache/datafusion-python/issues/665 is resolved. -interval = pa.scalar((0, 0, 365), type=pa.month_day_nano_interval()) +interval = pa.scalar((0, 365, 0), type=pa.month_day_nano_interval()) # Filter down dataframes df_nation = df_nation.filter(col("n_name") == lit(NATION_OF_INTEREST)) diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index d0b823bbd..846b1a459 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -53,7 +53,6 @@ ScalarVariable, Sort, TableScan, - GetIndexedField, Not, IsNotNull, IsTrue, @@ -116,7 +115,6 @@ "SimilarTo", "ScalarVariable", "Alias", - "GetIndexedField", "Not", "IsNotNull", "IsTrue", diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 493b6b6ba..449f706c3 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -461,7 +461,6 @@ def py_flatten(arr): pytest.param( lambda col: f.list_slice(col, literal(-1), literal(2)), lambda data: [arr[-1:2] for arr in data], - marks=pytest.mark.xfail, ), [ lambda col: f.array_intersect(col, literal([3.0, 4.0])), diff --git a/python/datafusion/tests/test_imports.py b/python/datafusion/tests/test_imports.py index 2a8a3de83..bd4e7c31d 100644 --- a/python/datafusion/tests/test_imports.py +++ b/python/datafusion/tests/test_imports.py @@ -53,7 +53,6 @@ SimilarTo, ScalarVariable, Alias, - GetIndexedField, Not, IsNotNull, IsTrue, @@ -126,7 +125,6 @@ def test_class_module_is_datafusion(): SimilarTo, ScalarVariable, Alias, - GetIndexedField, Not, IsNotNull, IsTrue, diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 49959c457..cd4f864bc 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -251,6 +251,7 @@ impl DataTypeMap { pub fn map_from_scalar_to_arrow(scalar_val: &ScalarValue) -> Result { match scalar_val { ScalarValue::Boolean(_) => Ok(DataType::Boolean), + ScalarValue::Float16(_) => Ok(DataType::Float16), ScalarValue::Float32(_) => Ok(DataType::Float32), ScalarValue::Float64(_) => Ok(DataType::Float64), ScalarValue::Decimal128(_, precision, scale) => { diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 4c9e2f3fe..8ef3a563e 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -164,7 +164,7 @@ impl ExecutionPlan for DatasetExec { self.schema.clone() } - fn children(&self) -> Vec> { + fn children(&self) -> Vec<&Arc> { // this is a leaf node and has no children vec![] } diff --git a/src/expr.rs b/src/expr.rs index 2f1477457..9fd29a591 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -23,12 +23,12 @@ use std::sync::Arc; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::arrow::pyarrow::PyArrowType; +use datafusion::functions::core::expr_ext::FieldAccessor; use datafusion::scalar::ScalarValue; use datafusion_expr::{ col, expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, - lit, Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, Operator, - TryCast, + lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast, }; use crate::common::data_type::{DataTypeMap, RexType}; @@ -71,7 +71,6 @@ pub mod filter; pub mod grouping_set; pub mod in_list; pub mod in_subquery; -pub mod indexed_field; pub mod join; pub mod like; pub mod limit; @@ -216,13 +215,7 @@ impl PyExpr { } fn __getitem__(&self, key: &str) -> PyResult { - Ok(Expr::GetIndexedField(GetIndexedField::new( - Box::new(self.expr.clone()), - GetFieldAccess::NamedStructField { - name: ScalarValue::Utf8(Some(key.to_string())), - }, - )) - .into()) + Ok(self.expr.clone().field(key).into()) } #[staticmethod] @@ -263,7 +256,7 @@ impl PyExpr { pub fn rex_type(&self) -> PyResult { Ok(match self.expr { Expr::Alias(..) => RexType::Alias, - Expr::Column(..) | Expr::GetIndexedField { .. } => RexType::Reference, + Expr::Column(..) => RexType::Reference, Expr::ScalarVariable(..) | Expr::Literal(..) => RexType::Literal, Expr::BinaryExpr { .. } | Expr::Not(..) @@ -314,6 +307,11 @@ impl PyExpr { ), )), ScalarValue::Boolean(v) => Ok(v.into_py(py)), + ScalarValue::Float16(_) => Err(py_datafusion_err( + datafusion_common::DataFusionError::NotImplemented( + "ScalarValue::Float16".to_string(), + ), + )), ScalarValue::Float32(v) => Ok(v.into_py(py)), ScalarValue::Float64(v) => Ok(v.into_py(py)), ScalarValue::Decimal128(v, _, _) => Ok(v.into_py(py)), @@ -355,8 +353,10 @@ impl PyExpr { ScalarValue::TimestampMicrosecond(v, _) => Ok(v.into_py(py)), ScalarValue::TimestampNanosecond(v, _) => Ok(v.into_py(py)), ScalarValue::IntervalYearMonth(v) => Ok(v.into_py(py)), - ScalarValue::IntervalDayTime(v) => Ok(v.into_py(py)), - ScalarValue::IntervalMonthDayNano(v) => Ok(v.into_py(py)), + ScalarValue::IntervalDayTime(v) => Ok(ScalarValue::IntervalDayTime(*v).into_py(py)), + ScalarValue::IntervalMonthDayNano(v) => { + Ok(ScalarValue::IntervalMonthDayNano(*v).into_py(py)) + } ScalarValue::DurationSecond(v) => Ok(v.into_py(py)), ScalarValue::DurationMicrosecond(v) => Ok(v.into_py(py)), ScalarValue::DurationNanosecond(v) => Ok(v.into_py(py)), @@ -417,7 +417,6 @@ impl PyExpr { | Expr::IsNotFalse(expr) | Expr::IsNotUnknown(expr) | Expr::Negative(expr) - | Expr::GetIndexedField(GetIndexedField { expr, .. }) | Expr::Cast(Cast { expr, .. }) | Expr::TryCast(TryCast { expr, .. }) | Expr::Sort(Sort { expr, .. }) @@ -513,9 +512,7 @@ impl PyExpr { op, right: _, }) => format!("{op}"), - Expr::ScalarFunction(ScalarFunction { func_def, args: _ }) => { - func_def.name().to_string() - } + Expr::ScalarFunction(ScalarFunction { func, args: _ }) => func.name().to_string(), Expr::Cast { .. } => "cast".to_string(), Expr::Between { .. } => "between".to_string(), Expr::Case { .. } => "case".to_string(), @@ -674,7 +671,6 @@ pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/expr/literal.rs b/src/expr/literal.rs index bce987f76..0333432f6 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -137,12 +137,7 @@ impl PyLiteral { pub fn value_interval_day_time(&self) -> PyResult> { match &self.value { - ScalarValue::IntervalDayTime(Some(iv)) => { - let interval = *iv as u64; - let days = (interval >> 32) as i32; - let ms = interval as i32; - Ok(Some((days, ms))) - } + ScalarValue::IntervalDayTime(Some(iv)) => Ok(Some((iv.days, iv.milliseconds))), ScalarValue::IntervalDayTime(None) => Ok(None), other => Err(unexpected_literal_value(other)), } diff --git a/src/expr/signature.rs b/src/expr/signature.rs index 2f194982e..7882cebed 100644 --- a/src/expr/signature.rs +++ b/src/expr/signature.rs @@ -20,7 +20,6 @@ use pyo3::prelude::*; #[allow(dead_code)] #[pyclass(name = "Signature", module = "datafusion.expr", subclass)] -#[allow(dead_code)] #[derive(Clone)] pub struct PySignature { type_signature: TypeSignature, diff --git a/src/functions.rs b/src/functions.rs index a4bd9860f..09cdee619 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -36,32 +36,84 @@ use datafusion_expr::{ }; #[pyfunction] -#[pyo3(signature = (y, x, distinct = false, filter = None, order_by = None))] -pub fn covar_samp( - y: PyExpr, - x: PyExpr, +pub fn sum(args: PyExpr) -> PyExpr { + functions_aggregate::expr_fn::sum(args.expr).into() +} + +#[pyfunction] +pub fn covar_samp(y: PyExpr, x: PyExpr) -> PyExpr { + functions_aggregate::expr_fn::covar_samp(y.expr, x.expr).into() +} + +#[pyfunction] +pub fn covar_pop(y: PyExpr, x: PyExpr) -> PyExpr { + functions_aggregate::expr_fn::covar_pop(y.expr, x.expr).into() +} + +#[pyfunction] +pub fn median(arg: PyExpr) -> PyExpr { + functions_aggregate::expr_fn::median(arg.expr).into() +} + +#[pyfunction] +pub fn covar(y: PyExpr, x: PyExpr) -> PyExpr { + // alias for covar_samp + covar_samp(y, x) +} + +#[pyfunction] +pub fn var_samp(expression: PyExpr) -> PyExpr { + functions_aggregate::expr_fn::var_sample(expression.expr).into() +} + +#[pyfunction] +/// Alias for [`var_samp`] +pub fn var(y: PyExpr) -> PyExpr { + var_samp(y) +} + +#[pyfunction] +#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None))] +pub fn first_value( + args: Vec, distinct: bool, filter: Option, order_by: Option>, - // null_treatment: Option, ) -> PyExpr { - let filter = filter.map(|x| Box::new(x.expr)); + // TODO: allow user to select null_treatment + let null_treatment = None; + let args = args.into_iter().map(|x| x.expr).collect::>(); let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); - functions_aggregate::expr_fn::covar_samp(y.expr, x.expr, distinct, filter, order_by, None) - .into() + functions_aggregate::expr_fn::first_value( + args, + distinct, + filter.map(|x| Box::new(x.expr)), + order_by, + null_treatment, + ) + .into() } #[pyfunction] -#[pyo3(signature = (y, x, distinct = false, filter = None, order_by = None))] -pub fn covar( - y: PyExpr, - x: PyExpr, +#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None))] +pub fn last_value( + args: Vec, distinct: bool, filter: Option, order_by: Option>, ) -> PyExpr { - // alias for covar_samp - covar_samp(y, x, distinct, filter, order_by) + // TODO: allow user to select null_treatment + let null_treatment = None; + let args = args.into_iter().map(|x| x.expr).collect::>(); + let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); + functions_aggregate::expr_fn::last_value( + args, + distinct, + filter.map(|x| Box::new(x.expr)), + order_by, + null_treatment, + ) + .into() } #[pyfunction] @@ -131,17 +183,20 @@ fn list_indexof(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { } #[pyfunction] -#[pyo3(signature = (array, begin, end, stride = 1))] -fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { - let stride = ScalarValue::Int64(stride); - let stride = Expr::Literal(stride); - datafusion_functions_array::expr_fn::array_slice(array.into(), begin.into(), end.into(), stride) - .into() +#[pyo3(signature = (array, begin, end, stride = None))] +fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { + datafusion_functions_array::expr_fn::array_slice( + array.into(), + begin.into(), + end.into(), + stride.map(Into::into), + ) + .into() } #[pyfunction] -#[pyo3(signature = (array, begin, end, stride = 1))] -fn list_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { +#[pyo3(signature = (array, begin, end, stride = None))] +fn list_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { // alias of array_slice array_slice(array, begin, end, stride) } @@ -176,6 +231,28 @@ fn concat_ws(sep: String, args: Vec) -> PyResult { Ok(functions::string::expr_fn::concat_ws(lit(sep), args).into()) } +#[pyfunction] +#[pyo3(signature = (values, regex, flags = None))] +fn regexp_match(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { + Ok(functions::expr_fn::regexp_match(values.expr, regex.expr, flags.map(|x| x.expr)).into()) +} + +#[pyfunction] +/// Replaces substring(s) matching a POSIX regular expression. +fn regexp_replace( + string: PyExpr, + pattern: PyExpr, + replacement: PyExpr, + flags: Option, +) -> PyResult { + Ok(functions::expr_fn::regexp_replace( + string.into(), + pattern.into(), + replacement.into(), + flags.map(|x| x.expr), + ) + .into()) +} /// Creates a new Sort Expr #[pyfunction] fn order_by(expr: PyExpr, asc: Option, nulls_first: Option) -> PyResult { @@ -435,17 +512,11 @@ expr_fn!( "Returns x if x is not NaN otherwise returns y." ); expr_fn!(nullif, arg_1 arg_2); -expr_fn_vec!(octet_length, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); +expr_fn!(octet_length, args, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); expr_fn!(pi); expr_fn!(power, base exponent); expr_fn!(pow, power, base exponent); expr_fn!(radians, num); -expr_fn!(regexp_match, input_arg1 input_arg2); -expr_fn!( - regexp_replace, - arg1 arg2 arg3 arg4, - "Replaces substring(s) matching a POSIX regular expression." -); expr_fn!(repeat, string n, "Repeats string the specified number of times."); expr_fn!( replace, @@ -576,19 +647,14 @@ aggregate_function!(array_agg, ArrayAgg); aggregate_function!(avg, Avg); aggregate_function!(corr, Correlation); aggregate_function!(count, Count); -aggregate_function!(covar_pop, CovariancePop); aggregate_function!(grouping, Grouping); aggregate_function!(max, Max); aggregate_function!(mean, Avg); -aggregate_function!(median, Median); aggregate_function!(min, Min); -aggregate_function!(sum, Sum); aggregate_function!(stddev, Stddev); aggregate_function!(stddev_pop, StddevPop); aggregate_function!(stddev_samp, Stddev); -aggregate_function!(var, Variance); aggregate_function!(var_pop, VariancePop); -aggregate_function!(var_samp, Variance); aggregate_function!(regr_avgx, RegrAvgx); aggregate_function!(regr_avgy, RegrAvgy); aggregate_function!(regr_count, RegrCount); @@ -598,8 +664,6 @@ aggregate_function!(regr_slope, RegrSlope); aggregate_function!(regr_sxx, RegrSXX); aggregate_function!(regr_sxy, RegrSXY); aggregate_function!(regr_syy, RegrSYY); -aggregate_function!(first_value, FirstValue); -aggregate_function!(last_value, LastValue); aggregate_function!(bit_and, BitAnd); aggregate_function!(bit_or, BitOr); aggregate_function!(bit_xor, BitXor); diff --git a/src/physical_plan.rs b/src/physical_plan.rs index 51032f2d4..c97c1a96e 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -40,7 +40,7 @@ impl PyExecutionPlan { self.plan .children() .iter() - .map(|p| p.to_owned().into()) + .map(|&p| p.to_owned().into()) .collect() } From 1f49d4615ebc464394d38b2dea0fa130749ea55f Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sun, 16 Jun 2024 10:53:00 -0500 Subject: [PATCH 154/413] use ScalarValue::to_pyarrow to convert to python object (#731) Closes #729 --- Cargo.lock | 1 + Cargo.toml | 1 + src/expr.rs | 91 ++--------------------------------------------------- 3 files changed, 5 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb674d0f2..2e437a03f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1032,6 +1032,7 @@ dependencies = [ name = "datafusion-python" version = "39.0.0" dependencies = [ + "arrow", "async-trait", "datafusion", "datafusion-common", diff --git a/Cargo.toml b/Cargo.toml index 85a19d1e5..7285cf3eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38", "gil-refs"] } +arrow = { version = "52", feature = ["pyarrow"] } datafusion = { version = "39.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } datafusion-common = { version = "39.0.0", features = ["pyarrow"] } datafusion-expr = "39.0.0" diff --git a/src/expr.rs b/src/expr.rs index 9fd29a591..09a773c4d 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -21,6 +21,7 @@ use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use std::sync::Arc; +use arrow::pyarrow::ToPyArrow; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::functions::core::expr_ext::FieldAccessor; @@ -32,7 +33,7 @@ use datafusion_expr::{ }; use crate::common::data_type::{DataTypeMap, RexType}; -use crate::errors::{py_datafusion_err, py_runtime_err, py_type_err, DataFusionError}; +use crate::errors::{py_runtime_err, py_type_err, DataFusionError}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; @@ -300,93 +301,7 @@ impl PyExpr { /// Extracts the Expr value into a PyObject that can be shared with Python pub fn python_value(&self, py: Python) -> PyResult { match &self.expr { - Expr::Literal(scalar_value) => match scalar_value { - ScalarValue::Null => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::Null".to_string(), - ), - )), - ScalarValue::Boolean(v) => Ok(v.into_py(py)), - ScalarValue::Float16(_) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::Float16".to_string(), - ), - )), - ScalarValue::Float32(v) => Ok(v.into_py(py)), - ScalarValue::Float64(v) => Ok(v.into_py(py)), - ScalarValue::Decimal128(v, _, _) => Ok(v.into_py(py)), - ScalarValue::Decimal256(_, _, _) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::Decimal256".to_string(), - ), - )), - ScalarValue::Int8(v) => Ok(v.into_py(py)), - ScalarValue::Int16(v) => Ok(v.into_py(py)), - ScalarValue::Int32(v) => Ok(v.into_py(py)), - ScalarValue::Int64(v) => Ok(v.into_py(py)), - ScalarValue::UInt8(v) => Ok(v.into_py(py)), - ScalarValue::UInt16(v) => Ok(v.into_py(py)), - ScalarValue::UInt32(v) => Ok(v.into_py(py)), - ScalarValue::UInt64(v) => Ok(v.into_py(py)), - ScalarValue::Utf8(v) => Ok(v.clone().into_py(py)), - ScalarValue::LargeUtf8(v) => Ok(v.clone().into_py(py)), - ScalarValue::Binary(v) => Ok(v.clone().into_py(py)), - ScalarValue::FixedSizeBinary(_, _) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::FixedSizeBinary".to_string(), - ), - )), - ScalarValue::LargeBinary(v) => Ok(v.clone().into_py(py)), - ScalarValue::List(_) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::List".to_string(), - ), - )), - ScalarValue::Date32(v) => Ok(v.into_py(py)), - ScalarValue::Date64(v) => Ok(v.into_py(py)), - ScalarValue::Time32Second(v) => Ok(v.into_py(py)), - ScalarValue::Time32Millisecond(v) => Ok(v.into_py(py)), - ScalarValue::Time64Microsecond(v) => Ok(v.into_py(py)), - ScalarValue::Time64Nanosecond(v) => Ok(v.into_py(py)), - ScalarValue::TimestampSecond(v, _) => Ok(v.into_py(py)), - ScalarValue::TimestampMillisecond(v, _) => Ok(v.into_py(py)), - ScalarValue::TimestampMicrosecond(v, _) => Ok(v.into_py(py)), - ScalarValue::TimestampNanosecond(v, _) => Ok(v.into_py(py)), - ScalarValue::IntervalYearMonth(v) => Ok(v.into_py(py)), - ScalarValue::IntervalDayTime(v) => Ok(ScalarValue::IntervalDayTime(*v).into_py(py)), - ScalarValue::IntervalMonthDayNano(v) => { - Ok(ScalarValue::IntervalMonthDayNano(*v).into_py(py)) - } - ScalarValue::DurationSecond(v) => Ok(v.into_py(py)), - ScalarValue::DurationMicrosecond(v) => Ok(v.into_py(py)), - ScalarValue::DurationNanosecond(v) => Ok(v.into_py(py)), - ScalarValue::DurationMillisecond(v) => Ok(v.into_py(py)), - ScalarValue::Struct(_) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::Struct".to_string(), - ), - )), - ScalarValue::Dictionary(_, _) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::Dictionary".to_string(), - ), - )), - ScalarValue::FixedSizeList(_) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::FixedSizeList".to_string(), - ), - )), - ScalarValue::LargeList(_) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::LargeList".to_string(), - ), - )), - ScalarValue::Union(_, _, _) => Err(py_datafusion_err( - datafusion_common::DataFusionError::NotImplemented( - "ScalarValue::Union".to_string(), - ), - )), - }, + Expr::Literal(scalar_value) => Ok(scalar_value.to_pyarrow(py)?), _ => Err(py_type_err(format!( "Non Expr::Literal encountered in types: {:?}", &self.expr From c7ea90d3cc17d72e26ef67c12efb4a2bd7bef942 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 18 Jun 2024 09:37:22 -0500 Subject: [PATCH 155/413] Pyo3 `Bound<'py, T>` api (#734) * remove gil-refs feature from pyo3 * migrate module instantiation to Bound api * migrate utils.rs to Bound api * migrate config.rs to Bound api * migrate context.rs to Bound api * migrate udaf.rs to Bound api * migrate pyarrow_filter_expression to Bound api * migrate dataframe.rs to Bound api * migrade dataset and dataset_exec to Bound api * migrate substrait.rs to Bound api --- Cargo.toml | 3 +- src/common.rs | 2 +- src/config.rs | 2 +- src/context.rs | 23 ++++++----- src/dataframe.rs | 67 +++++++++++++++++++------------- src/dataset.rs | 13 ++++--- src/dataset_exec.rs | 36 ++++++++--------- src/expr.rs | 2 +- src/functions.rs | 2 +- src/lib.rs | 36 ++++++++--------- src/pyarrow_filter_expression.rs | 23 +++++------ src/store.rs | 2 +- src/substrait.rs | 8 ++-- src/udaf.rs | 28 ++++++------- src/utils.rs | 4 +- 15 files changed, 136 insertions(+), 115 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7285cf3eb..78b46e449 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,7 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" -pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38", "gil-refs"] } +pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "52", feature = ["pyarrow"] } datafusion = { version = "39.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } datafusion-common = { version = "39.0.0", features = ["pyarrow"] } @@ -67,3 +67,4 @@ crate-type = ["cdylib", "rlib"] [profile.release] lto = true codegen-units = 1 + \ No newline at end of file diff --git a/src/common.rs b/src/common.rs index 682639aca..44c557ce7 100644 --- a/src/common.rs +++ b/src/common.rs @@ -23,7 +23,7 @@ pub mod function; pub mod schema; /// Initializes the `common` module to match the pattern of `datafusion-common` https://docs.rs/datafusion-common/18.0.0/datafusion_common/index.html -pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { +pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/config.rs b/src/config.rs index 228f95a0b..82a4f93ab 100644 --- a/src/config.rs +++ b/src/config.rs @@ -65,7 +65,7 @@ impl PyConfig { /// Get all configuration options pub fn get_all(&mut self, py: Python) -> PyResult { - let dict = PyDict::new(py); + let dict = PyDict::new_bound(py); let options = self.config.to_owned(); for entry in options.entries() { dict.set_item(entry.key, entry.value.clone().into_py(py))?; diff --git a/src/context.rs b/src/context.rs index 9462d0b86..ec63adb26 100644 --- a/src/context.rs +++ b/src/context.rs @@ -291,11 +291,11 @@ impl PySessionContext { pub fn register_object_store( &mut self, scheme: &str, - store: &PyAny, + store: &Bound<'_, PyAny>, host: Option<&str>, ) -> PyResult<()> { let res: Result<(Arc, String), PyErr> = - match StorageContexts::extract(store) { + match StorageContexts::extract_bound(store) { Ok(store) => match store { StorageContexts::AmazonS3(s3) => Ok((s3.inner, s3.bucket_name)), StorageContexts::GoogleCloudStorage(gcs) => Ok((gcs.inner, gcs.bucket_name)), @@ -443,8 +443,8 @@ impl PySessionContext { ) -> PyResult { Python::with_gil(|py| { // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import("pyarrow")?.getattr("Table")?; - let args = PyTuple::new(py, &[data]); + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[data]); let table = table_class.call_method1("from_pylist", args)?.into(); // Convert Arrow Table to datafusion DataFrame @@ -463,8 +463,8 @@ impl PySessionContext { ) -> PyResult { Python::with_gil(|py| { // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import("pyarrow")?.getattr("Table")?; - let args = PyTuple::new(py, &[data]); + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[data]); let table = table_class.call_method1("from_pydict", args)?.into(); // Convert Arrow Table to datafusion DataFrame @@ -507,8 +507,8 @@ impl PySessionContext { ) -> PyResult { Python::with_gil(|py| { // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import("pyarrow")?.getattr("Table")?; - let args = PyTuple::new(py, &[data]); + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[data]); let table = table_class.call_method1("from_pandas", args)?.into(); // Convert Arrow Table to datafusion DataFrame @@ -710,7 +710,12 @@ impl PySessionContext { } // Registers a PyArrow.Dataset - pub fn register_dataset(&self, name: &str, dataset: &PyAny, py: Python) -> PyResult<()> { + pub fn register_dataset( + &self, + name: &str, + dataset: &Bound<'_, PyAny>, + py: Python, + ) -> PyResult<()> { let table: Arc = Arc::new(Dataset::new(dataset, py)?); self.ctx diff --git a/src/dataframe.rs b/src/dataframe.rs index 8f4514398..1b91067d5 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -28,6 +28,7 @@ use datafusion::prelude::*; use datafusion_common::UnnestOptions; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; +use pyo3::pybacked::PyBackedStr; use pyo3::types::PyTuple; use tokio::task::JoinHandle; @@ -56,23 +57,25 @@ impl PyDataFrame { #[pymethods] impl PyDataFrame { - fn __getitem__(&self, key: PyObject) -> PyResult { - Python::with_gil(|py| { - if let Ok(key) = key.extract::<&str>(py) { - self.select_columns(vec![key]) - } else if let Ok(tuple) = key.extract::<&PyTuple>(py) { - let keys = tuple - .iter() - .map(|item| item.extract::<&str>()) - .collect::>>()?; - self.select_columns(keys) - } else if let Ok(keys) = key.extract::>(py) { - self.select_columns(keys) - } else { - let message = "DataFrame can only be indexed by string index or indices"; - Err(PyTypeError::new_err(message)) - } - }) + /// Enable selection for `df[col]`, `df[col1, col2, col3]`, and `df[[col1, col2, col3]]` + fn __getitem__(&self, key: Bound<'_, PyAny>) -> PyResult { + if let Ok(key) = key.extract::() { + // df[col] + self.select_columns(vec![key]) + } else if let Ok(tuple) = key.extract::<&PyTuple>() { + // df[col1, col2, col3] + let keys = tuple + .iter() + .map(|item| item.extract::()) + .collect::>>()?; + self.select_columns(keys) + } else if let Ok(keys) = key.extract::>() { + // df[[col1, col2, col3]] + self.select_columns(keys) + } else { + let message = "DataFrame can only be indexed by string index or indices"; + Err(PyTypeError::new_err(message)) + } } fn __repr__(&self, py: Python) -> PyResult { @@ -98,7 +101,8 @@ impl PyDataFrame { } #[pyo3(signature = (*args))] - fn select_columns(&self, args: Vec<&str>) -> PyResult { + fn select_columns(&self, args: Vec) -> PyResult { + let args = args.iter().map(|s| s.as_ref()).collect::>(); let df = self.df.as_ref().clone().select_columns(&args)?; Ok(Self::new(df)) } @@ -194,7 +198,7 @@ impl PyDataFrame { fn join( &self, right: PyDataFrame, - join_keys: (Vec<&str>, Vec<&str>), + join_keys: (Vec, Vec), how: &str, ) -> PyResult { let join_type = match how { @@ -212,11 +216,22 @@ impl PyDataFrame { } }; + let left_keys = join_keys + .0 + .iter() + .map(|s| s.as_ref()) + .collect::>(); + let right_keys = join_keys + .1 + .iter() + .map(|s| s.as_ref()) + .collect::>(); + let df = self.df.as_ref().clone().join( right.df.as_ref().clone(), join_type, - &join_keys.0, - &join_keys.1, + &left_keys, + &right_keys, None, )?; Ok(Self::new(df)) @@ -414,8 +429,8 @@ impl PyDataFrame { Python::with_gil(|py| { // Instantiate pyarrow Table object and use its from_batches method - let table_class = py.import("pyarrow")?.getattr("Table")?; - let args = PyTuple::new(py, &[batches, schema]); + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[batches, schema]); let table: PyObject = table_class.call_method1("from_batches", args)?.into(); Ok(table) }) @@ -489,8 +504,8 @@ impl PyDataFrame { let table = self.to_arrow_table(py)?; Python::with_gil(|py| { - let dataframe = py.import("polars")?.getattr("DataFrame")?; - let args = PyTuple::new(py, &[table]); + let dataframe = py.import_bound("polars")?.getattr("DataFrame")?; + let args = PyTuple::new_bound(py, &[table]); let result: PyObject = dataframe.call1(args)?.into(); Ok(result) }) @@ -514,7 +529,7 @@ fn print_dataframe(py: Python, df: DataFrame) -> PyResult<()> { // Import the Python 'builtins' module to access the print function // Note that println! does not print to the Python debug console and is not visible in notebooks for instance - let print = py.import("builtins")?.getattr("print")?; + let print = py.import_bound("builtins")?.getattr("print")?; print.call1((result,))?; Ok(()) } diff --git a/src/dataset.rs b/src/dataset.rs index fcbb503c0..724b4af76 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -46,13 +46,14 @@ pub(crate) struct Dataset { impl Dataset { // Creates a Python PyArrow.Dataset - pub fn new(dataset: &PyAny, py: Python) -> PyResult { + pub fn new(dataset: &Bound<'_, PyAny>, py: Python) -> PyResult { // Ensure that we were passed an instance of pyarrow.dataset.Dataset - let ds = PyModule::import(py, "pyarrow.dataset")?; - let ds_type: &PyType = ds.getattr("Dataset")?.downcast()?; + let ds = PyModule::import_bound(py, "pyarrow.dataset")?; + let ds_attr = ds.getattr("Dataset")?; + let ds_type = ds_attr.downcast::()?; if dataset.is_instance(ds_type)? { Ok(Dataset { - dataset: dataset.into(), + dataset: dataset.clone().unbind(), }) } else { Err(PyValueError::new_err( @@ -73,7 +74,7 @@ impl TableProvider for Dataset { /// Get a reference to the schema for this table fn schema(&self) -> SchemaRef { Python::with_gil(|py| { - let dataset = self.dataset.as_ref(py); + let dataset = self.dataset.bind(py); // This can panic but since we checked that self.dataset is a pyarrow.dataset.Dataset it should never Arc::new( dataset @@ -108,7 +109,7 @@ impl TableProvider for Dataset { ) -> DFResult> { Python::with_gil(|py| { let plan: Arc = Arc::new( - DatasetExec::new(py, self.dataset.as_ref(py), projection.cloned(), filters) + DatasetExec::new(py, self.dataset.bind(py), projection.cloned(), filters) .map_err(|err| DataFusionError::External(Box::new(err)))?, ); Ok(plan) diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 8ef3a563e..240c86486 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -53,7 +53,7 @@ impl Iterator for PyArrowBatchesAdapter { fn next(&mut self) -> Option { Python::with_gil(|py| { - let mut batches: &PyIterator = self.batches.as_ref(py); + let mut batches = self.batches.clone().into_bound(py); Some( batches .next()? @@ -79,7 +79,7 @@ pub(crate) struct DatasetExec { impl DatasetExec { pub fn new( py: Python, - dataset: &PyAny, + dataset: &Bound<'_, PyAny>, projection: Option>, filters: &[Expr], ) -> Result { @@ -103,7 +103,7 @@ impl DatasetExec { }) .transpose()?; - let kwargs = PyDict::new(py); + let kwargs = PyDict::new_bound(py); kwargs.set_item("columns", columns.clone())?; kwargs.set_item( @@ -111,7 +111,7 @@ impl DatasetExec { filter_expr.as_ref().map(|expr| expr.clone_ref(py)), )?; - let scanner = dataset.call_method("scanner", (), Some(kwargs))?; + let scanner = dataset.call_method("scanner", (), Some(&kwargs))?; let schema = Arc::new( scanner @@ -120,19 +120,17 @@ impl DatasetExec { .0, ); - let builtins = Python::import(py, "builtins")?; + let builtins = Python::import_bound(py, "builtins")?; let pylist = builtins.getattr("list")?; // Get the fragments or partitions of the dataset - let fragments_iterator: &PyAny = dataset.call_method1( + let fragments_iterator: Bound<'_, PyAny> = dataset.call_method1( "get_fragments", (filter_expr.as_ref().map(|expr| expr.clone_ref(py)),), )?; - let fragments: &PyList = pylist - .call1((fragments_iterator,))? - .downcast() - .map_err(PyErr::from)?; + let fragments_iter = pylist.call1((fragments_iterator,))?; + let fragments = fragments_iter.downcast::().map_err(PyErr::from)?; let projected_statistics = Statistics::new_unknown(&schema); let plan_properties = datafusion::physical_plan::PlanProperties::new( @@ -142,9 +140,9 @@ impl DatasetExec { ); Ok(DatasetExec { - dataset: dataset.into(), + dataset: dataset.clone().unbind(), schema, - fragments: fragments.into(), + fragments: fragments.clone().unbind(), columns, filter_expr, projected_statistics, @@ -183,8 +181,8 @@ impl ExecutionPlan for DatasetExec { ) -> DFResult { let batch_size = context.session_config().batch_size(); Python::with_gil(|py| { - let dataset = self.dataset.as_ref(py); - let fragments = self.fragments.as_ref(py); + let dataset = self.dataset.bind(py); + let fragments = self.fragments.bind(py); let fragment = fragments .get_item(partition) .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; @@ -193,7 +191,7 @@ impl ExecutionPlan for DatasetExec { let dataset_schema = dataset .getattr("schema") .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; - let kwargs = PyDict::new(py); + let kwargs = PyDict::new_bound(py); kwargs .set_item("columns", self.columns.clone()) .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; @@ -207,7 +205,7 @@ impl ExecutionPlan for DatasetExec { .set_item("batch_size", batch_size) .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; let scanner = fragment - .call_method("scanner", (dataset_schema,), Some(kwargs)) + .call_method("scanner", (dataset_schema,), Some(&kwargs)) .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; let schema: SchemaRef = Arc::new( scanner @@ -215,7 +213,7 @@ impl ExecutionPlan for DatasetExec { .and_then(|schema| Ok(schema.extract::>()?.0)) .map_err(|err| InnerDataFusionError::External(Box::new(err)))?, ); - let record_batches: &PyIterator = scanner + let record_batches: Bound<'_, PyIterator> = scanner .call_method0("to_batches") .map_err(|err| InnerDataFusionError::External(Box::new(err)))? .iter() @@ -264,7 +262,7 @@ impl ExecutionPlanProperties for DatasetExec { impl DisplayAs for DatasetExec { fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { Python::with_gil(|py| { - let number_of_fragments = self.fragments.as_ref(py).len(); + let number_of_fragments = self.fragments.bind(py).len(); match t { DisplayFormatType::Default | DisplayFormatType::Verbose => { let projected_columns: Vec = self @@ -274,7 +272,7 @@ impl DisplayAs for DatasetExec { .map(|x| x.name().to_owned()) .collect(); if let Some(filter_expr) = &self.filter_expr { - let filter_expr = filter_expr.as_ref(py).str().or(Err(std::fmt::Error))?; + let filter_expr = filter_expr.bind(py).str().or(Err(std::fmt::Error))?; write!( f, "DatasetExec: number_of_fragments={}, filter_expr={}, projection=[{}]", diff --git a/src/expr.rs b/src/expr.rs index 09a773c4d..dc1de669b 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -553,7 +553,7 @@ impl PyExpr { } /// Initializes the `expr` module to match the pattern of `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/ -pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { +pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/functions.rs b/src/functions.rs index 09cdee619..8e395ae4f 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -670,7 +670,7 @@ aggregate_function!(bit_xor, BitXor); aggregate_function!(bool_and, BoolAnd); aggregate_function!(bool_or, BoolOr); -pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { +pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(abs))?; m.add_wrapped(wrap_pyfunction!(acos))?; m.add_wrapped(wrap_pyfunction!(acosh))?; diff --git a/src/lib.rs b/src/lib.rs index a696ebff4..71c27e1ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,7 @@ pub(crate) struct TokioRuntime(tokio::runtime::Runtime); /// The higher-level public API is defined in pure python files under the /// datafusion directory. #[pymodule] -fn _internal(py: Python, m: &PyModule) -> PyResult<()> { +fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { // Register the Tokio Runtime as a module attribute so we can reuse it m.add( "runtime", @@ -94,35 +94,35 @@ fn _internal(py: Python, m: &PyModule) -> PyResult<()> { m.add_class::()?; // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/ - let common = PyModule::new(py, "common")?; - common::init_module(common)?; - m.add_submodule(common)?; + let common = PyModule::new_bound(py, "common")?; + common::init_module(&common)?; + m.add_submodule(&common)?; // Register `expr` as a submodule. Matching `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/ - let expr = PyModule::new(py, "expr")?; - expr::init_module(expr)?; - m.add_submodule(expr)?; + let expr = PyModule::new_bound(py, "expr")?; + expr::init_module(&expr)?; + m.add_submodule(&expr)?; // Register the functions as a submodule - let funcs = PyModule::new(py, "functions")?; - functions::init_module(funcs)?; - m.add_submodule(funcs)?; + let funcs = PyModule::new_bound(py, "functions")?; + functions::init_module(&funcs)?; + m.add_submodule(&funcs)?; - let store = PyModule::new(py, "object_store")?; - store::init_module(store)?; - m.add_submodule(store)?; + let store = PyModule::new_bound(py, "object_store")?; + store::init_module(&store)?; + m.add_submodule(&store)?; // Register substrait as a submodule #[cfg(feature = "substrait")] - setup_substrait_module(py, m)?; + setup_substrait_module(py, &m)?; Ok(()) } #[cfg(feature = "substrait")] -fn setup_substrait_module(py: Python, m: &PyModule) -> PyResult<()> { - let substrait = PyModule::new(py, "substrait")?; - substrait::init_module(substrait)?; - m.add_submodule(substrait)?; +fn setup_substrait_module(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { + let substrait = PyModule::new_bound(py, "substrait")?; + substrait::init_module(&substrait)?; + m.add_submodule(&substrait)?; Ok(()) } diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index 64124fb12..fca885121 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -32,9 +32,9 @@ pub(crate) struct PyArrowFilterExpression(PyObject); fn operator_to_py<'py>( operator: &Operator, - op: &'py PyModule, -) -> Result<&'py PyAny, DataFusionError> { - let py_op: &PyAny = match operator { + op: &Bound<'py, PyModule>, +) -> Result, DataFusionError> { + let py_op: Bound<'_, PyAny> = match operator { Operator::Eq => op.getattr("eq")?, Operator::NotEq => op.getattr("ne")?, Operator::Lt => op.getattr("lt")?, @@ -96,9 +96,9 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { // https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Expression.html#pyarrow-dataset-expression fn try_from(expr: &Expr) -> Result { Python::with_gil(|py| { - let pc = Python::import(py, "pyarrow.compute")?; - let op_module = Python::import(py, "operator")?; - let pc_expr: Result<&PyAny, DataFusionError> = match expr { + let pc = Python::import_bound(py, "pyarrow.compute")?; + let op_module = Python::import_bound(py, "operator")?; + let pc_expr: Result, DataFusionError> = match expr { Expr::Column(Column { name, .. }) => Ok(pc.getattr("field")?.call1((name,))?), Expr::Literal(v) => match v { ScalarValue::Boolean(Some(b)) => Ok(pc.getattr("scalar")?.call1((*b,))?), @@ -118,7 +118,7 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { ))), }, Expr::BinaryExpr(BinaryExpr { left, op, right }) => { - let operator = operator_to_py(op, op_module)?; + let operator = operator_to_py(op, &op_module)?; let left = PyArrowFilterExpression::try_from(left.as_ref())?.0; let right = PyArrowFilterExpression::try_from(right.as_ref())?.0; Ok(operator.call1((left, right))?) @@ -131,14 +131,15 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { Expr::IsNotNull(expr) => { let py_expr = PyArrowFilterExpression::try_from(expr.as_ref())? .0 - .into_ref(py); + .into_bound(py); Ok(py_expr.call_method0("is_valid")?) } Expr::IsNull(expr) => { let expr = PyArrowFilterExpression::try_from(expr.as_ref())? .0 - .into_ref(py); - Ok(expr.call_method1("is_null", (expr,))?) + .into_bound(py); + // TODO: this expression does not seems like it should be `call_method0` + Ok(expr.clone().call_method1("is_null", (expr,))?) } Expr::Between(Between { expr, @@ -168,7 +169,7 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { }) => { let expr = PyArrowFilterExpression::try_from(expr.as_ref())? .0 - .into_ref(py); + .into_bound(py); let scalars = extract_scalar_list(list, py)?; let ret = expr.call_method1("isin", (scalars,))?; let invert = op_module.getattr("invert")?; diff --git a/src/store.rs b/src/store.rs index 542cfa925..846d96a6d 100644 --- a/src/store.rs +++ b/src/store.rs @@ -219,7 +219,7 @@ impl PyAmazonS3Context { } } -pub(crate) fn init_module(m: &PyModule) -> PyResult<()> { +pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/substrait.rs b/src/substrait.rs index ff83f6f79..1e9e16c7b 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -40,7 +40,7 @@ impl PyPlan { self.plan .encode(&mut proto_bytes) .map_err(DataFusionError::EncodeError)?; - Ok(PyBytes::new(py, &proto_bytes).into()) + Ok(PyBytes::new_bound(py, &proto_bytes).unbind().into()) } } @@ -76,7 +76,7 @@ impl PySubstraitSerializer { pub fn serialize_to_plan(sql: &str, ctx: PySessionContext, py: Python) -> PyResult { match PySubstraitSerializer::serialize_bytes(sql, ctx, py) { Ok(proto_bytes) => { - let proto_bytes: &PyBytes = proto_bytes.as_ref(py).downcast().unwrap(); + let proto_bytes = proto_bytes.bind(py).downcast::().unwrap(); PySubstraitSerializer::deserialize_bytes(proto_bytes.as_bytes().to_vec(), py) } Err(e) => Err(py_datafusion_err(e)), @@ -87,7 +87,7 @@ impl PySubstraitSerializer { pub fn serialize_bytes(sql: &str, ctx: PySessionContext, py: Python) -> PyResult { let proto_bytes: Vec = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx)) .map_err(DataFusionError::from)?; - Ok(PyBytes::new(py, &proto_bytes).into()) + Ok(PyBytes::new_bound(py, &proto_bytes).unbind().into()) } #[staticmethod] @@ -140,7 +140,7 @@ impl PySubstraitConsumer { } } -pub fn init_module(m: &PyModule) -> PyResult<()> { +pub fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/udaf.rs b/src/udaf.rs index 9aea761cd..7b5e03668 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use pyo3::{prelude::*, types::PyBool, types::PyTuple}; +use pyo3::{prelude::*, types::PyTuple}; use datafusion::arrow::array::{Array, ArrayRef}; use datafusion::arrow::datatypes::DataType; @@ -42,12 +42,12 @@ impl RustAccumulator { impl Accumulator for RustAccumulator { fn state(&mut self) -> Result> { - Python::with_gil(|py| self.accum.as_ref(py).call_method0("state")?.extract()) + Python::with_gil(|py| self.accum.bind(py).call_method0("state")?.extract()) .map_err(|e| DataFusionError::Execution(format!("{e}"))) } fn evaluate(&mut self) -> Result { - Python::with_gil(|py| self.accum.as_ref(py).call_method0("evaluate")?.extract()) + Python::with_gil(|py| self.accum.bind(py).call_method0("evaluate")?.extract()) .map_err(|e| DataFusionError::Execution(format!("{e}"))) } @@ -58,11 +58,11 @@ impl Accumulator for RustAccumulator { .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); - let py_args = PyTuple::new(py, py_args); + let py_args = PyTuple::new_bound(py, py_args); // 2. call function self.accum - .as_ref(py) + .bind(py) .call_method1("update", py_args) .map_err(|e| DataFusionError::Execution(format!("{e}")))?; @@ -82,7 +82,7 @@ impl Accumulator for RustAccumulator { // 2. call merge self.accum - .as_ref(py) + .bind(py) .call_method1("merge", (state,)) .map_err(|e| DataFusionError::Execution(format!("{e}")))?; @@ -101,11 +101,11 @@ impl Accumulator for RustAccumulator { .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); - let py_args = PyTuple::new(py, py_args); + let py_args = PyTuple::new_bound(py, py_args); // 2. call function self.accum - .as_ref(py) + .bind(py) .call_method1("retract_batch", py_args) .map_err(|e| DataFusionError::Execution(format!("{e}")))?; @@ -114,12 +114,12 @@ impl Accumulator for RustAccumulator { } fn supports_retract_batch(&self) -> bool { - Python::with_gil(|py| { - let x: Result<&PyAny, PyErr> = - self.accum.as_ref(py).call_method0("supports_retract_batch"); - let x: &PyAny = x.unwrap_or(PyBool::new(py, false)); - x.extract().unwrap_or(false) - }) + Python::with_gil( + |py| match self.accum.bind(py).call_method0("supports_retract_batch") { + Ok(x) => x.extract().unwrap_or(false), + Err(_) => false, + }, + ) } } diff --git a/src/utils.rs b/src/utils.rs index 62cf07d9e..4334f86cd 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -24,13 +24,13 @@ use tokio::runtime::Runtime; /// Utility to get the Tokio Runtime from Python pub(crate) fn get_tokio_runtime(py: Python) -> PyRef { - let datafusion = py.import("datafusion._internal").unwrap(); + let datafusion = py.import_bound("datafusion._internal").unwrap(); let tmp = datafusion.getattr("runtime").unwrap(); match tmp.extract::>() { Ok(runtime) => runtime, Err(_e) => { let rt = TokioRuntime(tokio::runtime::Runtime::new().unwrap()); - let obj: &PyAny = Py::new(py, rt).unwrap().into_ref(py); + let obj: Bound<'_, TokioRuntime> = Py::new(py, rt).unwrap().into_bound(py); obj.extract().unwrap() } } From 532dc38dd0372447eeaee1e911caa38ccb7b450b Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Wed, 19 Jun 2024 11:17:50 -0500 Subject: [PATCH 156/413] github test action: drop python 3.7, add python 3.12 (#736) * add locked test requirements for python 3.12 * update github test action to drop 3.7 and add 3.12 * remove requirements-37.txt * fix test by using np.nan instead of np.NaN numpy 2.0 removed np.NaN. Ref: https://numpy.org/devdocs/release/2.0.0-notes.html#numpy-2-0-python-api-removals * update python-310 requirements.txt --- .github/workflows/test.yaml | 12 +- python/datafusion/tests/generic.py | 2 +- requirements-310.txt | 131 +++++++------- requirements-312.txt | 184 ++++++++++++++++++++ requirements-37.txt | 264 ----------------------------- 5 files changed, 260 insertions(+), 333 deletions(-) create mode 100644 requirements-312.txt delete mode 100644 requirements-37.txt diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 897c384c6..4f47dc984 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,12 +35,10 @@ jobs: python-version: - "3.10" - "3.11" + - "3.12" toolchain: - "stable" - # build stable for only 3.7 - include: - - python-version: "3.7" - toolchain: "stable" + steps: - uses: actions/checkout@v4 @@ -82,12 +80,12 @@ jobs: command: clippy args: --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure - - name: Create Virtualenv (3.7) - if: ${{ matrix.python-version == '3.7' }} + - name: Create Virtualenv (3.12) + if: ${{ matrix.python-version == '3.12' }} run: | python -m venv venv source venv/bin/activate - pip install -r requirements-37.txt + pip install -r requirements-312.txt - name: Create Virtualenv (3.10) if: ${{ matrix.python-version == '3.10' }} diff --git a/python/datafusion/tests/generic.py b/python/datafusion/tests/generic.py index 07399792c..0177e2df0 100644 --- a/python/datafusion/tests/generic.py +++ b/python/datafusion/tests/generic.py @@ -40,7 +40,7 @@ def data_with_nans(): np.random.seed(0) data = np.random.normal(0, 0.01, size=50) mask = np.random.randint(0, 2, size=50) - data[mask == 0] = np.NaN + data[mask == 0] = np.nan return data diff --git a/requirements-310.txt b/requirements-310.txt index 9c3b6e141..d7d25f3f1 100644 --- a/requirements-310.txt +++ b/requirements-310.txt @@ -60,49 +60,58 @@ mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 # via mypy -numpy==1.26.4 \ - --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ - --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ - --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ - --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ - --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ - --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ - --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ - --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ - --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ - --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ - --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ - --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ - --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ - --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ - --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ - --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ - --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ - --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ - --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ - --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ - --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ - --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ - --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ - --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ - --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ - --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ - --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ - --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ - --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ - --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ - --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ - --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ - --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ - --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ - --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ - --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f +numpy==2.0.0 \ + --hash=sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f \ + --hash=sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238 \ + --hash=sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f \ + --hash=sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95 \ + --hash=sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a \ + --hash=sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a \ + --hash=sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2 \ + --hash=sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2 \ + --hash=sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f \ + --hash=sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609 \ + --hash=sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f \ + --hash=sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad \ + --hash=sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86 \ + --hash=sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65 \ + --hash=sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb \ + --hash=sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995 \ + --hash=sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a \ + --hash=sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85 \ + --hash=sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4 \ + --hash=sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275 \ + --hash=sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1 \ + --hash=sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196 \ + --hash=sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d \ + --hash=sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e \ + --hash=sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514 \ + --hash=sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f \ + --hash=sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6 \ + --hash=sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4 \ + --hash=sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44 \ + --hash=sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df \ + --hash=sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581 \ + --hash=sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787 \ + --hash=sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5 \ + --hash=sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc \ + --hash=sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871 \ + --hash=sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54 \ + --hash=sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2 \ + --hash=sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98 \ + --hash=sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9 \ + --hash=sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864 \ + --hash=sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de \ + --hash=sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289 \ + --hash=sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b \ + --hash=sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c \ + --hash=sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9 # via # -r requirements.in # pyarrow -packaging==24.0 \ - --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ - --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 # via pytest pluggy==1.5.0 \ --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ @@ -150,24 +159,24 @@ pytest==8.2.2 \ --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 # via -r requirements.in -ruff==0.4.8 \ - --hash=sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780 \ - --hash=sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268 \ - --hash=sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb \ - --hash=sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed \ - --hash=sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9 \ - --hash=sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a \ - --hash=sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b \ - --hash=sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883 \ - --hash=sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066 \ - --hash=sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199 \ - --hash=sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764 \ - --hash=sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3 \ - --hash=sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45 \ - --hash=sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d \ - --hash=sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa \ - --hash=sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913 \ - --hash=sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc +ruff==0.4.9 \ + --hash=sha256:06b60f91bfa5514bb689b500a25ba48e897d18fea14dce14b48a0c40d1635893 \ + --hash=sha256:0e8e7b95673f22e0efd3571fb5b0cf71a5eaaa3cc8a776584f3b2cc878e46bff \ + --hash=sha256:2d45ddc6d82e1190ea737341326ecbc9a61447ba331b0a8962869fcada758505 \ + --hash=sha256:4555056049d46d8a381f746680db1c46e67ac3b00d714606304077682832998e \ + --hash=sha256:5d5460f789ccf4efd43f265a58538a2c24dbce15dbf560676e430375f20a8198 \ + --hash=sha256:673bddb893f21ab47a8334c8e0ea7fd6598ecc8e698da75bcd12a7b9d0a3206e \ + --hash=sha256:732dd550bfa5d85af8c3c6cbc47ba5b67c6aed8a89e2f011b908fc88f87649db \ + --hash=sha256:784d3ec9bd6493c3b720a0b76f741e6c2d7d44f6b2be87f5eef1ae8cc1d54c84 \ + --hash=sha256:78de3fdb95c4af084087628132336772b1c5044f6e710739d440fc0bccf4d321 \ + --hash=sha256:8064590fd1a50dcf4909c268b0e7c2498253273309ad3d97e4a752bb9df4f521 \ + --hash=sha256:88bffe9c6a454bf8529f9ab9091c99490578a593cc9f9822b7fc065ee0712a06 \ + --hash=sha256:8c1aff58c31948cc66d0b22951aa19edb5af0a3af40c936340cd32a8b1ab7438 \ + --hash=sha256:98ec2775fd2d856dc405635e5ee4ff177920f2141b8e2d9eb5bd6efd50e80317 \ + --hash=sha256:b262ed08d036ebe162123170b35703aaf9daffecb698cd367a8d585157732991 \ + --hash=sha256:e0a22c4157e53d006530c902107c7f550b9233e9706313ab57b892d7197d8e52 \ + --hash=sha256:e91175fbe48f8a2174c9aad70438fe9cb0a5732c4159b2a10a3565fea2d94cde \ + --hash=sha256:f1cb0828ac9533ba0135d148d214e284711ede33640465e706772645483427e3 # via -r requirements.in toml==0.10.2 \ --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ @@ -180,7 +189,7 @@ tomli==2.0.1 \ # maturin # mypy # pytest -typing-extensions==4.12.1 \ - --hash=sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a \ - --hash=sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1 +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 # via mypy diff --git a/requirements-312.txt b/requirements-312.txt new file mode 100644 index 000000000..e4de5a5d2 --- /dev/null +++ b/requirements-312.txt @@ -0,0 +1,184 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --generate-hashes --output-file=requirements-312.txt +# +iniconfig==2.0.0 \ + --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ + --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 + # via pytest +maturin==1.6.0 \ + --hash=sha256:16ef860df20028618b5a064da06b02c1c47acba064a4d25aaf84662a459ec599 \ + --hash=sha256:337899784955934dd67b30497d1dd5fab22da89f60bb079dbaf2eaa446b97a10 \ + --hash=sha256:4e931c92037128ade49cd26dd040d9c46ad8092d8170cc44f5c3a0b4a052d576 \ + --hash=sha256:50133965e52d8b5b969381fee3fde111ae2383905cdaba7650f256e08ccddcd4 \ + --hash=sha256:a2a2436628c36d98dabd79b52256df7e12fc4fd1b122984d9373fdf918fd4609 \ + --hash=sha256:aa4eb7dca7d246b466392f21016f67ff09a9aff2305fa714ca25a2344e4639e7 \ + --hash=sha256:b955025c24c8babc808db49e0ff90db8b4b1320dcc16b14eb26132841737230d \ + --hash=sha256:bd85edcb1b8e2bcddc1b7d16ce58ce00a66aa80c422745c8ad9e132ac40d4b48 \ + --hash=sha256:c87d1a7596c42b589099adb831343a56e02373588366e4cede96cbdf8bd68f9d \ + --hash=sha256:d67ca8dc7f3b2314bd3bf83c4de52645e220ee312fd526e53acc6a735f233fad \ + --hash=sha256:d8620970bd0b6a0acb99dbd0b1c2ebb7a69909d25f6023bdff9635a39001aa51 \ + --hash=sha256:d92b045e90ed919a8a2520dda64e3f384e5e746ea51e1498cc6ac3e9e5c76054 \ + --hash=sha256:dbbbf25dc3c207b0a7bd4f3aea1df33d4f22b8508592796a6f36f4d8ed216db0 + # via -r requirements.in +mypy==1.10.0 \ + --hash=sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061 \ + --hash=sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99 \ + --hash=sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de \ + --hash=sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a \ + --hash=sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9 \ + --hash=sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec \ + --hash=sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1 \ + --hash=sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131 \ + --hash=sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f \ + --hash=sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821 \ + --hash=sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5 \ + --hash=sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee \ + --hash=sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e \ + --hash=sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746 \ + --hash=sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2 \ + --hash=sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0 \ + --hash=sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b \ + --hash=sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53 \ + --hash=sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30 \ + --hash=sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda \ + --hash=sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051 \ + --hash=sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2 \ + --hash=sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7 \ + --hash=sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee \ + --hash=sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727 \ + --hash=sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976 \ + --hash=sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4 + # via -r requirements.in +mypy-extensions==1.0.0 \ + --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ + --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 + # via mypy +numpy==2.0.0 \ + --hash=sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f \ + --hash=sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238 \ + --hash=sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f \ + --hash=sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95 \ + --hash=sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a \ + --hash=sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a \ + --hash=sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2 \ + --hash=sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2 \ + --hash=sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f \ + --hash=sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609 \ + --hash=sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f \ + --hash=sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad \ + --hash=sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86 \ + --hash=sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65 \ + --hash=sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb \ + --hash=sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995 \ + --hash=sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a \ + --hash=sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85 \ + --hash=sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4 \ + --hash=sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275 \ + --hash=sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1 \ + --hash=sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196 \ + --hash=sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d \ + --hash=sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e \ + --hash=sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514 \ + --hash=sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f \ + --hash=sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6 \ + --hash=sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4 \ + --hash=sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44 \ + --hash=sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df \ + --hash=sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581 \ + --hash=sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787 \ + --hash=sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5 \ + --hash=sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc \ + --hash=sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871 \ + --hash=sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54 \ + --hash=sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2 \ + --hash=sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98 \ + --hash=sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9 \ + --hash=sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864 \ + --hash=sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de \ + --hash=sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289 \ + --hash=sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b \ + --hash=sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c \ + --hash=sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9 + # via + # -r requirements.in + # pyarrow +packaging==24.1 \ + --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ + --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 + # via pytest +pluggy==1.5.0 \ + --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ + --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 + # via pytest +pyarrow==16.1.0 \ + --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ + --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ + --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ + --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ + --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ + --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ + --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ + --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ + --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ + --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ + --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ + --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ + --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ + --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ + --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ + --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ + --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ + --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ + --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ + --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ + --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ + --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ + --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ + --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ + --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ + --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ + --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ + --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ + --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ + --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ + --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ + --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ + --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ + --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ + --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ + --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 + # via -r requirements.in +pytest==8.2.2 \ + --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ + --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 + # via -r requirements.in +ruff==0.4.9 \ + --hash=sha256:06b60f91bfa5514bb689b500a25ba48e897d18fea14dce14b48a0c40d1635893 \ + --hash=sha256:0e8e7b95673f22e0efd3571fb5b0cf71a5eaaa3cc8a776584f3b2cc878e46bff \ + --hash=sha256:2d45ddc6d82e1190ea737341326ecbc9a61447ba331b0a8962869fcada758505 \ + --hash=sha256:4555056049d46d8a381f746680db1c46e67ac3b00d714606304077682832998e \ + --hash=sha256:5d5460f789ccf4efd43f265a58538a2c24dbce15dbf560676e430375f20a8198 \ + --hash=sha256:673bddb893f21ab47a8334c8e0ea7fd6598ecc8e698da75bcd12a7b9d0a3206e \ + --hash=sha256:732dd550bfa5d85af8c3c6cbc47ba5b67c6aed8a89e2f011b908fc88f87649db \ + --hash=sha256:784d3ec9bd6493c3b720a0b76f741e6c2d7d44f6b2be87f5eef1ae8cc1d54c84 \ + --hash=sha256:78de3fdb95c4af084087628132336772b1c5044f6e710739d440fc0bccf4d321 \ + --hash=sha256:8064590fd1a50dcf4909c268b0e7c2498253273309ad3d97e4a752bb9df4f521 \ + --hash=sha256:88bffe9c6a454bf8529f9ab9091c99490578a593cc9f9822b7fc065ee0712a06 \ + --hash=sha256:8c1aff58c31948cc66d0b22951aa19edb5af0a3af40c936340cd32a8b1ab7438 \ + --hash=sha256:98ec2775fd2d856dc405635e5ee4ff177920f2141b8e2d9eb5bd6efd50e80317 \ + --hash=sha256:b262ed08d036ebe162123170b35703aaf9daffecb698cd367a8d585157732991 \ + --hash=sha256:e0a22c4157e53d006530c902107c7f550b9233e9706313ab57b892d7197d8e52 \ + --hash=sha256:e91175fbe48f8a2174c9aad70438fe9cb0a5732c4159b2a10a3565fea2d94cde \ + --hash=sha256:f1cb0828ac9533ba0135d148d214e284711ede33640465e706772645483427e3 + # via -r requirements.in +toml==0.10.2 \ + --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ + --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f + # via -r requirements.in +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 + # via mypy diff --git a/requirements-37.txt b/requirements-37.txt deleted file mode 100644 index 6eff3f822..000000000 --- a/requirements-37.txt +++ /dev/null @@ -1,264 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.7 -# To update, run: -# -# pip-compile --generate-hashes -# -attrs==21.4.0 \ - --hash=sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4 \ - --hash=sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd - # via pytest -black==22.3.0 \ - --hash=sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b \ - --hash=sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176 \ - --hash=sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09 \ - --hash=sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a \ - --hash=sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015 \ - --hash=sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79 \ - --hash=sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb \ - --hash=sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20 \ - --hash=sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464 \ - --hash=sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968 \ - --hash=sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82 \ - --hash=sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21 \ - --hash=sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0 \ - --hash=sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265 \ - --hash=sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b \ - --hash=sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a \ - --hash=sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72 \ - --hash=sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce \ - --hash=sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0 \ - --hash=sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a \ - --hash=sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163 \ - --hash=sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad \ - --hash=sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d - # via -r requirements.in -click==8.1.3 \ - --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \ - --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48 - # via black -flake8==4.0.1 \ - --hash=sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d \ - --hash=sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d - # via -r requirements.in -importlib-metadata==4.2.0 ; python_version < "3.8" \ - --hash=sha256:057e92c15bc8d9e8109738a48db0ccb31b4d9d5cfbee5a8670879a30be66304b \ - --hash=sha256:b7e52a1f8dec14a75ea73e0891f3060099ca1d8e6a462a4dff11c3e119ea1b31 - # via - # -r requirements.in - # click - # flake8 - # pluggy - # pytest -iniconfig==1.1.1 \ - --hash=sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3 \ - --hash=sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32 - # via pytest -isort==5.10.1 \ - --hash=sha256:6f62d78e2f89b4500b080fe3a81690850cd254227f27f75c3a0c491a1f351ba7 \ - --hash=sha256:e8443a5e7a020e9d7f97f1d7d9cd17c88bcb3bc7e218bf9cf5095fe550be2951 - # via -r requirements.in -maturin==0.14.2 \ - --hash=sha256:065a0b48259624af1330bbc2d9bd447615ff07603710ad4a839478aa8e9e98cd \ - --hash=sha256:2966ed9b2e66760ffa5e476dab20c9563dd14efa93ed40b706cbe0a03907d824 \ - --hash=sha256:50a7a15d02f0b407eb5cf4cbc072d27e302993d5734af71bb6d6c567da8c6396 \ - --hash=sha256:539545c666506d34474c1133a11e90b4e6b4b6a7e590f5727ae18a49d3b647d2 \ - --hash=sha256:9f843736fc1c065ffba2dcfc581811c2eb7032c7119ff23012ccde68607e549c \ - --hash=sha256:ace60890f19a87e4a421fd9753e42e5fb46b11e770feacd8ff6749d3bf114764 \ - --hash=sha256:ae3a3a754c3b935420eeb1ec34d98d1693251537331feb43277bae7892b7c51a \ - --hash=sha256:bbaa0689a87124c99adaff99e93dcf415e36f7068d5442f311b49ede4bb7b6c3 \ - --hash=sha256:c65b00f4a4773bc56ca690d93abd09c85cc59b161a313fba3d03aca1ffbba7ca \ - --hash=sha256:daefa75e3f03e914924cb5913b40ada47ff646562c3b1874ddaf5b1e4b0f41ef \ - --hash=sha256:e0b808d1a0b78532c1bcbeb2d01d90281585dc460a966f1949fd337a3fb614c0 \ - --hash=sha256:ebc722e9661a40c3d36a06ecc0d1c637ae1af3335ea0ab69230f46a71e344088 \ - --hash=sha256:fa01eeacfd1798cde3ba1c7a0b8d7a889d2ed0e207f4860beb819c5ddd351f43 - # via -r requirements.in -mccabe==0.6.1 \ - --hash=sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42 \ - --hash=sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f - # via flake8 -mypy==0.950 \ - --hash=sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d \ - --hash=sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8 \ - --hash=sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de \ - --hash=sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038 \ - --hash=sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed \ - --hash=sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334 \ - --hash=sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff \ - --hash=sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2 \ - --hash=sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22 \ - --hash=sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2 \ - --hash=sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2 \ - --hash=sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605 \ - --hash=sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb \ - --hash=sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519 \ - --hash=sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0 \ - --hash=sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc \ - --hash=sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b \ - --hash=sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f \ - --hash=sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075 \ - --hash=sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef \ - --hash=sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb \ - --hash=sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a \ - --hash=sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b - # via -r requirements.in -mypy-extensions==0.4.3 \ - --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ - --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 - # via - # black - # mypy -numpy==1.21.6 \ - --hash=sha256:1dbe1c91269f880e364526649a52eff93ac30035507ae980d2fed33aaee633ac \ - --hash=sha256:357768c2e4451ac241465157a3e929b265dfac85d9214074985b1786244f2ef3 \ - --hash=sha256:3820724272f9913b597ccd13a467cc492a0da6b05df26ea09e78b171a0bb9da6 \ - --hash=sha256:4391bd07606be175aafd267ef9bea87cf1b8210c787666ce82073b05f202add1 \ - --hash=sha256:4aa48afdce4660b0076a00d80afa54e8a97cd49f457d68a4342d188a09451c1a \ - --hash=sha256:58459d3bad03343ac4b1b42ed14d571b8743dc80ccbf27444f266729df1d6f5b \ - --hash=sha256:5c3c8def4230e1b959671eb959083661b4a0d2e9af93ee339c7dada6759a9470 \ - --hash=sha256:5f30427731561ce75d7048ac254dbe47a2ba576229250fb60f0fb74db96501a1 \ - --hash=sha256:643843bcc1c50526b3a71cd2ee561cf0d8773f062c8cbaf9ffac9fdf573f83ab \ - --hash=sha256:67c261d6c0a9981820c3a149d255a76918278a6b03b6a036800359aba1256d46 \ - --hash=sha256:67f21981ba2f9d7ba9ade60c9e8cbaa8cf8e9ae51673934480e45cf55e953673 \ - --hash=sha256:6aaf96c7f8cebc220cdfc03f1d5a31952f027dda050e5a703a0d1c396075e3e7 \ - --hash=sha256:7c4068a8c44014b2d55f3c3f574c376b2494ca9cc73d2f1bd692382b6dffe3db \ - --hash=sha256:7c7e5fa88d9ff656e067876e4736379cc962d185d5cd808014a8a928d529ef4e \ - --hash=sha256:7f5ae4f304257569ef3b948810816bc87c9146e8c446053539947eedeaa32786 \ - --hash=sha256:82691fda7c3f77c90e62da69ae60b5ac08e87e775b09813559f8901a88266552 \ - --hash=sha256:8737609c3bbdd48e380d463134a35ffad3b22dc56295eff6f79fd85bd0eeeb25 \ - --hash=sha256:9f411b2c3f3d76bba0865b35a425157c5dcf54937f82bbeb3d3c180789dd66a6 \ - --hash=sha256:a6be4cb0ef3b8c9250c19cc122267263093eee7edd4e3fa75395dfda8c17a8e2 \ - --hash=sha256:bcb238c9c96c00d3085b264e5c1a1207672577b93fa666c3b14a45240b14123a \ - --hash=sha256:bf2ec4b75d0e9356edea834d1de42b31fe11f726a81dfb2c2112bc1eaa508fcf \ - --hash=sha256:d136337ae3cc69aa5e447e78d8e1514be8c3ec9b54264e680cf0b4bd9011574f \ - --hash=sha256:d4bf4d43077db55589ffc9009c0ba0a94fa4908b9586d6ccce2e0b164c86303c \ - --hash=sha256:d6a96eef20f639e6a97d23e57dd0c1b1069a7b4fd7027482a4c5c451cd7732f4 \ - --hash=sha256:d9caa9d5e682102453d96a0ee10c7241b72859b01a941a397fd965f23b3e016b \ - --hash=sha256:dd1c8f6bd65d07d3810b90d02eba7997e32abbdf1277a481d698969e921a3be0 \ - --hash=sha256:e31f0bb5928b793169b87e3d1e070f2342b22d5245c755e2b81caa29756246c3 \ - --hash=sha256:ecb55251139706669fdec2ff073c98ef8e9a84473e51e716211b41aa0f18e656 \ - --hash=sha256:ee5ec40fdd06d62fe5d4084bef4fd50fd4bb6bfd2bf519365f569dc470163ab0 \ - --hash=sha256:f17e562de9edf691a42ddb1eb4a5541c20dd3f9e65b09ded2beb0799c0cf29bb \ - --hash=sha256:fdffbfb6832cd0b300995a2b08b8f6fa9f6e856d562800fea9182316d99c4e8e - # via - # -r requirements.in - # pyarrow -packaging==21.3 \ - --hash=sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb \ - --hash=sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522 - # via pytest -pathspec==0.9.0 \ - --hash=sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a \ - --hash=sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1 - # via black -platformdirs==2.5.2 \ - --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ - --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 - # via black -pluggy==1.0.0 \ - --hash=sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159 \ - --hash=sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3 - # via pytest -py==1.11.0 \ - --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ - --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 - # via pytest -pyarrow==8.0.0 \ - --hash=sha256:03a10daad957970e914920b793f6a49416699e791f4c827927fd4e4d892a5d16 \ - --hash=sha256:15511ce2f50343f3fd5e9f7c30e4d004da9134e9597e93e9c96c3985928cbe82 \ - --hash=sha256:1dd482ccb07c96188947ad94d7536ab696afde23ad172df8e18944ec79f55055 \ - --hash=sha256:25a5f7c7f36df520b0b7363ba9f51c3070799d4b05d587c60c0adaba57763479 \ - --hash=sha256:3bd201af6e01f475f02be88cf1f6ee9856ab98c11d8bbb6f58347c58cd07be00 \ - --hash=sha256:3fee786259d986f8c046100ced54d63b0c8c9f7cdb7d1bbe07dc69e0f928141c \ - --hash=sha256:42b7982301a9ccd06e1dd4fabd2e8e5df74b93ce4c6b87b81eb9e2d86dc79871 \ - --hash=sha256:4a18a211ed888f1ac0b0ebcb99e2d9a3e913a481120ee9b1fe33d3fedb945d4e \ - --hash=sha256:51e58778fcb8829fca37fbfaea7f208d5ce7ea89ea133dd13d8ce745278ee6f0 \ - --hash=sha256:541e7845ce5f27a861eb5b88ee165d931943347eec17b9ff1e308663531c9647 \ - --hash=sha256:65c7f4cc2be195e3db09296d31a654bb6d8786deebcab00f0e2455fd109d7456 \ - --hash=sha256:69b043a3fce064ebd9fbae6abc30e885680296e5bd5e6f7353e6a87966cf2ad7 \ - --hash=sha256:6ea2c54e6b5ecd64e8299d2abb40770fe83a718f5ddc3825ddd5cd28e352cce1 \ - --hash=sha256:78a6ac39cd793582998dac88ab5c1c1dd1e6503df6672f064f33a21937ec1d8d \ - --hash=sha256:81b87b782a1366279411f7b235deab07c8c016e13f9af9f7c7b0ee564fedcc8f \ - --hash=sha256:8392b9a1e837230090fe916415ed4c3433b2ddb1a798e3f6438303c70fbabcfc \ - --hash=sha256:863be6bad6c53797129610930794a3e797cb7d41c0a30e6794a2ac0e42ce41b8 \ - --hash=sha256:8cd86e04a899bef43e25184f4b934584861d787cf7519851a8c031803d45c6d8 \ - --hash=sha256:95c7822eb37663e073da9892f3499fe28e84f3464711a3e555e0c5463fd53a19 \ - --hash=sha256:98c13b2e28a91b0fbf24b483df54a8d7814c074c2623ecef40dce1fa52f6539b \ - --hash=sha256:ba2b7aa7efb59156b87987a06f5241932914e4d5bbb74a465306b00a6c808849 \ - --hash=sha256:c9c97c8e288847e091dfbcdf8ce51160e638346f51919a9e74fe038b2e8aee62 \ - --hash=sha256:cb06cacc19f3b426681f2f6803cc06ff481e7fe5b3a533b406bc5b2138843d4f \ - --hash=sha256:ce64bc1da3109ef5ab9e4c60316945a7239c798098a631358e9ab39f6e5529e9 \ - --hash=sha256:d5ef4372559b191cafe7db8932801eee252bfc35e983304e7d60b6954576a071 \ - --hash=sha256:d6f1e1040413651819074ef5b500835c6c42e6c446532a1ddef8bc5054e8dba5 \ - --hash=sha256:deb400df8f19a90b662babceb6dd12daddda6bb357c216e558b207c0770c7654 \ - --hash=sha256:ea132067ec712d1b1116a841db1c95861508862b21eddbcafefbce8e4b96b867 \ - --hash=sha256:ece333706a94c1221ced8b299042f85fd88b5db802d71be70024433ddf3aecab \ - --hash=sha256:edad25522ad509e534400d6ab98cf1872d30c31bc5e947712bfd57def7af15bb - # via -r requirements.in -pycodestyle==2.8.0 \ - --hash=sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20 \ - --hash=sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f - # via flake8 -pyflakes==2.4.0 \ - --hash=sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c \ - --hash=sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e - # via flake8 -pyparsing==3.0.9 \ - --hash=sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb \ - --hash=sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc - # via packaging -pytest==7.1.2 \ - --hash=sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c \ - --hash=sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45 - # via -r requirements.in -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via -r requirements.in -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via - # black - # maturin - # mypy - # pytest -typed-ast==1.5.3 \ - --hash=sha256:20d5118e494478ef2d3a2702d964dae830aedd7b4d3b626d003eea526be18718 \ - --hash=sha256:27e46cdd01d6c3a0dd8f728b6a938a6751f7bd324817501c15fb056307f918c6 \ - --hash=sha256:27f25232e2dd0edfe1f019d6bfaaf11e86e657d9bdb7b0956db95f560cceb2b3 \ - --hash=sha256:3042bfc9ca118712c9809201f55355479cfcdc17449f9f8db5e744e9625c6805 \ - --hash=sha256:37e5349d1d5de2f4763d534ccb26809d1c24b180a477659a12c4bde9dd677d74 \ - --hash=sha256:4fff9fdcce59dc61ec1b317bdb319f8f4e6b69ebbe61193ae0a60c5f9333dc49 \ - --hash=sha256:542cd732351ba8235f20faa0fc7398946fe1a57f2cdb289e5497e1e7f48cfedb \ - --hash=sha256:5dc2c11ae59003d4a26dda637222d9ae924387f96acae9492df663843aefad55 \ - --hash=sha256:8831479695eadc8b5ffed06fdfb3e424adc37962a75925668deeb503f446c0a3 \ - --hash=sha256:8cdf91b0c466a6c43f36c1964772918a2c04cfa83df8001ff32a89e357f8eb06 \ - --hash=sha256:8e0b8528838ffd426fea8d18bde4c73bcb4167218998cc8b9ee0a0f2bfe678a6 \ - --hash=sha256:8ef1d96ad05a291f5c36895d86d1375c0ee70595b90f6bb5f5fdbee749b146db \ - --hash=sha256:9ad3b48cf2b487be140072fb86feff36801487d4abb7382bb1929aaac80638ea \ - --hash=sha256:9cc9e1457e1feb06b075c8ef8aeb046a28ec351b1958b42c7c31c989c841403a \ - --hash=sha256:9e237e74fd321a55c90eee9bc5d44be976979ad38a29bbd734148295c1ce7617 \ - --hash=sha256:c9f1a27592fac87daa4e3f16538713d705599b0a27dfe25518b80b6b017f0a6d \ - --hash=sha256:d64dabc6336ddc10373922a146fa2256043b3b43e61f28961caec2a5207c56d5 \ - --hash=sha256:e20d196815eeffb3d76b75223e8ffed124e65ee62097e4e73afb5fec6b993e7a \ - --hash=sha256:e34f9b9e61333ecb0f7d79c21c28aa5cd63bec15cb7e1310d7d3da6ce886bc9b \ - --hash=sha256:ed44e81517364cb5ba367e4f68fca01fba42a7a4690d40c07886586ac267d9b9 \ - --hash=sha256:ee852185964744987609b40aee1d2eb81502ae63ee8eef614558f96a56c1902d \ - --hash=sha256:f60d9de0d087454c91b3999a296d0c4558c1666771e3460621875021bf899af9 \ - --hash=sha256:f818c5b81966d4728fec14caa338e30a70dfc3da577984d38f97816c4b3071ec \ - --hash=sha256:fd5df1313915dbd70eaaa88c19030b441742e8b05e6103c631c83b75e0435ccc - # via - # black - # mypy -typing-extensions==4.2.0 \ - --hash=sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708 \ - --hash=sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376 - # via - # black - # importlib-metadata - # mypy -zipp==3.8.0 \ - --hash=sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad \ - --hash=sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099 - # via importlib-metadata From 45a684445e25032961a7bb44ced3ce06f5ed9e6d Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Wed, 19 Jun 2024 11:20:39 -0500 Subject: [PATCH 157/413] Pyarrow filter pushdowns (#735) * fix pushdown for pyarrow filter IsNull The conversion was incorrectly passing in the expression itself as the `nan_as_null` argument. This caused the pushdown to silently fail. * expand the Expr::Literal's that can be used in PyArrowFilterExpression Closes #703 --- python/datafusion/tests/test_context.py | 54 +++++++++++++++++++++++++ src/pyarrow_filter_expression.rs | 29 +++++-------- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index df7e1813b..abc324db8 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -16,6 +16,7 @@ # under the License. import gzip import os +import datetime as dt import pyarrow as pa import pyarrow.dataset as ds @@ -303,6 +304,59 @@ def test_dataset_filter(ctx, capfd): assert result[0].column(1) == pa.array([-3]) +def test_pyarrow_predicate_pushdown_is_null(ctx, capfd): + """Ensure that pyarrow filter gets pushed down for `IsNull`""" + # create a RecordBatch and register it as a pyarrow.dataset.Dataset + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6]), pa.array([7, None, 9])], + names=["a", "b", "c"], + ) + dataset = ds.dataset([batch]) + ctx.register_dataset("t", dataset) + # Make sure the filter was pushed down in Physical Plan + df = ctx.sql("SELECT a FROM t WHERE c is NULL") + df.explain() + captured = capfd.readouterr() + assert "filter_expr=is_null(c, {nan_is_null=false})" in captured.out + + result = df.collect() + assert result[0].column(0) == pa.array([2]) + + +def test_pyarrow_predicate_pushdown_timestamp(ctx, tmpdir, capfd): + """Ensure that pyarrow filter gets pushed down for timestamp""" + # Ref: https://github.com/apache/datafusion-python/issues/703 + + # create pyarrow dataset with no actual files + col_type = pa.timestamp("ns", "+00:00") + nyd_2000 = pa.scalar(dt.datetime(2000, 1, 1, tzinfo=dt.timezone.utc), col_type) + pa_dataset_fs = pa.fs.SubTreeFileSystem(str(tmpdir), pa.fs.LocalFileSystem()) + pa_dataset_format = pa.dataset.ParquetFileFormat() + pa_dataset_partition = pa.dataset.field("a") <= nyd_2000 + fragments = [ + # NOTE: we never actually make this file. + # Working predicate pushdown means it never gets accessed + pa_dataset_format.make_fragment( + "1.parquet", + filesystem=pa_dataset_fs, + partition_expression=pa_dataset_partition, + ) + ] + pa_dataset = pa.dataset.FileSystemDataset( + fragments, + pa.schema([pa.field("a", col_type)]), + pa_dataset_format, + pa_dataset_fs, + ) + + ctx.register_dataset("t", pa_dataset) + + # the partition for our only fragment is for a < 2000-01-01. + # so querying for a > 2024-01-01 should not touch any files + df = ctx.sql("SELECT * FROM t WHERE a > '2024-01-01T00:00:00+00:00'") + assert df.collect() == [] + + def test_dataset_filter_nested_data(ctx): # create Arrow StructArrays to test nested data types data = pa.StructArray.from_arrays( diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index fca885121..ff447e1ab 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -21,6 +21,7 @@ use pyo3::prelude::*; use std::convert::TryFrom; use std::result::Result; +use arrow::pyarrow::ToPyArrow; use datafusion_common::{Column, ScalarValue}; use datafusion_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; @@ -56,6 +57,7 @@ fn extract_scalar_list(exprs: &[Expr], py: Python) -> Result, Data let ret: Result, DataFusionError> = exprs .iter() .map(|expr| match expr { + // TODO: should we also leverage `ScalarValue::to_pyarrow` here? Expr::Literal(v) => match v { ScalarValue::Boolean(Some(b)) => Ok(b.into_py(py)), ScalarValue::Int8(Some(i)) => Ok(i.into_py(py)), @@ -100,23 +102,7 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { let op_module = Python::import_bound(py, "operator")?; let pc_expr: Result, DataFusionError> = match expr { Expr::Column(Column { name, .. }) => Ok(pc.getattr("field")?.call1((name,))?), - Expr::Literal(v) => match v { - ScalarValue::Boolean(Some(b)) => Ok(pc.getattr("scalar")?.call1((*b,))?), - ScalarValue::Int8(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::Int16(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::Int32(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::Int64(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::UInt8(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::UInt16(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::UInt32(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::UInt64(Some(i)) => Ok(pc.getattr("scalar")?.call1((*i,))?), - ScalarValue::Float32(Some(f)) => Ok(pc.getattr("scalar")?.call1((*f,))?), - ScalarValue::Float64(Some(f)) => Ok(pc.getattr("scalar")?.call1((*f,))?), - ScalarValue::Utf8(Some(s)) => Ok(pc.getattr("scalar")?.call1((s,))?), - _ => Err(DataFusionError::Common(format!( - "PyArrow can't handle ScalarValue: {v:?}" - ))), - }, + Expr::Literal(scalar) => Ok(scalar.to_pyarrow(py)?.into_bound(py)), Expr::BinaryExpr(BinaryExpr { left, op, right }) => { let operator = operator_to_py(op, &op_module)?; let left = PyArrowFilterExpression::try_from(left.as_ref())?.0; @@ -138,8 +124,13 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { let expr = PyArrowFilterExpression::try_from(expr.as_ref())? .0 .into_bound(py); - // TODO: this expression does not seems like it should be `call_method0` - Ok(expr.clone().call_method1("is_null", (expr,))?) + + // https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Expression.html#pyarrow.dataset.Expression.is_null + // Whether floating-point NaNs are considered null. + let nan_is_null = false; + + let res = expr.call_method1("is_null", (nan_is_null,))?; + Ok(res) } Expr::Between(Between { expr, From faa26b2ce7f93cb3328d7362d5847a29c40b3142 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Jun 2024 13:24:39 -0700 Subject: [PATCH 158/413] build(deps): bump syn from 2.0.66 to 2.0.67 (#738) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.66 to 2.0.67. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.66...2.0.67) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e437a03f..41742da47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,7 +376,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -387,7 +387,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -1051,7 +1051,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.66", + "syn 2.0.67", "tokio", "url", "uuid", @@ -1239,7 +1239,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2138,7 +2138,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2178,7 +2178,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2217,7 +2217,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.66", + "syn 2.0.67", "tempfile", ] @@ -2231,7 +2231,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2299,7 +2299,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2312,7 +2312,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2622,7 +2622,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2686,7 +2686,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2697,7 +2697,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2720,7 +2720,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2842,7 +2842,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2876,7 +2876,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2889,7 +2889,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2912,7 +2912,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.66", + "syn 2.0.67", "typify", "walkdir", ] @@ -2936,9 +2936,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" dependencies = [ "proc-macro2", "quote", @@ -2986,7 +2986,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -3049,7 +3049,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -3122,7 +3122,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -3167,7 +3167,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -3201,7 +3201,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.66", + "syn 2.0.67", "thiserror", "unicode-ident", ] @@ -3219,7 +3219,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.66", + "syn 2.0.67", "typify-impl", ] @@ -3347,7 +3347,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", "wasm-bindgen-shared", ] @@ -3381,7 +3381,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3608,7 +3608,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 78b46e449..4e3821127 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ futures = "0.3" object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" -syn = "2.0.43" +syn = "2.0.67" url = "2.2" [build-dependencies] From 32d69752e4fb2399265b4a75c058bda63da099a7 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 25 Jun 2024 19:19:51 -0500 Subject: [PATCH 159/413] Pyo3 refactorings (#740) * let pyo3 convert the StorageContexts argument in PySessionContext::register_object_store * clean PySessionContext methods from_pylist and from_pydict * clean PySessionContext metehods from_polars, from_pandas, from_arrow_table * prefer bound Python token over Python::with_gil When available, using an already bound python token is zero-cost. Python::with_gil carries a runtime check. Ref: https://github.com/PyO3/pyo3/pull/4274 --- src/context.rs | 146 ++++++++++++++++++++------------------------- src/dataframe.rs | 55 +++++++---------- src/sql/logical.rs | 4 +- 3 files changed, 89 insertions(+), 116 deletions(-) diff --git a/src/context.rs b/src/context.rs index ec63adb26..50c4a1994 100644 --- a/src/context.rs +++ b/src/context.rs @@ -60,7 +60,7 @@ use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; use datafusion_common::ScalarValue; -use pyo3::types::PyTuple; +use pyo3::types::{PyDict, PyList, PyTuple}; use tokio::task::JoinHandle; /// Configuration options for a SessionContext @@ -291,24 +291,17 @@ impl PySessionContext { pub fn register_object_store( &mut self, scheme: &str, - store: &Bound<'_, PyAny>, + store: StorageContexts, host: Option<&str>, ) -> PyResult<()> { - let res: Result<(Arc, String), PyErr> = - match StorageContexts::extract_bound(store) { - Ok(store) => match store { - StorageContexts::AmazonS3(s3) => Ok((s3.inner, s3.bucket_name)), - StorageContexts::GoogleCloudStorage(gcs) => Ok((gcs.inner, gcs.bucket_name)), - StorageContexts::MicrosoftAzure(azure) => { - Ok((azure.inner, azure.container_name)) - } - StorageContexts::LocalFileSystem(local) => Ok((local.inner, "".to_string())), - }, - Err(_e) => Err(PyValueError::new_err("Invalid object store")), - }; - // for most stores the "host" is the bucket name and can be inferred from the store - let (store, upstream_host) = res?; + let (store, upstream_host): (Arc, String) = match store { + StorageContexts::AmazonS3(s3) => (s3.inner, s3.bucket_name), + StorageContexts::GoogleCloudStorage(gcs) => (gcs.inner, gcs.bucket_name), + StorageContexts::MicrosoftAzure(azure) => (azure.inner, azure.container_name), + StorageContexts::LocalFileSystem(local) => (local.inner, "".to_string()), + }; + // let users override the host to match the api signature from upstream let derived_host = if let Some(host) = host { host @@ -434,105 +427,96 @@ impl PySessionContext { } /// Construct datafusion dataframe from Python list - #[allow(clippy::wrong_self_convention)] pub fn from_pylist( &mut self, - data: PyObject, + data: Bound<'_, PyList>, name: Option<&str>, - _py: Python, ) -> PyResult { - Python::with_gil(|py| { - // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[data]); - let table = table_class.call_method1("from_pylist", args)?.into(); - - // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; - Ok(df) - }) + // Acquire GIL Token + let py = data.py(); + + // Instantiate pyarrow Table object & convert to Arrow Table + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[data]); + let table = table_class.call_method1("from_pylist", args)?; + + // Convert Arrow Table to datafusion DataFrame + let df = self.from_arrow_table(table, name, py)?; + Ok(df) } /// Construct datafusion dataframe from Python dictionary - #[allow(clippy::wrong_self_convention)] pub fn from_pydict( &mut self, - data: PyObject, + data: Bound<'_, PyDict>, name: Option<&str>, - _py: Python, ) -> PyResult { - Python::with_gil(|py| { - // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[data]); - let table = table_class.call_method1("from_pydict", args)?.into(); - - // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; - Ok(df) - }) + // Acquire GIL Token + let py = data.py(); + + // Instantiate pyarrow Table object & convert to Arrow Table + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[data]); + let table = table_class.call_method1("from_pydict", args)?; + + // Convert Arrow Table to datafusion DataFrame + let df = self.from_arrow_table(table, name, py)?; + Ok(df) } /// Construct datafusion dataframe from Arrow Table - #[allow(clippy::wrong_self_convention)] pub fn from_arrow_table( &mut self, - data: PyObject, + data: Bound<'_, PyAny>, name: Option<&str>, - _py: Python, + py: Python, ) -> PyResult { - Python::with_gil(|py| { - // Instantiate pyarrow Table object & convert to batches - let table = data.call_method0(py, "to_batches")?; - - let schema = data.getattr(py, "schema")?; - let schema = schema.extract::>(py)?; - - // Cast PyObject to RecordBatch type - // Because create_dataframe() expects a vector of vectors of record batches - // here we need to wrap the vector of record batches in an additional vector - let batches = table.extract::>>(py)?; - let list_of_batches = PyArrowType::from(vec![batches.0]); - self.create_dataframe(list_of_batches, name, Some(schema), py) - }) + // Instantiate pyarrow Table object & convert to batches + let table = data.call_method0("to_batches")?; + + let schema = data.getattr("schema")?; + let schema = schema.extract::>()?; + + // Cast PyAny to RecordBatch type + // Because create_dataframe() expects a vector of vectors of record batches + // here we need to wrap the vector of record batches in an additional vector + let batches = table.extract::>>()?; + let list_of_batches = PyArrowType::from(vec![batches.0]); + self.create_dataframe(list_of_batches, name, Some(schema), py) } /// Construct datafusion dataframe from pandas #[allow(clippy::wrong_self_convention)] pub fn from_pandas( &mut self, - data: PyObject, + data: Bound<'_, PyAny>, name: Option<&str>, - _py: Python, ) -> PyResult { - Python::with_gil(|py| { - // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[data]); - let table = table_class.call_method1("from_pandas", args)?.into(); - - // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; - Ok(df) - }) + // Obtain GIL token + let py = data.py(); + + // Instantiate pyarrow Table object & convert to Arrow Table + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[data]); + let table = table_class.call_method1("from_pandas", args)?; + + // Convert Arrow Table to datafusion DataFrame + let df = self.from_arrow_table(table, name, py)?; + Ok(df) } /// Construct datafusion dataframe from polars - #[allow(clippy::wrong_self_convention)] pub fn from_polars( &mut self, - data: PyObject, + data: Bound<'_, PyAny>, name: Option<&str>, - _py: Python, ) -> PyResult { - Python::with_gil(|py| { - // Convert Polars dataframe to Arrow Table - let table = data.call_method0(py, "to_arrow")?; + // Convert Polars dataframe to Arrow Table + let table = data.call_method0("to_arrow")?; - // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; - Ok(df) - }) + // Convert Arrow Table to datafusion DataFrame + let df = self.from_arrow_table(table, name, data.py())?; + Ok(df) } pub fn register_table(&mut self, name: &str, table: &PyTable) -> PyResult<()> { diff --git a/src/dataframe.rs b/src/dataframe.rs index 1b91067d5..9e36be2c4 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -423,17 +423,15 @@ impl PyDataFrame { /// Convert to Arrow Table /// Collect the batches and pass to Arrow Table - fn to_arrow_table(&self, py: Python) -> PyResult { + fn to_arrow_table(&self, py: Python<'_>) -> PyResult { let batches = self.collect(py)?.to_object(py); let schema: PyObject = self.schema().into_py(py); - Python::with_gil(|py| { - // Instantiate pyarrow Table object and use its from_batches method - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[batches, schema]); - let table: PyObject = table_class.call_method1("from_batches", args)?.into(); - Ok(table) - }) + // Instantiate pyarrow Table object and use its from_batches method + let table_class = py.import_bound("pyarrow")?.getattr("Table")?; + let args = PyTuple::new_bound(py, &[batches, schema]); + let table: PyObject = table_class.call_method1("from_batches", args)?.into(); + Ok(table) } fn execute_stream(&self, py: Python) -> PyResult { @@ -464,26 +462,22 @@ impl PyDataFrame { /// Convert to pandas dataframe with pyarrow /// Collect the batches, pass to Arrow Table & then convert to Pandas DataFrame - fn to_pandas(&self, py: Python) -> PyResult { + fn to_pandas(&self, py: Python<'_>) -> PyResult { let table = self.to_arrow_table(py)?; - Python::with_gil(|py| { - // See also: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pandas - let result = table.call_method0(py, "to_pandas")?; - Ok(result) - }) + // See also: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pandas + let result = table.call_method0(py, "to_pandas")?; + Ok(result) } /// Convert to Python list using pyarrow /// Each list item represents one row encoded as dictionary - fn to_pylist(&self, py: Python) -> PyResult { + fn to_pylist(&self, py: Python<'_>) -> PyResult { let table = self.to_arrow_table(py)?; - Python::with_gil(|py| { - // See also: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pylist - let result = table.call_method0(py, "to_pylist")?; - Ok(result) - }) + // See also: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pylist + let result = table.call_method0(py, "to_pylist")?; + Ok(result) } /// Convert to Python dictionary using pyarrow @@ -491,24 +485,19 @@ impl PyDataFrame { fn to_pydict(&self, py: Python) -> PyResult { let table = self.to_arrow_table(py)?; - Python::with_gil(|py| { - // See also: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pydict - let result = table.call_method0(py, "to_pydict")?; - Ok(result) - }) + // See also: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.to_pydict + let result = table.call_method0(py, "to_pydict")?; + Ok(result) } /// Convert to polars dataframe with pyarrow /// Collect the batches, pass to Arrow Table & then convert to polars DataFrame - fn to_polars(&self, py: Python) -> PyResult { + fn to_polars(&self, py: Python<'_>) -> PyResult { let table = self.to_arrow_table(py)?; - - Python::with_gil(|py| { - let dataframe = py.import_bound("polars")?.getattr("DataFrame")?; - let args = PyTuple::new_bound(py, &[table]); - let result: PyObject = dataframe.call1(args)?.into(); - Ok(result) - }) + let dataframe = py.import_bound("polars")?.getattr("DataFrame")?; + let args = PyTuple::new_bound(py, &[table]); + let result: PyObject = dataframe.call1(args)?.into(); + Ok(result) } // Executes this DataFrame to get the total number of rows. diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 62515c3dd..b1446b92a 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -63,7 +63,7 @@ impl PyLogicalPlan { impl PyLogicalPlan { /// Return the specific logical operator pub fn to_variant(&self, py: Python) -> PyResult { - Python::with_gil(|_| match self.plan.as_ref() { + match self.plan.as_ref() { LogicalPlan::Aggregate(plan) => PyAggregate::from(plan.clone()).to_variant(py), LogicalPlan::Analyze(plan) => PyAnalyze::from(plan.clone()).to_variant(py), LogicalPlan::CrossJoin(plan) => PyCrossJoin::from(plan.clone()).to_variant(py), @@ -85,7 +85,7 @@ impl PyLogicalPlan { "Cannot convert this plan to a LogicalNode: {:?}", other ))), - }) + } } /// Get the inputs to this plan From ec835abb4da5e34556a08f72cf30a27ddeadbc95 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 25 Jun 2024 19:20:39 -0500 Subject: [PATCH 160/413] UDAF `sum` workaround (#741) * provides workaround for half-migrated UDAF `sum` Ref #730 * provide compatibility for sqlparser::ast::NullTreatment This is now exposed as part of the API to `first_value` and `last_value` functions. If there's a more elegant way to achieve this, please let me know. --- Cargo.lock | 1 + Cargo.toml | 1 + examples/tpch/_tests.py | 1 - src/common.rs | 1 + src/common/data_type.rs | 30 ++++++++++++++++++++++++++++++ src/functions.rs | 33 ++++++++++++++++++++------------- 6 files changed, 53 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41742da47..f05c62e97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1051,6 +1051,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", + "sqlparser", "syn 2.0.67", "tokio", "url", diff --git a/Cargo.toml b/Cargo.toml index 4e3821127..e518449a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,6 +56,7 @@ parking_lot = "0.12" regex-syntax = "0.8.1" syn = "2.0.67" url = "2.2" +sqlparser = "0.47.0" [build-dependencies] pyo3-build-config = "0.21" diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index aa9491bfd..3f973d9f2 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -74,7 +74,6 @@ def check_q17(df): ("q10_returned_item_reporting", "q10"), pytest.param( "q11_important_stock_identification", "q11", - marks=pytest.mark.xfail # https://github.com/apache/datafusion-python/issues/730 ), ("q12_ship_mode_order_priority", "q12"), ("q13_customer_distribution", "q13"), diff --git a/src/common.rs b/src/common.rs index 44c557ce7..094e70c01 100644 --- a/src/common.rs +++ b/src/common.rs @@ -29,6 +29,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/common/data_type.rs b/src/common/data_type.rs index cd4f864bc..313318fc9 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -757,3 +757,33 @@ pub enum SqlType { VARBINARY, VARCHAR, } + +/// Specifies Ignore / Respect NULL within window functions. +/// For example +/// `FIRST_VALUE(column2) IGNORE NULLS OVER (PARTITION BY column1)` +#[allow(non_camel_case_types)] +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(name = "PythonType", module = "datafusion.common")] +pub enum NullTreatment { + IGNORE_NULLS, + RESPECT_NULLS, +} + +impl From for sqlparser::ast::NullTreatment { + fn from(null_treatment: NullTreatment) -> sqlparser::ast::NullTreatment { + match null_treatment { + NullTreatment::IGNORE_NULLS => sqlparser::ast::NullTreatment::IgnoreNulls, + NullTreatment::RESPECT_NULLS => sqlparser::ast::NullTreatment::RespectNulls, + } + } +} + +impl From for NullTreatment { + fn from(null_treatment: sqlparser::ast::NullTreatment) -> NullTreatment { + match null_treatment { + sqlparser::ast::NullTreatment::IgnoreNulls => NullTreatment::IGNORE_NULLS, + sqlparser::ast::NullTreatment::RespectNulls => NullTreatment::RESPECT_NULLS, + } + } +} diff --git a/src/functions.rs b/src/functions.rs index 8e395ae4f..b39d98b35 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -17,6 +17,7 @@ use pyo3::{prelude::*, wrap_pyfunction}; +use crate::common::data_type::NullTreatment; use crate::context::PySessionContext; use crate::errors::DataFusionError; use crate::expr::conditional_expr::PyCaseBuilder; @@ -73,15 +74,15 @@ pub fn var(y: PyExpr) -> PyExpr { } #[pyfunction] -#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None))] +#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None, null_treatment = None))] pub fn first_value( args: Vec, distinct: bool, filter: Option, order_by: Option>, + null_treatment: Option, ) -> PyExpr { - // TODO: allow user to select null_treatment - let null_treatment = None; + let null_treatment = null_treatment.map(Into::into); let args = args.into_iter().map(|x| x.expr).collect::>(); let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); functions_aggregate::expr_fn::first_value( @@ -95,15 +96,15 @@ pub fn first_value( } #[pyfunction] -#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None))] +#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None, null_treatment = None))] pub fn last_value( args: Vec, distinct: bool, filter: Option, order_by: Option>, + null_treatment: Option, ) -> PyExpr { - // TODO: allow user to select null_treatment - let null_treatment = None; + let null_treatment = null_treatment.map(Into::into); let args = args.into_iter().map(|x| x.expr).collect::>(); let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); functions_aggregate::expr_fn::last_value( @@ -320,14 +321,20 @@ fn window( window_frame: Option, ctx: Option, ) -> PyResult { - let fun = find_df_window_func(name).or_else(|| { - ctx.and_then(|ctx| { - ctx.ctx - .udaf(name) - .map(WindowFunctionDefinition::AggregateUDF) - .ok() + // workaround for https://github.com/apache/datafusion-python/issues/730 + let fun = if name == "sum" { + let sum_udf = functions_aggregate::sum::sum_udaf(); + Some(WindowFunctionDefinition::AggregateUDF(sum_udf)) + } else { + find_df_window_func(name).or_else(|| { + ctx.and_then(|ctx| { + ctx.ctx + .udaf(name) + .map(WindowFunctionDefinition::AggregateUDF) + .ok() + }) }) - }); + }; if fun.is_none() { return Err(DataFusionError::Common("window function not found".to_string()).into()); } From f374b44d5033a2d9aa235398f7091dc652c0a14b Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Tue, 25 Jun 2024 22:29:47 -0700 Subject: [PATCH 161/413] changelog (#742) --- CHANGELOG.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c25453534..32a74d825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,24 @@ # DataFusion Python Changelog -## [38.0.0](https://github.com/apache/datafusion-python/tree/38.0.0) (2024-05-16) +## [39.0.0](https://github.com/apache/datafusion-python/tree/39.0.0) (2024-06-25) + +**Merged pull requests:** + +- ci: add substrait feature to linux builds [#720](https://github.com/apache/datafusion-python/pull/720) (Michael-J-Ward) +- Docs deploy action [#721](https://github.com/apache/datafusion-python/pull/721) (Michael-J-Ward) +- update deps [#723](https://github.com/apache/datafusion-python/pull/723) (Michael-J-Ward) +- Upgrade maturin [#725](https://github.com/apache/datafusion-python/pull/725) (Michael-J-Ward) +- Upgrade datafusion 39 [#728](https://github.com/apache/datafusion-python/pull/728) (Michael-J-Ward) +- use ScalarValue::to_pyarrow to convert to python object [#731](https://github.com/apache/datafusion-python/pull/731) (Michael-J-Ward) +- Pyo3 `Bound<'py, T>` api [#734](https://github.com/apache/datafusion-python/pull/734) (Michael-J-Ward) +- github test action: drop python 3.7, add python 3.12 [#736](https://github.com/apache/datafusion-python/pull/736) (Michael-J-Ward) +- Pyarrow filter pushdowns [#735](https://github.com/apache/datafusion-python/pull/735) (Michael-J-Ward) +- build(deps): bump syn from 2.0.66 to 2.0.67 [#738](https://github.com/apache/datafusion-python/pull/738) (dependabot[bot]) +- Pyo3 refactorings [#740](https://github.com/apache/datafusion-python/pull/740) (Michael-J-Ward) +- UDAF `sum` workaround [#741](https://github.com/apache/datafusion-python/pull/741) (Michael-J-Ward) + +## [38.0.1](https://github.com/apache/datafusion-python/tree/38.0.1) (2024-05-25) **Implemented enhancements:** From 75896e5c2fbf12791c94eea633a48a1131727bd3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Jun 2024 14:12:33 -0700 Subject: [PATCH 162/413] build(deps): bump uuid from 1.8.0 to 1.9.1 (#744) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/1.8.0...1.9.1) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f05c62e97..f434c7d07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3288,9 +3288,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" dependencies = [ "getrandom", "serde", diff --git a/Cargo.toml b/Cargo.toml index e518449a8..790fd0949 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ datafusion-sql = "39.0.0" datafusion-substrait = { version = "39.0.0", optional = true } prost = "0.12" prost-types = "0.12" -uuid = { version = "1.8", features = ["v4"] } +uuid = { version = "1.9", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" From 594655f4cdcd590d610c2c5fc0272307011e58aa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Jun 2024 15:19:43 -0700 Subject: [PATCH 163/413] build(deps): bump mimalloc from 0.1.42 to 0.1.43 (#745) Bumps [mimalloc](https://github.com/purpleprotocol/mimalloc_rust) from 0.1.42 to 0.1.43. - [Release notes](https://github.com/purpleprotocol/mimalloc_rust/releases) - [Commits](https://github.com/purpleprotocol/mimalloc_rust/compare/v0.1.42...v0.1.43) --- updated-dependencies: - dependency-name: mimalloc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f434c7d07..b8d333d6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1696,9 +1696,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.38" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7bb23d733dfcc8af652a78b7bf232f0e967710d044732185e561e47c0336b6" +checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44" dependencies = [ "cc", "libc", @@ -1773,9 +1773,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9186d86b79b52f4a77af65604b51225e8db1d6ee7e3f41aec1e40829c71a176" +checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633" dependencies = [ "libmimalloc-sys", ] From a3908ed02ad32d8c9e2126aa578c2b49b9b72f05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Jun 2024 15:20:26 -0700 Subject: [PATCH 164/413] build(deps): bump syn from 2.0.67 to 2.0.68 (#746) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.67 to 2.0.68. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.67...2.0.68) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 58 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8d333d6f..3c282e071 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -376,7 +376,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -387,7 +387,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1052,7 +1052,7 @@ dependencies = [ "rand", "regex-syntax", "sqlparser", - "syn 2.0.67", + "syn 2.0.68", "tokio", "url", "uuid", @@ -1240,7 +1240,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2139,7 +2139,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2179,7 +2179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2218,7 +2218,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.67", + "syn 2.0.68", "tempfile", ] @@ -2232,7 +2232,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2300,7 +2300,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2313,7 +2313,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2623,7 +2623,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2687,7 +2687,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2698,7 +2698,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2721,7 +2721,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2843,7 +2843,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2877,7 +2877,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2890,7 +2890,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2913,7 +2913,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.67", + "syn 2.0.68", "typify", "walkdir", ] @@ -2937,9 +2937,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.67" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -2987,7 +2987,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3050,7 +3050,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3123,7 +3123,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3168,7 +3168,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3202,7 +3202,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.67", + "syn 2.0.68", "thiserror", "unicode-ident", ] @@ -3220,7 +3220,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.67", + "syn 2.0.68", "typify-impl", ] @@ -3348,7 +3348,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -3382,7 +3382,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3609,7 +3609,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 790fd0949..a77eca0c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,7 +54,7 @@ futures = "0.3" object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8.1" -syn = "2.0.67" +syn = "2.0.68" url = "2.2" sqlparser = "0.47.0" From faa5a3f5352342328e183cbeceb3219d7bd05ff8 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 6 Jul 2024 12:12:06 -0400 Subject: [PATCH 165/413] Tsaucer/find window fn (#747) * Add a search order when attempting to locate the appropriate window function * Remove unnecessary markings * Linting * Code cleanup --- examples/tpch/_tests.py | 31 ++++++--- examples/tpch/convert_data_to_parquet.py | 3 +- examples/tpch/q08_market_share.py | 4 +- .../tpch/q09_product_type_profit_measure.py | 4 +- examples/tpch/q13_customer_distribution.py | 4 +- examples/tpch/q14_promotion_effect.py | 4 +- .../tpch/q16_part_supplier_relationship.py | 3 +- examples/tpch/q17_small_quantity_order.py | 8 ++- examples/tpch/q20_potential_part_promotion.py | 4 +- examples/tpch/q22_global_sales_opportunity.py | 4 +- examples/tpch/util.py | 7 +- src/functions.rs | 64 +++++++++++++------ 12 files changed, 100 insertions(+), 40 deletions(-) diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 3f973d9f2..8804041b1 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -21,6 +21,7 @@ from datafusion import col, lit, functions as F from util import get_answer_file + def df_selection(col_name, col_type): if col_type == pa.float64() or isinstance(col_type, pa.Decimal128Type): return F.round(col(col_name), lit(2)).alias(col_name) @@ -29,6 +30,7 @@ def df_selection(col_name, col_type): else: return col(col_name) + def load_schema(col_name, col_type): if col_type == pa.int64() or col_type == pa.int32(): return col_name, pa.string() @@ -36,7 +38,8 @@ def load_schema(col_name, col_type): return col_name, pa.float64() else: return col_name, col_type - + + def expected_selection(col_name, col_type): if col_type == pa.int64() or col_type == pa.int32(): return F.trim(col(col_name)).cast(col_type).alias(col_name) @@ -45,20 +48,23 @@ def expected_selection(col_name, col_type): else: return col(col_name) + def selections_and_schema(original_schema): - columns = [ (c, original_schema.field(c).type) for c in original_schema.names ] + columns = [(c, original_schema.field(c).type) for c in original_schema.names] - df_selections = [ df_selection(c, t) for (c, t) in columns] - expected_schema = [ load_schema(c, t) for (c, t) in columns] - expected_selections = [ expected_selection(c, t) for (c, t) in columns] + df_selections = [df_selection(c, t) for (c, t) in columns] + expected_schema = [load_schema(c, t) for (c, t) in columns] + expected_selections = [expected_selection(c, t) for (c, t) in columns] return (df_selections, expected_schema, expected_selections) + def check_q17(df): raw_value = float(df.collect()[0]["avg_yearly"][0].as_py()) value = round(raw_value, 2) assert abs(value - 348406.05) < 0.001 + @pytest.mark.parametrize( ("query_code", "answer_file"), [ @@ -72,9 +78,7 @@ def check_q17(df): ("q08_market_share", "q8"), ("q09_product_type_profit_measure", "q9"), ("q10_returned_item_reporting", "q10"), - pytest.param( - "q11_important_stock_identification", "q11", - ), + ("q11_important_stock_identification", "q11"), ("q12_ship_mode_order_priority", "q12"), ("q13_customer_distribution", "q13"), ("q14_promotion_effect", "q14"), @@ -97,13 +101,20 @@ def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): if answer_file == "q17": return check_q17(df) - (df_selections, expected_schema, expected_selections) = selections_and_schema(df.schema()) + (df_selections, expected_schema, expected_selections) = selections_and_schema( + df.schema() + ) df = df.select(*df_selections) read_schema = pa.schema(expected_schema) - df_expected = module.ctx.read_csv(get_answer_file(answer_file), schema=read_schema, delimiter="|", file_extension=".out") + df_expected = module.ctx.read_csv( + get_answer_file(answer_file), + schema=read_schema, + delimiter="|", + file_extension=".out", + ) df_expected = df_expected.select(*expected_selections) diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py index d81ec290d..a8091a708 100644 --- a/examples/tpch/convert_data_to_parquet.py +++ b/examples/tpch/convert_data_to_parquet.py @@ -117,7 +117,6 @@ curr_dir = os.path.dirname(os.path.abspath(__file__)) for filename, curr_schema in all_schemas.items(): - # For convenience, go ahead and convert the schema column names to lowercase curr_schema = [(s[0].lower(), s[1]) for s in curr_schema] @@ -125,7 +124,7 @@ # in to handle the trailing | in the file output_cols = [r[0] for r in curr_schema] - curr_schema = [ pyarrow.field(r[0], r[1], nullable=False) for r in curr_schema] + curr_schema = [pyarrow.field(r[0], r[1], nullable=False) for r in curr_schema] # Trailing | requires extra field for in processing curr_schema.append(("some_null", pyarrow.null())) diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py index d13a71dff..cd6bc1fa9 100644 --- a/examples/tpch/q08_market_share.py +++ b/examples/tpch/q08_market_share.py @@ -47,7 +47,9 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_type") +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( + "p_partkey", "p_type" +) df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) diff --git a/examples/tpch/q09_product_type_profit_measure.py b/examples/tpch/q09_product_type_profit_measure.py index 29ffceed4..b4a7369f8 100644 --- a/examples/tpch/q09_product_type_profit_measure.py +++ b/examples/tpch/q09_product_type_profit_measure.py @@ -39,7 +39,9 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_name") +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( + "p_partkey", "p_name" +) df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( "s_suppkey", "s_nationkey" ) diff --git a/examples/tpch/q13_customer_distribution.py b/examples/tpch/q13_customer_distribution.py index 2b6e7e208..bc0a5bd1f 100644 --- a/examples/tpch/q13_customer_distribution.py +++ b/examples/tpch/q13_customer_distribution.py @@ -41,7 +41,9 @@ df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( "o_custkey", "o_comment" ) -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns("c_custkey") +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( + "c_custkey" +) # Use a regex to remove special cases df_orders = df_orders.filter( diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py index 75fa363ad..8cb1e4c5a 100644 --- a/examples/tpch/q14_promotion_effect.py +++ b/examples/tpch/q14_promotion_effect.py @@ -44,7 +44,9 @@ df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_partkey", "l_shipdate", "l_extendedprice", "l_discount" ) -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_type") +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( + "p_partkey", "p_type" +) # Check part type begins with PROMO diff --git a/examples/tpch/q16_part_supplier_relationship.py b/examples/tpch/q16_part_supplier_relationship.py index 0db2d1b84..fdcb5b4db 100644 --- a/examples/tpch/q16_part_supplier_relationship.py +++ b/examples/tpch/q16_part_supplier_relationship.py @@ -62,7 +62,8 @@ # Select the parts we are interested in df_part = df_part.filter(col("p_brand") != lit(BRAND)) df_part = df_part.filter( - F.substring(col("p_type"), lit(0), lit(len(TYPE_TO_IGNORE) + 1)) != lit(TYPE_TO_IGNORE) + F.substring(col("p_type"), lit(0), lit(len(TYPE_TO_IGNORE) + 1)) + != lit(TYPE_TO_IGNORE) ) # Python conversion of integer to literal casts it to int64 but the data for diff --git a/examples/tpch/q17_small_quantity_order.py b/examples/tpch/q17_small_quantity_order.py index 5880e7ed4..e0ee8bb90 100644 --- a/examples/tpch/q17_small_quantity_order.py +++ b/examples/tpch/q17_small_quantity_order.py @@ -56,7 +56,13 @@ # Find the average quantity window_frame = WindowFrame("rows", None, None) df = df.with_column( - "avg_quantity", F.window("avg", [col("l_quantity")], window_frame=window_frame, partition_by=[col("l_partkey")]) + "avg_quantity", + F.window( + "avg", + [col("l_quantity")], + window_frame=window_frame, + partition_by=[col("l_partkey")], + ), ) df = df.filter(col("l_quantity") < lit(0.2) * col("avg_quantity")) diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index 85e7226f7..05a267450 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -40,7 +40,9 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns("p_partkey", "p_name") +df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( + "p_partkey", "p_name" +) df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( "l_shipdate", "l_partkey", "l_suppkey", "l_quantity" ) diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py index dfde19cb4..622c1429f 100644 --- a/examples/tpch/q22_global_sales_opportunity.py +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -38,7 +38,9 @@ df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( "c_phone", "c_acctbal", "c_custkey" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns("o_custkey") +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( + "o_custkey" +) # The nation code is a two digit number, but we need to convert it to a string literal nation_codes = F.make_array(*[lit(str(n)) for n in NATION_CODES]) diff --git a/examples/tpch/util.py b/examples/tpch/util.py index 191fa6092..7e3d659dd 100644 --- a/examples/tpch/util.py +++ b/examples/tpch/util.py @@ -20,14 +20,17 @@ """ import os -from pathlib import Path + def get_data_path(filename: str) -> str: path = os.path.dirname(os.path.abspath(__file__)) return os.path.join(path, "data", filename) + def get_answer_file(answer_file: str) -> str: path = os.path.dirname(os.path.abspath(__file__)) - return os.path.join(path, "../../benchmarks/tpch/data/answers", f"{answer_file}.out") + return os.path.join( + path, "../../benchmarks/tpch/data/answers", f"{answer_file}.out" + ) diff --git a/src/functions.rs b/src/functions.rs index b39d98b35..0ec0d5ef1 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::functions_aggregate::all_default_aggregate_functions; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; @@ -311,6 +312,50 @@ fn case(expr: PyExpr) -> PyResult { }) } +/// Helper function to find the appropriate window function. First, if a session +/// context is defined check it's registered functions. If no context is defined, +/// attempt to find from all default functions. Lastly, as a fall back attempt +/// to use built in window functions, which are being deprecated. +fn find_window_fn(name: &str, ctx: Option) -> PyResult { + let mut maybe_fn = match &ctx { + Some(ctx) => { + let session_state = ctx.ctx.state(); + + match session_state.window_functions().contains_key(name) { + true => session_state + .window_functions() + .get(name) + .map(|f| WindowFunctionDefinition::WindowUDF(f.clone())), + false => session_state + .aggregate_functions() + .get(name) + .map(|f| WindowFunctionDefinition::AggregateUDF(f.clone())), + } + } + None => { + let default_aggregate_fns = all_default_aggregate_functions(); + + default_aggregate_fns + .iter() + .find(|v| v.aliases().contains(&name.to_string())) + .map(|f| WindowFunctionDefinition::AggregateUDF(f.clone())) + } + }; + + if maybe_fn.is_none() { + maybe_fn = find_df_window_func(name).or_else(|| { + ctx.and_then(|ctx| { + ctx.ctx + .udaf(name) + .map(WindowFunctionDefinition::AggregateUDF) + .ok() + }) + }); + } + + maybe_fn.ok_or(DataFusionError::Common("window function not found".to_string()).into()) +} + /// Creates a new Window function expression #[pyfunction] fn window( @@ -321,24 +366,7 @@ fn window( window_frame: Option, ctx: Option, ) -> PyResult { - // workaround for https://github.com/apache/datafusion-python/issues/730 - let fun = if name == "sum" { - let sum_udf = functions_aggregate::sum::sum_udaf(); - Some(WindowFunctionDefinition::AggregateUDF(sum_udf)) - } else { - find_df_window_func(name).or_else(|| { - ctx.and_then(|ctx| { - ctx.ctx - .udaf(name) - .map(WindowFunctionDefinition::AggregateUDF) - .ok() - }) - }) - }; - if fun.is_none() { - return Err(DataFusionError::Common("window function not found".to_string()).into()); - } - let fun = fun.unwrap(); + let fun = find_window_fn(name, ctx)?; let window_frame = window_frame .unwrap_or_else(|| PyWindowFrame::new("rows", None, Some(0)).unwrap()) .into(); From aa8aa9cd558ad0a0323db284ec7db5810d2abfa8 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 18 Jul 2024 09:05:50 -0400 Subject: [PATCH 166/413] Python wrapper classes for all user interfaces (#750) * Expose missing functions to python * Initial commit for creating wrapper classes and functions for all user facing python features * Remove extra level of python path that is no longer required * Move import to only happen for type checking for hints * Comment out classes from __all__ in the top level that are not currently exposed. * Add license comments * Add missing import * Functions now only has one level of depth * Applying google docstring formatting * Addressing PR request to add google formatted docstrings * Small docstring for ruff * Linting * Add docstring format checking to pre-commit stage * Set explicit return types on UDFs * Add options of passing either a path or a string * Switch to google docstring style * Update unit tests to include registering via path or string * Add py.typed file * Resolve deprecation warnings in unit tests * Add path to unit test * Expose an option in write_csv to include header and add unit test * Update write_parquet unit test to include paths or strings * Add unit test for write_json * Add unit test for substrait serialization to a file * Add unit tests for runtime config * Setting return type to typing_extensions.Self per PR recommendation * Correcting __next__ to not return None since it will raise an exception instead. * Add optiona parameter of decimal places to round and add unit test * Improve docstrings * Set default to None instead of empty dict * User request to allow passing multiple arguments to filter() * Enhance Expr comparison operators to accept any python value and attempt to convert it to a literal * Expose overlay and add unit test * Allow select() to take either str for column names or a full expr * Update comments on regexp and add unit tests * Remove TODO markings no longer applicable * Update udf documentation * Docstring formatting * Updating docstring formatting * Updating docstring formatting * Updating docstring formatting * Updating docstring formatting * Updating docstring formatting * Cleaning up docstring line lengths * Add pre-commit check of docstring line length * Do not emit doc entry for __init__ of some classes * Correct errors on code blocks generating in sphinx * Resolve conflict with * Add license info to py.typed * Clean up some docstring too long errors in CI * Correct ruff complain in unit tests * Temporarily install google test to get clippy to pass * Adding gmock to build step due to upstream error * Add type_extensions to conda meta file * Small comment suggestions from PR --- .github/workflows/build.yml | 4 + .github/workflows/test.yaml | 4 + benchmarks/db-benchmark/join-datafusion.py | 3 +- conda/recipes/meta.yaml | 1 + docs/source/api/functions.rst | 2 +- docs/source/conf.py | 21 + examples/substrait.py | 15 +- examples/tpch/_tests.py | 5 +- pyproject.toml | 18 + python/datafusion/__init__.py | 170 +-- python/datafusion/catalog.py | 76 + python/datafusion/common.py | 2 +- python/datafusion/context.py | 1003 +++++++++++++ python/datafusion/dataframe.py | 527 +++++++ python/datafusion/expr.py | 414 +++++- python/datafusion/functions.py | 1471 +++++++++++++++++++- python/datafusion/input/__init__.py | 5 + python/datafusion/input/base.py | 17 +- python/datafusion/input/location.py | 10 +- python/datafusion/object_store.py | 2 +- python/datafusion/py.typed | 16 + python/datafusion/record_batch.py | 74 + python/datafusion/substrait.py | 168 ++- python/datafusion/tests/conftest.py | 3 +- python/datafusion/tests/test_context.py | 87 +- python/datafusion/tests/test_dataframe.py | 90 +- python/datafusion/tests/test_expr.py | 25 +- python/datafusion/tests/test_functions.py | 66 +- python/datafusion/tests/test_imports.py | 15 +- python/datafusion/tests/test_sql.py | 61 +- python/datafusion/tests/test_substrait.py | 38 +- python/datafusion/tests/test_udaf.py | 4 +- python/datafusion/udf.py | 248 ++++ src/common.rs | 1 + src/common/data_type.rs | 2 +- src/dataframe.rs | 29 +- src/expr.rs | 1 + src/functions.rs | 27 +- src/lib.rs | 2 + src/substrait.rs | 2 +- 40 files changed, 4441 insertions(+), 288 deletions(-) create mode 100644 python/datafusion/catalog.py create mode 100644 python/datafusion/context.py create mode 100644 python/datafusion/dataframe.py create mode 100644 python/datafusion/py.typed create mode 100644 python/datafusion/record_batch.py create mode 100644 python/datafusion/udf.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 350be46d5..a37abe53a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,6 +89,10 @@ jobs: name: python-wheel-license path: . + # To remove once https://github.com/MaterializeInc/rust-protobuf-native/issues/20 is resolved + - name: Install gtest + uses: MarkusJx/googletest-installer@v1.1 + - name: Install Protoc uses: arduino/setup-protoc@v1 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4f47dc984..c9a365bbb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,6 +55,10 @@ jobs: version: '3.20.2' repo-token: ${{ secrets.GITHUB_TOKEN }} + # To remove once https://github.com/MaterializeInc/rust-protobuf-native/issues/20 is resolved + - name: Install gtest + uses: MarkusJx/googletest-installer@v1.1 + - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py index 4d59c7dc2..811ad8707 100755 --- a/benchmarks/db-benchmark/join-datafusion.py +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -74,7 +74,8 @@ def ans_shape(batches): ctx = df.SessionContext() print(ctx) -# TODO we should be applying projections to these table reads to crete relations of different sizes +# TODO we should be applying projections to these table reads to create relations +# of different sizes x_data = pacsv.read_csv( src_jn_x, convert_options=pacsv.ConvertOptions(auto_dict_encode=True) diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml index 72ac7f501..b0784253a 100644 --- a/conda/recipes/meta.yaml +++ b/conda/recipes/meta.yaml @@ -51,6 +51,7 @@ requirements: run: - python - pyarrow >=11.0.0 + - typing_extensions test: imports: diff --git a/docs/source/api/functions.rst b/docs/source/api/functions.rst index 958606df2..6f10d826e 100644 --- a/docs/source/api/functions.rst +++ b/docs/source/api/functions.rst @@ -24,4 +24,4 @@ Functions .. autosummary:: :toctree: ../generated/ - functions.functions + functions diff --git a/docs/source/conf.py b/docs/source/conf.py index c0da8b2cc..308069b6c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +"""Documenation generation.""" + # Configuration file for the Sphinx documentation builder. # # This file only contains a selection of the most common options. For a full @@ -78,6 +80,25 @@ autosummary_generate = True + +def autodoc_skip_member(app, what, name, obj, skip, options): + exclude_functions = "__init__" + exclude_classes = ("Expr", "DataFrame") + + class_name = "" + if hasattr(obj, "__qualname__"): + if obj.__qualname__ is not None: + class_name = obj.__qualname__.split(".")[0] + + should_exclude = name in exclude_functions and class_name in exclude_classes + + return True if should_exclude else None + + +def setup(app): + app.connect("autodoc-skip-member", autodoc_skip_member) + + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/examples/substrait.py b/examples/substrait.py index 23cd74649..fd4d0f9ca 100644 --- a/examples/substrait.py +++ b/examples/substrait.py @@ -18,16 +18,13 @@ from datafusion import SessionContext from datafusion import substrait as ss - # Create a DataFusion context ctx = SessionContext() # Register table with context ctx.register_csv("aggregate_test_data", "./testing/data/csv/aggregate_test_100.csv") -substrait_plan = ss.substrait.serde.serialize_to_plan( - "SELECT * FROM aggregate_test_data", ctx -) +substrait_plan = ss.Serde.serialize_to_plan("SELECT * FROM aggregate_test_data", ctx) # type(substrait_plan) -> # Encode it to bytes @@ -38,17 +35,15 @@ # Alternative serialization approaches # type(substrait_bytes) -> , at this point the bytes can be distributed to file, network, etc safely # where they could subsequently be deserialized on the receiving end. -substrait_bytes = ss.substrait.serde.serialize_bytes( - "SELECT * FROM aggregate_test_data", ctx -) +substrait_bytes = ss.Serde.serialize_bytes("SELECT * FROM aggregate_test_data", ctx) # Imagine here bytes would be read from network, file, etc ... for example brevity this is omitted and variable is simply reused # type(substrait_plan) -> -substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes) +substrait_plan = ss.Serde.deserialize_bytes(substrait_bytes) # type(df_logical_plan) -> -df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan) +df_logical_plan = ss.Consumer.from_substrait_plan(ctx, substrait_plan) # Back to Substrait Plan just for demonstration purposes # type(substrait_plan) -> -substrait_plan = ss.substrait.producer.to_substrait_plan(df_logical_plan) +substrait_plan = ss.Producer.to_substrait_plan(df_logical_plan) diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 8804041b1..903b53548 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -96,8 +96,9 @@ def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): module = import_module(query_code) df = module.df - # Treat q17 as a special case. The answer file does not match the spec. Running at - # scale factor 1, we have manually verified this result does match the expected value. + # Treat q17 as a special case. The answer file does not match the spec. + # Running at scale factor 1, we have manually verified this result does + # match the expected value. if answer_file == "q17": return check_q17(df) diff --git a/pyproject.toml b/pyproject.toml index b706065a4..a18ef0e5e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,3 +64,21 @@ exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true features = ["substrait"] + +# Enable docstring linting using the google style guide +[tool.ruff.lint] +select = ["E4", "E7", "E9", "F", "D", "W"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.pycodestyle] +max-doc-length = 88 + +# Disable docstring checking for these directories +[tool.ruff.lint.per-file-ignores] +"python/datafusion/tests/*" = ["D"] +"examples/*" = ["D", "W505"] +"dev/*" = ["D"] +"benchmarks/*" = ["D", "F"] +"docs/*" = ["D"] diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 846b1a459..59bc8e306 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -15,80 +15,44 @@ # specific language governing permissions and limitations # under the License. -from abc import ABCMeta, abstractmethod -from typing import List +"""DataFusion python package. + +This is a Python library that binds to Apache Arrow in-memory query engine DataFusion. +See https://datafusion.apache.org/python for more information. +""" try: import importlib.metadata as importlib_metadata except ImportError: import importlib_metadata -import pyarrow as pa - -from ._internal import ( - AggregateUDF, - Config, - DataFrame, +from .context import ( SessionContext, SessionConfig, RuntimeConfig, - ScalarUDF, SQLOptions, ) +# The following imports are okay to remain as opaque to the user. +from ._internal import Config + +from .udf import ScalarUDF, AggregateUDF, Accumulator + from .common import ( DFSchema, ) +from .dataframe import DataFrame + from .expr import ( - Alias, - Analyze, Expr, - Filter, - Limit, - Like, - ILike, - Projection, - SimilarTo, - ScalarVariable, - Sort, - TableScan, - Not, - IsNotNull, - IsTrue, - IsFalse, - IsUnknown, - IsNotTrue, - IsNotFalse, - IsNotUnknown, - Negative, - InList, - Exists, - Subquery, - InSubquery, - ScalarSubquery, - GroupingSet, - Placeholder, - Case, - Cast, - TryCast, - Between, - Explain, - CreateMemoryTable, - SubqueryAlias, - Extension, - CreateView, - Distinct, - DropTable, - Repartition, - Partitioning, - Window, WindowFrame, ) __version__ = importlib_metadata.version(__name__) __all__ = [ + "Accumulator", "Config", "DataFrame", "SessionContext", @@ -96,78 +60,16 @@ "SQLOptions", "RuntimeConfig", "Expr", - "AggregateUDF", "ScalarUDF", - "Window", "WindowFrame", "column", "literal", - "TableScan", - "Projection", "DFSchema", - "DFField", - "Analyze", - "Sort", - "Limit", - "Filter", - "Like", - "ILike", - "SimilarTo", - "ScalarVariable", - "Alias", - "Not", - "IsNotNull", - "IsTrue", - "IsFalse", - "IsUnknown", - "IsNotTrue", - "IsNotFalse", - "IsNotUnknown", - "Negative", - "ScalarFunction", - "BuiltinScalarFunction", - "InList", - "Exists", - "Subquery", - "InSubquery", - "ScalarSubquery", - "GroupingSet", - "Placeholder", - "Case", - "Cast", - "TryCast", - "Between", - "Explain", - "SubqueryAlias", - "Extension", - "CreateMemoryTable", - "CreateView", - "Distinct", - "DropTable", - "Repartition", - "Partitioning", ] -class Accumulator(metaclass=ABCMeta): - @abstractmethod - def state(self) -> List[pa.Scalar]: - pass - - @abstractmethod - def update(self, values: pa.Array) -> None: - pass - - @abstractmethod - def merge(self, states: pa.Array) -> None: - pass - - @abstractmethod - def evaluate(self) -> pa.Scalar: - pass - - -def column(value): +def column(value: str): + """Create a column expression.""" return Expr.column(value) @@ -175,46 +77,12 @@ def column(value): def literal(value): - if not isinstance(value, pa.Scalar): - value = pa.scalar(value) + """Create a literal expression.""" return Expr.literal(value) lit = literal +udf = ScalarUDF.udf -def udf(func, input_types, return_type, volatility, name=None): - """ - Create a new User Defined Function - """ - if not callable(func): - raise TypeError("`func` argument must be callable") - if name is None: - name = func.__qualname__.lower() - return ScalarUDF( - name=name, - func=func, - input_types=input_types, - return_type=return_type, - volatility=volatility, - ) - - -def udaf(accum, input_type, return_type, state_type, volatility, name=None): - """ - Create a new User Defined Aggregate Function - """ - if not issubclass(accum, Accumulator): - raise TypeError("`accum` must implement the abstract base class Accumulator") - if name is None: - name = accum.__qualname__.lower() - if isinstance(input_type, pa.lib.DataType): - input_type = [input_type] - return AggregateUDF( - name=name, - accumulator=accum, - input_type=input_type, - return_type=return_type, - state_type=state_type, - volatility=volatility, - ) +udaf = AggregateUDF.udaf diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py new file mode 100644 index 000000000..cec0be764 --- /dev/null +++ b/python/datafusion/catalog.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Data catalog providers.""" + +from __future__ import annotations + +import datafusion._internal as df_internal + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pyarrow + + +class Catalog: + """DataFusion data catalog.""" + + def __init__(self, catalog: df_internal.Catalog) -> None: + """This constructor is not typically called by the end user.""" + self.catalog = catalog + + def names(self) -> list[str]: + """Returns the list of databases in this catalog.""" + return self.catalog.names() + + def database(self, name: str = "public") -> Database: + """Returns the database with the given `name` from this catalog.""" + return Database(self.catalog.database(name)) + + +class Database: + """DataFusion Database.""" + + def __init__(self, db: df_internal.Database) -> None: + """This constructor is not typically called by the end user.""" + self.db = db + + def names(self) -> set[str]: + """Returns the list of all tables in this database.""" + return self.db.names() + + def table(self, name: str) -> Table: + """Return the table with the given `name` from this database.""" + return Table(self.db.table(name)) + + +class Table: + """DataFusion table.""" + + def __init__(self, table: df_internal.Table) -> None: + """This constructor is not typically called by the end user.""" + self.table = table + + def schema(self) -> pyarrow.Schema: + """Returns the schema associated with this table.""" + return self.table.schema() + + @property + def kind(self) -> str: + """Returns the kind of table.""" + return self.table.kind() diff --git a/python/datafusion/common.py b/python/datafusion/common.py index dd56640a4..2351845b8 100644 --- a/python/datafusion/common.py +++ b/python/datafusion/common.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +"""Common data types used throughout the DataFusion project.""" from ._internal import common diff --git a/python/datafusion/context.py b/python/datafusion/context.py new file mode 100644 index 000000000..a717db106 --- /dev/null +++ b/python/datafusion/context.py @@ -0,0 +1,1003 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Session Context and it's associated configuration.""" + +from __future__ import annotations + +from ._internal import SessionConfig as SessionConfigInternal +from ._internal import RuntimeConfig as RuntimeConfigInternal +from ._internal import SQLOptions as SQLOptionsInternal +from ._internal import SessionContext as SessionContextInternal +from ._internal import LogicalPlan, ExecutionPlan + +from datafusion._internal import AggregateUDF +from datafusion.catalog import Catalog, Table +from datafusion.dataframe import DataFrame +from datafusion.expr import Expr +from datafusion.record_batch import RecordBatchStream +from datafusion.udf import ScalarUDF + +from typing import Any, TYPE_CHECKING +from typing_extensions import deprecated + +if TYPE_CHECKING: + import pyarrow + import pandas + import polars + import pathlib + + +class SessionConfig: + """Session configuration options.""" + + def __init__(self, config_options: dict[str, str] | None = None) -> None: + """Create a new `SessionConfig` with the given configuration options. + + Args: + config_options: Configuration options. + """ + self.config_internal = SessionConfigInternal(config_options) + + def with_create_default_catalog_and_schema( + self, enabled: bool = True + ) -> SessionConfig: + """Control if the default catalog and schema will be automatically created. + + Args: + enabled: Whether the default catalog and schema will be + automatically created. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = ( + self.config_internal.with_create_default_catalog_and_schema(enabled) + ) + return self + + def with_default_catalog_and_schema( + self, catalog: str, schema: str + ) -> SessionConfig: + """Select a name for the default catalog and shcema. + + Args: + catalog: Catalog name. + schema: Schema name. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_default_catalog_and_schema( + catalog, schema + ) + return self + + def with_information_schema(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the inclusion of `information_schema` virtual tables. + + Args: + enabled: Whether to include `information_schema` virtual tables. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_information_schema(enabled) + return self + + def with_batch_size(self, batch_size: int) -> SessionConfig: + """Customize batch size. + + Args: + batch_size: Batch size. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_batch_size(batch_size) + return self + + def with_target_partitions(self, target_partitions: int) -> SessionConfig: + """Customize the number of target partitions for query execution. + + Increasing partitions can increase concurrency. + + Args: + target_partitions: Number of target partitions. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_target_partitions( + target_partitions + ) + return self + + def with_repartition_aggregations(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the use of repartitioning for aggregations. + + Enabling this improves parallelism. + + Args: + enabled: Whether to use repartitioning for aggregations. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_repartition_aggregations( + enabled + ) + return self + + def with_repartition_joins(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the use of repartitioning for joins to improve parallelism. + + Args: + enabled: Whether to use repartitioning for joins. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_repartition_joins(enabled) + return self + + def with_repartition_windows(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the use of repartitioning for window functions. + + This may improve parallelism. + + Args: + enabled: Whether to use repartitioning for window functions. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_repartition_windows(enabled) + return self + + def with_repartition_sorts(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the use of repartitioning for window functions. + + This may improve parallelism. + + Args: + enabled: Whether to use repartitioning for window functions. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_repartition_sorts(enabled) + return self + + def with_repartition_file_scans(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the use of repartitioning for file scans. + + Args: + enabled: Whether to use repartitioning for file scans. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_repartition_file_scans(enabled) + return self + + def with_repartition_file_min_size(self, size: int) -> SessionConfig: + """Set minimum file range size for repartitioning scans. + + Args: + size: Minimum file range size. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_repartition_file_min_size(size) + return self + + def with_parquet_pruning(self, enabled: bool = True) -> SessionConfig: + """Enable or disable the use of pruning predicate for parquet readers. + + Pruning predicates will enable the reader to skip row groups. + + Args: + enabled: Whether to use pruning predicate for parquet readers. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_parquet_pruning(enabled) + return self + + def set(self, key: str, value: str) -> SessionConfig: + """Set a configuration option. + + Args: + key: Option key. + value: Option value. + + Returns: + A new `SessionConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.set(key, value) + return self + + +class RuntimeConfig: + """Runtime configuration options.""" + + def __init__(self) -> None: + """Create a new `RuntimeConfig` with default values.""" + self.config_internal = RuntimeConfigInternal() + + def with_disk_manager_disabled(self) -> RuntimeConfig: + """Disable the disk manager, attempts to create temporary files will error. + + Returns: + A new `RuntimeConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_disk_manager_disabled() + return self + + def with_disk_manager_os(self) -> RuntimeConfig: + """Use the operating system's temporary directory for disk manager. + + Returns: + A new `RuntimeConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_disk_manager_os() + return self + + def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConfig: + """Use the specified paths for the disk manager's temporary files. + + Args: + paths: Paths to use for the disk manager's temporary files. + + Returns: + A new `RuntimeConfig` object with the updated setting. + """ + paths = [str(p) for p in paths] + self.config_internal = self.config_internal.with_disk_manager_specified(paths) + return self + + def with_unbounded_memory_pool(self) -> RuntimeConfig: + """Use an unbounded memory pool. + + Returns: + A new `RuntimeConfig` object with the updated setting. + """ + self.config_internal = self.config_internal.with_unbounded_memory_pool() + return self + + def with_fair_spill_pool(self, size: int) -> RuntimeConfig: + """Use a fair spill pool with the specified size. + + This pool works best when you know beforehand the query has multiple spillable + operators that will likely all need to spill. Sometimes it will cause spills + even when there was sufficient memory (reserved for other operators) to avoid + doing so:: + + ┌───────────────────────z──────────────────────z───────────────┐ + │ z z │ + │ z z │ + │ Spillable z Unspillable z Free │ + │ Memory z Memory z Memory │ + │ z z │ + │ z z │ + └───────────────────────z──────────────────────z───────────────┘ + + Args: + size: Size of the memory pool in bytes. + + Returns: + A new ``RuntimeConfig`` object with the updated setting. + + Examples usage:: + + config = RuntimeConfig().with_fair_spill_pool(1024) + """ + self.config_internal = self.config_internal.with_fair_spill_pool(size) + return self + + def with_greedy_memory_pool(self, size: int) -> RuntimeConfig: + """Use a greedy memory pool with the specified size. + + This pool works well for queries that do not need to spill or have a single + spillable operator. See `RuntimeConfig.with_fair_spill_pool` if there are + multiple spillable operators that all will spill. + + Args: + size: Size of the memory pool in bytes. + + Returns: + A new `RuntimeConfig` object with the updated setting. + + Example usage:: + + config = RuntimeConfig().with_greedy_memory_pool(1024) + """ + self.config_internal = self.config_internal.with_greedy_memory_pool(size) + return self + + def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeConfig: + """Use the specified path to create any needed temporary files. + + Args: + path: Path to use for temporary files. + + Returns: + A new `RuntimeConfig` object with the updated setting. + + Example usage:: + + config = RuntimeConfig().with_temp_file_path("/tmp") + """ + self.config_internal = self.config_internal.with_temp_file_path(str(path)) + return self + + +class SQLOptions: + """Options to be used when performing SQL queries on the ``SessionContext``.""" + + def __init__(self) -> None: + """Create a new `SQLOptions` with default values. + + The default values are: + - DDL commands are allowed + - DML commands are allowed + - Statements are allowed + """ + self.options_internal = SQLOptionsInternal() + + def with_allow_ddl(self, allow: bool = True) -> SQLOptions: + """Should DDL (Data Definition Language) commands be run? + + Examples of DDL commands include `CREATE TABLE` and `DROP TABLE`. + + Args: + allow: Allow DDL commands to be run. + + Returns: + A new `SQLOptions` object with the updated setting. + + Example usage:: + + options = SQLOptions().with_allow_ddl(True) + """ + self.options_internal = self.options_internal.with_allow_ddl(allow) + return self + + def with_allow_dml(self, allow: bool = True) -> SQLOptions: + """Should DML (Data Manipulation Language) commands be run? + + Examples of DML commands include `INSERT INTO` and `DELETE`. + + Args: + allow: Allow DML commands to be run. + + Returns: + A new `SQLOptions` object with the updated setting. + + Example usage:: + + options = SQLOptions().with_allow_dml(True) + """ + self.options_internal = self.options_internal.with_allow_dml(allow) + return self + + def with_allow_statements(self, allow: bool = True) -> SQLOptions: + """Should statements such as `SET VARIABLE` and `BEGIN TRANSACTION` be run? + + Args: + allow: Allow statements to be run. + + Returns: + A new `SQLOptions` object with the updated setting. + + Example usage:: + + options = SQLOptions().with_allow_statements(True) + """ + self.options_internal = self.options_internal.with_allow_statements(allow) + return self + + +class SessionContext: + """This is the main interface for executing queries and creating DataFrames. + + See https://datafusion.apache.org/python/user-guide/basics.html for + additional information. + """ + + def __init__( + self, config: SessionConfig | None = None, runtime: RuntimeConfig | None = None + ) -> None: + """Main interface for executing queries with DataFusion. + + Maintains the state of the connection between a user and an instance + of the connection between a user and an instance of the DataFusion + engine. + + Args: + config: Session configuration options. + runtime: Runtime configuration options. + + Example usage: + + The following example demostrates how to use the context to execute + a query against a CSV data source using the ``DataFrame`` API:: + + from datafusion import SessionContext + + ctx = SessionContext() + df = ctx.read_csv("data.csv") + """ + config = config.config_internal if config is not None else None + runtime = runtime.config_internal if config is not None else None + + self.ctx = SessionContextInternal(config, runtime) + + def register_object_store(self, schema: str, store: Any, host: str | None) -> None: + """Add a new object store into the session. + + Args: + schema: The data source schema. + store: The `ObjectStore` to register. + host: URL for the host. + """ + self.ctx.register_object_store(schema, store, host) + + def register_listing_table( + self, + name: str, + path: str | pathlib.Path, + table_partition_cols: list[tuple[str, str]] | None = None, + file_extension: str = ".parquet", + schema: pyarrow.Schema | None = None, + file_sort_order: list[list[Expr]] | None = None, + ) -> None: + """Register multiple files as a single table. + + Registers a `Table` that can assemble multiple files from locations in + an `ObjectStore` instance. + + Args: + name: Name of the resultant table. + path: Path to the file to register. + table_partition_cols: Partition columns. + file_extension: File extension of the provided table. + schema: The data source schema. + file_sort_order: Sort order for the file. + """ + if table_partition_cols is None: + table_partition_cols = [] + if file_sort_order is not None: + file_sort_order = [[x.expr for x in xs] for xs in file_sort_order] + self.ctx.register_listing_table( + name, + str(path), + table_partition_cols, + file_extension, + schema, + file_sort_order, + ) + + def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame: + """Create a `DataFrame` from SQL query text. + + Note: This API implements DDL statements such as `CREATE TABLE` and + `CREATE VIEW` and DML statements such as `INSERT INTO` with in-memory + default implementation. See `SessionContext.sql_with_options`. + + Args: + query: SQL query text. + options: If provided, the query will be validated against these options. + + Returns: + DataFrame representation of the SQL query. + """ + if options is None: + return DataFrame(self.ctx.sql(query)) + return DataFrame(self.ctx.sql_with_options(query, options.options_internal)) + + def sql_with_options(self, query: str, options: SQLOptions) -> DataFrame: + """Create a `DataFrame` from SQL query text. + + This function will first validating that the query is allowed by the + provided options. + + Args: + query: SQL query text. + options: SQL options. + + Returns: + DataFrame representation of the SQL query. + """ + return self.sql(query, options) + + def create_dataframe( + self, + partitions: list[list[pyarrow.RecordBatch]], + name: str | None = None, + schema: pyarrow.Schema | None = None, + ) -> DataFrame: + """Create and return a dataframe using the provided partitions. + + Args: + partitions: `RecordBatch` partitions to register. + name: Resultant dataframe name. + schema: Schema for the partitions. + + Returns: + DataFrame representation of the SQL query. + """ + return DataFrame(self.ctx.create_dataframe(partitions, name, schema)) + + def create_dataframe_from_logical_plan(self, plan: LogicalPlan) -> DataFrame: + """Create a `DataFrame` from an existing logical plan. + + Args: + plan: Logical plan. + + Returns: + DataFrame representation of the logical plan. + """ + return DataFrame(self.ctx.create_dataframe_from_logical_plan(plan)) + + def from_pylist( + self, data: list[dict[str, Any]], name: str | None = None + ) -> DataFrame: + """Create a `DataFrame` from a list of dictionaries. + + Args: + data: List of dictionaries. + name: Name of the DataFrame. + + Returns: + DataFrame representation of the list of dictionaries. + """ + return DataFrame(self.ctx.from_pylist(data, name)) + + def from_pydict( + self, data: dict[str, list[Any]], name: str | None = None + ) -> DataFrame: + """Create a `DataFrame` from a dictionary of lists. + + Args: + data: Dictionary of lists. + name: Name of the DataFrame. + + Returns: + DataFrame representation of the dictionary of lists. + """ + return DataFrame(self.ctx.from_pydict(data, name)) + + def from_arrow_table( + self, data: pyarrow.Table, name: str | None = None + ) -> DataFrame: + """Create a `DataFrame` from an Arrow table. + + Args: + data: Arrow table. + name: Name of the DataFrame. + + Returns: + DataFrame representation of the Arrow table. + """ + return DataFrame(self.ctx.from_arrow_table(data, name)) + + def from_pandas(self, data: pandas.DataFrame, name: str | None = None) -> DataFrame: + """Create a `DataFrame` from a Pandas DataFrame. + + Args: + data: Pandas DataFrame. + name: Name of the DataFrame. + + Returns: + DataFrame representation of the Pandas DataFrame. + """ + return DataFrame(self.ctx.from_pandas(data, name)) + + def from_polars(self, data: polars.DataFrame, name: str | None = None) -> DataFrame: + """Create a `DataFrame` from a Polars DataFrame. + + Args: + data: Polars DataFrame. + name: Name of the DataFrame. + + Returns: + DataFrame representation of the Polars DataFrame. + """ + return DataFrame(self.ctx.from_polars(data, name)) + + def register_table(self, name: str, table: pyarrow.Table) -> None: + """Register a table with the given name into the session. + + Args: + name: Name of the resultant table. + table: PyArrow table to add to the session context. + """ + self.ctx.register_table(name, table) + + def deregister_table(self, name: str) -> None: + """Remove a table from the session.""" + self.ctx.deregister_table(name) + + def register_record_batches( + self, name: str, partitions: list[list[pyarrow.RecordBatch]] + ) -> None: + """Register record batches as a table. + + This function will convert the provided partitions into a table and + register it into the session using the given name. + + Args: + name: Name of the resultant table. + partitions: Record batches to register as a table. + """ + self.ctx.register_record_batches(name, partitions) + + def register_parquet( + self, + name: str, + path: str | pathlib.Path, + table_partition_cols: list[tuple[str, str]] | None = None, + parquet_pruning: bool = True, + file_extension: str = ".parquet", + skip_metadata: bool = True, + schema: pyarrow.Schema | None = None, + file_sort_order: list[list[Expr]] | None = None, + ) -> None: + """Register a Parquet file as a table. + + The registered table can be referenced from SQL statement executed + against this context. + + Args: + name: Name of the table to register. + path: Path to the Parquet file. + table_partition_cols: Partition columns. + parquet_pruning: Whether the parquet reader should use the + predicate to prune row groups. + file_extension: File extension; only files with this extension are + selected for data input. + skip_metadata: Whether the parquet reader should skip any metadata + that may be in the file schema. This can help avoid schema + conflicts due to metadata. + schema: The data source schema. + file_sort_order: Sort order for the file. + """ + if table_partition_cols is None: + table_partition_cols = [] + self.ctx.register_parquet( + name, + str(path), + table_partition_cols, + parquet_pruning, + file_extension, + skip_metadata, + schema, + file_sort_order, + ) + + def register_csv( + self, + name: str, + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + has_header: bool = True, + delimiter: str = ",", + schema_infer_max_records: int = 1000, + file_extension: str = ".csv", + file_compression_type: str | None = None, + ) -> None: + """Register a CSV file as a table. + + The registered table can be referenced from SQL statement executed against. + + Args: + name: Name of the table to register. + path: Path to the CSV file. + schema: An optional schema representing the CSV file. If None, the + CSV reader will try to infer it based on data in file. + has_header: Whether the CSV file have a header. If schema inference + is run on a file with no headers, default column names are + created. + delimiter: An optional column delimiter. + schema_infer_max_records: Maximum number of rows to read from CSV + files for schema inference if needed. + file_extension: File extension; only files with this extension are + selected for data input. + file_compression_type: File compression type. + """ + self.ctx.register_csv( + name, + str(path), + schema, + has_header, + delimiter, + schema_infer_max_records, + file_extension, + file_compression_type, + ) + + def register_json( + self, + name: str, + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + schema_infer_max_records: int = 1000, + file_extension: str = ".json", + table_partition_cols: list[tuple[str, str]] | None = None, + file_compression_type: str | None = None, + ) -> None: + """Register a JSON file as a table. + + The registered table can be referenced from SQL statement executed + against this context. + + Args: + name: Name of the table to register. + path: Path to the JSON file. + schema: The data source schema. + schema_infer_max_records: Maximum number of rows to read from JSON + files for schema inference if needed. + file_extension: File extension; only files with this extension are + selected for data input. + table_partition_cols: Partition columns. + file_compression_type: File compression type. + """ + if table_partition_cols is None: + table_partition_cols = [] + self.ctx.register_json( + name, + str(path), + schema, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, + ) + + def register_avro( + self, + name: str, + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + file_extension: str = ".avro", + table_partition_cols: list[tuple[str, str]] | None = None, + ) -> None: + """Register an Avro file as a table. + + The registered table can be referenced from SQL statement executed against + this context. + + Args: + name: Name of the table to register. + path: Path to the Avro file. + schema: The data source schema. + file_extension: File extension to select. + table_partition_cols: Partition columns. + """ + if table_partition_cols is None: + table_partition_cols = [] + self.ctx.register_avro( + name, str(path), schema, file_extension, table_partition_cols + ) + + def register_dataset(self, name: str, dataset: pyarrow.dataset.Dataset) -> None: + """Register a `pyarrow.dataset.Dataset` as a table. + + Args: + name: Name of the table to register. + dataset: PyArrow dataset. + """ + self.ctx.register_dataset(name, dataset) + + def register_udf(self, udf: ScalarUDF) -> None: + """Register a user-defined function (UDF) with the context.""" + self.ctx.register_udf(udf.udf) + + def register_udaf(self, udaf: AggregateUDF) -> None: + """Register a user-defined aggregation function (UDAF) with the context.""" + self.ctx.register_udaf(udaf) + + def catalog(self, name: str = "datafusion") -> Catalog: + """Retrieve a catalog by name.""" + return self.ctx.catalog(name) + + @deprecated( + "Use the catalog provider interface `SessionContext.catalog` to " + "examine available catalogs, schemas and tables" + ) + def tables(self) -> set[str]: + """Deprecated.""" + return self.ctx.tables() + + def table(self, name: str) -> DataFrame: + """Retrieve a `DataFrame` representing a previously registered table.""" + return DataFrame(self.ctx.table(name)) + + def table_exist(self, name: str) -> bool: + """Return whether a table with the given name exists.""" + return self.ctx.table_exist(name) + + def empty_table(self) -> DataFrame: + """Create an empty `DataFrame`.""" + return DataFrame(self.ctx.empty_table()) + + def session_id(self) -> str: + """Retrun an id that uniquely identifies this `SessionContext`.""" + return self.ctx.session_id() + + def read_json( + self, + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + schema_infer_max_records: int = 1000, + file_extension: str = ".json", + table_partition_cols: list[tuple[str, str]] | None = None, + file_compression_type: str | None = None, + ) -> DataFrame: + """Create a `DataFrame` for reading a line-delimited JSON data source. + + Args: + path: Path to the JSON file. + schema: The data source schema. + schema_infer_max_records: Maximum number of rows to read from JSON + files for schema inference if needed. + file_extension: File extension; only files with this extension are + selected for data input. + table_partition_cols: Partition columns. + file_compression_type: File compression type. + + Returns: + DataFrame representation of the read JSON files. + """ + if table_partition_cols is None: + table_partition_cols = [] + return DataFrame( + self.ctx.read_json( + str(path), + schema, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, + ) + ) + + def read_csv( + self, + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + has_header: bool = True, + delimiter: str = ",", + schema_infer_max_records: int = 1000, + file_extension: str = ".csv", + table_partition_cols: list[tuple[str, str]] | None = None, + file_compression_type: str | None = None, + ) -> DataFrame: + """Create a `DataFrame` for reading a CSV data source. + + Args: + path: Path to the CSV file + schema: An optional schema representing the CSV files. If None, the + CSV reader will try to infer it based on data in file. + has_header: Whether the CSV file have a header. If schema inference + is run on a file with no headers, default column names are + created. + delimiter: An optional column delimiter. + schema_infer_max_records: Maximum number of rows to read from CSV + files for schema inference if needed. + file_extension: File extension; only files with this extension are + selected for data input. + table_partition_cols: Partition columns. + file_compression_type: File compression type. + + Returns: + DataFrame representation of the read CSV files + """ + if table_partition_cols is None: + table_partition_cols = [] + return DataFrame( + self.ctx.read_csv( + str(path), + schema, + has_header, + delimiter, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, + ) + ) + + def read_parquet( + self, + path: str | pathlib.Path, + table_partition_cols: list[tuple[str, str]] | None = None, + parquet_pruning: bool = True, + file_extension: str = ".parquet", + skip_metadata: bool = True, + schema: pyarrow.Schema | None = None, + file_sort_order: list[list[Expr]] | None = None, + ) -> DataFrame: + """Create a `DataFrame` for reading Parquet data source. + + Args: + path: Path to the Parquet file. + table_partition_cols: Partition columns. + parquet_pruning: Whether the parquet reader should use the predicate + to prune row groups. + file_extension: File extension; only files with this extension are + selected for data input. + skip_metadata: Whether the parquet reader should skip any metadata + that may be in the file schema. This can help avoid schema + conflicts due to metadata. + schema: An optional schema representing the parquet files. If None, + the parquet reader will try to infer it based on data in the + file. + file_sort_order: Sort order for the file. + + Returns: + DataFrame representation of the read Parquet files + """ + if table_partition_cols is None: + table_partition_cols = [] + return DataFrame( + self.ctx.read_parquet( + str(path), + table_partition_cols, + parquet_pruning, + file_extension, + skip_metadata, + schema, + file_sort_order, + ) + ) + + def read_avro( + self, + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + file_partition_cols: list[tuple[str, str]] | None = None, + file_extension: str = ".avro", + ) -> DataFrame: + """Create a ``DataFrame`` for reading Avro data source. + + Args: + path: Path to the Avro file. + schema: The data source schema. + file_partition_cols: Partition columns. + file_extension: File extension to select. + + Returns: + DataFrame representation of the read Avro file + """ + if file_partition_cols is None: + file_partition_cols = [] + return DataFrame( + self.ctx.read_avro(str(path), schema, file_partition_cols, file_extension) + ) + + def read_table(self, table: Table) -> DataFrame: + """Creates a ``DataFrame`` for a ``Table`` such as a ``ListingTable``.""" + return DataFrame(self.ctx.read_table(table)) + + def execute(self, plan: ExecutionPlan, partitions: int) -> RecordBatchStream: + """Execute the `plan` and return the results.""" + return RecordBatchStream(self.ctx.execute(plan, partitions)) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py new file mode 100644 index 000000000..68e6298f7 --- /dev/null +++ b/python/datafusion/dataframe.py @@ -0,0 +1,527 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""DataFrame is one of the core concepts in DataFusion. + +See https://datafusion.apache.org/python/user-guide/basics.html for more +information. +""" + +from __future__ import annotations + +from typing import Any, List, TYPE_CHECKING +from datafusion.record_batch import RecordBatchStream +from typing_extensions import deprecated + +if TYPE_CHECKING: + import pyarrow as pa + import pandas as pd + import polars as pl + import pathlib + +from datafusion._internal import DataFrame as DataFrameInternal +from datafusion.expr import Expr +from datafusion._internal import ( + LogicalPlan, + ExecutionPlan, +) + + +class DataFrame: + """Two dimensional table representation of data. + + See https://datafusion.apache.org/python/user-guide/basics.html for more + information. + """ + + def __init__(self, df: DataFrameInternal) -> None: + """This constructor is not to be used by the end user. + + See ``SessionContext`` for methods to create DataFrames. + """ + self.df = df + + def __getitem__(self, key: str | List[str]) -> DataFrame: + """Return a new `DataFrame` with the specified column or columns. + + Args: + key: Column name or list of column names to select. + + Returns: + DataFrame with the specified column or columns. + """ + return DataFrame(self.df.__getitem__(key)) + + def __repr__(self) -> str: + """Return a string representation of the DataFrame. + + Returns: + String representation of the DataFrame. + """ + return self.df.__repr__() + + def describe(self) -> DataFrame: + """Return a new `DataFrame` that has statistics for a DataFrame. + + Only summarized numeric datatypes at the moments and returns nulls + for non-numeric datatypes. + + The output format is modeled after pandas. + + Returns: + A summary DataFrame containing statistics. + """ + return DataFrame(self.df.describe()) + + def schema(self) -> pa.Schema: + """Return the `pyarrow.Schema` describing the output of this DataFrame. + + The output schema contains information on the name, data type, and + nullability for each column. + + Returns: + Describing schema of the DataFrame + """ + return self.df.schema() + + def select_columns(self, *args: str) -> DataFrame: + """Filter the DataFrame by columns. + + Returns: + DataFrame only containing the specified columns. + """ + return self.select(*args) + + def select(self, *exprs: Expr | str) -> DataFrame: + """Project arbitrary expressions into a new `DataFrame`. + + Args: + exprs: Either column names or `Expr` to select. + + Returns: + DataFrame after projection. It has one column for each expression. + + Example usage: + + The following example will return 3 columns from the original dataframe. + The first two columns will be the original column `a` and `b` since the + string "a" is assumed to refer to column selection. Also a duplicate of + column `a` will be returned with the column name `alternate_a`:: + + df = df.select("a", col("b"), col("a").alias("alternate_a")) + + """ + exprs = [ + arg.expr if isinstance(arg, Expr) else Expr.column(arg).expr + for arg in exprs + ] + return DataFrame(self.df.select(*exprs)) + + def filter(self, *predicates: Expr) -> DataFrame: + """Return a DataFrame for which `predicate` evaluates to `True`. + + Rows for which `predicate` evaluates to `False` or `None` are filtered + out. If more than one predicate is provided, these predicates will be + combined as a logical AND. If more complex logic is required, see the + logical operations in `datafusion.functions`. + + Args: + predicates: Predicate expression(s) to filter the DataFrame. + + Returns: + DataFrame after filtering. + """ + df = self.df + for p in predicates: + df = df.filter(p.expr) + return DataFrame(df) + + def with_column(self, name: str, expr: Expr) -> DataFrame: + """Add an additional column to the DataFrame. + + Args: + name: Name of the column to add. + expr: Expression to compute the column. + + Returns: + DataFrame with the new column. + """ + return DataFrame(self.df.with_column(name, expr.expr)) + + def with_column_renamed(self, old_name: str, new_name: str) -> DataFrame: + """Rename one column by applying a new projection. + + This is a no-op if the column to be renamed does not exist. + + The method supports case sensitive rename with wrapping column name + into one the following symbols (" or ' or `). + + Args: + old_name: Old column name. + new_name: New column name. + + Returns: + DataFrame with the column renamed. + """ + return DataFrame(self.df.with_column_renamed(old_name, new_name)) + + def aggregate(self, group_by: list[Expr], aggs: list[Expr]) -> DataFrame: + """Aggregates the rows of the current DataFrame. + + Args: + group_by: List of expressions to group by. + aggs: List of expressions to aggregate. + + Returns: + DataFrame after aggregation. + """ + group_by = [e.expr for e in group_by] + aggs = [e.expr for e in aggs] + return DataFrame(self.df.aggregate(group_by, aggs)) + + def sort(self, *exprs: Expr) -> DataFrame: + """Sort the DataFrame by the specified sorting expressions. + + Note that any expression can be turned into a sort expression by + calling its `sort` method. + + Args: + exprs: Sort expressions, applied in order. + + Returns: + DataFrame after sorting. + """ + exprs = [expr.expr for expr in exprs] + return DataFrame(self.df.sort(*exprs)) + + def limit(self, count: int, offset: int = 0) -> DataFrame: + """Return a new `DataFrame` with a limited number of rows. + + Args: + count: Number of rows to limit the DataFrame to. + offset: Number of rows to skip. + + Returns: + DataFrame after limiting. + """ + return DataFrame(self.df.limit(count, offset)) + + def collect(self) -> list[pa.RecordBatch]: + """Execute this `DataFrame` and collect results into memory. + + Prior to calling `collect`, modifying a DataFrme simply updates a plan + (no actual computation is performed). Calling `collect` triggers the + computation. + + Returns: + List of `pyarrow.RecordBatch`es collected from the DataFrame. + """ + return self.df.collect() + + def cache(self) -> DataFrame: + """Cache the DataFrame as a memory table. + + Returns: + Cached DataFrame. + """ + return DataFrame(self.df.cache()) + + def collect_partitioned(self) -> list[list[pa.RecordBatch]]: + """Execute this DataFrame and collect all partitioned results. + + This operation returns ``RecordBatch`` maintaining the input + partitioning. + + Returns: + List of list of ``RecordBatch`` collected from the + DataFrame. + """ + return self.df.collect_partitioned() + + def show(self, num: int = 20) -> None: + """Execute the DataFrame and print the result to the console. + + Args: + num: Number of lines to show. + """ + self.df.show(num) + + def distinct(self) -> DataFrame: + """Return a new `DataFrame` with all duplicated rows removed. + + Returns: + DataFrame after removing duplicates. + """ + return DataFrame(self.df.distinct()) + + def join( + self, + right: DataFrame, + join_keys: tuple[list[str], list[str]], + how: str, + ) -> DataFrame: + """Join this `DataFrame` with another `DataFrame`. + + Join keys are a pair of lists of column names in the left and right + dataframes, respectively. These lists must have the same length. + + Args: + right: Other DataFrame to join with. + join_keys: Tuple of two lists of column names to join on. + how: Type of join to perform. Supported types are "inner", "left", + "right", "full", "semi", "anti". + + Returns: + DataFrame after join. + """ + return DataFrame(self.df.join(right.df, join_keys, how)) + + def explain(self, verbose: bool = False, analyze: bool = False) -> DataFrame: + """Return a DataFrame with the explanation of its plan so far. + + If `analyze` is specified, runs the plan and reports metrics. + + Args: + verbose: If `True`, more details will be included. + analyze: If `True`, the plan will run and metrics reported. + + Returns: + DataFrame with the explanation of its plan. + """ + return DataFrame(self.df.explain(verbose, analyze)) + + def logical_plan(self) -> LogicalPlan: + """Return the unoptimized `LogicalPlan` that comprises this `DataFrame`. + + Returns: + Unoptimized logical plan. + """ + return self.df.logical_plan() + + def optimized_logical_plan(self) -> LogicalPlan: + """Return the optimized `LogicalPlan` that comprises this `DataFrame`. + + Returns: + Optimized logical plan. + """ + return self.df.optimized_logical_plan() + + def execution_plan(self) -> ExecutionPlan: + """Return the execution/physical plan that comprises this `DataFrame`. + + Returns: + Execution plan. + """ + return self.df.execution_plan() + + def repartition(self, num: int) -> DataFrame: + """Repartition a DataFrame into `num` partitions. + + The batches allocation uses a round-robin algorithm. + + Args: + num: Number of partitions to repartition the DataFrame into. + + Returns: + Repartitioned DataFrame. + """ + return DataFrame(self.df.repartition(num)) + + def repartition_by_hash(self, *exprs: Expr, num: int) -> DataFrame: + """Repartition a DataFrame using a hash partitioning scheme. + + Args: + exprs: Expressions to evaluate and perform hashing on. + num: Number of partitions to repartition the DataFrame into. + + Returns: + Repartitioned DataFrame. + """ + exprs = [expr.expr for expr in exprs] + return DataFrame(self.df.repartition_by_hash(*exprs, num=num)) + + def union(self, other: DataFrame, distinct: bool = False) -> DataFrame: + """Calculate the union of two `DataFrame`s. + + The two `DataFrame`s must have exactly the same schema. + + Args: + other: DataFrame to union with. + distinct: If `True`, duplicate rows will be removed. + + Returns: + DataFrame after union. + """ + return DataFrame(self.df.union(other.df, distinct)) + + def union_distinct(self, other: DataFrame) -> DataFrame: + """Calculate the distinct union of two `DataFrame`s. + + The two `DataFrame`s must have exactly the same schema. + Any duplicate rows are discarded. + + Args: + other: DataFrame to union with. + + Returns: + DataFrame after union. + """ + return DataFrame(self.df.union_distinct(other.df)) + + def intersect(self, other: DataFrame) -> DataFrame: + """Calculate the intersection of two `DataFrame`s. + + The two `DataFrame`s must have exactly the same schema. + + Args: + other: DataFrame to intersect with. + + Returns: + DataFrame after intersection. + """ + return DataFrame(self.df.intersect(other.df)) + + def except_all(self, other: DataFrame) -> DataFrame: + """Calculate the exception of two `DataFrame`s. + + The two `DataFrame`s must have exactly the same schema. + + Args: + other: DataFrame to calculate exception with. + + Returns: + DataFrame after exception. + """ + return DataFrame(self.df.except_all(other.df)) + + def write_csv(self, path: str | pathlib.Path, with_header: bool = False) -> None: + """Execute the `DataFrame` and write the results to a CSV file. + + Args: + path: Path of the CSV file to write. + with_header: If true, output the CSV header row. + """ + self.df.write_csv(str(path), with_header) + + def write_parquet( + self, + path: str | pathlib.Path, + compression: str = "uncompressed", + compression_level: int | None = None, + ) -> None: + """Execute the `DataFrame` and write the results to a Parquet file. + + Args: + path: Path of the Parquet file to write. + compression: Compression type to use. + compression_level: Compression level to use. + """ + self.df.write_parquet(str(path), compression, compression_level) + + def write_json(self, path: str | pathlib.Path) -> None: + """Execute the `DataFrame` and write the results to a JSON file. + + Args: + path: Path of the JSON file to write. + """ + self.df.write_json(str(path)) + + def to_arrow_table(self) -> pa.Table: + """Execute the `DataFrame` and convert it into an Arrow Table. + + Returns: + Arrow Table. + """ + return self.df.to_arrow_table() + + def execute_stream(self) -> RecordBatchStream: + """Executes this DataFrame and returns a stream over a single partition. + + Returns: + Record Batch Stream over a single partition. + """ + return RecordBatchStream(self.df.execute_stream()) + + def execute_stream_partitioned(self) -> list[RecordBatchStream]: + """Executes this DataFrame and returns a stream for each partition. + + Returns: + One record batch stream per partition. + """ + streams = self.df.execute_stream_partitioned() + return [RecordBatchStream(rbs) for rbs in streams] + + def to_pandas(self) -> pd.DataFrame: + """Execute the `DataFrame` and convert it into a Pandas DataFrame. + + Returns: + Pandas DataFrame. + """ + return self.df.to_pandas() + + def to_pylist(self) -> list[dict[str, Any]]: + """Execute the `DataFrame` and convert it into a list of dictionaries. + + Returns: + List of dictionaries. + """ + return self.df.to_pylist() + + def to_pydict(self) -> dict[str, list[Any]]: + """Execute the `DataFrame` and convert it into a dictionary of lists. + + Returns: + Dictionary of lists. + """ + return self.df.to_pydict() + + def to_polars(self) -> pl.DataFrame: + """Execute the `DataFrame` and convert it into a Polars DataFrame. + + Returns: + Polars DataFrame. + """ + return self.df.to_polars() + + def count(self) -> int: + """Return the total number of rows in this `DataFrame`. + + Note that this method will actually run a plan to calculate the + count, which may be slow for large or complicated DataFrames. + + Returns: + Number of rows in the DataFrame. + """ + return self.df.count() + + @deprecated("Use :func:`unnest_columns` instead.") + def unnest_column(self, column: str, preserve_nulls: bool = True) -> DataFrame: + """See ``unnest_columns``.""" + return DataFrame(self.df.unnest_column(column, preserve_nulls=preserve_nulls)) + + def unnest_columns(self, *columns: str, preserve_nulls: bool = True) -> DataFrame: + """Expand columns of arrays into a single row per array element. + + Args: + columns: Column names to perform unnest operation on. + preserve_nulls: If False, rows with null entries will not be + returned. + + Returns: + A DataFrame with the columns expanded. + """ + columns = [c for c in columns] + return DataFrame(self.df.unnest_columns(columns, preserve_nulls=preserve_nulls)) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index e914b85d7..c04a525a6 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -15,9 +15,417 @@ # specific language governing permissions and limitations # under the License. +"""This module supports expressions, one of the core concepts in DataFusion. -from ._internal import expr +See ``Expr`` for more details. +""" +from __future__ import annotations -def __getattr__(name): - return getattr(expr, name) +from ._internal import expr as expr_internal, LogicalPlan +from datafusion.common import RexType, DataTypeMap +from typing import Any +import pyarrow as pa + +# The following are imported from the internal representation. We may choose to +# give these all proper wrappers, or to simply leave as is. These were added +# in order to support passing the `test_imports` unit test. +# Tim Saucer note: It is not clear to me what the use case is for exposing +# these definitions to the end user. + +Alias = expr_internal.Alias +Analyze = expr_internal.Analyze +Aggregate = expr_internal.Aggregate +AggregateFunction = expr_internal.AggregateFunction +Between = expr_internal.Between +BinaryExpr = expr_internal.BinaryExpr +Case = expr_internal.Case +Cast = expr_internal.Cast +Column = expr_internal.Column +CreateMemoryTable = expr_internal.CreateMemoryTable +CreateView = expr_internal.CreateView +CrossJoin = expr_internal.CrossJoin +Distinct = expr_internal.Distinct +DropTable = expr_internal.DropTable +Exists = expr_internal.Exists +Explain = expr_internal.Explain +Extension = expr_internal.Extension +Filter = expr_internal.Filter +GroupingSet = expr_internal.GroupingSet +Join = expr_internal.Join +ILike = expr_internal.ILike +InList = expr_internal.InList +InSubquery = expr_internal.InSubquery +IsFalse = expr_internal.IsFalse +IsNotTrue = expr_internal.IsNotTrue +IsTrue = expr_internal.IsTrue +IsUnknown = expr_internal.IsUnknown +IsNotFalse = expr_internal.IsNotFalse +IsNotNull = expr_internal.IsNotNull +IsNotUnknown = expr_internal.IsNotUnknown +JoinConstraint = expr_internal.JoinConstraint +JoinType = expr_internal.JoinType +Like = expr_internal.Like +Limit = expr_internal.Limit +Literal = expr_internal.Literal +Negative = expr_internal.Negative +Not = expr_internal.Not +Partitioning = expr_internal.Partitioning +Placeholder = expr_internal.Placeholder +Projection = expr_internal.Projection +Repartition = expr_internal.Repartition +ScalarSubquery = expr_internal.ScalarSubquery +ScalarVariable = expr_internal.ScalarVariable +SimilarTo = expr_internal.SimilarTo +Sort = expr_internal.Sort +Subquery = expr_internal.Subquery +SubqueryAlias = expr_internal.SubqueryAlias +TableScan = expr_internal.TableScan +TryCast = expr_internal.TryCast +Union = expr_internal.Union + + +class Expr: + """Expression object. + + Expressions are one of the core concepts in DataFusion. See + https://datafusion.apache.org/python/user-guide/common-operations/expressions.html + for more information. + """ + + def __init__(self, expr: expr_internal.Expr) -> None: + """This constructor should not be called by the end user.""" + self.expr = expr + + def to_variant(self) -> Any: + """Convert this expression into a python object if possible.""" + return self.expr.to_variant() + + def display_name(self) -> str: + """Returns the name of this expression as it should appear in a schema. + + This name will not include any CAST expressions. + """ + return self.expr.display_name() + + def canonical_name(self) -> str: + """Returns a complete string representation of this expression.""" + return self.expr.canonical_name() + + def variant_name(self) -> str: + """Returns the name of the Expr variant. + + Ex: ``IsNotNull``, ``Literal``, ``BinaryExpr``, etc + """ + return self.expr.variant_name() + + def __richcmp__(self, other: Expr, op: int) -> Expr: + """Comparison operator.""" + return Expr(self.expr.__richcmp__(other, op)) + + def __repr__(self) -> str: + """Generate a string representation of this expression.""" + return self.expr.__repr__() + + def __add__(self, rhs: Any) -> Expr: + """Addition operator. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__add__(rhs.expr)) + + def __sub__(self, rhs: Any) -> Expr: + """Subtraction operator. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__sub__(rhs.expr)) + + def __truediv__(self, rhs: Any) -> Expr: + """Division operator. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__truediv__(rhs.expr)) + + def __mul__(self, rhs: Any) -> Expr: + """Multiplication operator. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__mul__(rhs.expr)) + + def __mod__(self, rhs: Any) -> Expr: + """Modulo operator (%). + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__mod__(rhs.expr)) + + def __and__(self, rhs: Expr) -> Expr: + """Logical AND.""" + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__and__(rhs.expr)) + + def __or__(self, rhs: Expr) -> Expr: + """Logical OR.""" + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__or__(rhs.expr)) + + def __invert__(self) -> Expr: + """Binary not (~).""" + return Expr(self.expr.__invert__()) + + def __getitem__(self, key: str) -> Expr: + """For struct data types, return the field indicated by ``key``.""" + return Expr(self.expr.__getitem__(key)) + + def __eq__(self, rhs: Any) -> Expr: + """Equal to. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__eq__(rhs.expr)) + + def __ne__(self, rhs: Any) -> Expr: + """Not equal to. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__ne__(rhs.expr)) + + def __ge__(self, rhs: Any) -> Expr: + """Greater than or equal to. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__ge__(rhs.expr)) + + def __gt__(self, rhs: Any) -> Expr: + """Greater than. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__gt__(rhs.expr)) + + def __le__(self, rhs: Any) -> Expr: + """Less than or equal to. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__le__(rhs.expr)) + + def __lt__(self, rhs: Any) -> Expr: + """Less than. + + Accepts either an expression or any valid PyArrow scalar literal value. + """ + if not isinstance(rhs, Expr): + rhs = Expr.literal(rhs) + return Expr(self.expr.__lt__(rhs.expr)) + + @staticmethod + def literal(value: Any) -> Expr: + """Creates a new expression representing a scalar value. + + `value` must be a valid PyArrow scalar value or easily castable to one. + """ + if not isinstance(value, pa.Scalar): + value = pa.scalar(value) + return Expr(expr_internal.Expr.literal(value)) + + @staticmethod + def column(value: str) -> Expr: + """Creates a new expression representing a column in a ``DataFrame``.""" + return Expr(expr_internal.Expr.column(value)) + + def alias(self, name: str) -> Expr: + """Assign a name to the expression.""" + return Expr(self.expr.alias(name)) + + def sort(self, ascending: bool = True, nulls_first: bool = True) -> Expr: + """Creates a sort ``Expr`` from an existing ``Expr``. + + Args: + ascending: If true, sort in ascending order. + nulls_first: Return null values first. + """ + return Expr(self.expr.sort(ascending=ascending, nulls_first=nulls_first)) + + def is_null(self) -> Expr: + """Returns ``True`` if this expression is null.""" + return Expr(self.expr.is_null()) + + def cast(self, to: pa.DataType[Any]) -> Expr: + """Cast to a new data type.""" + return Expr(self.expr.cast(to)) + + def rex_type(self) -> RexType: + """Return the Rex Type of this expression. + + A Rex (Row Expression) specifies a single row of data.That specification + could include user defined functions or types. RexType identifies the + row as one of the possible valid ``RexType``(s). + """ + return self.expr.rex_type() + + def types(self) -> DataTypeMap: + """Return the ``DataTypeMap``. + + Returns: + DataTypeMap which represents the PythonType, Arrow DataType, and + SqlType Enum which this expression represents. + """ + return self.expr.types() + + def python_value(self) -> Any: + """Extracts the Expr value into a PyObject. + + This is only valid for literal expressions. + + Returns: + Python object representing literal value of the expression. + """ + return self.expr.python_value() + + def rex_call_operands(self) -> list[Expr]: + """Return the operands of the expression based on it's variant type. + + Row expressions, Rex(s), operate on the concept of operands. Different + variants of Expressions, Expr(s), store those operands in different + datastructures. This function examines the Expr variant and returns + the operands to the calling logic. + """ + return [Expr(e) for e in self.expr.rex_call_operands()] + + def rex_call_operator(self) -> str: + """Extracts the operator associated with a row expression type call.""" + return self.expr.rex_call_operator() + + def column_name(self, plan: LogicalPlan) -> str: + """Compute the output column name based on the provided logical plan.""" + return self.expr.column_name(plan) + + +class WindowFrame: + """Defines a window frame for performing window operations.""" + + def __init__( + self, units: str, start_bound: int | None, end_bound: int | None + ) -> None: + """Construct a window frame using the given parameters. + + Args: + units: Should be one of `rows`, `range`, or `groups`. + start_bound: Sets the preceeding bound. Must be >= 0. If none, this + will be set to unbounded. If unit type is `groups`, this + parameter must be set. + end_bound: Sets the following bound. Must be >= 0. If none, this + will be set to unbounded. If unit type is `groups`, this + parameter must be set. + """ + self.window_frame = expr_internal.WindowFrame(units, start_bound, end_bound) + + def get_frame_units(self) -> str: + """Returns the window frame units for the bounds.""" + return self.window_frame.get_frame_units() + + def get_lower_bound(self) -> WindowFrameBound: + """Returns starting bound.""" + return WindowFrameBound(self.window_frame.get_lower_bound()) + + def get_upper_bound(self): + """Returns end bound.""" + return WindowFrameBound(self.window_frame.get_upper_bound()) + + +class WindowFrameBound: + """Defines a single window frame bound. + + ``WindowFrame`` typically requires a start and end bound. + """ + + def __init__(self, frame_bound: expr_internal.WindowFrameBound) -> None: + """Constructs a window frame bound.""" + self.frame_bound = frame_bound + + def get_offset(self) -> int | None: + """Returns the offset of the window frame.""" + return self.frame_bound.get_offset() + + def is_current_row(self) -> bool: + """Returns if the frame bound is current row.""" + return self.frame_bound.is_current_row() + + def is_following(self) -> bool: + """Returns if the frame bound is following.""" + return self.frame_bound.is_following() + + def is_preceding(self) -> bool: + """Returns if the frame bound is preceding.""" + return self.frame_bound.is_preceding() + + def is_unbounded(self) -> bool: + """Returns if the frame bound is unbounded.""" + return self.frame_bound.is_unbounded() + + +class CaseBuilder: + """Builder class for constructing case statements. + + An example usage would be as follows:: + + import datafusion.functions as f + from datafusion import lit, col + df.select( + f.case(col("column_a") + .when(lit(1), lit("One")) + .when(lit(2), lit("Two")) + .otherwise(lit("Unknown")) + ) + """ + + def __init__(self, case_builder: expr_internal.CaseBuilder) -> None: + """Constructs a case builder. + + This is not typically called by the end user directly. See + ``datafusion.functions.case`` instead. + """ + self.case_builder = case_builder + + def when(self, when_expr: Expr, then_expr: Expr) -> CaseBuilder: + """Add a case to match against.""" + return CaseBuilder(self.case_builder.when(when_expr.expr, then_expr.expr)) + + def otherwise(self, else_expr: Expr) -> Expr: + """Set a default value for the case statement.""" + return Expr(self.case_builder.otherwise(else_expr.expr)) + + def end(self) -> Expr: + """Finish building a case statement. + + Any non-matching cases will end in a `null` value. + """ + return Expr(self.case_builder.end()) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 782ecba22..ad77712ed 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -14,10 +14,1475 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +"""This module contains the user functions for operating on ``Expr``.""" +from __future__ import annotations -from ._internal import functions +# from datafusion._internal.context import SessionContext +# from datafusion._internal.expr import Expr +# from datafusion._internal.expr.conditional_expr import CaseBuilder +# from datafusion._internal.expr.window import WindowFrame +from datafusion._internal import functions as f, common +from datafusion.expr import CaseBuilder, Expr, WindowFrame +from datafusion.context import SessionContext -def __getattr__(name): - return getattr(functions, name) + +def isnan(expr: Expr) -> Expr: + """Returns true if a given number is +NaN or -NaN otherwise returns false.""" + return Expr(f.isnan(expr.expr)) + + +def nullif(expr1: Expr, expr2: Expr) -> Expr: + """Returns NULL if expr1 equals expr2; otherwise it returns expr1. + + This can be used to perform the inverse operation of the COALESCE expression. + """ + return Expr(f.nullif(expr1.expr, expr2.expr)) + + +def encode(input: Expr, encoding: Expr) -> Expr: + """Encode the `input`, using the `encoding`. encoding can be base64 or hex.""" + return Expr(f.encode(input.expr, encoding.expr)) + + +def decode(input: Expr, encoding: Expr) -> Expr: + """Decode the `input`, using the `encoding`. encoding can be base64 or hex.""" + return Expr(f.decode(input.expr, encoding.expr)) + + +def array_to_string(expr: Expr, delimiter: Expr) -> Expr: + """Converts each element to its text representation.""" + return Expr(f.array_to_string(expr.expr, delimiter.expr)) + + +def array_join(expr: Expr, delimiter: Expr) -> Expr: + """Converts each element to its text representation. + + This is an alias for :func:`array_to_string`. + """ + return array_to_string(expr, delimiter) + + +def list_to_string(expr: Expr, delimiter: Expr) -> Expr: + """Converts each element to its text representation. + + This is an alias for :func:`array_to_string`. + """ + return array_to_string(expr, delimiter) + + +def list_join(expr: Expr, delimiter: Expr) -> Expr: + """Converts each element to its text representation. + + This is an alias for :func:`array_to_string`. + """ + return array_to_string(expr, delimiter) + + +def in_list(arg: Expr, values: list[Expr], negated: bool = False) -> Expr: + """Returns whether the argument is contained within the list `values`.""" + values = [v.expr for v in values] + return Expr(f.in_list(arg.expr, values, negated)) + + +def digest(value: Expr, method: Expr) -> Expr: + """Computes the binary hash of an expression using the specified algorithm. + + Standard algorithms are md5, sha224, sha256, sha384, sha512, blake2s, + blake2b, and blake3. + """ + return Expr(f.digest(value.expr, method.expr)) + + +def concat(*args: Expr) -> Expr: + """Concatenates the text representations of all the arguments. + + NULL arguments are ignored. + """ + args = [arg.expr for arg in args] + return Expr(f.concat(*args)) + + +def concat_ws(separator: str, *args: Expr) -> Expr: + """Concatenates the list `args` with the separator. + + `NULL` arugments are ignored. `separator` should not be `NULL`. + """ + args = [arg.expr for arg in args] + return Expr(f.concat_ws(separator, *args)) + + +def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> Expr: + """Creates a new sort expression.""" + return Expr(f.order_by(expr.expr, ascending, nulls_first)) + + +def alias(expr: Expr, name: str) -> Expr: + """Creates an alias expression.""" + return Expr(f.alias(expr.expr, name)) + + +def col(name: str) -> Expr: + """Creates a column reference expression.""" + return Expr(f.col(name)) + + +def count_star() -> Expr: + """Create a COUNT(1) aggregate expression.""" + return Expr(f.count_star()) + + +def case(expr: Expr) -> CaseBuilder: + """Create a ``CaseBuilder`` to match cases for the expression ``expr``. + + See ``datafusion.expr.CaseBuilder`` for detailed usage of ``CaseBuilder``. + """ + return CaseBuilder(f.case(expr.expr)) + + +def window( + name: str, + args: list[Expr], + partition_by: list[Expr] | None = None, + order_by: list[Expr] | None = None, + window_frame: WindowFrame | None = None, + ctx: SessionContext | None = None, +) -> Expr: + """Creates a new Window function expression.""" + args = [a.expr for a in args] + partition_by = [e.expr for e in partition_by] if partition_by is not None else None + order_by = [o.expr for o in order_by] if order_by is not None else None + window_frame = window_frame.window_frame if window_frame is not None else None + return Expr(f.window(name, args, partition_by, order_by, window_frame, ctx)) + + +# scalar functions +def abs(arg: Expr) -> Expr: + """Return the absolute value of a given number. + + Returns: + -------- + Expr + A new expression representing the absolute value of the input expression. + """ + return Expr(f.abs(arg.expr)) + + +def acos(arg: Expr) -> Expr: + """Returns the arc cosine or inverse cosine of a number. + + Returns: + -------- + Expr + A new expression representing the arc cosine of the input expression. + """ + return Expr(f.acos(arg.expr)) + + +def acosh(arg: Expr) -> Expr: + """Returns inverse hyperbolic cosine.""" + return Expr(f.acosh(arg.expr)) + + +def ascii(arg: Expr) -> Expr: + """Returns the numeric code of the first character of the argument.""" + return Expr(f.ascii(arg.expr)) + + +def asin(arg: Expr) -> Expr: + """Returns the arc sine or inverse sine of a number.""" + return Expr(f.asin(arg.expr)) + + +def asinh(arg: Expr) -> Expr: + """Returns inverse hyperbolic sine.""" + return Expr(f.asinh(arg.expr)) + + +def atan(arg: Expr) -> Expr: + """Returns inverse tangent of a number.""" + return Expr(f.atan(arg.expr)) + + +def atanh(arg: Expr) -> Expr: + """Returns inverse hyperbolic tangent.""" + return Expr(f.atanh(arg.expr)) + + +def atan2(y: Expr, x: Expr) -> Expr: + """Returns inverse tangent of a division given in the argument.""" + return Expr(f.atan2(y.expr, x.expr)) + + +def bit_length(arg: Expr) -> Expr: + """Returns the number of bits in the string argument.""" + return Expr(f.bit_length(arg.expr)) + + +def btrim(arg: Expr) -> Expr: + """Removes all characters, spaces by default, from both sides of a string.""" + return Expr(f.btrim(arg.expr)) + + +def cbrt(arg: Expr) -> Expr: + """Returns the cube root of a number.""" + return Expr(f.cbrt(arg.expr)) + + +def ceil(arg: Expr) -> Expr: + """Returns the nearest integer greater than or equal to argument.""" + return Expr(f.ceil(arg.expr)) + + +def character_length(arg: Expr) -> Expr: + """Returns the number of characters in the argument.""" + return Expr(f.character_length(arg.expr)) + + +def length(string: Expr) -> Expr: + """The number of characters in the `string`.""" + return Expr(f.length(string.expr)) + + +def char_length(string: Expr) -> Expr: + """The number of characters in the `string`.""" + return Expr(f.char_length(string.expr)) + + +def chr(arg: Expr) -> Expr: + """Converts the Unicode code point to a UTF8 character.""" + return Expr(f.chr(arg.expr)) + + +def coalesce(*args: Expr) -> Expr: + """Returns the value of the first expr in `args` which is not NULL.""" + args = [arg.expr for arg in args] + return Expr(f.coalesce(*args)) + + +def cos(arg: Expr) -> Expr: + """Returns the cosine of the argument.""" + return Expr(f.cos(arg.expr)) + + +def cosh(arg: Expr) -> Expr: + """Returns the hyperbolic cosine of the argument.""" + return Expr(f.cosh(arg.expr)) + + +def cot(arg: Expr) -> Expr: + """Returns the cotangent of the argument.""" + return Expr(f.cot(arg.expr)) + + +def degrees(arg: Expr) -> Expr: + """Converts the argument from radians to degrees.""" + return Expr(f.degrees(arg.expr)) + + +def ends_with(arg: Expr, suffix: Expr) -> Expr: + """Returns true if the `string` ends with the `suffix`, false otherwise.""" + return Expr(f.ends_with(arg.expr, suffix.expr)) + + +def exp(arg: Expr) -> Expr: + """Returns the exponential of the arugment.""" + return Expr(f.exp(arg.expr)) + + +def factorial(arg: Expr) -> Expr: + """Returns the factorial of the argument.""" + return Expr(f.factorial(arg.expr)) + + +def find_in_set(string: Expr, string_list: Expr) -> Expr: + """Find a string in a list of strings. + + Returns a value in the range of 1 to N if the string is in the string list + `string_list` consisting of N substrings. + + The string list is a string composed of substrings separated by `,` characters. + """ + return Expr(f.find_in_set(string.expr, string_list.expr)) + + +def floor(arg: Expr) -> Expr: + """Returns the nearest integer less than or equal to the argument.""" + return Expr(f.floor(arg.expr)) + + +def gcd(x: Expr, y: Expr) -> Expr: + """Returns the greatest common divisor.""" + return Expr(f.gcd(x.expr, y.expr)) + + +def initcap(string: Expr) -> Expr: + """Set the initial letter of each word to capital. + + Converts the first letter of each word in `string` to uppercase and the remaining + characters to lowercase. + """ + return Expr(f.initcap(string.expr)) + + +def instr(string: Expr, substring: Expr) -> Expr: + """Finds the position from where the `substring` matches the `string`. + + This is an alias for :func:`strpos`. + """ + return strpos(string, substring) + + +def iszero(arg: Expr) -> Expr: + """Returns true if a given number is +0.0 or -0.0 otherwise returns false.""" + return Expr(f.iszero(arg.expr)) + + +def lcm(x: Expr, y: Expr) -> Expr: + """Returns the least common multiple.""" + return Expr(f.lcm(x.expr, y.expr)) + + +def left(string: Expr, n: Expr) -> Expr: + """Returns the first `n` characters in the `string`.""" + return Expr(f.left(string.expr, n.expr)) + + +def levenshtein(string1: Expr, string2: Expr) -> Expr: + """Returns the Levenshtein distance between the two given strings.""" + return Expr(f.levenshtein(string1.expr, string2.expr)) + + +def ln(arg: Expr) -> Expr: + """Returns the natural logarithm (base e) of the argument.""" + return Expr(f.ln(arg.expr)) + + +def log(base: Expr, num: Expr) -> Expr: + """Returns the logarithm of a number for a particular `base`.""" + return Expr(f.log(base.expr, num.expr)) + + +def log10(arg: Expr) -> Expr: + """Base 10 logarithm of the argument.""" + return Expr(f.log10(arg.expr)) + + +def log2(arg: Expr) -> Expr: + """Base 2 logarithm of the argument.""" + return Expr(f.log2(arg.expr)) + + +def lower(arg: Expr) -> Expr: + """Converts a string to lowercase.""" + return Expr(f.lower(arg.expr)) + + +def lpad(string: Expr, count: Expr, characters: Expr | None = None) -> Expr: + """Add left padding to a string. + + Extends the string to length length by prepending the characters fill (a + space by default). If the string is already longer than length then it is + truncated (on the right). + """ + characters = characters if characters is not None else Expr.literal(" ") + return Expr(f.lpad(string.expr, count.expr, characters.expr)) + + +def ltrim(arg: Expr) -> Expr: + """Removes all characters, spaces by default, from the beginning of a string.""" + return Expr(f.ltrim(arg.expr)) + + +def md5(arg: Expr) -> Expr: + """Computes an MD5 128-bit checksum for a string expression.""" + return Expr(f.md5(arg.expr)) + + +def nanvl(x: Expr, y: Expr) -> Expr: + """Returns `x` if `x` is not `NaN`. Otherwise returns `y`.""" + return Expr(f.nanvl(x.expr, y.expr)) + + +def octet_length(arg: Expr) -> Expr: + """Returns the number of bytes of a string.""" + return Expr(f.octet_length(arg.expr)) + + +def overlay( + string: Expr, substring: Expr, start: Expr, length: Expr | None = None +) -> Expr: + """Replace a substring with a new substring. + + Replace the substring of string that starts at the `start`'th character and + extends for `length` characters with new substring. + """ + if length is None: + return Expr(f.overlay(string.expr, substring.expr, start.expr)) + return Expr(f.overlay(string.expr, substring.expr, start.expr, length.expr)) + + +def pi() -> Expr: + """Returns an approximate value of π.""" + return Expr(f.pi()) + + +def position(string: Expr, substring: Expr) -> Expr: + """Finds the position from where the `substring` matches the `string`. + + This is an alias for :func:`strpos`. + """ + return strpos(string, substring) + + +def power(base: Expr, exponent: Expr) -> Expr: + """Returns `base` raised to the power of `exponent`.""" + return Expr(f.power(base.expr, exponent.expr)) + + +def pow(base: Expr, exponent: Expr) -> Expr: + """Returns `base` raised to the power of `exponent`. + + This is an alias of `power`. + """ + return power(base, exponent) + + +def radians(arg: Expr) -> Expr: + """Converts the argument from degrees to radians.""" + return Expr(f.radians(arg.expr)) + + +def regexp_like(string: Expr, regex: Expr, flags: Expr | None = None) -> Expr: + """Find if any regular expression (regex) matches exist. + + Tests a string using a regular expression returning true if at least one match, + false otherwise. + """ + if flags is not None: + flags = flags.expr + return Expr(f.regexp_like(string.expr, regex.expr, flags)) + + +def regexp_match(string: Expr, regex: Expr, flags: Expr | None = None) -> Expr: + """Perform regular expression (regex) matching. + + Returns an array with each element containing the leftmost-first match of the + corresponding index in `regex` to string in `string`. + """ + if flags is not None: + flags = flags.expr + return Expr(f.regexp_match(string.expr, regex.expr, flags)) + + +def regexp_replace( + string: Expr, pattern: Expr, replacement: Expr, flags: Expr | None = None +) -> Expr: + """Replaces substring(s) matching a PCRE-like regular expression. + + The full list of supported features and syntax can be found at + + + Supported flags with the addition of 'g' can be found at + + """ + if flags is not None: + flags = flags.expr + return Expr(f.regexp_replace(string.expr, pattern.expr, replacement.expr, flags)) + + +def repeat(string: Expr, n: Expr) -> Expr: + """Repeats the `string` to `n` times.""" + return Expr(f.repeat(string.expr, n.expr)) + + +def replace(string: Expr, from_val: Expr, to_val: Expr) -> Expr: + """Replaces all occurrences of `from` with `to` in the `string`.""" + return Expr(f.replace(string.expr, from_val.expr, to_val.expr)) + + +def reverse(arg: Expr) -> Expr: + """Reverse the string argument.""" + return Expr(f.reverse(arg.expr)) + + +def right(string: Expr, n: Expr) -> Expr: + """Returns the last `n` characters in the `string`.""" + return Expr(f.right(string.expr, n.expr)) + + +def round(value: Expr, decimal_places: Expr = Expr.literal(0)) -> Expr: + """Round the argument to the nearest integer. + + If the optional ``decimal_places`` is specified, round to the nearest number of + decimal places. You can specify a negative number of decimal places. For example + `round(lit(125.2345), lit(-2))` would yield a value of `100.0`. + """ + return Expr(f.round(value.expr, decimal_places.expr)) + + +def rpad(string: Expr, count: Expr, characters: Expr | None = None) -> Expr: + """Add right padding to a string. + + Extends the string to length length by appending the characters fill (a space + by default). If the string is already longer than length then it is truncated. + """ + characters = characters if characters is not None else Expr.literal(" ") + return Expr(f.rpad(string.expr, count.expr, characters.expr)) + + +def rtrim(arg: Expr) -> Expr: + """Removes all characters, spaces by default, from the end of a string.""" + return Expr(f.rtrim(arg.expr)) + + +def sha224(arg: Expr) -> Expr: + """Computes the SHA-224 hash of a binary string.""" + return Expr(f.sha224(arg.expr)) + + +def sha256(arg: Expr) -> Expr: + """Computes the SHA-256 hash of a binary string.""" + return Expr(f.sha256(arg.expr)) + + +def sha384(arg: Expr) -> Expr: + """Computes the SHA-384 hash of a binary string.""" + return Expr(f.sha384(arg.expr)) + + +def sha512(arg: Expr) -> Expr: + """Computes the SHA-512 hash of a binary string.""" + return Expr(f.sha512(arg.expr)) + + +def signum(arg: Expr) -> Expr: + """Returns the sign of the argument (-1, 0, +1).""" + return Expr(f.signum(arg.expr)) + + +def sin(arg: Expr) -> Expr: + """Returns the sine of the argument.""" + return Expr(f.sin(arg.expr)) + + +def sinh(arg: Expr) -> Expr: + """Returns the hyperbolic sine of the argument.""" + return Expr(f.sinh(arg.expr)) + + +def split_part(string: Expr, delimiter: Expr, index: Expr) -> Expr: + """Split a string and return one part. + + Splits a string based on a delimiter and picks out the desired field based + on the index. + """ + return Expr(f.split_part(string.expr, delimiter.expr, index.expr)) + + +def sqrt(arg: Expr) -> Expr: + """Returns the square root of the argument.""" + return Expr(f.sqrt(arg.expr)) + + +def starts_with(string: Expr, prefix: Expr) -> Expr: + """Returns true if string starts with prefix.""" + return Expr(f.starts_with(string.expr, prefix.expr)) + + +def strpos(string: Expr, substring: Expr) -> Expr: + """Finds the position from where the `substring` matches the `string`.""" + return Expr(f.strpos(string.expr, substring.expr)) + + +def substr(string: Expr, position: Expr) -> Expr: + """Substring from the `position` to the end.""" + return Expr(f.substr(string.expr, position.expr)) + + +def substr_index(string: Expr, delimiter: Expr, count: Expr) -> Expr: + """Returns the substring from `string` before `count` occurrences of `delimiter`.""" + return Expr(f.substr_index(string.expr, delimiter.expr, count.expr)) + + +def substring(string: Expr, position: Expr, length: Expr) -> Expr: + """Substring from the `position` with `length` characters.""" + return Expr(f.substring(string.expr, position.expr, length.expr)) + + +def tan(arg: Expr) -> Expr: + """Returns the tangent of the argument.""" + return Expr(f.tan(arg.expr)) + + +def tanh(arg: Expr) -> Expr: + """Returns the hyperbolic tangent of the argument.""" + return Expr(f.tanh(arg.expr)) + + +def to_hex(arg: Expr) -> Expr: + """Converts an integer to a hexadecimal string.""" + return Expr(f.to_hex(arg.expr)) + + +def now() -> Expr: + """Returns the current timestamp in nanoseconds. + + This will use the same value for all instances of now() in same statement. + """ + return Expr(f.now()) + + +def to_timestamp(arg: Expr, *formatters: Expr) -> Expr: + """Converts a string and optional formats to a `Timestamp` in nanoseconds. + + For usage of ``formatters`` see the rust chrono package ``strftime`` package. + + [Documentation here.](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) + """ + if formatters is None: + return f.to_timestamp(arg.expr) + + formatters = [f.expr for f in formatters] + return Expr(f.to_timestamp(arg.expr, *formatters)) + + +def to_timestamp_millis(arg: Expr, *formatters: Expr) -> Expr: + """Converts a string and optional formats to a `Timestamp` in milliseconds. + + See `to_timestamp` for a description on how to use formatters. + """ + return Expr(f.to_timestamp_millis(arg.expr, *formatters)) + + +def to_timestamp_micros(arg: Expr, *formatters: Expr) -> Expr: + """Converts a string and optional formats to a `Timestamp` in microseconds. + + See `to_timestamp` for a description on how to use formatters. + """ + return Expr(f.to_timestamp_micros(arg.expr, *formatters)) + + +def to_timestamp_nanos(arg: Expr, *formatters: Expr) -> Expr: + """Converts a string and optional formats to a `Timestamp` in nanoseconds. + + See `to_timestamp` for a description on how to use formatters. + """ + return Expr(f.to_timestamp_nanos(arg.expr, *formatters)) + + +def to_timestamp_seconds(arg: Expr, *formatters: Expr) -> Expr: + """Converts a string and optional formats to a `Timestamp` in seconds. + + See `to_timestamp` for a description on how to use formatters. + """ + return Expr(f.to_timestamp_seconds(arg.expr, *formatters)) + + +def to_unixtime(string: Expr, *format_arguments: Expr) -> Expr: + """Converts a string and optional formats to a Unixtime.""" + args = [f.expr for f in format_arguments] + return Expr(f.to_unixtime(string.expr, *args)) + + +def current_date() -> Expr: + """Returns current UTC date as a Date32 value.""" + return Expr(f.current_date()) + + +def current_time() -> Expr: + """Returns current UTC time as a Time64 value.""" + return Expr(f.current_time()) + + +def datepart(part: Expr, date: Expr) -> Expr: + """Return a specified part of a date. + + This is an alias for `date_part`. + """ + return date_part(part, date) + + +def date_part(part: Expr, date: Expr) -> Expr: + """Extracts a subfield from the date.""" + return Expr(f.date_part(part.expr, date.expr)) + + +def date_trunc(part: Expr, date: Expr) -> Expr: + """Truncates the date to a specified level of precision.""" + return Expr(f.date_trunc(part.expr, date.expr)) + + +def datetrunc(part: Expr, date: Expr) -> Expr: + """Truncates the date to a specified level of precision. + + This is an alias for `date_trunc`. + """ + return date_trunc(part, date) + + +def date_bin(stride: Expr, source: Expr, origin: Expr) -> Expr: + """Coerces an arbitrary timestamp to the start of the nearest specified interval.""" + return Expr(f.date_bin(stride.expr, source.expr, origin.expr)) + + +def make_date(year: Expr, month: Expr, day: Expr) -> Expr: + """Make a date from year, month and day component parts.""" + return Expr(f.make_date(year.expr, month.expr, day.expr)) + + +def translate(string: Expr, from_val: Expr, to_val: Expr) -> Expr: + """Replaces the characters in `from_val` with the counterpart in `to_val`.""" + return Expr(f.translate(string.expr, from_val.expr, to_val.expr)) + + +def trim(arg: Expr) -> Expr: + """Removes all characters, spaces by default, from both sides of a string.""" + return Expr(f.trim(arg.expr)) + + +def trunc(num: Expr, precision: Expr | None = None) -> Expr: + """Truncate the number toward zero with optional precision.""" + if precision is not None: + return Expr(f.trunc(num.expr, precision.expr)) + return Expr(f.trunc(num.expr)) + + +def upper(arg: Expr) -> Expr: + """Converts a string to uppercase.""" + return Expr(f.upper(arg.expr)) + + +def make_array(*args: Expr) -> Expr: + """Returns an array using the specified input expressions.""" + args = [arg.expr for arg in args] + return Expr(f.make_array(*args)) + + +def array(*args: Expr) -> Expr: + """Returns an array using the specified input expressions. + + This is an alias for `make_array`. + """ + return make_array(args) + + +def range(start: Expr, stop: Expr, step: Expr) -> Expr: + """Create a list of values in the range between start and stop.""" + return Expr(f.range(start.expr, stop.expr, step.expr)) + + +def uuid(arg: Expr) -> Expr: + """Returns uuid v4 as a string value.""" + return Expr(f.uuid(arg.expr)) + + +def struct(*args: Expr) -> Expr: + """Returns a struct with the given arguments.""" + args = [arg.expr for arg in args] + return Expr(f.struct(*args)) + + +def named_struct(name_pairs: list[(str, Expr)]) -> Expr: + """Returns a struct with the given names and arguments pairs.""" + name_pairs = [[Expr.literal(pair[0]), pair[1]] for pair in name_pairs] + + # flatten + name_pairs = [x.expr for xs in name_pairs for x in xs] + return Expr(f.named_struct(*name_pairs)) + + +def from_unixtime(arg: Expr) -> Expr: + """Converts an integer to RFC3339 timestamp format string.""" + return Expr(f.from_unixtime(arg.expr)) + + +def arrow_typeof(arg: Expr) -> Expr: + """Returns the Arrow type of the expression.""" + return Expr(f.arrow_typeof(arg.expr)) + + +def random() -> Expr: + """Returns a random value in the range `0.0 <= x < 1.0`.""" + return Expr(f.random()) + + +def array_append(array: Expr, element: Expr) -> Expr: + """Appends an element to the end of an array.""" + return Expr(f.array_append(array.expr, element.expr)) + + +def array_push_back(array: Expr, element: Expr) -> Expr: + """Appends an element to the end of an array. + + This is an alias for `array_append`. + """ + return array_append(array, element) + + +def list_append(array: Expr, element: Expr) -> Expr: + """Appends an element to the end of an array. + + This is an alias for `array_append`. + """ + return array_append(array, element) + + +def list_push_back(array: Expr, element: Expr) -> Expr: + """Appends an element to the end of an array. + + This is an alias for `array_append`. + """ + return array_append(array, element) + + +def array_concat(*args: Expr) -> Expr: + """Concatenates the input arrays.""" + args = [arg.expr for arg in args] + return Expr(f.array_concat(*args)) + + +def array_cat(*args: Expr) -> Expr: + """Concatenates the input arrays. + + This is an alias for `array_concat`. + """ + return array_concat(*args) + + +def array_dims(array: Expr) -> Expr: + """Returns an array of the array's dimensions.""" + return Expr(f.array_dims(array.expr)) + + +def array_distinct(array: Expr) -> Expr: + """Returns distinct values from the array after removing duplicates.""" + return Expr(f.array_distinct(array.expr)) + + +def list_distinct(array: Expr) -> Expr: + """Returns distinct values from the array after removing duplicates. + + This is an alias for `array_distinct`. + """ + return array_distinct(array) + + +def list_dims(array: Expr) -> Expr: + """Returns an array of the array's dimensions. + + This is an alias for `array_dims`. + """ + return array_dims(array) + + +def array_element(array: Expr, n: Expr) -> Expr: + """Extracts the element with the index n from the array.""" + return Expr(f.array_element(array.expr, n.expr)) + + +def array_extract(array: Expr, n: Expr) -> Expr: + """Extracts the element with the index n from the array. + + This is an alias for `array_element`. + """ + return array_element(array, n) + + +def list_element(array: Expr, n: Expr) -> Expr: + """Extracts the element with the index n from the array. + + This is an alias for `array_element`. + """ + return array_element(array, n) + + +def list_extract(array: Expr, n: Expr) -> Expr: + """Extracts the element with the index n from the array. + + This is an alias for `array_element`. + """ + return array_element(array, n) + + +def array_length(array: Expr) -> Expr: + """Returns the length of the array.""" + return Expr(f.array_length(array.expr)) + + +def list_length(array: Expr) -> Expr: + """Returns the length of the array. + + This is an alias for `array_length`. + """ + return array_length(array) + + +def array_has(first_array: Expr, second_array: Expr) -> Expr: + """Returns true if the element appears in the first array, otherwise false.""" + return Expr(f.array_has(first_array.expr, second_array.expr)) + + +def array_has_all(first_array: Expr, second_array: Expr) -> Expr: + """Determines if there is complete overlap ``second_array`` in ``first_array``. + + Returns true if each element of the second array appears in the first array. + Otherwise, it returns false. + """ + return Expr(f.array_has_all(first_array.expr, second_array.expr)) + + +def array_has_any(first_array: Expr, second_array: Expr) -> Expr: + """Determine if there is an overlap between ``first_array`` and ``second_array``. + + Returns true if at least one element of the second array appears in the first + array. Otherwise, it returns false. + """ + return Expr(f.array_has_any(first_array.expr, second_array.expr)) + + +def array_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: + """Return the position of the first occurrence of ``element`` in ``array``.""" + return Expr(f.array_position(array.expr, element.expr, index)) + + +def array_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: + """Return the position of the first occurrence of ``element`` in ``array``. + + This is an alias for `array_position`. + """ + return array_position(array, element, index) + + +def list_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: + """Return the position of the first occurrence of ``element`` in ``array``. + + This is an alias for `array_position`. + """ + return array_position(array, element, index) + + +def list_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: + """Return the position of the first occurrence of ``element`` in ``array``. + + This is an alias for `array_position`. + """ + return array_position(array, element, index) + + +def array_positions(array: Expr, element: Expr) -> Expr: + """Searches for an element in the array and returns all occurrences.""" + return Expr(f.array_positions(array.expr, element.expr)) + + +def list_positions(array: Expr, element: Expr) -> Expr: + """Searches for an element in the array and returns all occurrences. + + This is an alias for `array_positions`. + """ + return array_positions(array, element) + + +def array_ndims(array: Expr) -> Expr: + """Returns the number of dimensions of the array.""" + return Expr(f.array_ndims(array.expr)) + + +def list_ndims(array: Expr) -> Expr: + """Returns the number of dimensions of the array. + + This is an alias for `array_ndims`. + """ + return array_ndims(array) + + +def array_prepend(element: Expr, array: Expr) -> Expr: + """Prepends an element to the beginning of an array.""" + return Expr(f.array_prepend(element.expr, array.expr)) + + +def array_push_front(element: Expr, array: Expr) -> Expr: + """Prepends an element to the beginning of an array. + + This is an alias for `array_prepend`. + """ + return array_prepend(element, array) + + +def list_prepend(element: Expr, array: Expr) -> Expr: + """Prepends an element to the beginning of an array. + + This is an alias for `array_prepend`. + """ + return array_prepend(element, array) + + +def list_push_front(element: Expr, array: Expr) -> Expr: + """Prepends an element to the beginning of an array. + + This is an alias for `array_prepend`. + """ + return array_prepend(element, array) + + +def array_pop_back(array: Expr) -> Expr: + """Returns the array without the last element.""" + return Expr(f.array_pop_back(array.expr)) + + +def array_pop_front(array: Expr) -> Expr: + """Returns the array without the first element.""" + return Expr(f.array_pop_front(array.expr)) + + +def array_remove(array: Expr, element: Expr) -> Expr: + """Removes the first element from the array equal to the given value.""" + return Expr(f.array_remove(array.expr, element.expr)) + + +def list_remove(array: Expr, element: Expr) -> Expr: + """Removes the first element from the array equal to the given value. + + This is an alias for `array_remove`. + """ + return array_remove(array, element) + + +def array_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: + """Removes the first `max` elements from the array equal to the given value.""" + return Expr(f.array_remove_n(array.expr, element.expr, max.expr)) + + +def list_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: + """Removes the first `max` elements from the array equal to the given value. + + This is an alias for `array_remove_n`. + """ + return array_remove_n(array, element, max) + + +def array_remove_all(array: Expr, element: Expr) -> Expr: + """Removes all elements from the array equal to the given value.""" + return Expr(f.array_remove_all(array.expr, element.expr)) + + +def list_remove_all(array: Expr, element: Expr) -> Expr: + """Removes all elements from the array equal to the given value. + + This is an alias for `array_remove_all`. + """ + return array_remove_all(array, element) + + +def array_repeat(element: Expr, count: Expr) -> Expr: + """Returns an array containing `element` `count` times.""" + return Expr(f.array_repeat(element.expr, count.expr)) + + +def array_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: + """Replaces the first occurrence of ``from_val`` with ``to_val``.""" + return Expr(f.array_replace(array.expr, from_val.expr, to_val.expr)) + + +def list_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: + """Replaces the first occurrence of ``from_val`` with ``to_val``. + + This is an alias for `array_replace`. + """ + return array_replace(array, from_val, to_val) + + +def array_replace_n(array: Expr, from_val: Expr, to_val: Expr, max: Expr) -> Expr: + """Replace `n` occurrences of ``from_val`` with ``to_val``. + + Replaces the first `max` occurrences of the specified element with another + specified element. + """ + return Expr(f.array_replace_n(array.expr, from_val.expr, to_val.expr, max.expr)) + + +def list_replace_n(array: Expr, from_val: Expr, to_val: Expr, max: Expr) -> Expr: + """Replace `n` occurrences of ``from_val`` with ``to_val``. + + Replaces the first `max` occurrences of the specified element with another + specified element. + + This is an alias for `array_replace_n`. + """ + return array_replace_n(array, from_val, to_val, max) + + +def array_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: + """Replaces all occurrences of ``from_val`` with ``to_val``.""" + return Expr(f.array_replace_all(array.expr, from_val.expr, to_val.expr)) + + +def list_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: + """Replaces all occurrences of ``from_val`` with ``to_val``. + + This is an alias for `array_replace_all`. + """ + return array_replace_all(array, from_val, to_val) + + +def array_slice( + array: Expr, begin: Expr, end: Expr, stride: Expr | None = None +) -> Expr: + """Returns a slice of the array.""" + if stride is not None: + stride = stride.expr + return Expr(f.array_slice(array.expr, begin.expr, end.expr, stride)) + + +def list_slice(array: Expr, begin: Expr, end: Expr, stride: Expr | None = None) -> Expr: + """Returns a slice of the array. + + This is an alias for `array_slice`. + """ + return array_slice(array, begin, end, stride) + + +def array_intersect(array1: Expr, array2: Expr) -> Expr: + """Returns an array of the elements in the intersection of array1 and array2.""" + return Expr(f.array_intersect(array1.expr, array2.expr)) + + +def list_intersect(array1: Expr, array2: Expr) -> Expr: + """Returns an array of the elements in the intersection of `array1` and `array2`. + + This is an alias for `array_intersect`. + """ + return array_intersect(array1, array2) + + +def array_union(array1: Expr, array2: Expr) -> Expr: + """Returns an array of the elements in the union of array1 and array2. + + Duplicate rows will not be returned. + """ + return Expr(f.array_union(array1.expr, array2.expr)) + + +def list_union(array1: Expr, array2: Expr) -> Expr: + """Returns an array of the elements in the union of array1 and array2. + + Duplicate rows will not be returned. + + This is an alias for `array_union`. + """ + return array_union(array1, array2) + + +def array_except(array1: Expr, array2: Expr) -> Expr: + """Returns an array of the elements that appear in `array1` but not in `array2`.""" + return Expr(f.array_except(array1.expr, array2.expr)) + + +def list_except(array1: Expr, array2: Expr) -> Expr: + """Returns an array of the elements that appear in `array1` but not in the `array2`. + + This is an alias for `array_except`. + """ + return array_except(array1, array2) + + +def array_resize(array: Expr, size: Expr, value: Expr) -> Expr: + """Returns an array with the specified size filled. + + If `size` is greater than the `array` length, the additional entries will be filled + with the given `value`. + """ + return Expr(f.array_resize(array.expr, size.expr, value.expr)) + + +def list_resize(array: Expr, size: Expr, value: Expr) -> Expr: + """Returns an array with the specified size filled. + + If `size` is greater than the `array` length, the additional entries will be + filled with the given `value`. This is an alias for `array_resize`. + """ + return array_resize(array, size, value) + + +def flatten(array: Expr) -> Expr: + """Flattens an array of arrays into a single array.""" + return Expr(f.flatten(array.expr)) + + +# aggregate functions +def approx_distinct(arg: Expr) -> Expr: + """Returns the approximate number of distinct values.""" + return Expr(f.approx_distinct(arg.expr, distinct=True)) + + +def approx_median(arg: Expr, distinct: bool = False) -> Expr: + """Returns the approximate median value.""" + return Expr(f.approx_median(arg.expr, distinct=distinct)) + + +def approx_percentile_cont( + expr: Expr, + percentile: Expr, + num_centroids: int | None = None, + distinct: bool = False, +) -> Expr: + """Returns the value that is approximately at a given percentile of ``expr``.""" + if num_centroids is None: + return Expr( + f.approx_percentile_cont(expr.expr, percentile.expr, distinct=distinct) + ) + + return Expr( + f.approx_percentile_cont( + expr.expr, percentile.expr, num_centroids, distinct=distinct + ) + ) + + +def approx_percentile_cont_with_weight( + arg: Expr, weight: Expr, percentile: Expr, distinct: bool = False +) -> Expr: + """Returns the value of the approximate percentile. + + This function is similar to ``approx_percentile_cont`` except that it uses + the associated associated weights. + """ + return Expr( + f.approx_percentile_cont_with_weight( + arg.expr, weight.expr, percentile.expr, distinct=distinct + ) + ) + + +def array_agg(arg: Expr, distinct: bool = False) -> Expr: + """Aggregate values into an array.""" + return Expr(f.array_agg(arg.expr, distinct=distinct)) + + +def avg(arg: Expr, distinct: bool = False) -> Expr: + """Returns the average value.""" + return Expr(f.avg(arg.expr, distinct=distinct)) + + +def corr(value1: Expr, value2: Expr, distinct: bool = False) -> Expr: + """Returns the correlation coefficient between `value1` and `value2`.""" + return Expr(f.corr(value1.expr, value2.expr, distinct=distinct)) + + +def count(args: Expr | list[Expr] | None = None, distinct: bool = False) -> Expr: + """Returns the number of rows that match the given arguments.""" + if isinstance(args, list): + args = [arg.expr for arg in args] + elif isinstance(args, Expr): + args = [args.expr] + return Expr(f.count(*args, distinct=distinct)) + + +def covar(y: Expr, x: Expr) -> Expr: + """Computes the sample covariance. + + This is an alias for `covar_samp`. + """ + return Expr(f.covar(y.expr, x.expr)) + + +def covar_pop(y: Expr, x: Expr) -> Expr: + """Computes the population covariance.""" + return Expr(f.covar_pop(y.expr, x.expr)) + + +def covar_samp(y: Expr, x: Expr) -> Expr: + """Computes the sample covariance.""" + return Expr(f.covar_samp(y.expr, x.expr)) + + +def grouping(arg: Expr, distinct: bool = False) -> Expr: + """Indicates if the expression is aggregated or not. + + Returns 1 if the value of the argument is aggregated, 0 if not. + """ + return Expr(f.grouping([arg.expr], distinct=distinct)) + + +def max(arg: Expr, distinct: bool = False) -> Expr: + """Returns the maximum value of the arugment.""" + return Expr(f.max(arg.expr, distinct=distinct)) + + +def mean(arg: Expr, distinct: bool = False) -> Expr: + """Returns the average (mean) value of the argument. + + This is an alias for `avg`. + """ + return avg(arg, distinct) + + +def median(arg: Expr) -> Expr: + """Computes the median of a set of numbers.""" + return Expr(f.median(arg.expr)) + + +def min(arg: Expr, distinct: bool = False) -> Expr: + """Returns the minimum value of the argument.""" + return Expr(f.min(arg.expr, distinct=distinct)) + + +def sum(arg: Expr) -> Expr: + """Computes the sum of a set of numbers.""" + return Expr(f.sum(arg.expr)) + + +def stddev(arg: Expr, distinct: bool = False) -> Expr: + """Computes the standard deviation of the argument.""" + return Expr(f.stddev(arg.expr, distinct=distinct)) + + +def stddev_pop(arg: Expr, distinct: bool = False) -> Expr: + """Computes the population standard deviation of the argument.""" + return Expr(f.stddev_pop(arg.expr, distinct=distinct)) + + +def stddev_samp(arg: Expr, distinct: bool = False) -> Expr: + """Computes the sample standard deviation of the argument. + + This is an alias for `stddev`. + """ + return stddev(arg, distinct) + + +def var(arg: Expr) -> Expr: + """Computes the sample variance of the argument. + + This is an alias for `var_samp`. + """ + return var_samp(arg) + + +def var_pop(arg: Expr, distinct: bool = False) -> Expr: + """Computes the population variance of the argument.""" + return Expr(f.var_pop(arg.expr, distinct=distinct)) + + +def var_samp(arg: Expr) -> Expr: + """Computes the sample variance of the argument.""" + return Expr(f.var_samp(arg.expr)) + + +def regr_avgx(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the average of the independent variable `x`. + + Only non-null pairs of the inputs are evaluated. + """ + return Expr(f.regr_avgx[y.expr, x.expr], distinct) + + +def regr_avgy(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the average of the dependent variable ``y``. + + Only non-null pairs of the inputs are evaluated. + """ + return Expr(f.regr_avgy[y.expr, x.expr], distinct) + + +def regr_count(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Counts the number of rows in which both expressions are not null.""" + return Expr(f.regr_count[y.expr, x.expr], distinct) + + +def regr_intercept(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the intercept from the linear regression.""" + return Expr(f.regr_intercept[y.expr, x.expr], distinct) + + +def regr_r2(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the R-squared value from linear regression.""" + return Expr(f.regr_r2[y.expr, x.expr], distinct) + + +def regr_slope(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the slope from linear regression.""" + return Expr(f.regr_slope[y.expr, x.expr], distinct) + + +def regr_sxx(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the sum of squares of the independent variable `x`.""" + return Expr(f.regr_sxx[y.expr, x.expr], distinct) + + +def regr_sxy(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the sum of products of pairs of numbers.""" + return Expr(f.regr_sxy[y.expr, x.expr], distinct) + + +def regr_syy(y: Expr, x: Expr, distinct: bool = False) -> Expr: + """Computes the sum of squares of the dependent variable `y`.""" + return Expr(f.regr_syy[y.expr, x.expr], distinct) + + +def first_value( + arg: Expr, + distinct: bool = False, + filter: bool = None, + order_by: Expr | None = None, + null_treatment: common.NullTreatment | None = None, +) -> Expr: + """Returns the first value in a group of values.""" + return Expr( + f.first_value( + arg.expr, + distinct=distinct, + filter=filter, + order_by=order_by, + null_treatment=null_treatment, + ) + ) + + +def last_value( + arg: Expr, + distinct: bool = False, + filter: bool = None, + order_by: Expr | None = None, + null_treatment: common.NullTreatment | None = None, +) -> Expr: + """Returns the last value in a group of values.""" + return Expr( + f.last_value( + arg.expr, + distinct=distinct, + filter=filter, + order_by=order_by, + null_treatment=null_treatment, + ) + ) + + +def bit_and(*args: Expr, distinct: bool = False) -> Expr: + """Computes the bitwise AND of the argument.""" + args = [arg.expr for arg in args] + return Expr(f.bit_and(*args, distinct=distinct)) + + +def bit_or(*args: Expr, distinct: bool = False) -> Expr: + """Computes the bitwise OR of the argument.""" + args = [arg.expr for arg in args] + return Expr(f.bit_or(*args, distinct=distinct)) + + +def bit_xor(*args: Expr, distinct: bool = False) -> Expr: + """Computes the bitwise XOR of the argument.""" + args = [arg.expr for arg in args] + return Expr(f.bit_xor(*args, distinct=distinct)) + + +def bool_and(*args: Expr, distinct: bool = False) -> Expr: + """Computes the boolean AND of the arugment.""" + args = [arg.expr for arg in args] + return Expr(f.bool_and(*args, distinct=distinct)) + + +def bool_or(*args: Expr, distinct: bool = False) -> Expr: + """Computes the boolean OR of the arguement.""" + args = [arg.expr for arg in args] + return Expr(f.bool_or(*args, distinct=distinct)) diff --git a/python/datafusion/input/__init__.py b/python/datafusion/input/__init__.py index 27e39b8ca..f85ce21f0 100644 --- a/python/datafusion/input/__init__.py +++ b/python/datafusion/input/__init__.py @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +"""This package provides for input sources. + +The primary class used within DataFusion is ``LocationInputPlugin``. +""" + from .location import LocationInputPlugin __all__ = [ diff --git a/python/datafusion/input/base.py b/python/datafusion/input/base.py index efcaf7697..4eba19784 100644 --- a/python/datafusion/input/base.py +++ b/python/datafusion/input/base.py @@ -15,6 +15,11 @@ # specific language governing permissions and limitations # under the License. +"""This module provides ``BaseInputSource``. + +A user can extend this to provide a custom input source. +""" + from abc import ABC, abstractmethod from typing import Any @@ -22,18 +27,22 @@ class BaseInputSource(ABC): - """ - If a consuming library would like to provider their own InputSource - this is the class they should extend to write their own. Once - completed the Plugin InputSource can be registered with the + """Base Input Source class. + + If a consuming library would like to provider their own InputSource this is + the class they should extend to write their own. + + Once completed the Plugin InputSource can be registered with the SessionContext to ensure that it will be used in order to obtain the SqlTable information from the custom datasource. """ @abstractmethod def is_correct_input(self, input_item: Any, table_name: str, **kwargs) -> bool: + """Returns `True` if the input is valid.""" pass @abstractmethod def build_table(self, input_item: Any, table_name: str, **kwarg) -> SqlTable: + """Create a table from the input source.""" pass diff --git a/python/datafusion/input/location.py b/python/datafusion/input/location.py index 16e632d1b..566a63da9 100644 --- a/python/datafusion/input/location.py +++ b/python/datafusion/input/location.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +"""The default input source for DataFusion.""" + import os import glob from typing import Any @@ -24,12 +26,13 @@ class LocationInputPlugin(BaseInputSource): - """ - Input Plugin for everything, which can be read - in from a file (on disk, remote etc.) + """Input Plugin for everything. + + This can be read in from a file (on disk, remote etc.). """ def is_correct_input(self, input_item: Any, table_name: str, **kwargs): + """Returns `True` if the input is valid.""" return isinstance(input_item, str) def build_table( @@ -38,6 +41,7 @@ def build_table( table_name: str, **kwargs, ) -> SqlTable: + """Create a table from the input source.""" _, extension = os.path.splitext(input_file) format = extension.lstrip(".").lower() num_rows = 0 # Total number of rows in the file. Used for statistics diff --git a/python/datafusion/object_store.py b/python/datafusion/object_store.py index 70ecbd2bb..a9bb83d29 100644 --- a/python/datafusion/object_store.py +++ b/python/datafusion/object_store.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - +"""Object store functionality.""" from ._internal import object_store diff --git a/python/datafusion/py.typed b/python/datafusion/py.typed new file mode 100644 index 000000000..d216be4dd --- /dev/null +++ b/python/datafusion/py.typed @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file diff --git a/python/datafusion/record_batch.py b/python/datafusion/record_batch.py new file mode 100644 index 000000000..dcfd55485 --- /dev/null +++ b/python/datafusion/record_batch.py @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module provides the classes for handling record batches. + +These are typically the result of dataframe `execute_stream` operations. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import pyarrow + import datafusion._internal as df_internal + import typing_extensions + + +class RecordBatch: + """This class is essentially a wrapper for ``pyarrow.RecordBatch``.""" + + def __init__(self, record_batch: df_internal.RecordBatch) -> None: + """This constructor is generally not called by the end user. + + See the ``RecordBatchStream`` iterator for generating this class. + """ + self.record_batch = record_batch + + def to_pyarrow(self) -> pyarrow.RecordBatch: + """Convert to pyarrow ``RecordBatch``.""" + return self.record_batch.to_pyarrow() + + +class RecordBatchStream: + """This class represents a stream of record batches. + + These are typically the result of a ``DataFrame::execute_stream`` operation. + """ + + def __init__(self, record_batch_stream: df_internal.RecordBatchStream) -> None: + """This constructor is typically not called by the end user.""" + self.rbs = record_batch_stream + + def next(self) -> RecordBatch | None: + """See ``__next__`` for the iterator function.""" + try: + next_batch = next(self) + except StopIteration: + return None + + return next_batch + + def __next__(self) -> RecordBatch: + """Iterator function.""" + next_batch = next(self.rbs) + return RecordBatch(next_batch) + + def __iter__(self) -> typing_extensions.Self: + """Iterator function.""" + return self diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index eff809a0c..a199dd733 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -15,9 +15,171 @@ # specific language governing permissions and limitations # under the License. +"""This module provides support for using substrait with datafusion. -from ._internal import substrait +For additional information about substrait, see https://substrait.io/ for more +information about substrait. +""" +from __future__ import annotations -def __getattr__(name): - return getattr(substrait, name) +from ._internal import substrait as substrait_internal + +from typing import TYPE_CHECKING +from typing_extensions import deprecated +import pathlib + +if TYPE_CHECKING: + from datafusion.context import SessionContext + from datafusion._internal import LogicalPlan + + +class Plan: + """A class representing an encodable substrait plan.""" + + def __init__(self, plan: substrait_internal.Plan) -> None: + """Create a substrait plan. + + The user should not have to call this constructor directly. Rather, it + should be created via `Serde` or `Producer` classes in this module. + """ + self.plan_internal = plan + + def encode(self) -> bytes: + """Encode the plan to bytes. + + Returns: + Encoded plan. + """ + return self.plan_internal.encode() + + +@deprecated("Use `Plan` instead.") +class plan(Plan): + """See `Plan`.""" + + pass + + +class Serde: + """Provides the ``Substrait`` serialization and deserialization.""" + + @staticmethod + def serialize(sql: str, ctx: SessionContext, path: str | pathlib.Path) -> None: + """Serialize a SQL query to a Substrait plan and write it to a file. + + Args: + sql:SQL query to serialize. + ctx: SessionContext to use. + path: Path to write the Substrait plan to. + """ + return substrait_internal.serde.serialize(sql, ctx.ctx, str(path)) + + @staticmethod + def serialize_to_plan(sql: str, ctx: SessionContext) -> Plan: + """Serialize a SQL query to a Substrait plan. + + Args: + sql: SQL query to serialize. + ctx: SessionContext to use. + + Returns: + Substrait plan. + """ + return Plan(substrait_internal.serde.serialize_to_plan(sql, ctx.ctx)) + + @staticmethod + def serialize_bytes(sql: str, ctx: SessionContext) -> bytes: + """Serialize a SQL query to a Substrait plan as bytes. + + Args: + sql: SQL query to serialize. + ctx: SessionContext to use. + + Returns: + Substrait plan as bytes. + """ + return substrait_internal.serde.serialize_bytes(sql, ctx.ctx) + + @staticmethod + def deserialize(path: str | pathlib.Path) -> Plan: + """Deserialize a Substrait plan from a file. + + Args: + path: Path to read the Substrait plan from. + + Returns: + Substrait plan. + """ + return Plan(substrait_internal.serde.deserialize(str(path))) + + @staticmethod + def deserialize_bytes(proto_bytes: bytes) -> Plan: + """Deserialize a Substrait plan from bytes. + + Args: + proto_bytes: Bytes to read the Substrait plan from. + + Returns: + Substrait plan. + """ + return Plan(substrait_internal.serde.deserialize_bytes(proto_bytes)) + + +@deprecated("Use `Serde` instead.") +class serde(Serde): + """See `Serde` instead.""" + + pass + + +class Producer: + """Generates substrait plans from a logical plan.""" + + @staticmethod + def to_substrait_plan(logical_plan: LogicalPlan, ctx: SessionContext) -> Plan: + """Convert a DataFusion LogicalPlan to a Substrait plan. + + Args: + logical_plan: LogicalPlan to convert. + ctx: SessionContext to use. + + Returns: + Substrait plan. + """ + return Plan( + substrait_internal.producer.to_substrait_plan(logical_plan, ctx.ctx) + ) + + +@deprecated("Use `Producer` instead.") +class producer(Producer): + """Use `Producer` instead.""" + + pass + + +class Consumer: + """Generates a logical plan from a substrait plan.""" + + @staticmethod + def from_substrait_plan(ctx: SessionContext, plan: Plan) -> LogicalPlan: + """Convert a Substrait plan to a DataFusion LogicalPlan. + + Args: + ctx: SessionContext to use. + plan: Substrait plan to convert. + + Returns: + LogicalPlan. + """ + return substrait_internal.consumer.from_substrait_plan( + ctx.ctx, plan.plan_internal + ) + + +@deprecated("Use `Consumer` instead.") +class consumer(Consumer): + """Use `Consumer` instead.""" + + pass diff --git a/python/datafusion/tests/conftest.py b/python/datafusion/tests/conftest.py index a4eec41e2..1cc07e500 100644 --- a/python/datafusion/tests/conftest.py +++ b/python/datafusion/tests/conftest.py @@ -18,6 +18,7 @@ import pytest from datafusion import SessionContext import pyarrow as pa +from pyarrow.csv import write_csv @pytest.fixture @@ -37,7 +38,7 @@ def database(ctx, tmp_path): ], names=["int", "str", "float"], ) - pa.csv.write_csv(table, path) + write_csv(table, path) ctx.register_csv("csv", path) ctx.register_csv("csv1", str(path)) diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index abc324db8..8373659b0 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -17,6 +17,7 @@ import gzip import os import datetime as dt +import pathlib import pyarrow as pa import pyarrow.dataset as ds @@ -37,6 +38,36 @@ def test_create_context_no_args(): SessionContext() +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_runtime_configs(tmp_path, path_to_str): + path1 = tmp_path / "dir1" + path2 = tmp_path / "dir2" + + path1 = str(path1) if path_to_str else path1 + path2 = str(path2) if path_to_str else path2 + + runtime = RuntimeConfig().with_disk_manager_specified(path1, path2) + config = SessionConfig().with_default_catalog_and_schema("foo", "bar") + ctx = SessionContext(config, runtime) + assert ctx is not None + + db = ctx.catalog("foo").database("bar") + assert db is not None + + +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_temporary_files(tmp_path, path_to_str): + path = str(tmp_path) if path_to_str else tmp_path + + runtime = RuntimeConfig().with_temp_file_path(path) + config = SessionConfig().with_default_catalog_and_schema("foo", "bar") + ctx = SessionContext(config, runtime) + assert ctx is not None + + db = ctx.catalog("foo").database("bar") + assert db is not None + + def test_create_context_with_all_valid_args(): runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( @@ -68,7 +99,7 @@ def test_register_record_batches(ctx): ctx.register_record_batches("t", [[batch]]) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} result = ctx.sql("SELECT a+b, a-b FROM t").collect() @@ -84,7 +115,7 @@ def test_create_dataframe_registers_unique_table_name(ctx): ) df = ctx.create_dataframe([[batch]]) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 @@ -104,7 +135,7 @@ def test_create_dataframe_registers_with_defined_table_name(ctx): ) df = ctx.create_dataframe([[batch]], name="tbl") - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 @@ -118,11 +149,11 @@ def test_from_arrow_table(ctx): # convert to DataFrame df = ctx.from_arrow_table(table) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 - assert type(df) == DataFrame + assert isinstance(df, DataFrame) assert set(df.schema().names) == {"a", "b"} assert df.collect()[0].num_rows == 3 @@ -134,7 +165,7 @@ def test_from_arrow_table_with_name(ctx): # convert to DataFrame with optional name df = ctx.from_arrow_table(table, name="tbl") - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert tables[0] == "tbl" @@ -147,7 +178,7 @@ def test_from_arrow_table_empty(ctx): # convert to DataFrame df = ctx.from_arrow_table(table) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 @@ -162,7 +193,7 @@ def test_from_arrow_table_empty_no_schema(ctx): # convert to DataFrame df = ctx.from_arrow_table(table) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 @@ -180,11 +211,11 @@ def test_from_pylist(ctx): ] df = ctx.from_pylist(data) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 - assert type(df) == DataFrame + assert isinstance(df, DataFrame) assert set(df.schema().names) == {"a", "b"} assert df.collect()[0].num_rows == 3 @@ -194,11 +225,11 @@ def test_from_pydict(ctx): data = {"a": [1, 2, 3], "b": [4, 5, 6]} df = ctx.from_pydict(data) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 - assert type(df) == DataFrame + assert isinstance(df, DataFrame) assert set(df.schema().names) == {"a", "b"} assert df.collect()[0].num_rows == 3 @@ -210,11 +241,11 @@ def test_from_pandas(ctx): pandas_df = pd.DataFrame(data) df = ctx.from_pandas(pandas_df) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 - assert type(df) == DataFrame + assert isinstance(df, DataFrame) assert set(df.schema().names) == {"a", "b"} assert df.collect()[0].num_rows == 3 @@ -226,11 +257,11 @@ def test_from_polars(ctx): polars_df = pd.DataFrame(data) df = ctx.from_polars(polars_df) - tables = list(ctx.tables()) + tables = list(ctx.catalog().database().names()) assert df assert len(tables) == 1 - assert type(df) == DataFrame + assert isinstance(df, DataFrame) assert set(df.schema().names) == {"a", "b"} assert df.collect()[0].num_rows == 3 @@ -273,7 +304,7 @@ def test_register_dataset(ctx): dataset = ds.dataset([batch]) ctx.register_dataset("t", dataset) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} result = ctx.sql("SELECT a+b, a-b FROM t").collect() @@ -290,7 +321,7 @@ def test_dataset_filter(ctx, capfd): dataset = ds.dataset([batch]) ctx.register_dataset("t", dataset) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} df = ctx.sql("SELECT a+b, a-b FROM t WHERE a BETWEEN 2 and 3 AND b > 5") # Make sure the filter was pushed down in Physical Plan @@ -370,7 +401,7 @@ def test_dataset_filter_nested_data(ctx): dataset = ds.dataset([batch]) ctx.register_dataset("t", dataset) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} df = ctx.table("t") @@ -468,13 +499,23 @@ def test_read_csv_compressed(ctx, tmp_path): def test_read_parquet(ctx): - csv_df = ctx.read_parquet(path="parquet/data/alltypes_plain.parquet") - csv_df.show() + parquet_df = ctx.read_parquet(path="parquet/data/alltypes_plain.parquet") + parquet_df.show() + assert parquet_df is not None + + path = pathlib.Path.cwd() / "parquet/data/alltypes_plain.parquet" + parquet_df = ctx.read_parquet(path=path) + assert parquet_df is not None def test_read_avro(ctx): - csv_df = ctx.read_avro(path="testing/data/avro/alltypes_plain.avro") - csv_df.show() + avro_df = ctx.read_avro(path="testing/data/avro/alltypes_plain.avro") + avro_df.show() + assert avro_df is not None + + path = pathlib.Path.cwd() / "testing/data/avro/alltypes_plain.avro" + avro_df = ctx.read_avro(path=path) + assert avro_df is not None def test_create_sql_options(): diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index 2f6a818ea..25875da77 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -17,6 +17,7 @@ import os import pyarrow as pa +from pyarrow.csv import write_csv import pyarrow.parquet as pq import pytest @@ -96,6 +97,16 @@ def test_select(df): assert result.column(1) == pa.array([-3, -3, -3]) +def test_select_mixed_expr_string(df): + df = df.select_columns(column("b"), "a") + + # execute and collect the first (and only) batch + result = df.collect()[0] + + assert result.column(0) == pa.array([4, 5, 6]) + assert result.column(1) == pa.array([1, 2, 3]) + + def test_select_columns(df): df = df.select_columns("b", "a") @@ -107,17 +118,29 @@ def test_select_columns(df): def test_filter(df): - df = df.filter(column("a") > literal(2)).select( + df1 = df.filter(column("a") > literal(2)).select( column("a") + column("b"), column("a") - column("b"), ) # execute and collect the first (and only) batch - result = df.collect()[0] + result = df1.collect()[0] assert result.column(0) == pa.array([9]) assert result.column(1) == pa.array([-3]) + df.show() + # verify that if there is no filter applied, internal dataframe is unchanged + df2 = df.filter() + assert df.df == df2.df + + df3 = df.filter(column("a") > literal(1), column("b") != literal(6)) + result = df3.collect()[0] + + assert result.column(0) == pa.array([2]) + assert result.column(1) == pa.array([5]) + assert result.column(2) == pa.array([5]) + def test_sort(df): df = df.sort(column("b").sort(ascending=False)) @@ -175,7 +198,7 @@ def test_with_column_renamed(df): def test_unnest(nested_df): - nested_df = nested_df.unnest_column("a") + nested_df = nested_df.unnest_columns("a") # execute and collect the first (and only) batch result = nested_df.collect()[0] @@ -185,7 +208,7 @@ def test_unnest(nested_df): def test_unnest_without_nulls(nested_df): - nested_df = nested_df.unnest_column("a", preserve_nulls=False) + nested_df = nested_df.unnest_columns("a", preserve_nulls=False) # execute and collect the first (and only) batch result = nested_df.collect()[0] @@ -379,7 +402,7 @@ def test_get_dataframe(tmp_path): ], names=["int", "str", "float"], ) - pa.csv.write_csv(table, path) + write_csv(table, path) ctx.register_csv("csv", path) @@ -611,7 +634,7 @@ def test_to_pandas(df): # Convert datafusion dataframe to pandas dataframe pandas_df = df.to_pandas() - assert type(pandas_df) == pd.DataFrame + assert isinstance(pandas_df, pd.DataFrame) assert pandas_df.shape == (3, 3) assert set(pandas_df.columns) == {"a", "b", "c"} @@ -622,7 +645,7 @@ def test_empty_to_pandas(df): # Convert empty datafusion dataframe to pandas dataframe pandas_df = df.limit(0).to_pandas() - assert type(pandas_df) == pd.DataFrame + assert isinstance(pandas_df, pd.DataFrame) assert pandas_df.shape == (0, 3) assert set(pandas_df.columns) == {"a", "b", "c"} @@ -633,7 +656,7 @@ def test_to_polars(df): # Convert datafusion dataframe to polars dataframe polars_df = df.to_polars() - assert type(polars_df) == pl.DataFrame + assert isinstance(polars_df, pl.DataFrame) assert polars_df.shape == (3, 3) assert set(polars_df.columns) == {"a", "b", "c"} @@ -644,7 +667,7 @@ def test_empty_to_polars(df): # Convert empty datafusion dataframe to polars dataframe polars_df = df.limit(0).to_polars() - assert type(polars_df) == pl.DataFrame + assert isinstance(polars_df, pl.DataFrame) assert polars_df.shape == (0, 3) assert set(polars_df.columns) == {"a", "b", "c"} @@ -652,13 +675,15 @@ def test_empty_to_polars(df): def test_to_arrow_table(df): # Convert datafusion dataframe to pyarrow Table pyarrow_table = df.to_arrow_table() - assert type(pyarrow_table) == pa.Table + assert isinstance(pyarrow_table, pa.Table) assert pyarrow_table.shape == (3, 3) assert set(pyarrow_table.column_names) == {"a", "b", "c"} def test_execute_stream(df): stream = df.execute_stream() + for s in stream: + print(type(s)) assert all(batch is not None for batch in stream) assert not list(stream) # after one iteration the generator must be exhausted @@ -690,7 +715,7 @@ def test_execute_stream_partitioned(df): def test_empty_to_arrow_table(df): # Convert empty datafusion dataframe to pyarrow Table pyarrow_table = df.limit(0).to_arrow_table() - assert type(pyarrow_table) == pa.Table + assert isinstance(pyarrow_table, pa.Table) assert pyarrow_table.shape == (0, 3) assert set(pyarrow_table.column_names) == {"a", "b", "c"} @@ -736,8 +761,35 @@ def test_describe(df): } -def test_write_parquet(df, tmp_path): - path = tmp_path +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_write_csv(ctx, df, tmp_path, path_to_str): + path = str(tmp_path) if path_to_str else tmp_path + + df.write_csv(path, with_header=True) + + ctx.register_csv("csv", path) + result = ctx.table("csv").to_pydict() + expected = df.to_pydict() + + assert result == expected + + +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_write_json(ctx, df, tmp_path, path_to_str): + path = str(tmp_path) if path_to_str else tmp_path + + df.write_json(path) + + ctx.register_json("json", path) + result = ctx.table("json").to_pydict() + expected = df.to_pydict() + + assert result == expected + + +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_write_parquet(df, tmp_path, path_to_str): + path = str(tmp_path) if path_to_str else tmp_path df.write_parquet(str(path)) result = pq.read_table(str(path)).to_pydict() @@ -795,3 +847,15 @@ def test_write_compressed_parquet_missing_compression_level(df, tmp_path, compre with pytest.raises(ValueError): df.write_parquet(str(path), compression=compression) + + +# ctx = SessionContext() + +# # create a RecordBatch and a new DataFrame from it +# batch = pa.RecordBatch.from_arrays( +# [pa.array([1, 2, 3]), pa.array([4, 5, 6]), pa.array([8, 5, 8])], +# names=["a", "b", "c"], +# ) + +# df = ctx.create_dataframe([[batch]]) +# test_execute_stream(df) diff --git a/python/datafusion/tests/test_expr.py b/python/datafusion/tests/test_expr.py index 73f7d087a..c9f0e98d5 100644 --- a/python/datafusion/tests/test_expr.py +++ b/python/datafusion/tests/test_expr.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from datafusion import SessionContext +from datafusion import SessionContext, col from datafusion.expr import Column, Literal, BinaryExpr, AggregateFunction from datafusion.expr import ( Projection, @@ -25,6 +25,7 @@ Sort, TableScan, ) +import pyarrow import pytest @@ -116,3 +117,25 @@ def test_sort(test_ctx): plan = plan.to_variant() assert isinstance(plan, Sort) + + +def test_relational_expr(test_ctx): + ctx = SessionContext() + + batch = pyarrow.RecordBatch.from_arrays( + [pyarrow.array([1, 2, 3]), pyarrow.array(["alpha", "beta", "gamma"])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]], name="batch_array") + + assert df.filter(col("a") == 1).count() == 1 + assert df.filter(col("a") != 1).count() == 2 + assert df.filter(col("a") >= 1).count() == 3 + assert df.filter(col("a") > 1).count() == 2 + assert df.filter(col("a") <= 3).count() == 3 + assert df.filter(col("a") < 3).count() == 2 + + assert df.filter(col("b") == "beta").count() == 1 + assert df.filter(col("b") != "beta").count() == 2 + + assert df.filter(col("a") == "beta").count() == 0 diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 449f706c3..2384b6ab8 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -54,12 +54,11 @@ def test_named_struct(df): df = df.with_column( "d", f.named_struct( - literal("a"), - column("a"), - literal("b"), - column("b"), - literal("c"), - column("c"), + [ + ("a", column("a")), + ("b", column("b")), + ("c", column("c")), + ] ), ) @@ -97,9 +96,7 @@ def test_literal(df): def test_lit_arith(df): - """ - Test literals with arithmetic operations - """ + """Test literals with arithmetic operations""" df = df.select(literal(1) + column("b"), f.concat(column("a"), literal("!"))) result = df.collect() assert len(result) == 1 @@ -140,6 +137,7 @@ def test_math_functions(): f.power(col_v, literal(pa.scalar(3))), f.pow(col_v, literal(pa.scalar(4))), f.round(col_v), + f.round(col_v, literal(pa.scalar(3))), f.sqrt(col_v), f.signum(col_v), f.trunc(col_v), @@ -183,29 +181,30 @@ def test_math_functions(): np.testing.assert_array_almost_equal(result.column(15), np.power(values, 3)) np.testing.assert_array_almost_equal(result.column(16), np.power(values, 4)) np.testing.assert_array_almost_equal(result.column(17), np.round(values)) - np.testing.assert_array_almost_equal(result.column(18), np.sqrt(values)) - np.testing.assert_array_almost_equal(result.column(19), np.sign(values)) - np.testing.assert_array_almost_equal(result.column(20), np.trunc(values)) - np.testing.assert_array_almost_equal(result.column(21), np.arcsinh(values)) - np.testing.assert_array_almost_equal(result.column(22), np.arccosh(values)) - np.testing.assert_array_almost_equal(result.column(23), np.arctanh(values)) - np.testing.assert_array_almost_equal(result.column(24), np.cbrt(values)) - np.testing.assert_array_almost_equal(result.column(25), np.cosh(values)) - np.testing.assert_array_almost_equal(result.column(26), np.degrees(values)) - np.testing.assert_array_almost_equal(result.column(27), np.gcd(9, 3)) - np.testing.assert_array_almost_equal(result.column(28), np.lcm(6, 4)) + np.testing.assert_array_almost_equal(result.column(18), np.round(values, 3)) + np.testing.assert_array_almost_equal(result.column(19), np.sqrt(values)) + np.testing.assert_array_almost_equal(result.column(20), np.sign(values)) + np.testing.assert_array_almost_equal(result.column(21), np.trunc(values)) + np.testing.assert_array_almost_equal(result.column(22), np.arcsinh(values)) + np.testing.assert_array_almost_equal(result.column(23), np.arccosh(values)) + np.testing.assert_array_almost_equal(result.column(24), np.arctanh(values)) + np.testing.assert_array_almost_equal(result.column(25), np.cbrt(values)) + np.testing.assert_array_almost_equal(result.column(26), np.cosh(values)) + np.testing.assert_array_almost_equal(result.column(27), np.degrees(values)) + np.testing.assert_array_almost_equal(result.column(28), np.gcd(9, 3)) + np.testing.assert_array_almost_equal(result.column(29), np.lcm(6, 4)) np.testing.assert_array_almost_equal( - result.column(29), np.where(np.isnan(na_values), 5, na_values) + result.column(30), np.where(np.isnan(na_values), 5, na_values) ) - np.testing.assert_array_almost_equal(result.column(30), np.pi) - np.testing.assert_array_almost_equal(result.column(31), np.radians(values)) - np.testing.assert_array_almost_equal(result.column(32), np.sinh(values)) - np.testing.assert_array_almost_equal(result.column(33), np.tanh(values)) - np.testing.assert_array_almost_equal(result.column(34), math.factorial(6)) - np.testing.assert_array_almost_equal(result.column(35), np.isnan(na_values)) - np.testing.assert_array_almost_equal(result.column(36), na_values == 0) + np.testing.assert_array_almost_equal(result.column(31), np.pi) + np.testing.assert_array_almost_equal(result.column(32), np.radians(values)) + np.testing.assert_array_almost_equal(result.column(33), np.sinh(values)) + np.testing.assert_array_almost_equal(result.column(34), np.tanh(values)) + np.testing.assert_array_almost_equal(result.column(35), math.factorial(6)) + np.testing.assert_array_almost_equal(result.column(36), np.isnan(na_values)) + np.testing.assert_array_almost_equal(result.column(37), na_values == 0) np.testing.assert_array_almost_equal( - result.column(37), np.emath.logn(3, values + 1.0) + result.column(38), np.emath.logn(3, values + 1.0) ) @@ -591,7 +590,12 @@ def test_string_functions(df): f.trim(column("c")), f.upper(column("c")), f.ends_with(column("a"), literal("llo")), + f.overlay(column("a"), literal("--"), literal(2)), + f.regexp_like(column("a"), literal("(ell|orl)")), + f.regexp_match(column("a"), literal("(ell|orl)")), + f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), ) + result = df.collect() assert len(result) == 1 result = result[0] @@ -632,6 +636,10 @@ def test_string_functions(df): assert result.column(26) == pa.array(["hello", "world", "!"]) assert result.column(27) == pa.array(["HELLO ", " WORLD ", " !"]) assert result.column(28) == pa.array([True, False, False]) + assert result.column(29) == pa.array(["H--lo", "W--ld", "--"]) + assert result.column(30) == pa.array([True, True, False]) + assert result.column(31) == pa.array([["ell"], ["orl"], None]) + assert result.column(32) == pa.array(["H-o", "W-d", "!"]) def test_hash_functions(df): diff --git a/python/datafusion/tests/test_imports.py b/python/datafusion/tests/test_imports.py index bd4e7c31d..3d324fb62 100644 --- a/python/datafusion/tests/test_imports.py +++ b/python/datafusion/tests/test_imports.py @@ -94,13 +94,24 @@ def test_datafusion_python_version(): def test_class_module_is_datafusion(): + # context for klass in [ SessionContext, + ]: + assert klass.__module__ == "datafusion.context" + + # dataframe + for klass in [ DataFrame, - ScalarUDF, + ]: + assert klass.__module__ == "datafusion.dataframe" + + # udf + for klass in [ AggregateUDF, + ScalarUDF, ]: - assert klass.__module__ == "datafusion" + assert klass.__module__ == "datafusion.udf" # expressions for klass in [Expr, Column, Literal, BinaryExpr, AggregateFunction]: diff --git a/python/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py index 8ec2ffb12..d85f380e7 100644 --- a/python/datafusion/tests/test_sql.py +++ b/python/datafusion/tests/test_sql.py @@ -19,6 +19,7 @@ import numpy as np import pyarrow as pa +from pyarrow.csv import write_csv import pyarrow.dataset as ds import pytest from datafusion.object_store import LocalFileSystem @@ -45,7 +46,7 @@ def test_register_csv(ctx, tmp_path): ], names=["int", "str", "float"], ) - pa.csv.write_csv(table, path) + write_csv(table, path) with open(path, "rb") as csv_file: with gzip.open(gzip_path, "wb") as gzipped_file: @@ -76,7 +77,13 @@ def test_register_csv(ctx, tmp_path): ) ctx.register_csv("csv3", path, schema=alternative_schema) - assert ctx.tables() == {"csv", "csv1", "csv2", "csv3", "csv_gzip"} + assert ctx.catalog().database().names() == { + "csv", + "csv1", + "csv2", + "csv3", + "csv_gzip", + } for table in ["csv", "csv1", "csv2", "csv_gzip"]: result = ctx.sql(f"SELECT COUNT(int) AS cnt FROM {table}").collect() @@ -100,14 +107,16 @@ def test_register_csv(ctx, tmp_path): def test_register_parquet(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) ctx.register_parquet("t", path) - assert ctx.tables() == {"t"} + ctx.register_parquet("t1", str(path)) + assert ctx.catalog().database().names() == {"t", "t1"} result = ctx.sql("SELECT COUNT(a) AS cnt FROM t").collect() result = pa.Table.from_batches(result) assert result.to_pydict() == {"cnt": [100]} -def test_register_parquet_partitioned(ctx, tmp_path): +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_register_parquet_partitioned(ctx, tmp_path, path_to_str): dir_root = tmp_path / "dataset_parquet_partitioned" dir_root.mkdir(exist_ok=False) (dir_root / "grp=a").mkdir(exist_ok=False) @@ -124,14 +133,16 @@ def test_register_parquet_partitioned(ctx, tmp_path): pa.parquet.write_table(table.slice(0, 3), dir_root / "grp=a/file.parquet") pa.parquet.write_table(table.slice(3, 4), dir_root / "grp=b/file.parquet") + dir_root = str(dir_root) if path_to_str else dir_root + ctx.register_parquet( "datapp", - str(dir_root), + dir_root, table_partition_cols=[("grp", "string")], parquet_pruning=True, file_extension=".parquet", ) - assert ctx.tables() == {"datapp"} + assert ctx.catalog().database().names() == {"datapp"} result = ctx.sql("SELECT grp, COUNT(*) AS cnt FROM datapp GROUP BY grp").collect() result = pa.Table.from_batches(result) @@ -140,12 +151,14 @@ def test_register_parquet_partitioned(ctx, tmp_path): assert dict(zip(rd["grp"], rd["cnt"])) == {"a": 3, "b": 1} -def test_register_dataset(ctx, tmp_path): +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_register_dataset(ctx, tmp_path, path_to_str): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) + path = str(path) if path_to_str else path dataset = ds.dataset(path, format="parquet") ctx.register_dataset("t", dataset) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} result = ctx.sql("SELECT COUNT(a) AS cnt FROM t").collect() result = pa.Table.from_batches(result) @@ -174,6 +187,12 @@ def test_register_json(ctx, tmp_path): file_extension="gz", file_compression_type="gzip", ) + ctx.register_json( + "json_gzip1", + str(gzip_path), + file_extension="gz", + file_compression_type="gzip", + ) alternative_schema = pa.schema( [ @@ -184,7 +203,14 @@ def test_register_json(ctx, tmp_path): ) ctx.register_json("json3", path, schema=alternative_schema) - assert ctx.tables() == {"json", "json1", "json2", "json3", "json_gzip"} + assert ctx.catalog().database().names() == { + "json", + "json1", + "json2", + "json3", + "json_gzip", + "json_gzip1", + } for table in ["json", "json1", "json2", "json_gzip"]: result = ctx.sql(f'SELECT COUNT("B") AS cnt FROM {table}').collect() @@ -234,7 +260,7 @@ def test_execute(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", pa.array(data)) ctx.register_parquet("t", path) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} # count result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").collect() @@ -280,9 +306,7 @@ def test_execute(ctx, tmp_path): def test_cast(ctx, tmp_path): - """ - Verify that we can cast - """ + """Verify that we can cast""" path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) ctx.register_parquet("t", path) @@ -379,7 +403,10 @@ def test_simple_select(ctx, tmp_path, arr): @pytest.mark.parametrize("file_sort_order", (None, [[col("int").sort(True, True)]])) @pytest.mark.parametrize("pass_schema", (True, False)) -def test_register_listing_table(ctx, tmp_path, pass_schema, file_sort_order): +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_register_listing_table( + ctx, tmp_path, pass_schema, file_sort_order, path_to_str +): dir_root = tmp_path / "dataset_parquet_partitioned" dir_root.mkdir(exist_ok=False) (dir_root / "grp=a/date_id=20201005").mkdir(exist_ok=False, parents=True) @@ -404,16 +431,18 @@ def test_register_listing_table(ctx, tmp_path, pass_schema, file_sort_order): table.slice(5, 10), dir_root / "grp=b/date_id=20201005/file.parquet" ) + dir_root = f"file://{dir_root}/" if path_to_str else dir_root + ctx.register_object_store("file://local", LocalFileSystem(), None) ctx.register_listing_table( "my_table", - f"file://{dir_root}/", + dir_root, table_partition_cols=[("grp", "string"), ("date_id", "int")], file_extension=".parquet", schema=table.schema if pass_schema else None, file_sort_order=file_sort_order, ) - assert ctx.tables() == {"my_table"} + assert ctx.catalog().database().names() == {"my_table"} result = ctx.sql( "SELECT grp, COUNT(*) AS count FROM my_table GROUP BY grp" diff --git a/python/datafusion/tests/test_substrait.py b/python/datafusion/tests/test_substrait.py index 62f6413a3..2071c8f3b 100644 --- a/python/datafusion/tests/test_substrait.py +++ b/python/datafusion/tests/test_substrait.py @@ -35,17 +35,43 @@ def test_substrait_serialization(ctx): ctx.register_record_batches("t", [[batch]]) - assert ctx.tables() == {"t"} + assert ctx.catalog().database().names() == {"t"} # For now just make sure the method calls blow up - substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM t", ctx) + substrait_plan = ss.Serde.serialize_to_plan("SELECT * FROM t", ctx) substrait_bytes = substrait_plan.encode() assert isinstance(substrait_bytes, bytes) - substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM t", ctx) - substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes) - logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan) + substrait_bytes = ss.Serde.serialize_bytes("SELECT * FROM t", ctx) + substrait_plan = ss.Serde.deserialize_bytes(substrait_bytes) + logical_plan = ss.Consumer.from_substrait_plan(ctx, substrait_plan) # demonstrate how to create a DataFrame from a deserialized logical plan df = ctx.create_dataframe_from_logical_plan(logical_plan) - substrait_plan = ss.substrait.producer.to_substrait_plan(df.logical_plan(), ctx) + substrait_plan = ss.Producer.to_substrait_plan(df.logical_plan(), ctx) + + +@pytest.mark.parametrize("path_to_str", (True, False)) +def test_substrait_file_serialization(ctx, tmp_path, path_to_str): + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], + names=["a", "b"], + ) + + ctx.register_record_batches("t", [[batch]]) + + assert ctx.catalog().database().names() == {"t"} + + path = tmp_path / "substrait_plan" + path = str(path) if path_to_str else path + + sql_command = "SELECT * FROM T" + ss.Serde.serialize(sql_command, ctx, path) + + expected_plan = ss.Serde.serialize_to_plan(sql_command, ctx) + actual_plan = ss.Serde.deserialize(path) + + expected_logical_plan = ss.Consumer.from_substrait_plan(ctx, expected_plan) + expected_actual_plan = ss.Consumer.from_substrait_plan(ctx, actual_plan) + + assert str(expected_logical_plan) == str(expected_actual_plan) diff --git a/python/datafusion/tests/test_udaf.py b/python/datafusion/tests/test_udaf.py index c2b29d199..81194927c 100644 --- a/python/datafusion/tests/test_udaf.py +++ b/python/datafusion/tests/test_udaf.py @@ -25,9 +25,7 @@ class Summarize(Accumulator): - """ - Interface of a user-defined accumulation. - """ + """Interface of a user-defined accumulation.""" def __init__(self): self._sum = pa.scalar(0.0) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py new file mode 100644 index 000000000..4bfbabe69 --- /dev/null +++ b/python/datafusion/udf.py @@ -0,0 +1,248 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Provides the user defined functions for evaluation of dataframes.""" + +from __future__ import annotations + +import datafusion._internal as df_internal +from datafusion.expr import Expr +from typing import Callable, TYPE_CHECKING, TypeVar +from abc import ABCMeta, abstractmethod +from typing import List +from enum import Enum +import pyarrow + +if TYPE_CHECKING: + _R = TypeVar("_R", bound=pyarrow.DataType) + + +class Volatility(Enum): + """Defines how stable or volatile a function is. + + When setting the volatility of a function, you can either pass this + enumeration or a `str`. The `str` equivalent is the lower case value of the + name (`"immutable"`, `"stable"`, or `"volatile"`). + """ + + Immutable = 1 + """An immutable function will always return the same output when given the + same input. + + DataFusion will attempt to inline immutable functions during planning. + """ + + Stable = 2 + """ + Returns the same value for a given input within a single queries. + + A stable function may return different values given the same input across + different queries but must return the same value for a given input within a + query. An example of this is the `Now` function. DataFusion will attempt to + inline `Stable` functions during planning, when possible. For query + `select col1, now() from t1`, it might take a while to execute but `now()` + column will be the same for each output row, which is evaluated during + planning. + """ + + Volatile = 3 + """A volatile function may change the return value from evaluation to + evaluation. + + Multiple invocations of a volatile function may return different results + when used in the same query. An example of this is the random() function. + DataFusion can not evaluate such functions during planning. In the query + `select col1, random() from t1`, `random()` function will be evaluated + for each output row, resulting in a unique random value for each row. + """ + + def __str__(self): + """Returns the string equivalent.""" + return self.name.lower() + + +class ScalarUDF: + """Class for performing scalar user defined functions (UDF). + + Scalar UDFs operate on a row by row basis. See also ``AggregateUDF`` for + operating on a group of rows. + """ + + def __init__( + self, + name: str | None, + func: Callable[..., _R], + input_types: list[pyarrow.DataType], + return_type: _R, + volatility: Volatility | str, + ) -> None: + """Instantiate a scalar user defined function (UDF). + + See helper method ``udf`` for argument details. + """ + self.udf = df_internal.ScalarUDF( + name, func, input_types, return_type, str(volatility) + ) + + def __call__(self, *args: Expr) -> Expr: + """Execute the UDF. + + This function is not typically called by an end user. These calls will + occur during the evaluation of the dataframe. + """ + args = [arg.expr for arg in args] + return Expr(self.udf.__call__(*args)) + + @staticmethod + def udf( + func: Callable[..., _R], + input_types: list[pyarrow.DataType], + return_type: _R, + volatility: Volatility | str, + name: str | None = None, + ) -> ScalarUDF: + """Create a new User Defined Function. + + Args: + func: A callable python function. + input_types: The data types of the arguments to `func`. This list + must be of the same length as the number of arguments. + return_type: The data type of the return value from the python + function. + volatility: See ``Volatility`` for allowed values. + name: A descriptive name for the function. + + Returns: + A user defined aggregate function, which can be used in either data + aggregation or window function calls. + """ + if not callable(func): + raise TypeError("`func` argument must be callable") + if name is None: + name = func.__qualname__.lower() + return ScalarUDF( + name=name, + func=func, + input_types=input_types, + return_type=return_type, + volatility=volatility, + ) + + +class Accumulator(metaclass=ABCMeta): + """Defines how an `AggregateUDF` accumulates values during an evaluation.""" + + @abstractmethod + def state(self) -> List[pyarrow.Scalar]: + """Return the current state.""" + pass + + @abstractmethod + def update(self, values: pyarrow.Array) -> None: + """Evalute an array of values and update state.""" + pass + + @abstractmethod + def merge(self, states: pyarrow.Array) -> None: + """Merge a set of states.""" + pass + + @abstractmethod + def evaluate(self) -> pyarrow.Scalar: + """Return the resultant value.""" + pass + + +if TYPE_CHECKING: + _A = TypeVar("_A", bound=(Callable[..., _R], Accumulator)) + + +class AggregateUDF: + """Class for performing scalar user defined functions (UDF). + + Aggregate UDFs operate on a group of rows and return a single value. See + also ``ScalarUDF`` for operating on a row by row basis. + """ + + def __init__( + self, + name: str | None, + accumulator: _A, + input_types: list[pyarrow.DataType], + return_type: _R, + state_type: list[pyarrow.DataType], + volatility: Volatility | str, + ) -> None: + """Instantiate a user defined aggregate function (UDAF). + + See ``Aggregate::udaf`` for a convenience function and arugment + descriptions. + """ + self.udf = df_internal.AggregateUDF( + name, accumulator, input_types, return_type, state_type, str(volatility) + ) + + def __call__(self, *args: Expr) -> Expr: + """Execute the UDAF. + + This function is not typically called by an end user. These calls will + occur during the evaluation of the dataframe. + """ + args = [arg.expr for arg in args] + return Expr(self.udf.__call__(*args)) + + @staticmethod + def udaf( + accum: _A, + input_types: list[pyarrow.DataType], + return_type: _R, + state_type: list[pyarrow.DataType], + volatility: Volatility | str, + name: str | None = None, + ) -> AggregateUDF: + """Create a new User Defined Aggregate Function. + + The accumulator function must be callable and implement `Accumulator`. + + Args: + accum: The accumulator python function. + input_types: The data types of the arguments to `accum`. + return_type: The data type of the return value. + state_type: The data types of the intermediate accumulation. + volatility: See `Volatility` for allowed values. + name: A descriptive name for the function. + + Returns: + A user defined aggregate function, which can be used in either data + aggregation or window function calls. + """ + if not issubclass(accum, Accumulator): + raise TypeError( + "`accum` must implement the abstract base class Accumulator" + ) + if name is None: + name = accum.__qualname__.lower() + if isinstance(input_types, pyarrow.lib.DataType): + input_types = [input_types] + return AggregateUDF( + name=name, + accumulator=accum, + input_types=input_types, + return_type=return_type, + state_type=state_type, + volatility=volatility, + ) diff --git a/src/common.rs b/src/common.rs index 094e70c01..453bf67a4 100644 --- a/src/common.rs +++ b/src/common.rs @@ -27,6 +27,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 313318fc9..3299a46f7 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -764,7 +764,7 @@ pub enum SqlType { #[allow(non_camel_case_types)] #[allow(clippy::upper_case_acronyms)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -#[pyclass(name = "PythonType", module = "datafusion.common")] +#[pyclass(name = "NullTreatment", module = "datafusion.common")] pub enum NullTreatment { IGNORE_NULLS, RESPECT_NULLS, diff --git a/src/dataframe.rs b/src/dataframe.rs index 9e36be2c4..4db59d4fe 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; -use datafusion::config::TableParquetOptions; +use datafusion::config::{CsvOptions, TableParquetOptions}; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; @@ -320,6 +320,18 @@ impl PyDataFrame { Ok(Self::new(df)) } + #[pyo3(signature = (columns, preserve_nulls=true))] + fn unnest_columns(&self, columns: Vec, preserve_nulls: bool) -> PyResult { + let unnest_options = UnnestOptions { preserve_nulls }; + let cols = columns.iter().map(|s| s.as_ref()).collect::>(); + let df = self + .df + .as_ref() + .clone() + .unnest_columns_with_options(&cols, unnest_options)?; + Ok(Self::new(df)) + } + /// Calculate the intersection of two `DataFrame`s. The two `DataFrame`s must have exactly the same schema fn intersect(&self, py_df: PyDataFrame) -> PyResult { let new_df = self @@ -337,13 +349,18 @@ impl PyDataFrame { } /// Write a `DataFrame` to a CSV file. - fn write_csv(&self, path: &str, py: Python) -> PyResult<()> { + fn write_csv(&self, path: &str, with_header: bool, py: Python) -> PyResult<()> { + let csv_options = CsvOptions { + has_header: Some(with_header), + ..Default::default() + }; wait_for_future( py, - self.df - .as_ref() - .clone() - .write_csv(path, DataFrameWriteOptions::new(), None), + self.df.as_ref().clone().write_csv( + path, + DataFrameWriteOptions::new(), + Some(csv_options), + ), )?; Ok(()) } diff --git a/src/expr.rs b/src/expr.rs index dc1de669b..aab0daa6f 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -583,6 +583,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/functions.rs b/src/functions.rs index 0ec0d5ef1..1e75dd5fb 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -233,6 +233,12 @@ fn concat_ws(sep: String, args: Vec) -> PyResult { Ok(functions::string::expr_fn::concat_ws(lit(sep), args).into()) } +#[pyfunction] +#[pyo3(signature = (values, regex, flags = None))] +fn regexp_like(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { + Ok(functions::expr_fn::regexp_like(values.expr, regex.expr, flags.map(|x| x.expr)).into()) +} + #[pyfunction] #[pyo3(signature = (values, regex, flags = None))] fn regexp_match(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { @@ -257,12 +263,12 @@ fn regexp_replace( } /// Creates a new Sort Expr #[pyfunction] -fn order_by(expr: PyExpr, asc: Option, nulls_first: Option) -> PyResult { +fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { Ok(PyExpr { expr: datafusion_expr::Expr::Sort(Sort { expr: Box::new(expr.expr), - asc: asc.unwrap_or(true), - nulls_first: nulls_first.unwrap_or(true), + asc, + nulls_first, }), }) } @@ -516,6 +522,7 @@ expr_fn!(chr, arg, "Returns the character with the given code."); expr_fn_vec!(coalesce); expr_fn!(cos, num); expr_fn!(cosh, num); +expr_fn!(cot, num); expr_fn!(degrees, num); expr_fn!(decode, input encoding); expr_fn!(encode, input encoding); @@ -527,6 +534,7 @@ expr_fn!(gcd, x y); expr_fn!(initcap, string, "Converts the first letter of each word to upper case and the rest to lower case. Words are sequences of alphanumeric characters separated by non-alphanumeric characters."); expr_fn!(isnan, num); expr_fn!(iszero, num); +expr_fn!(levenshtein, string1 string2); expr_fn!(lcm, x y); expr_fn!(left, string n, "Returns first n characters in the string, or when n is negative, returns all but last |n| characters."); expr_fn!(ln, num); @@ -548,6 +556,7 @@ expr_fn!( ); expr_fn!(nullif, arg_1 arg_2); expr_fn!(octet_length, args, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); +expr_fn_vec!(overlay); expr_fn!(pi); expr_fn!(power, base exponent); expr_fn!(pow, power, base exponent); @@ -583,7 +592,9 @@ expr_fn!(sqrt, num); expr_fn!(starts_with, string prefix, "Returns true if string starts with prefix."); expr_fn!(strpos, string substring, "Returns starting index of specified substring within string, or zero if it's not present. (Same as position(substring in string), but note the reversed argument order.)"); expr_fn!(substr, string position); +expr_fn!(substr_index, string delimiter count); expr_fn!(substring, string position length); +expr_fn!(find_in_set, string string_list); expr_fn!(tan, num); expr_fn!(tanh, num); expr_fn!( @@ -596,6 +607,7 @@ expr_fn_vec!(to_timestamp); expr_fn_vec!(to_timestamp_millis); expr_fn_vec!(to_timestamp_micros); expr_fn_vec!(to_timestamp_seconds); +expr_fn_vec!(to_unixtime); expr_fn!(current_date); expr_fn!(current_time); expr_fn!(date_part, part date); @@ -603,6 +615,7 @@ expr_fn!(datepart, date_part, part date); expr_fn!(date_trunc, part date); expr_fn!(datetrunc, date_trunc, part date); expr_fn!(date_bin, stride source origin); +expr_fn!(make_date, year month day); expr_fn!(translate, string from to, "Replaces each character in string that matches a character in the from set with the corresponding character in the to set. If from is longer than to, occurrences of the extra characters in from are deleted."); expr_fn_vec!(trim, "Removes the longest string containing only characters in characters (a space by default) from the start, end, or both ends (BOTH is the default) of string."); @@ -740,6 +753,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(corr))?; m.add_wrapped(wrap_pyfunction!(cos))?; m.add_wrapped(wrap_pyfunction!(cosh))?; + m.add_wrapped(wrap_pyfunction!(cot))?; m.add_wrapped(wrap_pyfunction!(count))?; m.add_wrapped(wrap_pyfunction!(count_star))?; m.add_wrapped(wrap_pyfunction!(covar))?; @@ -753,6 +767,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(date_part))?; m.add_wrapped(wrap_pyfunction!(datetrunc))?; m.add_wrapped(wrap_pyfunction!(date_trunc))?; + m.add_wrapped(wrap_pyfunction!(make_date))?; m.add_wrapped(wrap_pyfunction!(digest))?; m.add_wrapped(wrap_pyfunction!(ends_with))?; m.add_wrapped(wrap_pyfunction!(exp))?; @@ -765,6 +780,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(initcap))?; m.add_wrapped(wrap_pyfunction!(isnan))?; m.add_wrapped(wrap_pyfunction!(iszero))?; + m.add_wrapped(wrap_pyfunction!(levenshtein))?; m.add_wrapped(wrap_pyfunction!(lcm))?; m.add_wrapped(wrap_pyfunction!(left))?; m.add_wrapped(wrap_pyfunction!(length))?; @@ -787,11 +803,13 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(nullif))?; m.add_wrapped(wrap_pyfunction!(octet_length))?; m.add_wrapped(wrap_pyfunction!(order_by))?; + m.add_wrapped(wrap_pyfunction!(overlay))?; m.add_wrapped(wrap_pyfunction!(pi))?; m.add_wrapped(wrap_pyfunction!(power))?; m.add_wrapped(wrap_pyfunction!(pow))?; m.add_wrapped(wrap_pyfunction!(radians))?; m.add_wrapped(wrap_pyfunction!(random))?; + m.add_wrapped(wrap_pyfunction!(regexp_like))?; m.add_wrapped(wrap_pyfunction!(regexp_match))?; m.add_wrapped(wrap_pyfunction!(regexp_replace))?; m.add_wrapped(wrap_pyfunction!(repeat))?; @@ -817,7 +835,9 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(strpos))?; m.add_wrapped(wrap_pyfunction!(r#struct))?; // Use raw identifier since struct is a keyword m.add_wrapped(wrap_pyfunction!(substr))?; + m.add_wrapped(wrap_pyfunction!(substr_index))?; m.add_wrapped(wrap_pyfunction!(substring))?; + m.add_wrapped(wrap_pyfunction!(find_in_set))?; m.add_wrapped(wrap_pyfunction!(sum))?; m.add_wrapped(wrap_pyfunction!(tan))?; m.add_wrapped(wrap_pyfunction!(tanh))?; @@ -826,6 +846,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(to_timestamp_millis))?; m.add_wrapped(wrap_pyfunction!(to_timestamp_micros))?; m.add_wrapped(wrap_pyfunction!(to_timestamp_seconds))?; + m.add_wrapped(wrap_pyfunction!(to_unixtime))?; m.add_wrapped(wrap_pyfunction!(translate))?; m.add_wrapped(wrap_pyfunction!(trim))?; m.add_wrapped(wrap_pyfunction!(trunc))?; diff --git a/src/lib.rs b/src/lib.rs index 71c27e1ac..357eaacd9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,6 +92,8 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/ let common = PyModule::new_bound(py, "common")?; diff --git a/src/substrait.rs b/src/substrait.rs index 1e9e16c7b..60a523800 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -27,7 +27,7 @@ use datafusion_substrait::serializer; use datafusion_substrait::substrait::proto::Plan; use prost::Message; -#[pyclass(name = "plan", module = "datafusion.substrait", subclass)] +#[pyclass(name = "Plan", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub struct PyPlan { pub plan: Plan, From f00b8ee1959f354950660b0b0c784d4f48f420c6 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 20 Jul 2024 11:21:46 -0400 Subject: [PATCH 167/413] Expose array sort (#764) --- python/datafusion/functions.py | 22 ++++++++++++++++++++++ python/datafusion/tests/test_functions.py | 8 ++++++++ src/functions.rs | 4 ++++ 3 files changed, 34 insertions(+) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index ad77712ed..46d2a2f04 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -1126,6 +1126,28 @@ def list_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: return array_replace_all(array, from_val, to_val) +def array_sort(array: Expr, descending: bool = False, null_first: bool = False) -> Expr: + """Sort an array. + + Args: + array: The input array to sort. + descending: If True, sorts in descending order. + null_first: If True, nulls will be returned at the beginning of the array. + """ + desc = "DESC" if descending else "ASC" + nulls_first = "NULLS FIRST" if null_first else "NULLS LAST" + return Expr( + f.array_sort( + array.expr, Expr.literal(desc).expr, Expr.literal(nulls_first).expr + ) + ) + + +def list_sort(array: Expr, descending: bool = False, null_first: bool = False) -> Expr: + """This is an alias for ``array_sort``.""" + return array_sort(array, descending=descending, null_first=null_first) + + def array_slice( array: Expr, begin: Expr, end: Expr, stride: Expr | None = None ) -> Expr: diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 2384b6ab8..25d7de14a 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -453,6 +453,14 @@ def py_flatten(arr): lambda col: f.list_replace_all(col, literal(3.0), literal(4.0)), lambda data: [py_arr_replace(arr, 3.0, 4.0) for arr in data], ], + [ + lambda col: f.array_sort(col, descending=True, null_first=True), + lambda data: [np.sort(arr)[::-1] for arr in data], + ], + [ + lambda col: f.list_sort(col, descending=False, null_first=False), + lambda data: [np.sort(arr) for arr in data], + ], [ lambda col: f.array_slice(col, literal(2), literal(4)), lambda data: [arr[1:4] for arr in data], diff --git a/src/functions.rs b/src/functions.rs index 1e75dd5fb..74eb48a62 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -673,6 +673,8 @@ array_fn!(array_replace_n, array from to max); array_fn!(list_replace_n, array_replace_n, array from to max); array_fn!(array_replace_all, array from to); array_fn!(list_replace_all, array_replace_all, array from to); +array_fn!(array_sort, array desc null_first); +array_fn!(list_sort, array_sort, array desc null_first); array_fn!(array_intersect, first_array second_array); array_fn!(list_intersect, array_intersect, first_array second_array); array_fn!(array_union, array1 array2); @@ -936,6 +938,8 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(list_replace_n))?; m.add_wrapped(wrap_pyfunction!(array_replace_all))?; m.add_wrapped(wrap_pyfunction!(list_replace_all))?; + m.add_wrapped(wrap_pyfunction!(array_sort))?; + m.add_wrapped(wrap_pyfunction!(list_sort))?; m.add_wrapped(wrap_pyfunction!(array_slice))?; m.add_wrapped(wrap_pyfunction!(list_slice))?; m.add_wrapped(wrap_pyfunction!(flatten))?; From fd6b4df6d19a2176b3cd84ecf27004b5ba301d36 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Thu, 25 Jul 2024 09:14:09 -0500 Subject: [PATCH 168/413] Upgrade protobuf and remove GH Action googletest-installer (#773) * update deps to force new protobuf-src The new version **should** remove the need for the blocked GH action. Ref: https://github.com/MaterializeInc/rust-protobuf-native/issues/20 Ref: https://github.com/apache/datafusion-python/issues/763 * remove googletest-installer github action Fixes https://github.com/apache/datafusion-python/issues/763 * remove needless Clone of TimeUnit type which is Copy --- .github/workflows/build.yml | 4 - .github/workflows/test.yaml | 4 - Cargo.lock | 516 ++++++++++++++++++++---------------- src/common/data_type.rs | 8 +- 4 files changed, 293 insertions(+), 239 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a37abe53a..350be46d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,10 +89,6 @@ jobs: name: python-wheel-license path: . - # To remove once https://github.com/MaterializeInc/rust-protobuf-native/issues/20 is resolved - - name: Install gtest - uses: MarkusJx/googletest-installer@v1.1 - - name: Install Protoc uses: arduino/setup-protoc@v1 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c9a365bbb..4f47dc984 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,10 +55,6 @@ jobs: version: '3.20.2' repo-token: ${{ secrets.GITHUB_TOKEN }} - # To remove once https://github.com/MaterializeInc/rust-protobuf-native/issues/20 is resolved - - name: Install gtest - uses: MarkusJx/googletest-installer@v1.1 - - name: Setup Python uses: actions/setup-python@v5 with: diff --git a/Cargo.lock b/Cargo.lock index 3c282e071..0979306ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" [[package]] name = "arrayvec" @@ -130,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" +checksum = "6127ea5e585a12ec9f742232442828ebaf264dfa5eefdd71282376c599562b77" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" +checksum = "7add7f39210b7d726e2a8efc0083e7bf06e8f2d15bdb4896b564dce4410fbf5d" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" +checksum = "81c16ec702d3898c2f5cfdc148443c6cd7dbe5bac28399859eb0a3d38f072827" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" +checksum = "cae6970bab043c4fbc10aee1660ceb5b306d0c42c8cc5f6ae564efcd9759b663" dependencies = [ "bytes", "half", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" +checksum = "1c7ef44f26ef4f8edc392a048324ed5d757ad09135eff6d5509e6450d39e0398" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" +checksum = "5f843490bd258c5182b66e888161bb6f198f49f3792f7c7f98198b924ae0f564" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" +checksum = "a769666ffac256dd301006faca1ca553d0ae7cffcf4cd07095f73f95eb226514" dependencies = [ "arrow-buffer", "arrow-schema", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" +checksum = "dbf9c3fb57390a1af0b7bb3b5558c1ee1f63905f3eccf49ae7676a8d1e6e5a72" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" +checksum = "654e7f3724176b66ddfacba31af397c48e106fbe4d281c8144e7d237df5acfd7" dependencies = [ "arrow-array", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" +checksum = "e8008370e624e8e3c68174faaf793540287106cfda8ad1da862fdc53d8e096b4" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" +checksum = "ca5e3a6b7fda8d9fe03f3b18a2d946354ea7f3c8e4076dbdb502ad50d9d44824" dependencies = [ "ahash", "arrow-array", @@ -312,18 +312,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" +checksum = "dab1c12b40e29d9f3b699e0203c2a73ba558444c05e388a4377208f8f9c97eee" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" +checksum = "e80159088ffe8c48965cb9b1a7c968b2729f29f37363df7eca177fc3281fe7c3" dependencies = [ "ahash", "arrow-array", @@ -335,9 +335,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" +checksum = "0fd04a6ea7de183648edbcb7a6dd925bbd04c210895f6384c780e27a9b54afcd" dependencies = [ "arrow-array", "arrow-buffer", @@ -352,9 +352,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "bzip2", "flate2", @@ -364,8 +364,8 @@ dependencies = [ "pin-project-lite", "tokio", "xz2", - "zstd 0.13.0", - "zstd-safe 7.0.0", + "zstd 0.13.2", + "zstd-safe 7.2.0", ] [[package]] @@ -376,18 +376,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -413,9 +413,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -446,9 +446,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake2" @@ -461,9 +461,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.1" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cca6d3674597c30ddf2c587bf8d9d65c9a84d2326d941cc79c9842dfe0ef52" +checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" dependencies = [ "arrayref", "arrayvec", @@ -516,9 +516,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "bzip2" @@ -543,13 +543,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.98" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" dependencies = [ "jobserver", "libc", - "once_cell", ] [[package]] @@ -568,7 +567,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -608,7 +607,7 @@ version = "7.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" dependencies = [ - "strum 0.26.2", + "strum 0.26.3", "strum_macros 0.26.4", "unicode-width", ] @@ -791,7 +790,7 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.0", + "zstd 0.13.2", ] [[package]] @@ -863,7 +862,7 @@ dependencies = [ "paste", "serde_json", "sqlparser", - "strum 0.26.2", + "strum 0.26.3", "strum_macros 0.26.4", ] @@ -1052,7 +1051,7 @@ dependencies = [ "rand", "regex-syntax", "sqlparser", - "syn 2.0.68", + "syn 2.0.72", "tokio", "url", "uuid", @@ -1072,7 +1071,7 @@ dependencies = [ "log", "regex", "sqlparser", - "strum 0.26.2", + "strum 0.26.3", ] [[package]] @@ -1117,9 +1116,9 @@ checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "equivalent" @@ -1240,7 +1239,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -1383,9 +1382,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http", @@ -1393,12 +1392,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http", "http-body", "pin-project-lite", @@ -1406,9 +1405,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "humantime" @@ -1418,9 +1417,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -1438,15 +1437,16 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http", "hyper", "hyper-util", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1455,9 +1455,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", @@ -1564,6 +1564,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1572,9 +1581,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -1590,9 +1599,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" @@ -1722,9 +1731,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lz4_flex" @@ -1758,9 +1767,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memoffset" @@ -1788,22 +1797,23 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1828,9 +1838,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -1898,18 +1908,18 @@ dependencies = [ [[package]] name = "object" -version = "0.35.0" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" dependencies = [ "async-trait", "base64 0.22.1", @@ -1918,7 +1928,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.12.1", + "itertools 0.13.0", "md-5", "parking_lot", "percent-encoding", @@ -1977,14 +1987,14 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] name = "parquet" -version = "52.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" +checksum = "0f22ba0d95db56dde8685e3fadcb915cdaadda31ab8abbe3ff7f0ad1ef333267" dependencies = [ "ahash", "arrow-array", @@ -2012,7 +2022,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.0", + "zstd 0.13.2", "zstd-sys", ] @@ -2139,7 +2149,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2162,9 +2172,9 @@ checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "portable-atomic" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" +checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "ppv-lite86" @@ -2179,14 +2189,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -2218,7 +2228,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.68", + "syn 2.0.72", "tempfile", ] @@ -2232,7 +2242,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2246,9 +2256,9 @@ dependencies = [ [[package]] name = "protobuf-src" -version = "2.0.1+26.1" +version = "2.1.0+27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ba1cfa4b9dc098926b8cce388bf434b93516db3ecf6e8b1a37eb643d733ee7" +checksum = "a7edafa3bcc668fa93efafcbdf58d7821bbda0f4b458ac7fae3d57ec0fec8167" dependencies = [ "cmake", ] @@ -2300,7 +2310,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2313,7 +2323,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2324,14 +2334,60 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quinn" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -2373,18 +2429,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -2394,9 +2450,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -2405,15 +2461,15 @@ dependencies = [ [[package]] name = "regex-lite" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "regress" @@ -2427,9 +2483,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "base64 0.22.1", "bytes", @@ -2449,6 +2505,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", "rustls", "rustls-native-certs", "rustls-pemfile", @@ -2496,6 +2553,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2511,7 +2574,7 @@ version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", @@ -2520,11 +2583,11 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ - "log", + "once_cell", "ring", "rustls-pki-types", "rustls-webpki", @@ -2534,9 +2597,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -2563,9 +2626,9 @@ checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring", "rustls-pki-types", @@ -2623,7 +2686,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2634,11 +2697,11 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -2647,9 +2710,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -2672,22 +2735,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2698,14 +2761,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", @@ -2714,14 +2777,14 @@ dependencies = [ [[package]] name = "serde_tokenstream" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a00ffd23fd882d096f09fcaae2a9de8329a328628e86027e049ee051dc1621f" +checksum = "8790a7c3fe883e443eaa2af6f705952bc5d6e8671a220b9335c8cae92c037e74" dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2843,7 +2906,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2860,9 +2923,9 @@ checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros 0.26.4", ] @@ -2877,7 +2940,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2890,7 +2953,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -2913,16 +2976,16 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.68", + "syn 2.0.72", "typify", "walkdir", ] [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -2937,9 +3000,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -2948,15 +3011,15 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "target-lexicon" -version = "0.12.14" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" [[package]] name = "tempfile" @@ -2972,22 +3035,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -3012,9 +3075,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -3027,37 +3090,36 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ "rustls", "rustls-pki-types", @@ -3123,7 +3185,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -3168,7 +3230,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -3202,7 +3264,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.68", + "syn 2.0.72", "thiserror", "unicode-ident", ] @@ -3220,7 +3282,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.68", + "syn 2.0.72", "typify-impl", ] @@ -3277,9 +3339,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -3288,9 +3350,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", "serde", @@ -3348,7 +3410,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -3382,7 +3444,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3431,7 +3493,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3449,7 +3511,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -3469,18 +3531,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -3491,9 +3553,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -3503,9 +3565,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -3515,15 +3577,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -3533,9 +3595,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -3545,9 +3607,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -3557,9 +3619,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -3569,9 +3631,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" @@ -3594,22 +3656,22 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.34" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.72", ] [[package]] @@ -3629,11 +3691,11 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.0" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.0.0", + "zstd-safe 7.2.0", ] [[package]] @@ -3648,18 +3710,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.0.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.11+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" dependencies = [ "cc", "pkg-config", diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 3299a46f7..42c5aefe4 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -132,7 +132,7 @@ impl DataTypeMap { SqlType::FLOAT, )), DataType::Timestamp(unit, tz) => Ok(DataTypeMap::new( - DataType::Timestamp(unit.clone(), tz.clone()), + DataType::Timestamp(*unit, tz.clone()), PythonType::Datetime, SqlType::DATE, )), @@ -147,12 +147,12 @@ impl DataTypeMap { SqlType::DATE, )), DataType::Time32(unit) => Ok(DataTypeMap::new( - DataType::Time32(unit.clone()), + DataType::Time32(*unit), PythonType::Datetime, SqlType::DATE, )), DataType::Time64(unit) => Ok(DataTypeMap::new( - DataType::Time64(unit.clone()), + DataType::Time64(*unit), PythonType::Datetime, SqlType::DATE, )), @@ -160,7 +160,7 @@ impl DataTypeMap { format!("{:?}", arrow_type), ))), DataType::Interval(interval_unit) => Ok(DataTypeMap::new( - DataType::Interval(interval_unit.clone()), + DataType::Interval(*interval_unit), PythonType::Datetime, match interval_unit { IntervalUnit::DayTime => SqlType::INTERVAL_DAY, From f5801557e0804df68ef39de6a422835cf82ac9d3 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Tue, 30 Jul 2024 22:47:56 -0500 Subject: [PATCH 169/413] Upgrade Datafusion 40 (#771) * chore: update datafusion deps * feat: impl ExecutionPlan::static_name() for DatasetExec This required trait method was added upstream [0] and recommends to simply forward to `static_name`. [0]: https://github.com/apache/datafusion/pull/10266 * feat: update first_value and last_value wrappers. Upstream signatures were changed for the new new `AggregateBuilder` api [0]. This simply gets the code to work. We should better incorporate that API into `datafusion-python`. [0] https://github.com/apache/datafusion/pull/10560 * migrate count to UDAF Builtin Count was removed upstream. TBD whether we want to re-implement `count_star` with new API. Ref: https://github.com/apache/datafusion/pull/10893 * migrate approx_percentile_cont, approx_distinct, and approx_median to UDAF Ref: approx_distinct https://github.com/apache/datafusion/pull/10851 Ref: approx_median https://github.com/apache/datafusion/pull/10840 Ref: approx_percentile_cont and _with_weight https://github.com/apache/datafusion/pull/10917 * migrate avg to UDAF Ref: https://github.com/apache/datafusion/pull/10964 * migrage corr to UDAF Ref: https://github.com/apache/datafusion/issues/10884 * migrate grouping to UDAF Ref: https://github.com/apache/datafusion/issues/10906 * add alias `mean` for UDAF `avg` * migrate stddev to UDAF Ref: https://github.com/apache/datafusion/issues/10827 * remove rust alias for stddev The python wrapper now provides stddev_samp alias. * migrage var_pop to UDAF Ref: https://github.com/apache/datafusion/pull/10836 * migrate regr_* functions to UDAF Ref: https://github.com/apache/datafusion/pull/10898 * migrate bitwise functions to UDAF The functions now take a single expression instead of a Vec<_>. Ref: https://github.com/apache/datafusion/pull/10930 * add missing variants for ScalarValue with todo * fix typo in approx_percentile_cont * add distinct arg to count * comment out failing test `approx_percentile_cont` is now returning a DoubleArray instead of an IntArray. This may be a bug upstream; it requires further investigation. * update tests to expect lowercase `sum` in query plans This was changed upstream. Ref: https://github.com/apache/datafusion/pull/10831 * update ScalarType data_type map * add docs dependency pickleshare * re-implement count_star * lint: ruff python lint * lint: rust cargo fmt * include name of window function in error for find_window_fn * refactor `find_window_fn` for debug clarity * search default aggregate functions by both name and aliases The alias list no longer includes the name of the function. Ref: https://github.com/apache/datafusion/issues/10658 * fix markdown in find_window_fn docs * parameterize test_window_functions `first_value` and `last_value` are currently failing and marked as xfail. * add test ids to test_simple_select tests marked xfail * update find_window_fn to search built-ins first The behavior of `first_value` and `last_value` UDAFs currently does not match the built-in behavior. This allowed me to remove `marks=pytest.xfail` from the window tests. * improve first_call and last_call use of the builder API * remove trailing todos * fix examples/substrait.py * chore: remove explicit aliases from functions.rs Ref: https://github.com/apache/datafusion-python/issues/779 * remove `array_fn!` aliases * remove alias rules for `expr_fn_vec!` * remove alias rules from `expr_fn!` macro * remove unnecessary pyo3 var-arg signatures in functions.rs * remove pyo3 signatures that provided defaults for first_value and last_value * parametrize test_string_functions * test regr_ function wrappers Closes #778 --- Cargo.lock | 69 ++- Cargo.toml | 16 +- docs/requirements.txt | 3 +- examples/substrait.py | 2 +- python/datafusion/functions.py | 68 +-- python/datafusion/tests/test_aggregation.py | 3 +- python/datafusion/tests/test_dataframe.py | 116 ++-- python/datafusion/tests/test_functions.py | 184 +++--- python/datafusion/tests/test_sql.py | 17 +- src/common/data_type.rs | 5 + src/dataset_exec.rs | 5 + src/functions.rs | 646 ++++++++++++-------- 12 files changed, 635 insertions(+), 499 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0979306ab..c41ef771a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -739,9 +739,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f92d2d7a9cba4580900b32b009848d9eb35f1028ac84cdd6ddcf97612cd0068" +checksum = "ab9d55a9cd2634818953809f75ebe5248b00dd43c3227efb2a51a2d5feaad54e" dependencies = [ "ahash", "apache-avro", @@ -795,9 +795,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "effed030d2c1667eb1e11df5372d4981eaf5d11a521be32220b3985ae5ba6971" +checksum = "def66b642959e7f96f5d2da22e1f43d3bd35598f821e5ce351a0553e0f1b7367" dependencies = [ "ahash", "apache-avro", @@ -819,18 +819,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0091318129dad1359f08e4c6c71f855163c35bba05d1dbf983196f727857894" +checksum = "f104bb9cb44c06c9badf8a0d7e0855e5f7fa5e395b887d7f835e8a9457dc1352" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8385aba84fc4a06d3ebccfbcbf9b4f985e80c762fac634b49079f7cc14933fb1" +checksum = "2ac0fd8b5d80bbca3fc3b6f40da4e9f6907354824ec3b18bbd83fee8cf5c3c3e" dependencies = [ "arrow", "chrono", @@ -849,9 +849,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb192f0055d2ce64e38ac100abc18e4e6ae9734d3c28eee522bbbd6a32108a3" +checksum = "2103d2cc16fb11ef1fa993a6cac57ed5cb028601db4b97566c90e5fa77aa1e68" dependencies = [ "ahash", "arrow", @@ -868,9 +868,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27c081ae5b7edd712b92767fb8ed5c0e32755682f8075707666cd70835807c0b" +checksum = "a369332afd0ef5bd565f6db2139fb9f1dfdd0afa75a7f70f000b74208d76994f" dependencies = [ "arrow", "base64 0.22.1", @@ -880,7 +880,6 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", - "datafusion-physical-expr", "hashbrown", "hex", "itertools 0.12.1", @@ -895,9 +894,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb28a4ea52c28a26990646986a27c4052829a2a2572386258679e19263f8b78" +checksum = "92718db1aff70c47e5abf9fc975768530097059e5db7c7b78cd64b5e9a11fc77" dependencies = [ "ahash", "arrow", @@ -913,9 +912,9 @@ dependencies = [ [[package]] name = "datafusion-functions-array" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b17c02a74cdc87380a56758ec27e7d417356bf806f33062700908929aedb8a" +checksum = "30bb80f46ff3dcf4bb4510209c2ba9b8ce1b716ac8b7bf70c6bf7dca6260c831" dependencies = [ "arrow", "arrow-array", @@ -926,6 +925,7 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions", + "datafusion-functions-aggregate", "itertools 0.12.1", "log", "paste", @@ -933,9 +933,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12172f2a6c9eb4992a51e62d709eeba5dedaa3b5369cce37ff6c2260e100ba76" +checksum = "82f34692011bec4fdd6fc18c264bf8037b8625d801e6dd8f5111af15cb6d71d3" dependencies = [ "arrow", "async-trait", @@ -947,14 +947,15 @@ dependencies = [ "indexmap", "itertools 0.12.1", "log", + "paste", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a3fce531b623e94180f6cd33d620ef01530405751b6ddd2fd96250cdbd78e2e" +checksum = "45538630defedb553771434a437f7ca8f04b9b3e834344aafacecb27dc65d5e5" dependencies = [ "ahash", "arrow", @@ -968,7 +969,6 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", "datafusion-physical-expr-common", "half", "hashbrown", @@ -983,21 +983,23 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046400b6a2cc3ed57a7c576f5ae6aecc77804ac8e0186926b278b189305b2a77" +checksum = "9d8a72b0ca908e074aaeca52c14ddf5c28d22361e9cb6bc79bb733cd6661b536" dependencies = [ + "ahash", "arrow", "datafusion-common", "datafusion-expr", + "hashbrown", "rand", ] [[package]] name = "datafusion-physical-plan" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aed47f5a2ad8766260befb375b201592e86a08b260256e168ae4311426a2bff" +checksum = "b504eae6107a342775e22e323e9103f7f42db593ec6103b28605b7b7b1405c4a" dependencies = [ "ahash", "arrow", @@ -1029,7 +1031,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "39.0.0" +version = "40.0.0" dependencies = [ "arrow", "async-trait", @@ -1059,9 +1061,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fa92bb1fd15e46ce5fb6f1c85f3ac054592560f294429a28e392b5f9cd4255e" +checksum = "e5db33f323f41b95ae201318ba654a9bf11113e58a51a1dff977b1a836d3d889" dependencies = [ "arrow", "arrow-array", @@ -1076,9 +1078,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8249d69665c1cd32e07789ed6dd1da6528a23019ef16d3483db52952b6f9f68a" +checksum = "434e52fbff22e6e04e6c787f603a6aba4961a7e249a29c743c5d4f609ec2dcef" dependencies = [ "arrow-buffer", "async-recursion", @@ -1089,6 +1091,7 @@ dependencies = [ "pbjson-types", "prost", "substrait", + "url", ] [[package]] @@ -2958,9 +2961,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.34.1" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04c77dec9b6c4e48ac828937bbe7cf473b0933168c5d76d51a5816ace7046be9" +checksum = "b1ee6e584c8bf37104b7eb51c25eae07a9321b0e01379bec3b7c462d2f42afbf" dependencies = [ "heck 0.5.0", "pbjson", diff --git a/Cargo.toml b/Cargo.toml index a77eca0c3..d05a617a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "39.0.0" +version = "40.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -38,13 +38,13 @@ tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync rand = "0.8" pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "52", feature = ["pyarrow"] } -datafusion = { version = "39.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-common = { version = "39.0.0", features = ["pyarrow"] } -datafusion-expr = "39.0.0" -datafusion-functions-array = "39.0.0" -datafusion-optimizer = "39.0.0" -datafusion-sql = "39.0.0" -datafusion-substrait = { version = "39.0.0", optional = true } +datafusion = { version = "40.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-common = { version = "40.0.0", features = ["pyarrow"] } +datafusion-expr = "40.0.0" +datafusion-functions-array = "40.0.0" +datafusion-optimizer = "40.0.0" +datafusion-sql = "40.0.0" +datafusion-substrait = { version = "40.0.0", optional = true } prost = "0.12" prost-types = "0.12" uuid = { version = "1.9", features = ["v4"] } diff --git a/docs/requirements.txt b/docs/requirements.txt index 67f1ec6ac..42bc4e517 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -21,4 +21,5 @@ myst-parser maturin jinja2 ipython -pandas \ No newline at end of file +pandas +pickleshare \ No newline at end of file diff --git a/examples/substrait.py b/examples/substrait.py index fd4d0f9ca..fa6f77912 100644 --- a/examples/substrait.py +++ b/examples/substrait.py @@ -46,4 +46,4 @@ # Back to Substrait Plan just for demonstration purposes # type(substrait_plan) -> -substrait_plan = ss.Producer.to_substrait_plan(df_logical_plan) +substrait_plan = ss.Producer.to_substrait_plan(df_logical_plan, ctx) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 46d2a2f04..ca41f5ff7 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -101,7 +101,7 @@ def concat(*args: Expr) -> Expr: NULL arguments are ignored. """ args = [arg.expr for arg in args] - return Expr(f.concat(*args)) + return Expr(f.concat(args)) def concat_ws(separator: str, *args: Expr) -> Expr: @@ -110,7 +110,7 @@ def concat_ws(separator: str, *args: Expr) -> Expr: `NULL` arugments are ignored. `separator` should not be `NULL`. """ args = [arg.expr for arg in args] - return Expr(f.concat_ws(separator, *args)) + return Expr(f.concat_ws(separator, args)) def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> Expr: @@ -757,7 +757,7 @@ def upper(arg: Expr) -> Expr: def make_array(*args: Expr) -> Expr: """Returns an array using the specified input expressions.""" args = [arg.expr for arg in args] - return Expr(f.make_array(*args)) + return Expr(f.make_array(args)) def array(*args: Expr) -> Expr: @@ -840,7 +840,7 @@ def list_push_back(array: Expr, element: Expr) -> Expr: def array_concat(*args: Expr) -> Expr: """Concatenates the input arrays.""" args = [arg.expr for arg in args] - return Expr(f.array_concat(*args)) + return Expr(f.array_concat(args)) def array_cat(*args: Expr) -> Expr: @@ -1233,9 +1233,9 @@ def flatten(array: Expr) -> Expr: # aggregate functions -def approx_distinct(arg: Expr) -> Expr: +def approx_distinct(expression: Expr) -> Expr: """Returns the approximate number of distinct values.""" - return Expr(f.approx_distinct(arg.expr, distinct=True)) + return Expr(f.approx_distinct(expression.expr)) def approx_median(arg: Expr, distinct: bool = False) -> Expr: @@ -1244,20 +1244,21 @@ def approx_median(arg: Expr, distinct: bool = False) -> Expr: def approx_percentile_cont( - expr: Expr, + expression: Expr, percentile: Expr, - num_centroids: int | None = None, distinct: bool = False, ) -> Expr: """Returns the value that is approximately at a given percentile of ``expr``.""" + # Re-enable num_centroids: https://github.com/apache/datafusion-python/issues/777 + num_centroids = None if num_centroids is None: return Expr( - f.approx_percentile_cont(expr.expr, percentile.expr, distinct=distinct) + f.approx_percentile_cont(expression.expr, percentile.expr, distinct=distinct) ) return Expr( f.approx_percentile_cont( - expr.expr, percentile.expr, num_centroids, distinct=distinct + expression.expr, percentile.expr, distinct=distinct ) ) @@ -1306,7 +1307,7 @@ def covar(y: Expr, x: Expr) -> Expr: This is an alias for `covar_samp`. """ - return Expr(f.covar(y.expr, x.expr)) + return covar_samp(y, x) def covar_pop(y: Expr, x: Expr) -> Expr: @@ -1324,7 +1325,7 @@ def grouping(arg: Expr, distinct: bool = False) -> Expr: Returns 1 if the value of the argument is aggregated, 0 if not. """ - return Expr(f.grouping([arg.expr], distinct=distinct)) + return Expr(f.grouping(arg.expr, distinct=distinct)) def max(arg: Expr, distinct: bool = False) -> Expr: @@ -1396,7 +1397,7 @@ def regr_avgx(y: Expr, x: Expr, distinct: bool = False) -> Expr: Only non-null pairs of the inputs are evaluated. """ - return Expr(f.regr_avgx[y.expr, x.expr], distinct) + return Expr(f.regr_avgx(y.expr, x.expr, distinct)) def regr_avgy(y: Expr, x: Expr, distinct: bool = False) -> Expr: @@ -1404,42 +1405,42 @@ def regr_avgy(y: Expr, x: Expr, distinct: bool = False) -> Expr: Only non-null pairs of the inputs are evaluated. """ - return Expr(f.regr_avgy[y.expr, x.expr], distinct) + return Expr(f.regr_avgy(y.expr, x.expr, distinct)) def regr_count(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Counts the number of rows in which both expressions are not null.""" - return Expr(f.regr_count[y.expr, x.expr], distinct) + return Expr(f.regr_count(y.expr, x.expr, distinct)) def regr_intercept(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Computes the intercept from the linear regression.""" - return Expr(f.regr_intercept[y.expr, x.expr], distinct) + return Expr(f.regr_intercept(y.expr, x.expr, distinct)) def regr_r2(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Computes the R-squared value from linear regression.""" - return Expr(f.regr_r2[y.expr, x.expr], distinct) + return Expr(f.regr_r2(y.expr, x.expr, distinct)) def regr_slope(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Computes the slope from linear regression.""" - return Expr(f.regr_slope[y.expr, x.expr], distinct) + return Expr(f.regr_slope(y.expr, x.expr, distinct)) def regr_sxx(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Computes the sum of squares of the independent variable `x`.""" - return Expr(f.regr_sxx[y.expr, x.expr], distinct) + return Expr(f.regr_sxx(y.expr, x.expr, distinct)) def regr_sxy(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Computes the sum of products of pairs of numbers.""" - return Expr(f.regr_sxy[y.expr, x.expr], distinct) + return Expr(f.regr_sxy(y.expr, x.expr, distinct)) def regr_syy(y: Expr, x: Expr, distinct: bool = False) -> Expr: """Computes the sum of squares of the dependent variable `y`.""" - return Expr(f.regr_syy[y.expr, x.expr], distinct) + return Expr(f.regr_syy(y.expr, x.expr, distinct)) def first_value( @@ -1480,31 +1481,26 @@ def last_value( ) -def bit_and(*args: Expr, distinct: bool = False) -> Expr: +def bit_and(arg: Expr, distinct: bool = False) -> Expr: """Computes the bitwise AND of the argument.""" - args = [arg.expr for arg in args] - return Expr(f.bit_and(*args, distinct=distinct)) + return Expr(f.bit_and(arg.expr, distinct=distinct)) -def bit_or(*args: Expr, distinct: bool = False) -> Expr: +def bit_or(arg: Expr, distinct: bool = False) -> Expr: """Computes the bitwise OR of the argument.""" - args = [arg.expr for arg in args] - return Expr(f.bit_or(*args, distinct=distinct)) + return Expr(f.bit_or(arg.expr, distinct=distinct)) -def bit_xor(*args: Expr, distinct: bool = False) -> Expr: +def bit_xor(arg: Expr, distinct: bool = False) -> Expr: """Computes the bitwise XOR of the argument.""" - args = [arg.expr for arg in args] - return Expr(f.bit_xor(*args, distinct=distinct)) + return Expr(f.bit_xor(arg.expr, distinct=distinct)) -def bool_and(*args: Expr, distinct: bool = False) -> Expr: +def bool_and(arg: Expr, distinct: bool = False) -> Expr: """Computes the boolean AND of the arugment.""" - args = [arg.expr for arg in args] - return Expr(f.bool_and(*args, distinct=distinct)) + return Expr(f.bool_and(arg.expr, distinct=distinct)) -def bool_or(*args: Expr, distinct: bool = False) -> Expr: +def bool_or(arg: Expr, distinct: bool = False) -> Expr: """Computes the boolean OR of the arguement.""" - args = [arg.expr for arg in args] - return Expr(f.bool_or(*args, distinct=distinct)) + return Expr(f.bool_or(arg.expr, distinct=distinct)) diff --git a/python/datafusion/tests/test_aggregation.py b/python/datafusion/tests/test_aggregation.py index 99a470b6b..c10e5f36c 100644 --- a/python/datafusion/tests/test_aggregation.py +++ b/python/datafusion/tests/test_aggregation.py @@ -79,7 +79,8 @@ def test_built_in_aggregation(df): assert result.column(0) == pa.array([2], type=pa.uint64()) assert result.column(1) == pa.array([4]) assert result.column(2) == pa.array([4]) - assert result.column(3) == pa.array([6]) + # Ref: https://github.com/apache/datafusion-python/issues/777 + # assert result.column(3) == pa.array([6]) assert result.column(4) == pa.array([[4, 4, 6]]) np.testing.assert_array_almost_equal(result.column(5), np.average(values_a)) np.testing.assert_array_almost_equal( diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index 25875da77..6444d9321 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -278,63 +278,48 @@ def test_distinct(): assert df_a.collect() == df_b.collect() -def test_window_functions(df): +data_test_window_functions = [ + ("row", f.window("row_number", [], order_by=[f.order_by(column("c"))]), [2, 1, 3]), + ("rank", f.window("rank", [], order_by=[f.order_by(column("c"))]), [2, 1, 2]), + ("dense_rank", f.window("dense_rank", [], order_by=[f.order_by(column("c"))]), [2, 1, 2] ), + ("percent_rank", f.window("percent_rank", [], order_by=[f.order_by(column("c"))]), [0.5, 0, 0.5]), + ("cume_dist", f.window("cume_dist", [], order_by=[f.order_by(column("b"))]), [0.3333333333333333, 0.6666666666666666, 1.0]), + ("ntile", f.window("ntile", [literal(2)], order_by=[f.order_by(column("c"))]), [1, 1, 2]), + ("next", f.window("lead", [column("b")], order_by=[f.order_by(column("b"))]), [5, 6, None]), + ("previous", f.window("lag", [column("b")], order_by=[f.order_by(column("b"))]), [None, 4, 5]), + pytest.param( + "first_value", + f.window( + "first_value", + [column("a")], + order_by=[f.order_by(column("b"))] + ), + [1, 1, 1], + ), + pytest.param( + "last_value", + f.window("last_value", [column("b")], order_by=[f.order_by(column("b"))]), + [4, 5, 6], + ), + pytest.param( + "2nd_value", + f.window( + "nth_value", + [column("b"), literal(2)], + order_by=[f.order_by(column("b"))], + ), + [None, 5, 5], + ), +] + + +@pytest.mark.parametrize("name,expr,result", data_test_window_functions) +def test_window_functions(df, name, expr, result): df = df.select( column("a"), column("b"), column("c"), - f.alias( - f.window("row_number", [], order_by=[f.order_by(column("c"))]), - "row", - ), - f.alias( - f.window("rank", [], order_by=[f.order_by(column("c"))]), - "rank", - ), - f.alias( - f.window("dense_rank", [], order_by=[f.order_by(column("c"))]), - "dense_rank", - ), - f.alias( - f.window("percent_rank", [], order_by=[f.order_by(column("c"))]), - "percent_rank", - ), - f.alias( - f.window("cume_dist", [], order_by=[f.order_by(column("b"))]), - "cume_dist", - ), - f.alias( - f.window("ntile", [literal(2)], order_by=[f.order_by(column("c"))]), - "ntile", - ), - f.alias( - f.window("lag", [column("b")], order_by=[f.order_by(column("b"))]), - "previous", - ), - f.alias( - f.window("lead", [column("b")], order_by=[f.order_by(column("b"))]), - "next", - ), - f.alias( - f.window( - "first_value", - [column("a")], - order_by=[f.order_by(column("b"))], - ), - "first_value", - ), - f.alias( - f.window("last_value", [column("b")], order_by=[f.order_by(column("b"))]), - "last_value", - ), - f.alias( - f.window( - "nth_value", - [column("b"), literal(2)], - order_by=[f.order_by(column("b"))], - ), - "2nd_value", - ), + f.alias(expr, name) ) table = pa.Table.from_batches(df.collect()) @@ -343,18 +328,9 @@ def test_window_functions(df): "a": [1, 2, 3], "b": [4, 5, 6], "c": [8, 5, 8], - "row": [2, 1, 3], - "rank": [2, 1, 2], - "dense_rank": [2, 1, 2], - "percent_rank": [0.5, 0, 0.5], - "cume_dist": [0.3333333333333333, 0.6666666666666666, 1.0], - "ntile": [1, 1, 2], - "next": [5, 6, None], - "previous": [None, 4, 5], - "first_value": [1, 1, 1], - "last_value": [4, 5, 6], - "2nd_value": [None, 5, 5], + name: result } + assert table.sort_by("a").to_pydict() == expected @@ -434,13 +410,13 @@ def test_explain(df): def test_logical_plan(aggregate_df): plan = aggregate_df.logical_plan() - expected = "Projection: test.c1, SUM(test.c2)" + expected = "Projection: test.c1, sum(test.c2)" assert expected == plan.display() expected = ( - "Projection: test.c1, SUM(test.c2)\n" - " Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" + "Projection: test.c1, sum(test.c2)\n" + " Aggregate: groupBy=[[test.c1]], aggr=[[sum(test.c2)]]\n" " TableScan: test" ) @@ -450,12 +426,12 @@ def test_logical_plan(aggregate_df): def test_optimized_logical_plan(aggregate_df): plan = aggregate_df.optimized_logical_plan() - expected = "Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]" + expected = "Aggregate: groupBy=[[test.c1]], aggr=[[sum(test.c2)]]" assert expected == plan.display() expected = ( - "Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" + "Aggregate: groupBy=[[test.c1]], aggr=[[sum(test.c2)]]\n" " TableScan: test projection=[c1, c2]" ) @@ -466,7 +442,7 @@ def test_execution_plan(aggregate_df): plan = aggregate_df.execution_plan() expected = ( - "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[SUM(test.c2)]\n" # noqa: E501 + "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[sum(test.c2)]\n" # noqa: E501 ) assert expected == plan.display() diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 25d7de14a..293912321 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -567,87 +567,51 @@ def test_array_function_obj_tests(stmt, py_expr): assert a == b -def test_string_functions(df): - df = df.select( - f.ascii(column("a")), - f.bit_length(column("a")), - f.btrim(literal(" World ")), - f.character_length(column("a")), - f.chr(literal(68)), - f.concat_ws("-", column("a"), literal("test")), - f.concat(column("a"), literal("?")), - f.initcap(column("c")), - f.left(column("a"), literal(3)), - f.length(column("c")), - f.lower(column("a")), - f.lpad(column("a"), literal(7)), - f.ltrim(column("c")), - f.md5(column("a")), - f.octet_length(column("a")), - f.repeat(column("a"), literal(2)), - f.replace(column("a"), literal("l"), literal("?")), - f.reverse(column("a")), - f.right(column("a"), literal(4)), - f.rpad(column("a"), literal(8)), - f.rtrim(column("c")), - f.split_part(column("a"), literal("l"), literal(1)), - f.starts_with(column("a"), literal("Wor")), - f.strpos(column("a"), literal("o")), - f.substr(column("a"), literal(3)), - f.translate(column("a"), literal("or"), literal("ld")), - f.trim(column("c")), - f.upper(column("c")), - f.ends_with(column("a"), literal("llo")), - f.overlay(column("a"), literal("--"), literal(2)), - f.regexp_like(column("a"), literal("(ell|orl)")), - f.regexp_match(column("a"), literal("(ell|orl)")), - f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), - ) - +@pytest.mark.parametrize("function, expected_result", [ + (f.ascii(column("a")), pa.array([72, 87, 33], type=pa.int32())), # H = 72; W = 87; ! = 33 + (f.bit_length(column("a")), pa.array([40, 40, 8], type=pa.int32())), + (f.btrim(literal(" World ")), pa.array(["World", "World", "World"])), + (f.character_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), + (f.chr(literal(68)), pa.array(["D", "D", "D"])), + (f.concat_ws("-", column("a"), literal("test")), pa.array(["Hello-test", "World-test", "!-test"])), + (f.concat(column("a"), literal("?")), pa.array(["Hello?", "World?", "!?"])), + (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), + (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), + (f.length(column("c")), pa.array([6, 7, 2], type=pa.int32())), + (f.lower(column("a")), pa.array(["hello", "world", "!"])), + (f.lpad(column("a"), literal(7)), pa.array([" Hello", " World", " !"])), + (f.ltrim(column("c")), pa.array(["hello ", "world ", "!"])), + (f.md5(column("a")), pa.array([ + "8b1a9953c4611296a827abf8c47804d7", + "f5a7924e621e84c9280a9a27e1bcb7f6", + "9033e0e305f247c0c3c80d0c7848c8b3", + ])), + (f.octet_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), + (f.repeat(column("a"), literal(2)), pa.array(["HelloHello", "WorldWorld", "!!"])), + (f.replace(column("a"), literal("l"), literal("?")), pa.array(["He??o", "Wor?d", "!"])), + (f.reverse(column("a")), pa.array(["olleH", "dlroW", "!"])), + (f.right(column("a"), literal(4)), pa.array(["ello", "orld", "!"])), + (f.rpad(column("a"), literal(8)), pa.array(["Hello ", "World ", "! "])), + (f.rtrim(column("c")), pa.array(["hello", " world", " !"])), + (f.split_part(column("a"), literal("l"), literal(1)), pa.array(["He", "Wor", "!"])), + (f.starts_with(column("a"), literal("Wor")), pa.array([False, True, False])), + (f.strpos(column("a"), literal("o")), pa.array([5, 2, 0], type=pa.int32())), + (f.substr(column("a"), literal(3)), pa.array(["llo", "rld", ""])), + (f.translate(column("a"), literal("or"), literal("ld")), pa.array(["Helll", "Wldld", "!"])), + (f.trim(column("c")), pa.array(["hello", "world", "!"])), + (f.upper(column("c")), pa.array(["HELLO ", " WORLD ", " !"])), + (f.ends_with(column("a"), literal("llo")), pa.array([True, False, False])), + (f.overlay(column("a"), literal("--"), literal(2)), pa.array(["H--lo", "W--ld", "--"])), + (f.regexp_like(column("a"), literal("(ell|orl)")), pa.array([True, True, False])), + (f.regexp_match(column("a"), literal("(ell|orl)")), pa.array([["ell"], ["orl"], None])), + (f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), pa.array(["H-o", "W-d", "!"])), +]) +def test_string_functions(df, function, expected_result): + df = df.select(function) result = df.collect() assert len(result) == 1 result = result[0] - assert result.column(0) == pa.array( - [72, 87, 33], type=pa.int32() - ) # H = 72; W = 87; ! = 33 - assert result.column(1) == pa.array([40, 40, 8], type=pa.int32()) - assert result.column(2) == pa.array(["World", "World", "World"]) - assert result.column(3) == pa.array([5, 5, 1], type=pa.int32()) - assert result.column(4) == pa.array(["D", "D", "D"]) - assert result.column(5) == pa.array(["Hello-test", "World-test", "!-test"]) - assert result.column(6) == pa.array(["Hello?", "World?", "!?"]) - assert result.column(7) == pa.array(["Hello ", " World ", " !"]) - assert result.column(8) == pa.array(["Hel", "Wor", "!"]) - assert result.column(9) == pa.array([6, 7, 2], type=pa.int32()) - assert result.column(10) == pa.array(["hello", "world", "!"]) - assert result.column(11) == pa.array([" Hello", " World", " !"]) - assert result.column(12) == pa.array(["hello ", "world ", "!"]) - assert result.column(13) == pa.array( - [ - "8b1a9953c4611296a827abf8c47804d7", - "f5a7924e621e84c9280a9a27e1bcb7f6", - "9033e0e305f247c0c3c80d0c7848c8b3", - ] - ) - assert result.column(14) == pa.array([5, 5, 1], type=pa.int32()) - assert result.column(15) == pa.array(["HelloHello", "WorldWorld", "!!"]) - assert result.column(16) == pa.array(["He??o", "Wor?d", "!"]) - assert result.column(17) == pa.array(["olleH", "dlroW", "!"]) - assert result.column(18) == pa.array(["ello", "orld", "!"]) - assert result.column(19) == pa.array(["Hello ", "World ", "! "]) - assert result.column(20) == pa.array(["hello", " world", " !"]) - assert result.column(21) == pa.array(["He", "Wor", "!"]) - assert result.column(22) == pa.array([False, True, False]) - assert result.column(23) == pa.array([5, 2, 0], type=pa.int32()) - assert result.column(24) == pa.array(["llo", "rld", ""]) - assert result.column(25) == pa.array(["Helll", "Wldld", "!"]) - assert result.column(26) == pa.array(["hello", "world", "!"]) - assert result.column(27) == pa.array(["HELLO ", " WORLD ", " !"]) - assert result.column(28) == pa.array([True, False, False]) - assert result.column(29) == pa.array(["H--lo", "W--ld", "--"]) - assert result.column(30) == pa.array([True, True, False]) - assert result.column(31) == pa.array([["ell"], ["orl"], None]) - assert result.column(32) == pa.array(["H-o", "W-d", "!"]) + assert result.column(0) == expected_result def test_hash_functions(df): @@ -831,7 +795,7 @@ def test_case(df): assert result.column(2) == pa.array(["Hola", "Mundo", None]) -def test_regr_funcs(df): +def test_regr_funcs_sql(df): # test case base on # https://github.com/apache/arrow-datafusion/blob/d1361d56b9a9e0c165d3d71a8df6795d2a5f51dd/datafusion/core/tests/sqllogictests/test_files/aggregate.slt#L2330 ctx = SessionContext() @@ -853,6 +817,68 @@ def test_regr_funcs(df): assert result[0].column(8) == pa.array([0], type=pa.float64()) +def test_regr_funcs_sql_2(): + # test case based on `regr_*() basic tests + # https://github.com/apache/datafusion/blob/d1361d56b9a9e0c165d3d71a8df6795d2a5f51dd/datafusion/core/tests/sqllogictests/test_files/aggregate.slt#L2358C1-L2374C1 + ctx = SessionContext() + + # Perform the regression functions using SQL + result_sql = ctx.sql( + "select " + "regr_slope(column2, column1), " + "regr_intercept(column2, column1), " + "regr_count(column2, column1), " + "regr_r2(column2, column1), " + "regr_avgx(column2, column1), " + "regr_avgy(column2, column1), " + "regr_sxx(column2, column1), " + "regr_syy(column2, column1), " + "regr_sxy(column2, column1) " + "from (values (1,2), (2,4), (3,6))" + ).collect() + + # Assertions for SQL results + assert result_sql[0].column(0) == pa.array([2], type=pa.float64()) + assert result_sql[0].column(1) == pa.array([0], type=pa.float64()) + assert result_sql[0].column(2) == pa.array([3], type=pa.float64()) # todo: i would not expect this to be float + assert result_sql[0].column(3) == pa.array([1], type=pa.float64()) + assert result_sql[0].column(4) == pa.array([2], type=pa.float64()) + assert result_sql[0].column(5) == pa.array([4], type=pa.float64()) + assert result_sql[0].column(6) == pa.array([2], type=pa.float64()) + assert result_sql[0].column(7) == pa.array([8], type=pa.float64()) + assert result_sql[0].column(8) == pa.array([4], type=pa.float64()) + + +@pytest.mark.parametrize("func, expected", [ + pytest.param(f.regr_slope, pa.array([2], type=pa.float64()), id="regr_slope"), + pytest.param(f.regr_intercept, pa.array([0], type=pa.float64()), id="regr_intercept"), + pytest.param(f.regr_count, pa.array([3], type=pa.float64()), id="regr_count"), # TODO: I would expect this to return an int array + pytest.param(f.regr_r2, pa.array([1], type=pa.float64()), id="regr_r2"), + pytest.param(f.regr_avgx, pa.array([2], type=pa.float64()), id="regr_avgx"), + pytest.param(f.regr_avgy, pa.array([4], type=pa.float64()), id="regr_avgy"), + pytest.param(f.regr_sxx, pa.array([2], type=pa.float64()), id="regr_sxx"), + pytest.param(f.regr_syy, pa.array([8], type=pa.float64()), id="regr_syy"), + pytest.param(f.regr_sxy, pa.array([4], type=pa.float64()), id="regr_sxy") +]) +def test_regr_funcs_df(func, expected): + + # test case based on `regr_*() basic tests + # https://github.com/apache/datafusion/blob/d1361d56b9a9e0c165d3d71a8df6795d2a5f51dd/datafusion/core/tests/sqllogictests/test_files/aggregate.slt#L2358C1-L2374C1 + + + ctx = SessionContext() + + # Create a DataFrame + data = {'column1': [1, 2, 3], 'column2': [2, 4, 6]} + df = ctx.from_pydict(data, name="test_table") + + # Perform the regression function using DataFrame API + result_df = df.aggregate([], [func(f.col("column2"), f.col("column1"))]).collect() + + # Assertion for DataFrame API result + assert result_df[0].column(0) == expected + + def test_first_last_value(df): df = df.aggregate( [], diff --git a/python/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py index d85f380e7..1505fb1e7 100644 --- a/python/datafusion/tests/test_sql.py +++ b/python/datafusion/tests/test_sql.py @@ -378,17 +378,18 @@ def test_udf( # C data interface missing pytest.param( pa.array([b"1111", b"2222", b"3333"], pa.binary(4), _null_mask), + id="binary4", marks=pytest.mark.xfail, ), - pytest.param(helpers.data_datetime("s"), marks=pytest.mark.xfail), - pytest.param(helpers.data_datetime("ms"), marks=pytest.mark.xfail), - pytest.param(helpers.data_datetime("us"), marks=pytest.mark.xfail), - pytest.param(helpers.data_datetime("ns"), marks=pytest.mark.xfail), + pytest.param(helpers.data_datetime("s"), id="datetime_s", marks=pytest.mark.xfail), + pytest.param(helpers.data_datetime("ms"), id="datetime_ms", marks=pytest.mark.xfail), + pytest.param(helpers.data_datetime("us"), id="datetime_us", marks=pytest.mark.xfail), + pytest.param(helpers.data_datetime("ns"), id="datetime_ns", marks=pytest.mark.xfail), # Not writtable to parquet - pytest.param(helpers.data_timedelta("s"), marks=pytest.mark.xfail), - pytest.param(helpers.data_timedelta("ms"), marks=pytest.mark.xfail), - pytest.param(helpers.data_timedelta("us"), marks=pytest.mark.xfail), - pytest.param(helpers.data_timedelta("ns"), marks=pytest.mark.xfail), + pytest.param(helpers.data_timedelta("s"), id="timedelta_s", marks=pytest.mark.xfail), + pytest.param(helpers.data_timedelta("ms"), id="timedelta_ms", marks=pytest.mark.xfail), + pytest.param(helpers.data_timedelta("us"), id="timedelta_us", marks=pytest.mark.xfail), + pytest.param(helpers.data_timedelta("ns"), id="timedelta_ns", marks=pytest.mark.xfail), ], ) def test_simple_select(ctx, tmp_path, arr): diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 42c5aefe4..469bb789a 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -326,6 +326,11 @@ impl DataTypeMap { ScalarValue::Union(_, _, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( "ScalarValue::LargeList".to_string(), ))), + ScalarValue::Utf8View(_) => Ok(DataType::Utf8View), + ScalarValue::BinaryView(_) => Ok(DataType::BinaryView), + ScalarValue::Map(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( + "ScalarValue::Map".to_string(), + ))), } } } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 240c86486..5fe1f4d1b 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -152,6 +152,11 @@ impl DatasetExec { } impl ExecutionPlan for DatasetExec { + fn name(&self) -> &str { + // [ExecutionPlan::name] docs recommends forwarding to `static_name` + Self::static_name() + } + /// Return a reference to Any that can be used for downcasting fn as_any(&self) -> &dyn Any { self diff --git a/src/functions.rs b/src/functions.rs index 74eb48a62..e60c63c8e 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,6 +16,7 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; +use datafusion_expr::AggregateExt; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; @@ -30,13 +31,141 @@ use datafusion::functions_aggregate; use datafusion_common::{Column, ScalarValue, TableReference}; use datafusion_expr::expr::Alias; use datafusion_expr::{ - aggregate_function, expr::{ find_df_window_func, AggregateFunction, AggregateFunctionDefinition, Sort, WindowFunction, }, lit, Expr, WindowFunctionDefinition, }; +#[pyfunction] +pub fn approx_distinct(expression: PyExpr) -> PyExpr { + functions_aggregate::expr_fn::approx_distinct::approx_distinct(expression.expr).into() +} + +#[pyfunction] +pub fn approx_median(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::approx_median(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn approx_percentile_cont( + expression: PyExpr, + percentile: PyExpr, + distinct: bool, +) -> PyResult { + let expr = + functions_aggregate::expr_fn::approx_percentile_cont(expression.expr, percentile.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn approx_percentile_cont_with_weight( + expression: PyExpr, + weight: PyExpr, + percentile: PyExpr, + distinct: bool, +) -> PyResult { + let expr = functions_aggregate::expr_fn::approx_percentile_cont_with_weight( + expression.expr, + weight.expr, + percentile.expr, + ); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn avg(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::avg(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn bit_and(expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::bit_and(expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn bit_or(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::bit_or(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn bit_xor(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::bit_xor(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn bool_and(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::bool_and(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn bool_or(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::bool_or(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn corr(y: PyExpr, x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::corr(y.expr, x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn grouping(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::grouping(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + #[pyfunction] pub fn sum(args: PyExpr) -> PyExpr { functions_aggregate::expr_fn::sum(args.expr).into() @@ -58,9 +187,23 @@ pub fn median(arg: PyExpr) -> PyExpr { } #[pyfunction] -pub fn covar(y: PyExpr, x: PyExpr) -> PyExpr { - // alias for covar_samp - covar_samp(y, x) +pub fn stddev(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::stddev(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn stddev_pop(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::stddev_pop(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } } #[pyfunction] @@ -69,53 +212,166 @@ pub fn var_samp(expression: PyExpr) -> PyExpr { } #[pyfunction] -/// Alias for [`var_samp`] -pub fn var(y: PyExpr) -> PyExpr { - var_samp(y) +pub fn var_pop(expression: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::var_pop(expression.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_avgx(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_avgx(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_avgy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_avgy(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_count(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_count(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_intercept(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_intercept(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_r2(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_r2(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_slope(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_slope(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_sxx(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_sxx(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_sxy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_sxy(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } +} + +#[pyfunction] +pub fn regr_syy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::regr_syy(expr_y.expr, expr_x.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } } #[pyfunction] -#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None, null_treatment = None))] pub fn first_value( - args: Vec, + expr: PyExpr, distinct: bool, filter: Option, order_by: Option>, null_treatment: Option, -) -> PyExpr { - let null_treatment = null_treatment.map(Into::into); - let args = args.into_iter().map(|x| x.expr).collect::>(); - let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); - functions_aggregate::expr_fn::first_value( - args, - distinct, - filter.map(|x| Box::new(x.expr)), - order_by, - null_treatment, - ) - .into() +) -> PyResult { + // If we initialize the UDAF with order_by directly, then it gets over-written by the builder + let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); + + // luckily, I can guarantee initializing a builder with an `order_by` default of empty vec + let order_by = order_by + .map(|x| x.into_iter().map(|x| x.expr).collect::>()) + .unwrap_or_default(); + let mut builder = agg_fn.order_by(order_by); + + if distinct { + builder = builder.distinct(); + } + + if let Some(filter) = filter { + builder = builder.filter(filter.expr); + } + + if let Some(null_treatment) = null_treatment { + builder = builder.null_treatment(null_treatment.into()) + } + + Ok(builder.build()?.into()) } #[pyfunction] -#[pyo3(signature = (*args, distinct = false, filter = None, order_by = None, null_treatment = None))] pub fn last_value( - args: Vec, + expr: PyExpr, distinct: bool, filter: Option, order_by: Option>, null_treatment: Option, -) -> PyExpr { - let null_treatment = null_treatment.map(Into::into); - let args = args.into_iter().map(|x| x.expr).collect::>(); - let order_by = order_by.map(|x| x.into_iter().map(|x| x.expr).collect::>()); - functions_aggregate::expr_fn::last_value( - args, - distinct, - filter.map(|x| Box::new(x.expr)), - order_by, - null_treatment, - ) - .into() +) -> PyResult { + let agg_fn = functions_aggregate::expr_fn::last_value(vec![expr.expr]); + + // luckily, I can guarantee initializing a builder with an `order_by` default of empty vec + let order_by = order_by + .map(|x| x.into_iter().map(|x| x.expr).collect::>()) + .unwrap_or_default(); + let mut builder = agg_fn.order_by(order_by); + + if distinct { + builder = builder.distinct(); + } + + if let Some(filter) = filter { + builder = builder.filter(filter.expr); + } + + if let Some(null_treatment) = null_treatment { + builder = builder.null_treatment(null_treatment.into()) + } + + Ok(builder.build()?.into()) } #[pyfunction] @@ -129,34 +385,23 @@ fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { } #[pyfunction] -#[pyo3(signature = (*exprs))] fn make_array(exprs: Vec) -> PyExpr { datafusion_functions_array::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) .into() } #[pyfunction] -#[pyo3(signature = (*exprs))] -fn array(exprs: Vec) -> PyExpr { - // alias for make_array - make_array(exprs) -} - -#[pyfunction] -#[pyo3(signature = (*exprs))] fn array_concat(exprs: Vec) -> PyExpr { let exprs = exprs.into_iter().map(|x| x.into()).collect(); datafusion_functions_array::expr_fn::array_concat(exprs).into() } #[pyfunction] -#[pyo3(signature = (*exprs))] fn array_cat(exprs: Vec) -> PyExpr { array_concat(exprs) } #[pyfunction] -#[pyo3(signature = (array, element, index = 1))] fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { let index = ScalarValue::Int64(index); let index = Expr::Literal(index); @@ -164,28 +409,6 @@ fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr } #[pyfunction] -#[pyo3(signature = (array, element, index = 1))] -fn array_indexof(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { - // alias of array_position - array_position(array, element, index) -} - -#[pyfunction] -#[pyo3(signature = (array, element, index = 1))] -fn list_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { - // alias of array_position - array_position(array, element, index) -} - -#[pyfunction] -#[pyo3(signature = (array, element, index = 1))] -fn list_indexof(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { - // alias of array_position - array_position(array, element, index) -} - -#[pyfunction] -#[pyo3(signature = (array, begin, end, stride = None))] fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { datafusion_functions_array::expr_fn::array_slice( array.into(), @@ -196,18 +419,10 @@ fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option .into() } -#[pyfunction] -#[pyo3(signature = (array, begin, end, stride = None))] -fn list_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { - // alias of array_slice - array_slice(array, begin, end, stride) -} - /// Computes a binary hash of the given data. type is the algorithm to use. /// Standard algorithms are md5, sha224, sha256, sha384, sha512, blake2s, blake2b, and blake3. // #[pyfunction(value, method)] #[pyfunction] -#[pyo3(signature = (value, method))] fn digest(value: PyExpr, method: PyExpr) -> PyExpr { PyExpr { expr: functions::expr_fn::digest(value.expr, method.expr), @@ -217,7 +432,6 @@ fn digest(value: PyExpr, method: PyExpr) -> PyExpr { /// Concatenates the text representations of all the arguments. /// NULL arguments are ignored. #[pyfunction] -#[pyo3(signature = (*args))] fn concat(args: Vec) -> PyResult { let args = args.into_iter().map(|e| e.expr).collect::>(); Ok(functions::string::expr_fn::concat(args).into()) @@ -227,20 +441,17 @@ fn concat(args: Vec) -> PyResult { /// The first argument is used as the separator string, and should not be NULL. /// Other NULL arguments are ignored. #[pyfunction] -#[pyo3(signature = (sep, *args))] fn concat_ws(sep: String, args: Vec) -> PyResult { let args = args.into_iter().map(|e| e.expr).collect::>(); Ok(functions::string::expr_fn::concat_ws(lit(sep), args).into()) } #[pyfunction] -#[pyo3(signature = (values, regex, flags = None))] fn regexp_like(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { Ok(functions::expr_fn::regexp_like(values.expr, regex.expr, flags.map(|x| x.expr)).into()) } #[pyfunction] -#[pyo3(signature = (values, regex, flags = None))] fn regexp_match(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { Ok(functions::expr_fn::regexp_match(values.expr, regex.expr, flags.map(|x| x.expr)).into()) } @@ -293,21 +504,23 @@ fn col(name: &str) -> PyResult { }) } +// TODO: should we just expose this in python? /// Create a COUNT(1) aggregate expression #[pyfunction] -fn count_star() -> PyResult { - Ok(PyExpr { - expr: Expr::AggregateFunction(AggregateFunction { - func_def: datafusion_expr::expr::AggregateFunctionDefinition::BuiltIn( - aggregate_function::AggregateFunction::Count, - ), - args: vec![lit(1)], - distinct: false, - filter: None, - order_by: None, - null_treatment: None, - }), - }) +fn count_star() -> PyExpr { + functions_aggregate::expr_fn::count(lit(1)).into() +} + +/// Wrapper for [`functions_aggregate::expr_fn::count`] +/// Count the number of non-null values in the column +#[pyfunction] +fn count(expr: PyExpr, distinct: bool) -> PyResult { + let expr = functions_aggregate::expr_fn::count(expr.expr); + if distinct { + Ok(expr.distinct().build()?.into()) + } else { + Ok(expr.into()) + } } /// Create a CASE WHEN statement with literal WHEN expressions for comparison to the base expression. @@ -318,48 +531,70 @@ fn case(expr: PyExpr) -> PyResult { }) } -/// Helper function to find the appropriate window function. First, if a session -/// context is defined check it's registered functions. If no context is defined, -/// attempt to find from all default functions. Lastly, as a fall back attempt -/// to use built in window functions, which are being deprecated. +/// Helper function to find the appropriate window function. +/// +/// Search procedure: +/// 1) Search built in window functions, which are being deprecated. +/// 1) If a session context is provided: +/// 1) search User Defined Aggregate Functions (UDAFs) +/// 1) search registered window functions +/// 1) search registered aggregate functions +/// 1) If no function has been found, search default aggregate functions. +/// +/// NOTE: we search the built-ins first because the `UDAF` versions currently do not have the same behavior. fn find_window_fn(name: &str, ctx: Option) -> PyResult { - let mut maybe_fn = match &ctx { - Some(ctx) => { - let session_state = ctx.ctx.state(); - - match session_state.window_functions().contains_key(name) { - true => session_state - .window_functions() - .get(name) - .map(|f| WindowFunctionDefinition::WindowUDF(f.clone())), - false => session_state - .aggregate_functions() - .get(name) - .map(|f| WindowFunctionDefinition::AggregateUDF(f.clone())), - } + // search built in window functions (soon to be deprecated) + let df_window_func = find_df_window_func(name); + if let Some(df_window_func) = df_window_func { + return Ok(df_window_func); + } + + if let Some(ctx) = ctx { + // search UDAFs + let udaf = ctx + .ctx + .udaf(name) + .map(WindowFunctionDefinition::AggregateUDF) + .ok(); + + if let Some(udaf) = udaf { + return Ok(udaf); } - None => { - let default_aggregate_fns = all_default_aggregate_functions(); - default_aggregate_fns - .iter() - .find(|v| v.aliases().contains(&name.to_string())) - .map(|f| WindowFunctionDefinition::AggregateUDF(f.clone())) + let session_state = ctx.ctx.state(); + + // search registered window functions + let window_fn = session_state + .window_functions() + .get(name) + .map(|f| WindowFunctionDefinition::WindowUDF(f.clone())); + + if let Some(window_fn) = window_fn { + return Ok(window_fn); } - }; - if maybe_fn.is_none() { - maybe_fn = find_df_window_func(name).or_else(|| { - ctx.and_then(|ctx| { - ctx.ctx - .udaf(name) - .map(WindowFunctionDefinition::AggregateUDF) - .ok() - }) - }); + // search registered aggregate functions + let agg_fn = session_state + .aggregate_functions() + .get(name) + .map(|f| WindowFunctionDefinition::AggregateUDF(f.clone())); + + if let Some(agg_fn) = agg_fn { + return Ok(agg_fn); + } } - maybe_fn.ok_or(DataFusionError::Common("window function not found".to_string()).into()) + // search default aggregate functions + let agg_fn = all_default_aggregate_functions() + .iter() + .find(|v| v.name() == name || v.aliases().contains(&name.to_string())) + .map(|f| WindowFunctionDefinition::AggregateUDF(f.clone())); + + if let Some(agg_fn) = agg_fn { + return Ok(agg_fn); + } + + Err(DataFusionError::Common(format!("window function `{name}` not found")).into()) } /// Creates a new Window function expression @@ -424,25 +659,19 @@ macro_rules! aggregate_function { /// /// These functions have explicit named arguments. macro_rules! expr_fn { - ($NAME: ident) => { - expr_fn!($NAME, $NAME, , stringify!($NAME)); - }; - ($NAME:ident, $($arg:ident)*) => { - expr_fn!($NAME, $NAME, $($arg)*, stringify!($FUNC)); - }; - ($NAME:ident, $FUNC:ident, $($arg:ident)*) => { - expr_fn!($NAME, $FUNC, $($arg)*, stringify!($FUNC)); + ($FUNC: ident) => { + expr_fn!($FUNC, , stringify!($FUNC)); }; - ($NAME: ident, $DOC: expr) => { - expr_fn!($NAME, $NAME, ,$DOC); + ($FUNC:ident, $($arg:ident)*) => { + expr_fn!($FUNC, $($arg)*, stringify!($FUNC)); }; - ($NAME: ident, $($arg:ident)*, $DOC: expr) => { - expr_fn!($NAME, $NAME, $($arg)* ,$DOC); + ($FUNC: ident, $DOC: expr) => { + expr_fn!($FUNC, ,$DOC); }; - ($NAME: ident, $FUNC: ident, $($arg:ident)*, $DOC: expr) => { + ($FUNC: ident, $($arg:ident)*, $DOC: expr) => { #[doc = $DOC] #[pyfunction] - fn $NAME($($arg: PyExpr),*) -> PyExpr { + fn $FUNC($($arg: PyExpr),*) -> PyExpr { functions::expr_fn::$FUNC($($arg.into()),*).into() } }; @@ -452,17 +681,14 @@ macro_rules! expr_fn { /// /// These functions take a single `Vec` argument using `pyo3(signature = (*args))`. macro_rules! expr_fn_vec { - ($NAME: ident) => { - expr_fn_vec!($NAME, $NAME, stringify!($NAME)); + ($FUNC: ident) => { + expr_fn_vec!($FUNC, stringify!($FUNC)); }; - ($NAME: ident, $DOC: expr) => { - expr_fn_vec!($NAME, $NAME, $DOC); - }; - ($NAME: ident, $FUNC: ident, $DOC: expr) => { + ($FUNC: ident, $DOC: expr) => { #[doc = $DOC] #[pyfunction] #[pyo3(signature = (*args))] - fn $NAME(args: Vec) -> PyExpr { + fn $FUNC(args: Vec) -> PyExpr { let args = args.into_iter().map(|e| e.into()).collect::>(); functions::expr_fn::$FUNC(args).into() } @@ -473,22 +699,19 @@ macro_rules! expr_fn_vec { /// /// These functions have explicit named arguments. macro_rules! array_fn { - ($NAME: ident) => { - array_fn!($NAME, $NAME, , stringify!($NAME)); + ($FUNC: ident) => { + array_fn!($FUNC, , stringify!($FUNC)); }; - ($NAME:ident, $($arg:ident)*) => { - array_fn!($NAME, $NAME, $($arg)*, stringify!($FUNC)); + ($FUNC:ident, $($arg:ident)*) => { + array_fn!($FUNC, $($arg)*, stringify!($FUNC)); }; - ($NAME: ident, $FUNC:ident, $($arg:ident)*) => { - array_fn!($NAME, $FUNC, $($arg)*, stringify!($FUNC)); + ($FUNC: ident, $DOC: expr) => { + array_fn!($FUNC, , $DOC); }; - ($NAME: ident, $DOC: expr) => { - array_fn!($NAME, $NAME, , $DOC); - }; - ($NAME: ident, $FUNC:ident, $($arg:ident)*, $DOC:expr) => { + ($FUNC: ident, $($arg:ident)*, $DOC:expr) => { #[doc = $DOC] #[pyfunction] - fn $NAME($($arg: PyExpr),*) -> PyExpr { + fn $FUNC($($arg: PyExpr),*) -> PyExpr { datafusion_functions_array::expr_fn::$FUNC($($arg.into()),*).into() } }; @@ -559,7 +782,6 @@ expr_fn!(octet_length, args, "Returns number of bytes in the string. Since this expr_fn_vec!(overlay); expr_fn!(pi); expr_fn!(power, base exponent); -expr_fn!(pow, power, base exponent); expr_fn!(radians, num); expr_fn!(repeat, string n, "Repeats string the specified number of times."); expr_fn!( @@ -611,9 +833,7 @@ expr_fn_vec!(to_unixtime); expr_fn!(current_date); expr_fn!(current_time); expr_fn!(date_part, part date); -expr_fn!(datepart, date_part, part date); expr_fn!(date_trunc, part date); -expr_fn!(datetrunc, date_trunc, part date); expr_fn!(date_bin, stride source origin); expr_fn!(make_date, year month day); @@ -630,95 +850,37 @@ expr_fn!(random); // Array Functions array_fn!(array_append, array element); -array_fn!(array_push_back, array_append, array element); array_fn!(array_to_string, array delimiter); -array_fn!(array_join, array_to_string, array delimiter); -array_fn!(list_to_string, array_to_string, array delimiter); -array_fn!(list_join, array_to_string, array delimiter); -array_fn!(list_append, array_append, array element); -array_fn!(list_push_back, array_append, array element); array_fn!(array_dims, array); array_fn!(array_distinct, array); -array_fn!(list_distinct, array_distinct, array); -array_fn!(list_dims, array_dims, array); array_fn!(array_element, array element); -array_fn!(array_extract, array_element, array element); -array_fn!(list_element, array_element, array element); -array_fn!(list_extract, array_element, array element); array_fn!(array_length, array); -array_fn!(list_length, array_length, array); array_fn!(array_has, first_array second_array); array_fn!(array_has_all, first_array second_array); array_fn!(array_has_any, first_array second_array); -array_fn!(array_positions, array_positions, array element); -array_fn!(list_positions, array_positions, array element); +array_fn!(array_positions, array element); array_fn!(array_ndims, array); -array_fn!(list_ndims, array_ndims, array); array_fn!(array_prepend, element array); -array_fn!(array_push_front, array_prepend, element array); -array_fn!(list_prepend, array_prepend, element array); -array_fn!(list_push_front, array_prepend, element array); array_fn!(array_pop_back, array); array_fn!(array_pop_front, array); array_fn!(array_remove, array element); -array_fn!(list_remove, array_remove, array element); array_fn!(array_remove_n, array element max); -array_fn!(list_remove_n, array_remove_n, array element max); array_fn!(array_remove_all, array element); -array_fn!(list_remove_all, array_remove_all, array element); array_fn!(array_repeat, element count); array_fn!(array_replace, array from to); -array_fn!(list_replace, array_replace, array from to); array_fn!(array_replace_n, array from to max); -array_fn!(list_replace_n, array_replace_n, array from to max); array_fn!(array_replace_all, array from to); -array_fn!(list_replace_all, array_replace_all, array from to); array_fn!(array_sort, array desc null_first); -array_fn!(list_sort, array_sort, array desc null_first); array_fn!(array_intersect, first_array second_array); -array_fn!(list_intersect, array_intersect, first_array second_array); array_fn!(array_union, array1 array2); -array_fn!(list_union, array_union, array1 array2); array_fn!(array_except, first_array second_array); -array_fn!(list_except, array_except, first_array second_array); array_fn!(array_resize, array size value); -array_fn!(list_resize, array_resize, array size value); array_fn!(flatten, array); array_fn!(range, start stop step); -aggregate_function!(approx_distinct, ApproxDistinct); -aggregate_function!(approx_median, ApproxMedian); -aggregate_function!(approx_percentile_cont, ApproxPercentileCont); -aggregate_function!( - approx_percentile_cont_with_weight, - ApproxPercentileContWithWeight -); aggregate_function!(array_agg, ArrayAgg); -aggregate_function!(avg, Avg); -aggregate_function!(corr, Correlation); -aggregate_function!(count, Count); -aggregate_function!(grouping, Grouping); aggregate_function!(max, Max); -aggregate_function!(mean, Avg); aggregate_function!(min, Min); -aggregate_function!(stddev, Stddev); -aggregate_function!(stddev_pop, StddevPop); -aggregate_function!(stddev_samp, Stddev); -aggregate_function!(var_pop, VariancePop); -aggregate_function!(regr_avgx, RegrAvgx); -aggregate_function!(regr_avgy, RegrAvgy); -aggregate_function!(regr_count, RegrCount); -aggregate_function!(regr_intercept, RegrIntercept); -aggregate_function!(regr_r2, RegrR2); -aggregate_function!(regr_slope, RegrSlope); -aggregate_function!(regr_sxx, RegrSXX); -aggregate_function!(regr_sxy, RegrSXY); -aggregate_function!(regr_syy, RegrSYY); -aggregate_function!(bit_and, BitAnd); -aggregate_function!(bit_or, BitOr); -aggregate_function!(bit_xor, BitXor); -aggregate_function!(bool_and, BoolAnd); -aggregate_function!(bool_or, BoolOr); pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(abs))?; @@ -729,7 +891,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(approx_median))?; m.add_wrapped(wrap_pyfunction!(approx_percentile_cont))?; m.add_wrapped(wrap_pyfunction!(approx_percentile_cont_with_weight))?; - m.add_wrapped(wrap_pyfunction!(array))?; m.add_wrapped(wrap_pyfunction!(range))?; m.add_wrapped(wrap_pyfunction!(array_agg))?; m.add_wrapped(wrap_pyfunction!(arrow_typeof))?; @@ -758,16 +919,13 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(cot))?; m.add_wrapped(wrap_pyfunction!(count))?; m.add_wrapped(wrap_pyfunction!(count_star))?; - m.add_wrapped(wrap_pyfunction!(covar))?; m.add_wrapped(wrap_pyfunction!(covar_pop))?; m.add_wrapped(wrap_pyfunction!(covar_samp))?; m.add_wrapped(wrap_pyfunction!(current_date))?; m.add_wrapped(wrap_pyfunction!(current_time))?; m.add_wrapped(wrap_pyfunction!(degrees))?; m.add_wrapped(wrap_pyfunction!(date_bin))?; - m.add_wrapped(wrap_pyfunction!(datepart))?; m.add_wrapped(wrap_pyfunction!(date_part))?; - m.add_wrapped(wrap_pyfunction!(datetrunc))?; m.add_wrapped(wrap_pyfunction!(date_trunc))?; m.add_wrapped(wrap_pyfunction!(make_date))?; m.add_wrapped(wrap_pyfunction!(digest))?; @@ -796,7 +954,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(max))?; m.add_wrapped(wrap_pyfunction!(make_array))?; m.add_wrapped(wrap_pyfunction!(md5))?; - m.add_wrapped(wrap_pyfunction!(mean))?; m.add_wrapped(wrap_pyfunction!(median))?; m.add_wrapped(wrap_pyfunction!(min))?; m.add_wrapped(wrap_pyfunction!(named_struct))?; @@ -808,7 +965,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(overlay))?; m.add_wrapped(wrap_pyfunction!(pi))?; m.add_wrapped(wrap_pyfunction!(power))?; - m.add_wrapped(wrap_pyfunction!(pow))?; m.add_wrapped(wrap_pyfunction!(radians))?; m.add_wrapped(wrap_pyfunction!(random))?; m.add_wrapped(wrap_pyfunction!(regexp_like))?; @@ -833,7 +989,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(starts_with))?; m.add_wrapped(wrap_pyfunction!(stddev))?; m.add_wrapped(wrap_pyfunction!(stddev_pop))?; - m.add_wrapped(wrap_pyfunction!(stddev_samp))?; m.add_wrapped(wrap_pyfunction!(strpos))?; m.add_wrapped(wrap_pyfunction!(r#struct))?; // Use raw identifier since struct is a keyword m.add_wrapped(wrap_pyfunction!(substr))?; @@ -854,7 +1009,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(trunc))?; m.add_wrapped(wrap_pyfunction!(upper))?; m.add_wrapped(wrap_pyfunction!(self::uuid))?; // Use self to avoid name collision - m.add_wrapped(wrap_pyfunction!(var))?; m.add_wrapped(wrap_pyfunction!(var_pop))?; m.add_wrapped(wrap_pyfunction!(var_samp))?; m.add_wrapped(wrap_pyfunction!(window))?; @@ -881,67 +1035,35 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { // Array Functions m.add_wrapped(wrap_pyfunction!(array_append))?; - m.add_wrapped(wrap_pyfunction!(array_push_back))?; - m.add_wrapped(wrap_pyfunction!(list_append))?; - m.add_wrapped(wrap_pyfunction!(list_push_back))?; m.add_wrapped(wrap_pyfunction!(array_concat))?; m.add_wrapped(wrap_pyfunction!(array_cat))?; m.add_wrapped(wrap_pyfunction!(array_dims))?; m.add_wrapped(wrap_pyfunction!(array_distinct))?; - m.add_wrapped(wrap_pyfunction!(list_distinct))?; - m.add_wrapped(wrap_pyfunction!(list_dims))?; m.add_wrapped(wrap_pyfunction!(array_element))?; - m.add_wrapped(wrap_pyfunction!(array_extract))?; - m.add_wrapped(wrap_pyfunction!(list_element))?; - m.add_wrapped(wrap_pyfunction!(list_extract))?; m.add_wrapped(wrap_pyfunction!(array_length))?; - m.add_wrapped(wrap_pyfunction!(list_length))?; m.add_wrapped(wrap_pyfunction!(array_has))?; m.add_wrapped(wrap_pyfunction!(array_has_all))?; m.add_wrapped(wrap_pyfunction!(array_has_any))?; m.add_wrapped(wrap_pyfunction!(array_position))?; - m.add_wrapped(wrap_pyfunction!(array_indexof))?; - m.add_wrapped(wrap_pyfunction!(list_position))?; - m.add_wrapped(wrap_pyfunction!(list_indexof))?; m.add_wrapped(wrap_pyfunction!(array_positions))?; - m.add_wrapped(wrap_pyfunction!(list_positions))?; m.add_wrapped(wrap_pyfunction!(array_to_string))?; m.add_wrapped(wrap_pyfunction!(array_intersect))?; - m.add_wrapped(wrap_pyfunction!(list_intersect))?; m.add_wrapped(wrap_pyfunction!(array_union))?; - m.add_wrapped(wrap_pyfunction!(list_union))?; m.add_wrapped(wrap_pyfunction!(array_except))?; - m.add_wrapped(wrap_pyfunction!(list_except))?; m.add_wrapped(wrap_pyfunction!(array_resize))?; - m.add_wrapped(wrap_pyfunction!(list_resize))?; - m.add_wrapped(wrap_pyfunction!(array_join))?; - m.add_wrapped(wrap_pyfunction!(list_to_string))?; - m.add_wrapped(wrap_pyfunction!(list_join))?; m.add_wrapped(wrap_pyfunction!(array_ndims))?; - m.add_wrapped(wrap_pyfunction!(list_ndims))?; m.add_wrapped(wrap_pyfunction!(array_prepend))?; - m.add_wrapped(wrap_pyfunction!(array_push_front))?; - m.add_wrapped(wrap_pyfunction!(list_prepend))?; - m.add_wrapped(wrap_pyfunction!(list_push_front))?; m.add_wrapped(wrap_pyfunction!(array_pop_back))?; m.add_wrapped(wrap_pyfunction!(array_pop_front))?; m.add_wrapped(wrap_pyfunction!(array_remove))?; - m.add_wrapped(wrap_pyfunction!(list_remove))?; m.add_wrapped(wrap_pyfunction!(array_remove_n))?; - m.add_wrapped(wrap_pyfunction!(list_remove_n))?; m.add_wrapped(wrap_pyfunction!(array_remove_all))?; - m.add_wrapped(wrap_pyfunction!(list_remove_all))?; m.add_wrapped(wrap_pyfunction!(array_repeat))?; m.add_wrapped(wrap_pyfunction!(array_replace))?; - m.add_wrapped(wrap_pyfunction!(list_replace))?; m.add_wrapped(wrap_pyfunction!(array_replace_n))?; - m.add_wrapped(wrap_pyfunction!(list_replace_n))?; m.add_wrapped(wrap_pyfunction!(array_replace_all))?; - m.add_wrapped(wrap_pyfunction!(list_replace_all))?; m.add_wrapped(wrap_pyfunction!(array_sort))?; - m.add_wrapped(wrap_pyfunction!(list_sort))?; m.add_wrapped(wrap_pyfunction!(array_slice))?; - m.add_wrapped(wrap_pyfunction!(list_slice))?; m.add_wrapped(wrap_pyfunction!(flatten))?; Ok(()) From 66bfe36d2b6d7f17446a4205aa98172abe439ee2 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 1 Aug 2024 10:53:04 -0400 Subject: [PATCH 170/413] Bugfix: Calling count with None arguments (#768) * Small bugfix. When arguments are None, we should use count_star * When no arguments are given to count, set argument to lit(1) so that it is similar to count_star but still enables you to select distinct=True. --- python/datafusion/functions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index ca41f5ff7..0cb5b0443 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -1295,6 +1295,8 @@ def corr(value1: Expr, value2: Expr, distinct: bool = False) -> Expr: def count(args: Expr | list[Expr] | None = None, distinct: bool = False) -> Expr: """Returns the number of rows that match the given arguments.""" + if args is None: + return count(Expr.literal(1), distinct=distinct) if isinstance(args, list): args = [arg.expr for arg in args] elif isinstance(args, Expr): From 951d6b92c45c39f0dccd27d179ebdb3ddf78add8 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 1 Aug 2024 10:53:19 -0400 Subject: [PATCH 171/413] Add in user example that compares a two different approaches to UDFs (#770) * Add in user example that compares a two different approaches to UDFs * add license --- examples/python-udf-comparisons.py | 186 +++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 examples/python-udf-comparisons.py diff --git a/examples/python-udf-comparisons.py b/examples/python-udf-comparisons.py new file mode 100644 index 000000000..e2d856749 --- /dev/null +++ b/examples/python-udf-comparisons.py @@ -0,0 +1,186 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datafusion import SessionContext, col, lit, udf, functions as F +import os +import pyarrow as pa +import pyarrow.compute as pc +import time + +path = os.path.dirname(os.path.abspath(__file__)) +filepath = os.path.join(path, "../tpch/data/lineitem.parquet") + +# This example serves to demonstrate alternate approaches to answering the +# question "return all of the rows that have a specific combination of these +# values". We have the combinations we care about provided as a python +# list of tuples. There is no built in function that supports this operation, +# but it can be explicilty specified via a single expression or we can +# use a user defined function. + +ctx = SessionContext() + +# These part keys and suppliers are chosen because there are +# cases where two suppliers each have two of the part keys +# but we are interested in these specific combinations. + +values_of_interest = [ + (1530, 4031, "N"), + (6530, 1531, "N"), + (5618, 619, "N"), + (8118, 8119, "N"), +] + +partkeys = [lit(r[0]) for r in values_of_interest] +suppkeys = [lit(r[1]) for r in values_of_interest] +returnflags = [lit(r[2]) for r in values_of_interest] + +df_lineitem = ctx.read_parquet(filepath).select( + "l_partkey", "l_suppkey", "l_returnflag" +) + +start_time = time.time() + +df_simple_filter = df_lineitem.filter( + F.in_list(col("l_partkey"), partkeys), + F.in_list(col("l_suppkey"), suppkeys), + F.in_list(col("l_returnflag"), returnflags), +) + +num_rows = df_simple_filter.count() +print( + f"Simple filtering has number {num_rows} rows and took {time.time() - start_time} s" +) +print("This is the incorrect number of rows!") +start_time = time.time() + +# Explicitly check for the combinations of interest. +# This works but is not scalable. + +filter_expr = ( + ( + (col("l_partkey") == values_of_interest[0][0]) + & (col("l_suppkey") == values_of_interest[0][1]) + & (col("l_returnflag") == values_of_interest[0][2]) + ) + | ( + (col("l_partkey") == values_of_interest[1][0]) + & (col("l_suppkey") == values_of_interest[1][1]) + & (col("l_returnflag") == values_of_interest[1][2]) + ) + | ( + (col("l_partkey") == values_of_interest[2][0]) + & (col("l_suppkey") == values_of_interest[2][1]) + & (col("l_returnflag") == values_of_interest[2][2]) + ) + | ( + (col("l_partkey") == values_of_interest[3][0]) + & (col("l_suppkey") == values_of_interest[3][1]) + & (col("l_returnflag") == values_of_interest[3][2]) + ) +) + +df_explicit_filter = df_lineitem.filter(filter_expr) + +num_rows = df_explicit_filter.count() +print( + f"Explicit filtering has number {num_rows} rows and took {time.time() - start_time} s" +) +start_time = time.time() + +# Instead try a python UDF + + +def is_of_interest_impl( + partkey_arr: pa.Array, + suppkey_arr: pa.Array, + returnflag_arr: pa.Array, +) -> pa.Array: + result = [] + for idx, partkey in enumerate(partkey_arr): + partkey = partkey.as_py() + suppkey = suppkey_arr[idx].as_py() + returnflag = returnflag_arr[idx].as_py() + value = (partkey, suppkey, returnflag) + result.append(value in values_of_interest) + + return pa.array(result) + + +is_of_interest = udf( + is_of_interest_impl, + [pa.int32(), pa.int32(), pa.utf8()], + pa.bool_(), + "stable", +) + +df_udf_filter = df_lineitem.filter( + is_of_interest(col("l_partkey"), col("l_suppkey"), col("l_returnflag")) +) + +num_rows = df_udf_filter.count() +print(f"UDF filtering has number {num_rows} rows and took {time.time() - start_time} s") +start_time = time.time() + +# Now use a user defined function but lean on the built in pyarrow array +# functions so we never convert rows to python objects. + +# To see other pyarrow compute functions see +# https://arrow.apache.org/docs/python/api/compute.html +# +# It is important that the number of rows in the returned array +# matches the original array, so we cannot use functions like +# filtered_partkey_arr.filter(filtered_suppkey_arr). + + +def udf_using_pyarrow_compute_impl( + partkey_arr: pa.Array, + suppkey_arr: pa.Array, + returnflag_arr: pa.Array, +) -> pa.Array: + results = None + for partkey, suppkey, returnflag in values_of_interest: + filtered_partkey_arr = pc.equal(partkey_arr, partkey) + filtered_suppkey_arr = pc.equal(suppkey_arr, suppkey) + filtered_returnflag_arr = pc.equal(returnflag_arr, returnflag) + + resultant_arr = pc.and_(filtered_partkey_arr, filtered_suppkey_arr) + resultant_arr = pc.and_(resultant_arr, filtered_returnflag_arr) + + if results is None: + results = resultant_arr + else: + results = pc.or_(results, resultant_arr) + + return results + + +udf_using_pyarrow_compute = udf( + udf_using_pyarrow_compute_impl, + [pa.int32(), pa.int32(), pa.utf8()], + pa.bool_(), + "stable", +) + +df_udf_pyarrow_compute = df_lineitem.filter( + udf_using_pyarrow_compute(col("l_partkey"), col("l_suppkey"), col("l_returnflag")) +) + +num_rows = df_udf_pyarrow_compute.count() +print( + f"UDF filtering using pyarrow compute has number {num_rows} rows and took {time.time() - start_time} s" +) +start_time = time.time() From 9a6805e99f5a4f9b872453ae640c2b4633c9dcbf Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 1 Aug 2024 10:53:38 -0400 Subject: [PATCH 172/413] Add missing exports for wrapper modules (#782) * Add imports in base file to match those in internal * Correct class capitalization and exports for substrait * Add exports for common to match internal * Add exports for Expr to match internal * Add __all__ to functions * Add exports for object store to match internal * Add pytest to ensure all pyo3 exposed objects are also exposed in our wrappers so we don't miss any functions or classes * Add license --- python/datafusion/__init__.py | 22 +- python/datafusion/common.py | 32 ++- python/datafusion/expr.py | 74 ++++++ python/datafusion/functions.py | 221 ++++++++++++++++++ python/datafusion/object_store.py | 12 + python/datafusion/substrait.py | 21 +- .../datafusion/tests/test_wrapper_coverage.py | 49 ++++ src/substrait.rs | 6 +- 8 files changed, 423 insertions(+), 14 deletions(-) create mode 100644 python/datafusion/tests/test_wrapper_coverage.py diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 59bc8e306..0569ac4b0 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -33,8 +33,12 @@ SQLOptions, ) +from .catalog import Catalog, Database, Table + # The following imports are okay to remain as opaque to the user. -from ._internal import Config +from ._internal import Config, LogicalPlan, ExecutionPlan, runtime + +from .record_batch import RecordBatchStream, RecordBatch from .udf import ScalarUDF, AggregateUDF, Accumulator @@ -49,6 +53,8 @@ WindowFrame, ) +from . import functions, object_store, substrait + __version__ = importlib_metadata.version(__name__) __all__ = [ @@ -65,6 +71,20 @@ "column", "literal", "DFSchema", + "runtime", + "Catalog", + "Database", + "Table", + "AggregateUDF", + "LogicalPlan", + "ExecutionPlan", + "RecordBatch", + "RecordBatchStream", + "common", + "expr", + "functions", + "object_store", + "substrait", ] diff --git a/python/datafusion/common.py b/python/datafusion/common.py index 2351845b8..225e33304 100644 --- a/python/datafusion/common.py +++ b/python/datafusion/common.py @@ -16,8 +16,34 @@ # under the License. """Common data types used throughout the DataFusion project.""" -from ._internal import common +from ._internal import common as common_internal +# TODO these should all have proper wrapper classes -def __getattr__(name): - return getattr(common, name) +DFSchema = common_internal.DFSchema +DataType = common_internal.DataType +DataTypeMap = common_internal.DataTypeMap +NullTreatment = common_internal.NullTreatment +PythonType = common_internal.PythonType +RexType = common_internal.RexType +SqlFunction = common_internal.SqlFunction +SqlSchema = common_internal.SqlSchema +SqlStatistics = common_internal.SqlStatistics +SqlTable = common_internal.SqlTable +SqlType = common_internal.SqlType +SqlView = common_internal.SqlView + +__all__ = [ + "DFSchema", + "DataType", + "DataTypeMap", + "RexType", + "PythonType", + "SqlType", + "NullTreatment", + "SqlTable", + "SqlSchema", + "SqlView", + "SqlStatistics", + "SqlFunction", +] diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index c04a525a6..318b8b9ae 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -47,6 +47,7 @@ CrossJoin = expr_internal.CrossJoin Distinct = expr_internal.Distinct DropTable = expr_internal.DropTable +EmptyRelation = expr_internal.EmptyRelation Exists = expr_internal.Exists Explain = expr_internal.Explain Extension = expr_internal.Extension @@ -58,6 +59,7 @@ InSubquery = expr_internal.InSubquery IsFalse = expr_internal.IsFalse IsNotTrue = expr_internal.IsNotTrue +IsNull = expr_internal.IsNull IsTrue = expr_internal.IsTrue IsUnknown = expr_internal.IsUnknown IsNotFalse = expr_internal.IsNotFalse @@ -83,6 +85,70 @@ TableScan = expr_internal.TableScan TryCast = expr_internal.TryCast Union = expr_internal.Union +Unnest = expr_internal.Unnest +Window = expr_internal.Window + +__all__ = [ + "Expr", + "Column", + "Literal", + "BinaryExpr", + "Literal", + "AggregateFunction", + "Not", + "IsNotNull", + "IsNull", + "IsTrue", + "IsFalse", + "IsUnknown", + "IsNotTrue", + "IsNotFalse", + "IsNotUnknown", + "Negative", + "Like", + "ILike", + "SimilarTo", + "ScalarVariable", + "Alias", + "InList", + "Exists", + "Subquery", + "InSubquery", + "ScalarSubquery", + "Placeholder", + "GroupingSet", + "Case", + "CaseBuilder", + "Cast", + "TryCast", + "Between", + "Explain", + "Limit", + "Aggregate", + "Sort", + "Analyze", + "EmptyRelation", + "Join", + "JoinType", + "JoinConstraint", + "CrossJoin", + "Union", + "Unnest", + "Extension", + "Filter", + "Projection", + "TableScan", + "CreateMemoryTable", + "CreateView", + "Distinct", + "SubqueryAlias", + "DropTable", + "Partitioning", + "Repartition", + "Window", + "WindowFrame", + "WindowFrameBound", +] class Expr: @@ -246,6 +312,14 @@ def __lt__(self, rhs: Any) -> Expr: rhs = Expr.literal(rhs) return Expr(self.expr.__lt__(rhs.expr)) + __radd__ = __add__ + __rand__ = __and__ + __rmod__ = __mod__ + __rmul__ = __mul__ + __ror__ = __or__ + __rsub__ = __sub__ + __rtruediv__ = __truediv__ + @staticmethod def literal(value: Any) -> Expr: """Creates a new expression representing a scalar value. diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 0cb5b0443..be83d359f 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -27,6 +27,227 @@ from datafusion.expr import CaseBuilder, Expr, WindowFrame from datafusion.context import SessionContext +__all__ = [ + "abs", + "acos", + "acosh", + "alias", + "approx_distinct", + "approx_median", + "approx_percentile_cont", + "approx_percentile_cont_with_weight", + "array", + "array_agg", + "array_append", + "array_cat", + "array_concat", + "array_dims", + "array_distinct", + "array_element", + "array_except", + "array_extract", + "array_has", + "array_has_all", + "array_has_any", + "array_indexof", + "array_intersect", + "array_join", + "array_length", + "array_ndims", + "array_pop_back", + "array_pop_front", + "array_position", + "array_positions", + "array_prepend", + "array_push_back", + "array_push_front", + "array_remove", + "array_remove_all", + "array_remove_n", + "array_repeat", + "array_replace", + "array_replace_all", + "array_replace_n", + "array_resize", + "array_slice", + "array_sort", + "array_to_string", + "array_union", + "arrow_typeof", + "ascii", + "asin", + "asinh", + "atan", + "atan2", + "atanh", + "avg", + "bit_and", + "bit_length", + "bit_or", + "bit_xor", + "bool_and", + "bool_or", + "btrim", + "case", + "cbrt", + "ceil", + "char_length", + "character_length", + "chr", + "coalesce", + "col", + "concat", + "concat_ws", + "corr", + "cos", + "cosh", + "cot", + "count", + "count_star", + "covar", + "covar_pop", + "covar_samp", + "current_date", + "current_time", + "date_bin", + "date_part", + "date_trunc", + "datepart", + "datetrunc", + "decode", + "degrees", + "digest", + "encode", + "ends_with", + "exp", + "factorial", + "find_in_set", + "first_value", + "flatten", + "floor", + "from_unixtime", + "gcd", + "grouping", + "in_list", + "initcap", + "isnan", + "iszero", + "last_value", + "lcm", + "left", + "length", + "levenshtein", + "list_append", + "list_dims", + "list_distinct", + "list_element", + "list_except", + "list_extract", + "list_indexof", + "list_intersect", + "list_join", + "list_length", + "list_ndims", + "list_position", + "list_positions", + "list_prepend", + "list_push_back", + "list_push_front", + "list_remove", + "list_remove_all", + "list_remove_n", + "list_replace", + "list_replace_all", + "list_replace_n", + "list_resize", + "list_slice", + "list_sort", + "list_to_string", + "list_union", + "ln", + "log", + "log10", + "log2", + "lower", + "lpad", + "ltrim", + "make_array", + "make_date", + "max", + "md5", + "mean", + "median", + "min", + "named_struct", + "nanvl", + "now", + "nullif", + "octet_length", + "order_by", + "overlay", + "pi", + "pow", + "power", + "radians", + "random", + "range", + "regexp_like", + "regexp_match", + "regexp_replace", + "regr_avgx", + "regr_avgy", + "regr_count", + "regr_intercept", + "regr_r2", + "regr_slope", + "regr_sxx", + "regr_sxy", + "regr_syy", + "repeat", + "replace", + "reverse", + "right", + "round", + "rpad", + "rtrim", + "sha224", + "sha256", + "sha384", + "sha512", + "signum", + "sin", + "sinh", + "split_part", + "sqrt", + "starts_with", + "stddev", + "stddev_pop", + "stddev_samp", + "strpos", + "struct", + "substr", + "substr_index", + "substring", + "sum", + "tan", + "tanh", + "to_hex", + "to_timestamp", + "to_timestamp_micros", + "to_timestamp_millis", + "to_timestamp_seconds", + "to_unixtime", + "translate", + "trim", + "trunc", + "upper", + "uuid", + "var", + "var_pop", + "var_samp", + "window", +] + def isnan(expr: Expr) -> Expr: """Returns true if a given number is +NaN or -NaN otherwise returns false.""" diff --git a/python/datafusion/object_store.py b/python/datafusion/object_store.py index a9bb83d29..c927e7614 100644 --- a/python/datafusion/object_store.py +++ b/python/datafusion/object_store.py @@ -18,6 +18,18 @@ from ._internal import object_store +AmazonS3 = object_store.AmazonS3 +GoogleCloud = object_store.GoogleCloud +LocalFileSystem = object_store.LocalFileSystem +MicrosoftAzure = object_store.MicrosoftAzure + +__all__ = [ + "AmazonS3", + "GoogleCloud", + "LocalFileSystem", + "MicrosoftAzure", +] + def __getattr__(name): return getattr(object_store, name) diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index a199dd733..4b44ad19b 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -33,6 +33,13 @@ from datafusion.context import SessionContext from datafusion._internal import LogicalPlan +__all__ = [ + "Plan", + "Consumer", + "Producer", + "Serde", +] + class Plan: """A class representing an encodable substrait plan.""" @@ -73,7 +80,7 @@ def serialize(sql: str, ctx: SessionContext, path: str | pathlib.Path) -> None: ctx: SessionContext to use. path: Path to write the Substrait plan to. """ - return substrait_internal.serde.serialize(sql, ctx.ctx, str(path)) + return substrait_internal.Serde.serialize(sql, ctx.ctx, str(path)) @staticmethod def serialize_to_plan(sql: str, ctx: SessionContext) -> Plan: @@ -86,7 +93,7 @@ def serialize_to_plan(sql: str, ctx: SessionContext) -> Plan: Returns: Substrait plan. """ - return Plan(substrait_internal.serde.serialize_to_plan(sql, ctx.ctx)) + return Plan(substrait_internal.Serde.serialize_to_plan(sql, ctx.ctx)) @staticmethod def serialize_bytes(sql: str, ctx: SessionContext) -> bytes: @@ -99,7 +106,7 @@ def serialize_bytes(sql: str, ctx: SessionContext) -> bytes: Returns: Substrait plan as bytes. """ - return substrait_internal.serde.serialize_bytes(sql, ctx.ctx) + return substrait_internal.Serde.serialize_bytes(sql, ctx.ctx) @staticmethod def deserialize(path: str | pathlib.Path) -> Plan: @@ -111,7 +118,7 @@ def deserialize(path: str | pathlib.Path) -> Plan: Returns: Substrait plan. """ - return Plan(substrait_internal.serde.deserialize(str(path))) + return Plan(substrait_internal.Serde.deserialize(str(path))) @staticmethod def deserialize_bytes(proto_bytes: bytes) -> Plan: @@ -123,7 +130,7 @@ def deserialize_bytes(proto_bytes: bytes) -> Plan: Returns: Substrait plan. """ - return Plan(substrait_internal.serde.deserialize_bytes(proto_bytes)) + return Plan(substrait_internal.Serde.deserialize_bytes(proto_bytes)) @deprecated("Use `Serde` instead.") @@ -148,7 +155,7 @@ def to_substrait_plan(logical_plan: LogicalPlan, ctx: SessionContext) -> Plan: Substrait plan. """ return Plan( - substrait_internal.producer.to_substrait_plan(logical_plan, ctx.ctx) + substrait_internal.Producer.to_substrait_plan(logical_plan, ctx.ctx) ) @@ -173,7 +180,7 @@ def from_substrait_plan(ctx: SessionContext, plan: Plan) -> LogicalPlan: Returns: LogicalPlan. """ - return substrait_internal.consumer.from_substrait_plan( + return substrait_internal.Consumer.from_substrait_plan( ctx.ctx, plan.plan_internal ) diff --git a/python/datafusion/tests/test_wrapper_coverage.py b/python/datafusion/tests/test_wrapper_coverage.py new file mode 100644 index 000000000..44b9ca831 --- /dev/null +++ b/python/datafusion/tests/test_wrapper_coverage.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import datafusion +import datafusion.functions +import datafusion.object_store +import datafusion.substrait + + +def missing_exports(internal_obj, wrapped_obj) -> None: + for attr in dir(internal_obj): + assert attr in dir(wrapped_obj) + + internal_attr = getattr(internal_obj, attr) + wrapped_attr = getattr(wrapped_obj, attr) + + assert wrapped_attr is not None if internal_attr is not None else True + + if attr in ["__self__", "__class__"]: + continue + if isinstance(internal_attr, list): + assert isinstance(wrapped_attr, list) + for val in internal_attr: + assert val in wrapped_attr + elif hasattr(internal_attr, "__dict__"): + missing_exports(internal_attr, wrapped_attr) + + +def test_datafusion_missing_exports() -> None: + """Check for any missing pythone exports. + + This test verifies that every exposed class, attribute, and function in + the internal (pyo3) module is also exposed in our python wrappers. + """ + missing_exports(datafusion._internal, datafusion) diff --git a/src/substrait.rs b/src/substrait.rs index 60a523800..f89b6b093 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -59,7 +59,7 @@ impl From for PyPlan { /// A PySubstraitSerializer is a representation of a Serializer that is capable of both serializing /// a `LogicalPlan` instance to Substrait Protobuf bytes and also deserialize Substrait Protobuf bytes /// to a valid `LogicalPlan` instance. -#[pyclass(name = "serde", module = "datafusion.substrait", subclass)] +#[pyclass(name = "Serde", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub struct PySubstraitSerializer; @@ -105,7 +105,7 @@ impl PySubstraitSerializer { } } -#[pyclass(name = "producer", module = "datafusion.substrait", subclass)] +#[pyclass(name = "Producer", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub struct PySubstraitProducer; @@ -121,7 +121,7 @@ impl PySubstraitProducer { } } -#[pyclass(name = "consumer", module = "datafusion.substrait", subclass)] +#[pyclass(name = "Consumer", module = "datafusion.substrait", subclass)] #[derive(Debug, Clone)] pub struct PySubstraitConsumer; From 3eb198b5171ebdd41a0279d1da059544429945ca Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sun, 4 Aug 2024 18:07:34 -0500 Subject: [PATCH 173/413] Add PyExpr to_variant conversions (#793) * make PyExpr::to_variant arms explicit * update PyInList to wrap expr::InList * update PyExists to wrap expr::Exists * update PyInSubquery to wrap expr::InSubquery * update Placeholder to wrap expr::Placeholder * make PyLogicalPlan::to_variant match arms explicit * add PySortExpr wrapper * add PyUnnestExpr wrapper * update PyAlias to wrap upstream Alias * return not implemented error for unimplemnted variants in PyExpr::to_variant * added to_variant python test from the GH issue * remove unused import * return unsupported_variants for unimplemented variants in PyLogicalPlan::to_variant --- python/datafusion/tests/test_expr.py | 28 +++++++++++ src/expr.rs | 51 +++++++++++++++++--- src/expr/alias.rs | 32 +++++++------ src/expr/exists.rs | 15 +++--- src/expr/in_list.rs | 22 ++++----- src/expr/in_subquery.rs | 22 ++++----- src/expr/placeholder.rs | 21 ++++---- src/expr/sort_expr.rs | 71 ++++++++++++++++++++++++++++ src/expr/unnest_expr.rs | 67 ++++++++++++++++++++++++++ src/sql/logical.rs | 15 ++++-- 10 files changed, 273 insertions(+), 71 deletions(-) create mode 100644 src/expr/sort_expr.rs create mode 100644 src/expr/unnest_expr.rs diff --git a/python/datafusion/tests/test_expr.py b/python/datafusion/tests/test_expr.py index c9f0e98d5..1a41120a5 100644 --- a/python/datafusion/tests/test_expr.py +++ b/python/datafusion/tests/test_expr.py @@ -139,3 +139,31 @@ def test_relational_expr(test_ctx): assert df.filter(col("b") != "beta").count() == 2 assert df.filter(col("a") == "beta").count() == 0 + + +def test_expr_to_variant(): + # Taken from https://github.com/apache/datafusion-python/issues/781 + from datafusion import SessionContext + from datafusion.expr import Filter + + + def traverse_logical_plan(plan): + cur_node = plan.to_variant() + if isinstance(cur_node, Filter): + return cur_node.predicate().to_variant() + if hasattr(plan, 'inputs'): + for input_plan in plan.inputs(): + res = traverse_logical_plan(input_plan) + if res is not None: + return res + + ctx = SessionContext() + data = {'id': [1, 2, 3], 'name': ['Alice', 'Bob', 'Charlie']} + ctx.from_pydict(data, name='table1') + query = "SELECT * FROM table1 t1 WHERE t1.name IN ('dfa', 'ad', 'dfre', 'vsa')" + logical_plan = ctx.sql(query).optimized_logical_plan() + variant = traverse_logical_plan(logical_plan) + assert variant is not None + assert variant.expr().to_variant().qualified_name() == 'table1.name' + assert str(variant.list()) == '[Expr(Utf8("dfa")), Expr(Utf8("ad")), Expr(Utf8("dfre")), Expr(Utf8("vsa"))]' + assert not variant.negated() diff --git a/src/expr.rs b/src/expr.rs index aab0daa6f..04bfc85c2 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -33,7 +33,7 @@ use datafusion_expr::{ }; use crate::common::data_type::{DataTypeMap, RexType}; -use crate::errors::{py_runtime_err, py_type_err, DataFusionError}; +use crate::errors::{py_runtime_err, py_type_err, py_unsupported_variant_err, DataFusionError}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; @@ -84,11 +84,13 @@ pub mod scalar_subquery; pub mod scalar_variable; pub mod signature; pub mod sort; +pub mod sort_expr; pub mod subquery; pub mod subquery_alias; pub mod table_scan; pub mod union; pub mod unnest; +pub mod unnest_expr; pub mod window; /// A PyExpr that can be used on a DataFrame @@ -119,8 +121,9 @@ pub fn py_expr_list(expr: &[Expr]) -> PyResult> { impl PyExpr { /// Return the specific expression fn to_variant(&self, py: Python) -> PyResult { - Python::with_gil(|_| match &self.expr { - Expr::Alias(alias) => Ok(PyAlias::new(&alias.expr, &alias.name).into_py(py)), + Python::with_gil(|_| { + match &self.expr { + Expr::Alias(alias) => Ok(PyAlias::from(alias.clone()).into_py(py)), Expr::Column(col) => Ok(PyColumn::from(col.clone()).into_py(py)), Expr::ScalarVariable(data_type, variables) => { Ok(PyScalarVariable::new(data_type, variables).into_py(py)) @@ -141,10 +144,44 @@ impl PyExpr { Expr::AggregateFunction(expr) => { Ok(PyAggregateFunction::from(expr.clone()).into_py(py)) } - other => Err(py_runtime_err(format!( - "Cannot convert this Expr to a Python object: {:?}", - other + Expr::SimilarTo(value) => Ok(PySimilarTo::from(value.clone()).into_py(py)), + Expr::Between(value) => Ok(between::PyBetween::from(value.clone()).into_py(py)), + Expr::Case(value) => Ok(case::PyCase::from(value.clone()).into_py(py)), + Expr::Cast(value) => Ok(cast::PyCast::from(value.clone()).into_py(py)), + Expr::TryCast(value) => Ok(cast::PyTryCast::from(value.clone()).into_py(py)), + Expr::Sort(value) => Ok(sort_expr::PySortExpr::from(value.clone()).into_py(py)), + Expr::ScalarFunction(value) => Err(py_unsupported_variant_err(format!( + "Converting Expr::ScalarFunction to a Python object is not implemented: {:?}", + value ))), + Expr::WindowFunction(value) => Err(py_unsupported_variant_err(format!( + "Converting Expr::WindowFunction to a Python object is not implemented: {:?}", + value + ))), + Expr::InList(value) => Ok(in_list::PyInList::from(value.clone()).into_py(py)), + Expr::Exists(value) => Ok(exists::PyExists::from(value.clone()).into_py(py)), + Expr::InSubquery(value) => { + Ok(in_subquery::PyInSubquery::from(value.clone()).into_py(py)) + } + Expr::ScalarSubquery(value) => { + Ok(scalar_subquery::PyScalarSubquery::from(value.clone()).into_py(py)) + } + Expr::Wildcard { qualifier } => Err(py_unsupported_variant_err(format!( + "Converting Expr::Wildcard to a Python object is not implemented : {:?}", + qualifier + ))), + Expr::GroupingSet(value) => { + Ok(grouping_set::PyGroupingSet::from(value.clone()).into_py(py)) + } + Expr::Placeholder(value) => { + Ok(placeholder::PyPlaceholder::from(value.clone()).into_py(py)) + } + Expr::OuterReferenceColumn(data_type, column) => Err(py_unsupported_variant_err(format!( + "Converting Expr::OuterReferenceColumn to a Python object is not implemented: {:?} - {:?}", + data_type, column + ))), + Expr::Unnest(value) => Ok(unnest_expr::PyUnnestExpr::from(value.clone()).into_py(py)), + } }) } @@ -599,6 +636,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -606,6 +644,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/expr/alias.rs b/src/expr/alias.rs index 2ce656342..3208800ad 100644 --- a/src/expr/alias.rs +++ b/src/expr/alias.rs @@ -19,13 +19,24 @@ use crate::expr::PyExpr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; -use datafusion_expr::Expr; +use datafusion_expr::expr::Alias; #[pyclass(name = "Alias", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyAlias { - expr: PyExpr, - alias_name: String, + alias: Alias, +} + +impl From for PyAlias { + fn from(alias: Alias) -> Self { + Self { alias } + } +} + +impl From for Alias { + fn from(py_alias: PyAlias) -> Self { + py_alias.alias + } } impl Display for PyAlias { @@ -35,29 +46,20 @@ impl Display for PyAlias { "Alias \nExpr: `{:?}` \nAlias Name: `{}`", - &self.expr, &self.alias_name + &self.alias.expr, &self.alias.name ) } } -impl PyAlias { - pub fn new(expr: &Expr, alias_name: &String) -> Self { - Self { - expr: expr.clone().into(), - alias_name: alias_name.to_owned(), - } - } -} - #[pymethods] impl PyAlias { /// Retrieve the "name" of the alias fn alias(&self) -> PyResult { - Ok(self.alias_name.clone()) + Ok(self.alias.name.clone()) } fn expr(&self) -> PyResult { - Ok(self.expr.clone()) + Ok((*self.alias.expr.clone()).into()) } /// Get a String representation of this column diff --git a/src/expr/exists.rs b/src/expr/exists.rs index 7df9a6e81..fd2aa8c2f 100644 --- a/src/expr/exists.rs +++ b/src/expr/exists.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Subquery; +use datafusion_expr::expr::Exists; use pyo3::prelude::*; use super::subquery::PySubquery; @@ -23,23 +23,22 @@ use super::subquery::PySubquery; #[pyclass(name = "Exists", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyExists { - subquery: Subquery, - negated: bool, + exists: Exists, } -impl PyExists { - pub fn new(subquery: Subquery, negated: bool) -> Self { - Self { subquery, negated } +impl From for PyExists { + fn from(exists: Exists) -> Self { + PyExists { exists } } } #[pymethods] impl PyExists { fn subquery(&self) -> PySubquery { - self.subquery.clone().into() + self.exists.subquery.clone().into() } fn negated(&self) -> bool { - self.negated + self.exists.negated } } diff --git a/src/expr/in_list.rs b/src/expr/in_list.rs index 840eee2ce..c1a99a3c8 100644 --- a/src/expr/in_list.rs +++ b/src/expr/in_list.rs @@ -16,38 +16,32 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::Expr; +use datafusion_expr::expr::InList; use pyo3::prelude::*; #[pyclass(name = "InList", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyInList { - expr: Box, - list: Vec, - negated: bool, + in_list: InList, } -impl PyInList { - pub fn new(expr: Box, list: Vec, negated: bool) -> Self { - Self { - expr, - list, - negated, - } +impl From for PyInList { + fn from(in_list: InList) -> Self { + PyInList { in_list } } } #[pymethods] impl PyInList { fn expr(&self) -> PyExpr { - (*self.expr).clone().into() + (*self.in_list.expr).clone().into() } fn list(&self) -> Vec { - self.list.iter().map(|e| e.clone().into()).collect() + self.in_list.list.iter().map(|e| e.clone().into()).collect() } fn negated(&self) -> bool { - self.negated + self.in_list.negated } } diff --git a/src/expr/in_subquery.rs b/src/expr/in_subquery.rs index 6cee4a1f0..7dfafdbf0 100644 --- a/src/expr/in_subquery.rs +++ b/src/expr/in_subquery.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::{Expr, Subquery}; +use datafusion_expr::expr::InSubquery; use pyo3::prelude::*; use super::{subquery::PySubquery, PyExpr}; @@ -23,32 +23,26 @@ use super::{subquery::PySubquery, PyExpr}; #[pyclass(name = "InSubquery", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyInSubquery { - expr: Box, - subquery: Subquery, - negated: bool, + in_subquery: InSubquery, } -impl PyInSubquery { - pub fn new(expr: Box, subquery: Subquery, negated: bool) -> Self { - Self { - expr, - subquery, - negated, - } +impl From for PyInSubquery { + fn from(in_subquery: InSubquery) -> Self { + PyInSubquery { in_subquery } } } #[pymethods] impl PyInSubquery { fn expr(&self) -> PyExpr { - (*self.expr).clone().into() + (*self.in_subquery.expr).clone().into() } fn subquery(&self) -> PySubquery { - self.subquery.clone().into() + self.in_subquery.subquery.clone().into() } fn negated(&self) -> bool { - self.negated + self.in_subquery.negated } } diff --git a/src/expr/placeholder.rs b/src/expr/placeholder.rs index e37c8b561..ca75ce37e 100644 --- a/src/expr/placeholder.rs +++ b/src/expr/placeholder.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion::arrow::datatypes::DataType; +use datafusion_expr::expr::Placeholder; use pyo3::prelude::*; use crate::common::data_type::PyDataType; @@ -23,26 +23,25 @@ use crate::common::data_type::PyDataType; #[pyclass(name = "Placeholder", module = "datafusion.expr", subclass)] #[derive(Clone)] pub struct PyPlaceholder { - id: String, - data_type: Option, + placeholder: Placeholder, } -impl PyPlaceholder { - pub fn new(id: String, data_type: DataType) -> Self { - Self { - id, - data_type: Some(data_type), - } +impl From for PyPlaceholder { + fn from(placeholder: Placeholder) -> Self { + PyPlaceholder { placeholder } } } #[pymethods] impl PyPlaceholder { fn id(&self) -> String { - self.id.clone() + self.placeholder.id.clone() } fn data_type(&self) -> Option { - self.data_type.as_ref().map(|e| e.clone().into()) + self.placeholder + .data_type + .as_ref() + .map(|e| e.clone().into()) } } diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs new file mode 100644 index 000000000..6a8a0cf0c --- /dev/null +++ b/src/expr/sort_expr.rs @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::expr::PyExpr; +use datafusion_expr::SortExpr; +use pyo3::prelude::*; +use std::fmt::{self, Display, Formatter}; + +#[pyclass(name = "SortExpr", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PySortExpr { + sort: SortExpr, +} + +impl From for SortExpr { + fn from(sort: PySortExpr) -> Self { + sort.sort + } +} + +impl From for PySortExpr { + fn from(sort: SortExpr) -> PySortExpr { + PySortExpr { sort } + } +} + +impl Display for PySortExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Sort + Expr: {:?} + Asc: {:?} + NullsFirst: {:?}", + &self.sort.expr, &self.sort.asc, &self.sort.nulls_first + ) + } +} + +#[pymethods] +impl PySortExpr { + fn expr(&self) -> PyResult { + Ok((*self.sort.expr).clone().into()) + } + + fn ascending(&self) -> PyResult { + Ok(self.sort.asc) + } + + fn nulls_first(&self) -> PyResult { + Ok(self.sort.nulls_first) + } + + fn __repr__(&self) -> String { + format!("{}", self) + } +} diff --git a/src/expr/unnest_expr.rs b/src/expr/unnest_expr.rs new file mode 100644 index 000000000..a2f8230cc --- /dev/null +++ b/src/expr/unnest_expr.rs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_expr::expr::Unnest; +use pyo3::prelude::*; +use std::fmt::{self, Display, Formatter}; + +use super::PyExpr; + +#[pyclass(name = "UnnestExpr", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyUnnestExpr { + unnest: Unnest, +} + +impl From for PyUnnestExpr { + fn from(unnest: Unnest) -> PyUnnestExpr { + PyUnnestExpr { unnest } + } +} + +impl From for Unnest { + fn from(unnest: PyUnnestExpr) -> Self { + unnest.unnest + } +} + +impl Display for PyUnnestExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "Unnest + Expr: {:?}", + &self.unnest.expr, + ) + } +} + +#[pymethods] +impl PyUnnestExpr { + /// Retrieves the expression that is being unnested + fn expr(&self) -> PyResult { + Ok((*self.unnest.expr).clone().into()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("UnnestExpr({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("UnnestExpr".to_string()) + } +} diff --git a/src/sql/logical.rs b/src/sql/logical.rs index b1446b92a..c4471f503 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -81,9 +81,18 @@ impl PyLogicalPlan { LogicalPlan::SubqueryAlias(plan) => PySubqueryAlias::from(plan.clone()).to_variant(py), LogicalPlan::Unnest(plan) => PyUnnest::from(plan.clone()).to_variant(py), LogicalPlan::Window(plan) => PyWindow::from(plan.clone()).to_variant(py), - other => Err(py_unsupported_variant_err(format!( - "Cannot convert this plan to a LogicalNode: {:?}", - other + LogicalPlan::Repartition(_) + | LogicalPlan::Union(_) + | LogicalPlan::Statement(_) + | LogicalPlan::Values(_) + | LogicalPlan::Prepare(_) + | LogicalPlan::Dml(_) + | LogicalPlan::Ddl(_) + | LogicalPlan::Copy(_) + | LogicalPlan::DescribeTable(_) + | LogicalPlan::RecursiveQuery(_) => Err(py_unsupported_variant_err(format!( + "Conversion of variant not implemented: {:?}", + self.plan ))), } } From 1d615482ff42960dfa6987de7d3572e393aa235c Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 5 Aug 2024 15:26:58 -0400 Subject: [PATCH 174/413] Add missing expressions to wrapper export (#795) --- python/datafusion/expr.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 318b8b9ae..955c4e736 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -80,12 +80,14 @@ ScalarVariable = expr_internal.ScalarVariable SimilarTo = expr_internal.SimilarTo Sort = expr_internal.Sort +SortExpr = expr_internal.SortExpr Subquery = expr_internal.Subquery SubqueryAlias = expr_internal.SubqueryAlias TableScan = expr_internal.TableScan TryCast = expr_internal.TryCast Union = expr_internal.Union Unnest = expr_internal.Unnest +UnnestExpr = expr_internal.UnnestExpr Window = expr_internal.Window __all__ = [ @@ -126,6 +128,7 @@ "Limit", "Aggregate", "Sort", + "SortExpr", "Analyze", "EmptyRelation", "Join", @@ -134,6 +137,7 @@ "CrossJoin", "Union", "Unnest", + "UnnestExpr", "Extension", "Filter", "Projection", From bd0e82088b8504b4f3683e085601e3bdd6f3aaa0 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 6 Aug 2024 10:30:32 -0400 Subject: [PATCH 175/413] Doc/cross reference (#791) * Update docstrings so that cross references work in online docs. Also switch from autosummary to autoapi in sphinx for building API reference documents * Update documentation to cross reference * Correct class names and internal attr * Revert changes that will end up coming in via PR #782 * Add autoapi to requirements file * Add git ignore for files retrieved during local site building * Remove unused portions of doc config * Reset substrait capitalization that was reverted during rebase * Small example changes --- docs/.gitignore | 2 + docs/requirements.txt | 3 +- docs/source/api.rst | 31 --- docs/source/api/dataframe.rst | 27 --- docs/source/api/execution_context.rst | 29 --- docs/source/api/expression.rst | 27 --- docs/source/api/functions.rst | 27 --- docs/source/api/object_store.rst | 27 --- docs/source/conf.py | 56 +++-- docs/source/index.rst | 2 - docs/source/user-guide/basics.rst | 14 +- .../common-operations/aggregations.rst | 2 +- .../common-operations/basic-info.rst | 8 +- .../common-operations/expressions.rst | 12 +- .../common-operations/functions.rst | 14 +- .../user-guide/common-operations/joins.rst | 4 +- .../common-operations/select-and-filter.rst | 10 +- .../common-operations/udf-and-udfa.rst | 4 +- .../user-guide/common-operations/windows.rst | 6 +- docs/source/user-guide/configuration.rst | 8 +- docs/source/user-guide/io/avro.rst | 2 +- docs/source/user-guide/io/csv.rst | 6 +- docs/source/user-guide/io/json.rst | 2 +- docs/source/user-guide/io/parquet.rst | 6 +- examples/python-udf-comparisons.py | 6 +- python/datafusion/__init__.py | 11 +- python/datafusion/catalog.py | 4 +- python/datafusion/context.py | 134 ++++++----- python/datafusion/dataframe.py | 109 +++++---- python/datafusion/expr.py | 23 +- python/datafusion/functions.py | 218 +++++++++--------- python/datafusion/record_batch.py | 14 +- python/datafusion/substrait.py | 3 +- python/datafusion/udf.py | 36 +-- 34 files changed, 370 insertions(+), 517 deletions(-) create mode 100644 docs/.gitignore delete mode 100644 docs/source/api.rst delete mode 100644 docs/source/api/dataframe.rst delete mode 100644 docs/source/api/execution_context.rst delete mode 100644 docs/source/api/expression.rst delete mode 100644 docs/source/api/functions.rst delete mode 100644 docs/source/api/object_store.rst diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 000000000..41e135341 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,2 @@ +pokemon.csv +yellow_trip_data.parquet diff --git a/docs/requirements.txt b/docs/requirements.txt index 42bc4e517..f5cece78e 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -22,4 +22,5 @@ maturin jinja2 ipython pandas -pickleshare \ No newline at end of file +pickleshare +sphinx-autoapi diff --git a/docs/source/api.rst b/docs/source/api.rst deleted file mode 100644 index d9f4a09dd..000000000 --- a/docs/source/api.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api: - -************* -API Reference -************* - -.. toctree:: - :maxdepth: 2 - - api/dataframe - api/execution_context - api/expression - api/functions - api/object_store diff --git a/docs/source/api/dataframe.rst b/docs/source/api/dataframe.rst deleted file mode 100644 index 0a3c4c8b1..000000000 --- a/docs/source/api/dataframe.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api.dataframe: -.. currentmodule:: datafusion - -DataFrame -========= - -.. autosummary:: - :toctree: ../generated/ - - DataFrame diff --git a/docs/source/api/execution_context.rst b/docs/source/api/execution_context.rst deleted file mode 100644 index a3bda76d7..000000000 --- a/docs/source/api/execution_context.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api.execution_context: -.. currentmodule:: datafusion - -SessionContext -============== - -.. autosummary:: - :toctree: ../generated/ - - SessionConfig - RuntimeConfig - SessionContext diff --git a/docs/source/api/expression.rst b/docs/source/api/expression.rst deleted file mode 100644 index 30137d135..000000000 --- a/docs/source/api/expression.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api.expression: -.. currentmodule:: datafusion - -Expr -========== - -.. autosummary:: - :toctree: ../generated/ - - Expr diff --git a/docs/source/api/functions.rst b/docs/source/api/functions.rst deleted file mode 100644 index 6f10d826e..000000000 --- a/docs/source/api/functions.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api.functions: -.. currentmodule:: datafusion - -Functions -========= - -.. autosummary:: - :toctree: ../generated/ - - functions diff --git a/docs/source/api/object_store.rst b/docs/source/api/object_store.rst deleted file mode 100644 index 8d78f0724..000000000 --- a/docs/source/api/object_store.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one -.. or more contributor license agreements. See the NOTICE file -.. distributed with this work for additional information -.. regarding copyright ownership. The ASF licenses this file -.. to you under the Apache License, Version 2.0 (the -.. "License"); you may not use this file except in compliance -.. with the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, -.. software distributed under the License is distributed on an -.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -.. KIND, either express or implied. See the License for the -.. specific language governing permissions and limitations -.. under the License. - -.. _api.object_store: -.. currentmodule:: datafusion.object_store - -ObjectStore -=========== - -.. autosummary:: - :toctree: ../generated/ - - object_store \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 308069b6c..d5084551e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -46,15 +46,11 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.autosummary", - "sphinx.ext.doctest", - "sphinx.ext.ifconfig", "sphinx.ext.mathjax", - "sphinx.ext.viewcode", "sphinx.ext.napoleon", "myst_parser", "IPython.sphinxext.ipython_directive", + "autoapi.extension", ] source_suffix = { @@ -70,33 +66,35 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [] -# Show members for classes in .. autosummary -autodoc_default_options = { - "members": None, - "undoc-members": None, - "show-inheritance": None, - "inherited-members": None, -} - -autosummary_generate = True - +autoapi_dirs = ["../../python"] +autoapi_ignore = ["*tests*"] +autoapi_member_order = "groupwise" +suppress_warnings = ["autoapi.python_import_resolution"] +autoapi_python_class_content = "both" -def autodoc_skip_member(app, what, name, obj, skip, options): - exclude_functions = "__init__" - exclude_classes = ("Expr", "DataFrame") - class_name = "" - if hasattr(obj, "__qualname__"): - if obj.__qualname__ is not None: - class_name = obj.__qualname__.split(".")[0] +def autoapi_skip_member_fn(app, what, name, obj, skip, options): + skip_contents = [ + # Re-exports + ("class", "datafusion.DataFrame"), + ("class", "datafusion.SessionContext"), + ("module", "datafusion.common"), + # Deprecated + ("class", "datafusion.substrait.serde"), + ("class", "datafusion.substrait.plan"), + ("class", "datafusion.substrait.producer"), + ("class", "datafusion.substrait.consumer"), + ("method", "datafusion.context.SessionContext.tables"), + ("method", "datafusion.dataframe.DataFrame.unnest_column"), + ] + if (what, name) in skip_contents: + skip = True - should_exclude = name in exclude_functions and class_name in exclude_classes + return skip - return True if should_exclude else None - -def setup(app): - app.connect("autodoc-skip-member", autodoc_skip_member) +def setup(sphinx): + sphinx.connect("autoapi-skip-member", autoapi_skip_member_fn) # -- Options for HTML output ------------------------------------------------- @@ -106,9 +104,7 @@ def setup(app): # html_theme = "pydata_sphinx_theme" -html_theme_options = { - "use_edit_page_button": True, -} +html_theme_options = {"use_edit_page_button": False, "show_toc_level": 2} html_context = { "github_user": "apache", diff --git a/docs/source/index.rst b/docs/source/index.rst index 16c88e033..b0103a336 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -104,5 +104,3 @@ Example :hidden: :maxdepth: 1 :caption: API - - api diff --git a/docs/source/user-guide/basics.rst b/docs/source/user-guide/basics.rst index 438b23199..3c97d1ef9 100644 --- a/docs/source/user-guide/basics.rst +++ b/docs/source/user-guide/basics.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _user_guide_concepts: + Concepts ======== @@ -52,7 +54,7 @@ The first statement group: # create a context ctx = datafusion.SessionContext() -creates a :code:`SessionContext`, that is, the main interface for executing queries with DataFusion. It maintains the state +creates a :py:class:`~datafusion.context.SessionContext`, that is, the main interface for executing queries with DataFusion. It maintains the state of the connection between a user and an instance of the DataFusion engine. Additionally it provides the following functionality: - Create a DataFrame from a CSV or Parquet data source. @@ -72,9 +74,9 @@ The second statement group creates a :code:`DataFrame`, df = ctx.create_dataframe([[batch]]) A DataFrame refers to a (logical) set of rows that share the same column names, similar to a `Pandas DataFrame `_. -DataFrames are typically created by calling a method on :code:`SessionContext`, such as :code:`read_csv`, and can then be modified by -calling the transformation methods, such as :meth:`.DataFrame.filter`, :meth:`.DataFrame.select`, :meth:`.DataFrame.aggregate`, -and :meth:`.DataFrame.limit` to build up a query definition. +DataFrames are typically created by calling a method on :py:class:`~datafusion.context.SessionContext`, such as :code:`read_csv`, and can then be modified by +calling the transformation methods, such as :py:func:`~datafusion.dataframe.DataFrame.filter`, :py:func:`~datafusion.dataframe.DataFrame.select`, :py:func:`~datafusion.dataframe.DataFrame.aggregate`, +and :py:func:`~datafusion.dataframe.DataFrame.limit` to build up a query definition. The third statement uses :code:`Expressions` to build up a query definition. @@ -85,5 +87,5 @@ The third statement uses :code:`Expressions` to build up a query definition. col("a") - col("b"), ) -Finally the :code:`collect` method converts the logical plan represented by the DataFrame into a physical plan and execute it, -collecting all results into a list of `RecordBatch `_. \ No newline at end of file +Finally the :py:func:`~datafusion.dataframe.DataFrame.collect` method converts the logical plan represented by the DataFrame into a physical plan and execute it, +collecting all results into a list of `RecordBatch `_. diff --git a/docs/source/user-guide/common-operations/aggregations.rst b/docs/source/user-guide/common-operations/aggregations.rst index 235d644e6..b9202129e 100644 --- a/docs/source/user-guide/common-operations/aggregations.rst +++ b/docs/source/user-guide/common-operations/aggregations.rst @@ -19,7 +19,7 @@ Aggregation ============ An aggregate or aggregation is a function where the values of multiple rows are processed together to form a single summary value. -For performing an aggregation, DataFusion provides the :meth:`.DataFrame.aggregate` +For performing an aggregation, DataFusion provides the :py:func:`~datafusion.dataframe.DataFrame.aggregate` .. ipython:: python diff --git a/docs/source/user-guide/common-operations/basic-info.rst b/docs/source/user-guide/common-operations/basic-info.rst index 424e1cc92..d48b49d5c 100644 --- a/docs/source/user-guide/common-operations/basic-info.rst +++ b/docs/source/user-guide/common-operations/basic-info.rst @@ -34,26 +34,26 @@ In this section, you will learn how to display essential details of DataFrames u }) df -Use :meth:`.DataFrame.limit` to view the top rows of the frame: +Use :py:func:`~datafusion.dataframe.DataFrame.limit` to view the top rows of the frame: .. ipython:: python df.limit(2) -Display the columns of the DataFrame using :meth:`.DataFrame.schema`: +Display the columns of the DataFrame using :py:func:`~datafusion.dataframe.DataFrame.schema`: .. ipython:: python df.schema() -The method :meth:`.DataFrame.to_pandas` uses pyarrow to convert to pandas DataFrame, by collecting the batches, +The method :py:func:`~datafusion.dataframe.DataFrame.to_pandas` uses pyarrow to convert to pandas DataFrame, by collecting the batches, passing them to an Arrow table, and then converting them to a pandas DataFrame. .. ipython:: python df.to_pandas() -:meth:`.DataFrame.describe` shows a quick statistic summary of your data: +:py:func:`~datafusion.dataframe.DataFrame.describe` shows a quick statistic summary of your data: .. ipython:: python diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index ebb514f14..c8f8b8f29 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _expressions: + Expressions =========== @@ -26,16 +28,16 @@ concept shared across most compilers and databases. Column ------ -The first expression most new users will interact with is the Column, which is created by calling :func:`col`. -This expression represents a column within a DataFrame. The function :func:`col` takes as in input a string +The first expression most new users will interact with is the Column, which is created by calling :py:func:`~datafusion.col`. +This expression represents a column within a DataFrame. The function :py:func:`~datafusion.col` takes as in input a string and returns an expression as it's output. Literal ------- Literal expressions represent a single value. These are helpful in a wide range of operations where -a specific, known value is of interest. You can create a literal expression using the function :func:`lit`. -The type of the object passed to the :func:`lit` function will be used to convert it to a known data type. +a specific, known value is of interest. You can create a literal expression using the function :py:func:`~datafusion.lit`. +The type of the object passed to the :py:func:`~datafusion.lit` function will be used to convert it to a known data type. In the following example we create expressions for the column named `color` and the literal scalar string `red`. The resultant variable `red_units` is itself also an expression. @@ -62,7 +64,7 @@ Functions --------- As mentioned before, most functions in DataFusion return an expression at their output. This allows us to create -a wide variety of expressions built up from other expressions. For example, :func:`.alias` is a function that takes +a wide variety of expressions built up from other expressions. For example, :py:func:`~datafusion.expr.Expr.alias` is a function that takes as it input a single expression and returns an expression in which the name of the expression has changed. The following example shows a series of expressions that are built up from functions operating on expressions. diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index d793314f7..a0b95c908 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -19,7 +19,7 @@ Functions ========= DataFusion provides a large number of built-in functions for performing complex queries without requiring user-defined functions. -In here we will cover some of the more popular use cases. If you want to view all the functions go to the :ref:`Functions` API Reference. +In here we will cover some of the more popular use cases. If you want to view all the functions go to the :py:mod:`Functions ` API Reference. We'll use the pokemon dataset in the following examples. @@ -40,7 +40,7 @@ We'll use the pokemon dataset in the following examples. Mathematical ------------ -DataFusion offers mathematical functions such as :func:`.pow` or :func:`.log` +DataFusion offers mathematical functions such as :py:func:`~datafusion.functions.pow` or :py:func:`~datafusion.functions.log` .. ipython:: python @@ -55,7 +55,7 @@ DataFusion offers mathematical functions such as :func:`.pow` or :func:`.log` Conditional ----------- -There 3 conditional functions in DataFusion :func:`.coalesce`, :func:`.nullif` and :func:`.case` (not available in Python) +There 3 conditional functions in DataFusion :py:func:`~datafusion.functions.coalesce`, :py:func:`~datafusion.functions.nullif` and :py:func:`~datafusion.functions.case`. .. ipython:: python @@ -66,13 +66,13 @@ There 3 conditional functions in DataFusion :func:`.coalesce`, :func:`.nullif` a Temporal -------- -For selecting the current time use :func:`.now` +For selecting the current time use :py:func:`~datafusion.functions.now` .. ipython:: python df.select(f.now()) -Convert to timestamps using :func:`.to_timestamp` +Convert to timestamps using :py:func:`~datafusion.functions.to_timestamp` .. ipython:: python @@ -92,7 +92,7 @@ DataFusion offers a range of helpful options. f.left(col('"Name"'), literal(4)).alias("code") ) -This also includes the functions for regular expressions like :func:`.regexp_replace` and :func:`.regexp_match` +This also includes the functions for regular expressions like :py:func:`~datafusion.functions.regexp_replace` and :py:func:`~datafusion.functions.regexp_match` .. ipython:: python @@ -105,7 +105,7 @@ This also includes the functions for regular expressions like :func:`.regexp_rep Other ----- -The function :func:`.in_list` allows to check a column for the presence of multiple values: +The function :py:func:`~datafusion.functions.in_list` allows to check a column for the presence of multiple values: .. ipython:: python diff --git a/docs/source/user-guide/common-operations/joins.rst b/docs/source/user-guide/common-operations/joins.rst index 128203116..09fa145a7 100644 --- a/docs/source/user-guide/common-operations/joins.rst +++ b/docs/source/user-guide/common-operations/joins.rst @@ -18,7 +18,7 @@ Joins ===== -DataFusion supports the following join variants via the method :meth:`.DataFrame.join` +DataFusion supports the following join variants via the method :py:func:`~datafusion.dataframe.DataFrame.join` - Inner Join - Left Join @@ -58,7 +58,7 @@ will be included in the resulting DataFrame. left.join(right, join_keys=(["customer_id"], ["id"]), how="inner") -The parameter :code:`join_keys` specifies the columns from the left DataFrame and right DataFrame that contains the values +The parameter ``join_keys`` specifies the columns from the left DataFrame and right DataFrame that contains the values that should match. Left Join diff --git a/docs/source/user-guide/common-operations/select-and-filter.rst b/docs/source/user-guide/common-operations/select-and-filter.rst index 8ede230e6..92b4841b2 100644 --- a/docs/source/user-guide/common-operations/select-and-filter.rst +++ b/docs/source/user-guide/common-operations/select-and-filter.rst @@ -18,7 +18,7 @@ Column Selections ================= -Use :meth:`.DataFrame.select_columns` for basic column selection. +Use :py:func:`~datafusion.dataframe.DataFrame.select` for basic column selection. DataFusion can work with several file types, to start simple we can use a subset of the `TLC Trip Record Data `_ @@ -35,8 +35,8 @@ DataFusion can work with several file types, to start simple we can use a subset df = ctx.read_parquet("yellow_trip_data.parquet") df.select_columns("trip_distance", "passenger_count") -For mathematical or logical operations use :func:`.col` to select columns, and give meaningful names to the resulting -operations using :func:`.alias` +For mathematical or logical operations use :py:func:`~datafusion.col` to select columns, and give meaningful names to the resulting +operations using :py:func:`~datafusion.expr.Expr.alias` .. ipython:: python @@ -48,7 +48,7 @@ operations using :func:`.alias` Please be aware that all identifiers are effectively made lower-case in SQL, so if your file has capital letters (ex: Name) you must put your column name in double quotes or the selection won’t work. As an alternative for simple - column selection use :meth:`.DataFrame.select_columns` without double quotes + column selection use :py:func:`~datafusion.dataframe.DataFrame.select_columns` without double quotes For selecting columns with capital letters use ``'"VendorID"'`` @@ -57,7 +57,7 @@ For selecting columns with capital letters use ``'"VendorID"'`` df.select(col('"VendorID"')) -To combine it with literal values use the :func:`.lit` +To combine it with literal values use the :py:func:`~datafusion.lit` .. ipython:: python diff --git a/docs/source/user-guide/common-operations/udf-and-udfa.rst b/docs/source/user-guide/common-operations/udf-and-udfa.rst index 62d249c7e..54c685794 100644 --- a/docs/source/user-guide/common-operations/udf-and-udfa.rst +++ b/docs/source/user-guide/common-operations/udf-and-udfa.rst @@ -19,7 +19,7 @@ User Defined Functions ====================== DataFusion provides powerful expressions and functions, reducing the need for custom Python functions. -However you can still incorporate your own functions, i.e. User-Defined Functions (UDFs), with the :func:`.udf` function. +However you can still incorporate your own functions, i.e. User-Defined Functions (UDFs), with the :py:func:`~datafusion.udf.ScalarUDF.udf` function. .. ipython:: python @@ -42,7 +42,7 @@ However you can still incorporate your own functions, i.e. User-Defined Function df.select(is_null_arr(col("a"))).to_pandas() -Additionally the :func:`.udaf` function allows you to define User-Defined Aggregate Functions (UDAFs) +Additionally the :py:func:`~datafusion.udf.AggregateUDF.udaf` function allows you to define User-Defined Aggregate Functions (UDAFs) .. code-block:: python diff --git a/docs/source/user-guide/common-operations/windows.rst b/docs/source/user-guide/common-operations/windows.rst index f884c7e0d..5ef3c986c 100644 --- a/docs/source/user-guide/common-operations/windows.rst +++ b/docs/source/user-guide/common-operations/windows.rst @@ -21,7 +21,7 @@ Window Functions In this section you will learn about window functions. A window function utilizes values from one or multiple rows to produce a result for each individual row, unlike an aggregate function that provides a single value for multiple rows. -The functionality of window functions in DataFusion is supported by the dedicated :func:`.window` function. +The functionality of window functions in DataFusion is supported by the dedicated :py:func:`~datafusion.functions.window` function. We'll use the pokemon dataset (from Ritchie Vink) in the following examples. @@ -40,7 +40,7 @@ We'll use the pokemon dataset (from Ritchie Vink) in the following examples. ctx = SessionContext() df = ctx.read_csv("pokemon.csv") -Here is an example that shows how to compare each pokemons’s attack power with the average attack power in its :code:`"Type 1"` +Here is an example that shows how to compare each pokemons’s attack power with the average attack power in its ``"Type 1"`` .. ipython:: python @@ -54,7 +54,7 @@ Here is an example that shows how to compare each pokemons’s attack power with ) You can also control the order in which rows are processed by window functions by providing -a list of :func:`.order_by` functions for the :code:`order_by` parameter. +a list of ``order_by`` functions for the ``order_by`` parameter. .. ipython:: python diff --git a/docs/source/user-guide/configuration.rst b/docs/source/user-guide/configuration.rst index 0c1a4818a..7d330019f 100644 --- a/docs/source/user-guide/configuration.rst +++ b/docs/source/user-guide/configuration.rst @@ -18,8 +18,8 @@ Configuration ============= -Let's look at how we can configure DataFusion. When creating a :code:`SessionContext`, you can pass in -a :code:`SessionConfig` and :code:`RuntimeConfig` object. These two cover a wide range of options. +Let's look at how we can configure DataFusion. When creating a :py:class:`~datafusion.context.SessionContext`, you can pass in +a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.context.RuntimeConfig` object. These two cover a wide range of options. .. code-block:: python @@ -47,5 +47,5 @@ a :code:`SessionConfig` and :code:`RuntimeConfig` object. These two cover a wide print(ctx) -You can read more about available :code:`SessionConfig` options `here `_, -and about :code:`RuntimeConfig` options `here `_. +You can read more about available :py:class:`~datafusion.context.SessionConfig` options in the `rust DataFusion Configuration guide `_, +and about :code:`RuntimeConfig` options in the rust `online API documentation `_. diff --git a/docs/source/user-guide/io/avro.rst b/docs/source/user-guide/io/avro.rst index 85d546e2a..5f1ff728e 100644 --- a/docs/source/user-guide/io/avro.rst +++ b/docs/source/user-guide/io/avro.rst @@ -19,7 +19,7 @@ Avro ==== `Avro `_ is a serialization format for record data. Reading an avro file is very straightforward -with :meth:`.SessionContext.read_avro` +with :py:func:`~datafusion.context.SessionContext.read_avro` .. code-block:: python diff --git a/docs/source/user-guide/io/csv.rst b/docs/source/user-guide/io/csv.rst index 3f95c54a5..d2a62bfec 100644 --- a/docs/source/user-guide/io/csv.rst +++ b/docs/source/user-guide/io/csv.rst @@ -18,7 +18,7 @@ CSV === -Reading a csv is very straightforward with :meth:`.SessionContext.read_csv` +Reading a csv is very straightforward with :py:func:`~datafusion.context.SessionContext.read_csv` .. code-block:: python @@ -28,9 +28,9 @@ Reading a csv is very straightforward with :meth:`.SessionContext.read_csv` ctx = SessionContext() df = ctx.read_csv("file.csv") -An alternative is to use :meth:`.SessionContext.register_csv` +An alternative is to use :py:func:`~datafusion.context.SessionContext.register_csv` .. code-block:: python ctx.register_csv("file", "file.csv") - df = ctx.table("file") \ No newline at end of file + df = ctx.table("file") diff --git a/docs/source/user-guide/io/json.rst b/docs/source/user-guide/io/json.rst index 1ee065c44..f9da3755a 100644 --- a/docs/source/user-guide/io/json.rst +++ b/docs/source/user-guide/io/json.rst @@ -18,7 +18,7 @@ JSON ==== `JSON `_ (JavaScript Object Notation) is a lightweight data-interchange format. -When it comes to reading a JSON file, using :meth:`.SessionContext.read_json` is a simple and easy +When it comes to reading a JSON file, using :py:func:`~datafusion.context.SessionContext.read_json` is a simple and easy .. code-block:: python diff --git a/docs/source/user-guide/io/parquet.rst b/docs/source/user-guide/io/parquet.rst index 78bba30c5..75bc981cc 100644 --- a/docs/source/user-guide/io/parquet.rst +++ b/docs/source/user-guide/io/parquet.rst @@ -18,7 +18,7 @@ Parquet ======= -It is quite simple to read a parquet file using the :meth:`.SessionContext.read_parquet` function. +It is quite simple to read a parquet file using the :py:func:`~datafusion.context.SessionContext.read_parquet` function. .. code-block:: python @@ -28,9 +28,9 @@ It is quite simple to read a parquet file using the :meth:`.SessionContext.read_ ctx = SessionContext() df = ctx.read_parquet("file.parquet") -An alternative is to use :meth:`.SessionContext.register_parquet` +An alternative is to use :py:func:`~datafusion.context.SessionContext.register_parquet` .. code-block:: python ctx.register_parquet("file", "file.parquet") - df = ctx.table("file") \ No newline at end of file + df = ctx.table("file") diff --git a/examples/python-udf-comparisons.py b/examples/python-udf-comparisons.py index e2d856749..5a6f548fb 100644 --- a/examples/python-udf-comparisons.py +++ b/examples/python-udf-comparisons.py @@ -22,7 +22,7 @@ import time path = os.path.dirname(os.path.abspath(__file__)) -filepath = os.path.join(path, "../tpch/data/lineitem.parquet") +filepath = os.path.join(path, "./tpch/data/lineitem.parquet") # This example serves to demonstrate alternate approaches to answering the # question "return all of the rows that have a specific combination of these @@ -122,7 +122,7 @@ def is_of_interest_impl( is_of_interest = udf( is_of_interest_impl, - [pa.int32(), pa.int32(), pa.utf8()], + [pa.int64(), pa.int64(), pa.utf8()], pa.bool_(), "stable", ) @@ -170,7 +170,7 @@ def udf_using_pyarrow_compute_impl( udf_using_pyarrow_compute = udf( udf_using_pyarrow_compute_impl, - [pa.int32(), pa.int32(), pa.utf8()], + [pa.int64(), pa.int64(), pa.utf8()], pa.bool_(), "stable", ) diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 0569ac4b0..08ca3fe02 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -69,7 +69,9 @@ "ScalarUDF", "WindowFrame", "column", + "col", "literal", + "lit", "DFSchema", "runtime", "Catalog", @@ -93,7 +95,9 @@ def column(value: str): return Expr.column(value) -col = column +def col(value: str): + """Create a column expression.""" + return Expr.column(value) def literal(value): @@ -101,7 +105,10 @@ def literal(value): return Expr.literal(value) -lit = literal +def lit(value): + """Create a literal expression.""" + return Expr.literal(value) + udf = ScalarUDF.udf diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py index cec0be764..acd28f33d 100644 --- a/python/datafusion/catalog.py +++ b/python/datafusion/catalog.py @@ -39,7 +39,7 @@ def names(self) -> list[str]: return self.catalog.names() def database(self, name: str = "public") -> Database: - """Returns the database with the given `name` from this catalog.""" + """Returns the database with the given ``name`` from this catalog.""" return Database(self.catalog.database(name)) @@ -55,7 +55,7 @@ def names(self) -> set[str]: return self.db.names() def table(self, name: str) -> Table: - """Return the table with the given `name` from this database.""" + """Return the table with the given ``name`` from this database.""" return Table(self.db.table(name)) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index a717db106..922cc87a3 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -46,7 +46,7 @@ class SessionConfig: """Session configuration options.""" def __init__(self, config_options: dict[str, str] | None = None) -> None: - """Create a new `SessionConfig` with the given configuration options. + """Create a new :py:class:`SessionConfig` with the given configuration options. Args: config_options: Configuration options. @@ -63,7 +63,7 @@ def with_create_default_catalog_and_schema( automatically created. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = ( self.config_internal.with_create_default_catalog_and_schema(enabled) @@ -80,7 +80,7 @@ def with_default_catalog_and_schema( schema: Schema name. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_default_catalog_and_schema( catalog, schema @@ -88,13 +88,13 @@ def with_default_catalog_and_schema( return self def with_information_schema(self, enabled: bool = True) -> SessionConfig: - """Enable or disable the inclusion of `information_schema` virtual tables. + """Enable or disable the inclusion of ``information_schema`` virtual tables. Args: - enabled: Whether to include `information_schema` virtual tables. + enabled: Whether to include ``information_schema`` virtual tables. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_information_schema(enabled) return self @@ -106,7 +106,7 @@ def with_batch_size(self, batch_size: int) -> SessionConfig: batch_size: Batch size. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_batch_size(batch_size) return self @@ -120,7 +120,7 @@ def with_target_partitions(self, target_partitions: int) -> SessionConfig: target_partitions: Number of target partitions. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_target_partitions( target_partitions @@ -136,7 +136,7 @@ def with_repartition_aggregations(self, enabled: bool = True) -> SessionConfig: enabled: Whether to use repartitioning for aggregations. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_repartition_aggregations( enabled @@ -150,7 +150,7 @@ def with_repartition_joins(self, enabled: bool = True) -> SessionConfig: enabled: Whether to use repartitioning for joins. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_repartition_joins(enabled) return self @@ -164,7 +164,7 @@ def with_repartition_windows(self, enabled: bool = True) -> SessionConfig: enabled: Whether to use repartitioning for window functions. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_repartition_windows(enabled) return self @@ -178,7 +178,7 @@ def with_repartition_sorts(self, enabled: bool = True) -> SessionConfig: enabled: Whether to use repartitioning for window functions. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_repartition_sorts(enabled) return self @@ -190,7 +190,7 @@ def with_repartition_file_scans(self, enabled: bool = True) -> SessionConfig: enabled: Whether to use repartitioning for file scans. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_repartition_file_scans(enabled) return self @@ -202,7 +202,7 @@ def with_repartition_file_min_size(self, size: int) -> SessionConfig: size: Minimum file range size. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_repartition_file_min_size(size) return self @@ -216,7 +216,7 @@ def with_parquet_pruning(self, enabled: bool = True) -> SessionConfig: enabled: Whether to use pruning predicate for parquet readers. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_parquet_pruning(enabled) return self @@ -229,7 +229,7 @@ def set(self, key: str, value: str) -> SessionConfig: value: Option value. Returns: - A new `SessionConfig` object with the updated setting. + A new :py:class:`SessionConfig` object with the updated setting. """ self.config_internal = self.config_internal.set(key, value) return self @@ -239,14 +239,14 @@ class RuntimeConfig: """Runtime configuration options.""" def __init__(self) -> None: - """Create a new `RuntimeConfig` with default values.""" + """Create a new :py:class:`RuntimeConfig` with default values.""" self.config_internal = RuntimeConfigInternal() def with_disk_manager_disabled(self) -> RuntimeConfig: """Disable the disk manager, attempts to create temporary files will error. Returns: - A new `RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_disk_manager_disabled() return self @@ -255,7 +255,7 @@ def with_disk_manager_os(self) -> RuntimeConfig: """Use the operating system's temporary directory for disk manager. Returns: - A new `RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_disk_manager_os() return self @@ -267,7 +267,7 @@ def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConf paths: Paths to use for the disk manager's temporary files. Returns: - A new `RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. """ paths = [str(p) for p in paths] self.config_internal = self.config_internal.with_disk_manager_specified(paths) @@ -277,7 +277,7 @@ def with_unbounded_memory_pool(self) -> RuntimeConfig: """Use an unbounded memory pool. Returns: - A new `RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. """ self.config_internal = self.config_internal.with_unbounded_memory_pool() return self @@ -303,7 +303,7 @@ def with_fair_spill_pool(self, size: int) -> RuntimeConfig: size: Size of the memory pool in bytes. Returns: - A new ``RuntimeConfig`` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. Examples usage:: @@ -316,14 +316,14 @@ def with_greedy_memory_pool(self, size: int) -> RuntimeConfig: """Use a greedy memory pool with the specified size. This pool works well for queries that do not need to spill or have a single - spillable operator. See `RuntimeConfig.with_fair_spill_pool` if there are + spillable operator. See :py:func:`with_fair_spill_pool` if there are multiple spillable operators that all will spill. Args: size: Size of the memory pool in bytes. Returns: - A new `RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. Example usage:: @@ -339,7 +339,7 @@ def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeConfig: path: Path to use for temporary files. Returns: - A new `RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeConfig` object with the updated setting. Example usage:: @@ -350,10 +350,10 @@ def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeConfig: class SQLOptions: - """Options to be used when performing SQL queries on the ``SessionContext``.""" + """Options to be used when performing SQL queries.""" def __init__(self) -> None: - """Create a new `SQLOptions` with default values. + """Create a new :py:class:`SQLOptions` with default values. The default values are: - DDL commands are allowed @@ -365,13 +365,13 @@ def __init__(self) -> None: def with_allow_ddl(self, allow: bool = True) -> SQLOptions: """Should DDL (Data Definition Language) commands be run? - Examples of DDL commands include `CREATE TABLE` and `DROP TABLE`. + Examples of DDL commands include ``CREATE TABLE`` and ``DROP TABLE``. Args: allow: Allow DDL commands to be run. Returns: - A new `SQLOptions` object with the updated setting. + A new :py:class:`SQLOptions` object with the updated setting. Example usage:: @@ -383,13 +383,13 @@ def with_allow_ddl(self, allow: bool = True) -> SQLOptions: def with_allow_dml(self, allow: bool = True) -> SQLOptions: """Should DML (Data Manipulation Language) commands be run? - Examples of DML commands include `INSERT INTO` and `DELETE`. + Examples of DML commands include ``INSERT INTO`` and ``DELETE``. Args: allow: Allow DML commands to be run. Returns: - A new `SQLOptions` object with the updated setting. + A new :py:class:`SQLOptions` object with the updated setting. Example usage:: @@ -399,13 +399,13 @@ def with_allow_dml(self, allow: bool = True) -> SQLOptions: return self def with_allow_statements(self, allow: bool = True) -> SQLOptions: - """Should statements such as `SET VARIABLE` and `BEGIN TRANSACTION` be run? + """Should statements such as ``SET VARIABLE`` and ``BEGIN TRANSACTION`` be run? Args: allow: Allow statements to be run. Returns: - A new `SQLOptions` object with the updated setting. + A new :py:class:SQLOptions` object with the updated setting. Example usage:: @@ -418,8 +418,7 @@ def with_allow_statements(self, allow: bool = True) -> SQLOptions: class SessionContext: """This is the main interface for executing queries and creating DataFrames. - See https://datafusion.apache.org/python/user-guide/basics.html for - additional information. + See :ref:`user_guide_concepts` in the online documentation for more information. """ def __init__( @@ -438,7 +437,7 @@ def __init__( Example usage: The following example demostrates how to use the context to execute - a query against a CSV data source using the ``DataFrame`` API:: + a query against a CSV data source using the :py:class:`DataFrame` API:: from datafusion import SessionContext @@ -455,7 +454,7 @@ def register_object_store(self, schema: str, store: Any, host: str | None) -> No Args: schema: The data source schema. - store: The `ObjectStore` to register. + store: The :py:class:`~datafusion.object_store.ObjectStore` to register. host: URL for the host. """ self.ctx.register_object_store(schema, store, host) @@ -471,8 +470,9 @@ def register_listing_table( ) -> None: """Register multiple files as a single table. - Registers a `Table` that can assemble multiple files from locations in - an `ObjectStore` instance. + Registers a :py:class:`~datafusion.catalog.Table` that can assemble multiple + files from locations in an :py:class:`~datafusion.object_store.ObjectStore` + instance. Args: name: Name of the resultant table. @@ -496,11 +496,12 @@ def register_listing_table( ) def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame: - """Create a `DataFrame` from SQL query text. + """Create a :py:class:`~datafusion.DataFrame` from SQL query text. - Note: This API implements DDL statements such as `CREATE TABLE` and - `CREATE VIEW` and DML statements such as `INSERT INTO` with in-memory - default implementation. See `SessionContext.sql_with_options`. + Note: This API implements DDL statements such as ``CREATE TABLE`` and + ``CREATE VIEW`` and DML statements such as ``INSERT INTO`` with in-memory + default implementation.See + :py:func:`~datafusion.context.SessionContext.sql_with_options`. Args: query: SQL query text. @@ -514,7 +515,7 @@ def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame: return DataFrame(self.ctx.sql_with_options(query, options.options_internal)) def sql_with_options(self, query: str, options: SQLOptions) -> DataFrame: - """Create a `DataFrame` from SQL query text. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from SQL query text. This function will first validating that the query is allowed by the provided options. @@ -537,7 +538,7 @@ def create_dataframe( """Create and return a dataframe using the provided partitions. Args: - partitions: `RecordBatch` partitions to register. + partitions: :py:class:`pyarrow.RecordBatch` partitions to register. name: Resultant dataframe name. schema: Schema for the partitions. @@ -547,7 +548,7 @@ def create_dataframe( return DataFrame(self.ctx.create_dataframe(partitions, name, schema)) def create_dataframe_from_logical_plan(self, plan: LogicalPlan) -> DataFrame: - """Create a `DataFrame` from an existing logical plan. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from an existing plan. Args: plan: Logical plan. @@ -560,7 +561,7 @@ def create_dataframe_from_logical_plan(self, plan: LogicalPlan) -> DataFrame: def from_pylist( self, data: list[dict[str, Any]], name: str | None = None ) -> DataFrame: - """Create a `DataFrame` from a list of dictionaries. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from a list. Args: data: List of dictionaries. @@ -574,7 +575,7 @@ def from_pylist( def from_pydict( self, data: dict[str, list[Any]], name: str | None = None ) -> DataFrame: - """Create a `DataFrame` from a dictionary of lists. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from a dictionary. Args: data: Dictionary of lists. @@ -588,7 +589,7 @@ def from_pydict( def from_arrow_table( self, data: pyarrow.Table, name: str | None = None ) -> DataFrame: - """Create a `DataFrame` from an Arrow table. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow table. Args: data: Arrow table. @@ -600,7 +601,7 @@ def from_arrow_table( return DataFrame(self.ctx.from_arrow_table(data, name)) def from_pandas(self, data: pandas.DataFrame, name: str | None = None) -> DataFrame: - """Create a `DataFrame` from a Pandas DataFrame. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from a Pandas DataFrame. Args: data: Pandas DataFrame. @@ -612,7 +613,7 @@ def from_pandas(self, data: pandas.DataFrame, name: str | None = None) -> DataFr return DataFrame(self.ctx.from_pandas(data, name)) def from_polars(self, data: polars.DataFrame, name: str | None = None) -> DataFrame: - """Create a `DataFrame` from a Polars DataFrame. + """Create a :py:class:`~datafusion.dataframe.DataFrame` from a Polars DataFrame. Args: data: Polars DataFrame. @@ -799,7 +800,7 @@ def register_avro( ) def register_dataset(self, name: str, dataset: pyarrow.dataset.Dataset) -> None: - """Register a `pyarrow.dataset.Dataset` as a table. + """Register a :py:class:`pyarrow.dataset.Dataset` as a table. Args: name: Name of the table to register. @@ -809,18 +810,18 @@ def register_dataset(self, name: str, dataset: pyarrow.dataset.Dataset) -> None: def register_udf(self, udf: ScalarUDF) -> None: """Register a user-defined function (UDF) with the context.""" - self.ctx.register_udf(udf.udf) + self.ctx.register_udf(udf._udf) def register_udaf(self, udaf: AggregateUDF) -> None: """Register a user-defined aggregation function (UDAF) with the context.""" - self.ctx.register_udaf(udaf) + self.ctx.register_udaf(udaf._udaf) def catalog(self, name: str = "datafusion") -> Catalog: """Retrieve a catalog by name.""" return self.ctx.catalog(name) @deprecated( - "Use the catalog provider interface `SessionContext.catalog` to " + "Use the catalog provider interface ``SessionContext.Catalog`` to " "examine available catalogs, schemas and tables" ) def tables(self) -> set[str]: @@ -828,7 +829,7 @@ def tables(self) -> set[str]: return self.ctx.tables() def table(self, name: str) -> DataFrame: - """Retrieve a `DataFrame` representing a previously registered table.""" + """Retrieve a previously registered table by name.""" return DataFrame(self.ctx.table(name)) def table_exist(self, name: str) -> bool: @@ -836,11 +837,11 @@ def table_exist(self, name: str) -> bool: return self.ctx.table_exist(name) def empty_table(self) -> DataFrame: - """Create an empty `DataFrame`.""" + """Create an empty :py:class:`~datafusion.dataframe.DataFrame`.""" return DataFrame(self.ctx.empty_table()) def session_id(self) -> str: - """Retrun an id that uniquely identifies this `SessionContext`.""" + """Retrun an id that uniquely identifies this :py:class:`SessionContext`.""" return self.ctx.session_id() def read_json( @@ -852,7 +853,7 @@ def read_json( table_partition_cols: list[tuple[str, str]] | None = None, file_compression_type: str | None = None, ) -> DataFrame: - """Create a `DataFrame` for reading a line-delimited JSON data source. + """Read a line-delimited JSON data source. Args: path: Path to the JSON file. @@ -891,7 +892,7 @@ def read_csv( table_partition_cols: list[tuple[str, str]] | None = None, file_compression_type: str | None = None, ) -> DataFrame: - """Create a `DataFrame` for reading a CSV data source. + """Read a CSV data source. Args: path: Path to the CSV file @@ -936,7 +937,7 @@ def read_parquet( schema: pyarrow.Schema | None = None, file_sort_order: list[list[Expr]] | None = None, ) -> DataFrame: - """Create a `DataFrame` for reading Parquet data source. + """Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. Args: path: Path to the Parquet file. @@ -977,7 +978,7 @@ def read_avro( file_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = ".avro", ) -> DataFrame: - """Create a ``DataFrame`` for reading Avro data source. + """Create a :py:class:`DataFrame` for reading Avro data source. Args: path: Path to the Avro file. @@ -995,9 +996,14 @@ def read_avro( ) def read_table(self, table: Table) -> DataFrame: - """Creates a ``DataFrame`` for a ``Table`` such as a ``ListingTable``.""" + """Creates a :py:class:`~datafusion.dataframe.DataFrame` from a table. + + For a :py:class:`~datafusion.catalog.Table` such as a + :py:class:`~datafusion.catalog.ListingTable`, create a + :py:class:`~datafusion.dataframe.DataFrame`. + """ return DataFrame(self.ctx.read_table(table)) def execute(self, plan: ExecutionPlan, partitions: int) -> RecordBatchStream: - """Execute the `plan` and return the results.""" + """Execute the ``plan`` and return the results.""" return RecordBatchStream(self.ctx.execute(plan, partitions)) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 68e6298f7..fa7398442 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -14,10 +14,9 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""DataFrame is one of the core concepts in DataFusion. +""":py:class:`DataFrame` is one of the core concepts in DataFusion. -See https://datafusion.apache.org/python/user-guide/basics.html for more -information. +See :ref:`user_guide_concepts` in the online documentation for more information. """ from __future__ import annotations @@ -43,19 +42,19 @@ class DataFrame: """Two dimensional table representation of data. - See https://datafusion.apache.org/python/user-guide/basics.html for more - information. + See :ref:`user_guide_concepts` in the online documentation for more information. """ def __init__(self, df: DataFrameInternal) -> None: """This constructor is not to be used by the end user. - See ``SessionContext`` for methods to create DataFrames. + See :py:class:`~datafusion.context.SessionContext` for methods to + create a :py:class:`DataFrame`. """ self.df = df def __getitem__(self, key: str | List[str]) -> DataFrame: - """Return a new `DataFrame` with the specified column or columns. + """Return a new :py:class`DataFrame` with the specified column or columns. Args: key: Column name or list of column names to select. @@ -74,7 +73,7 @@ def __repr__(self) -> str: return self.df.__repr__() def describe(self) -> DataFrame: - """Return a new `DataFrame` that has statistics for a DataFrame. + """Return the statistics for this DataFrame. Only summarized numeric datatypes at the moments and returns nulls for non-numeric datatypes. @@ -87,7 +86,7 @@ def describe(self) -> DataFrame: return DataFrame(self.df.describe()) def schema(self) -> pa.Schema: - """Return the `pyarrow.Schema` describing the output of this DataFrame. + """Return the :py:class:`pyarrow.Schema` of this DataFrame. The output schema contains information on the name, data type, and nullability for each column. @@ -106,10 +105,10 @@ def select_columns(self, *args: str) -> DataFrame: return self.select(*args) def select(self, *exprs: Expr | str) -> DataFrame: - """Project arbitrary expressions into a new `DataFrame`. + """Project arbitrary expressions into a new :py:class:`DataFrame`. Args: - exprs: Either column names or `Expr` to select. + exprs: Either column names or :py:class:`~datafusion.expr.Expr` to select. Returns: DataFrame after projection. It has one column for each expression. @@ -117,9 +116,9 @@ def select(self, *exprs: Expr | str) -> DataFrame: Example usage: The following example will return 3 columns from the original dataframe. - The first two columns will be the original column `a` and `b` since the + The first two columns will be the original column ``a`` and ``b`` since the string "a" is assumed to refer to column selection. Also a duplicate of - column `a` will be returned with the column name `alternate_a`:: + column ``a`` will be returned with the column name ``alternate_a``:: df = df.select("a", col("b"), col("a").alias("alternate_a")) @@ -131,12 +130,12 @@ def select(self, *exprs: Expr | str) -> DataFrame: return DataFrame(self.df.select(*exprs)) def filter(self, *predicates: Expr) -> DataFrame: - """Return a DataFrame for which `predicate` evaluates to `True`. + """Return a DataFrame for which ``predicate`` evaluates to ``True``. - Rows for which `predicate` evaluates to `False` or `None` are filtered + Rows for which ``predicate`` evaluates to ``False`` or ``None`` are filtered out. If more than one predicate is provided, these predicates will be combined as a logical AND. If more complex logic is required, see the - logical operations in `datafusion.functions`. + logical operations in :py:mod:`~datafusion.functions`. Args: predicates: Predicate expression(s) to filter the DataFrame. @@ -162,12 +161,12 @@ def with_column(self, name: str, expr: Expr) -> DataFrame: return DataFrame(self.df.with_column(name, expr.expr)) def with_column_renamed(self, old_name: str, new_name: str) -> DataFrame: - """Rename one column by applying a new projection. + r"""Rename one column by applying a new projection. This is a no-op if the column to be renamed does not exist. The method supports case sensitive rename with wrapping column name - into one the following symbols (" or ' or `). + into one the following symbols (" or ' or \`). Args: old_name: Old column name. @@ -196,7 +195,7 @@ def sort(self, *exprs: Expr) -> DataFrame: """Sort the DataFrame by the specified sorting expressions. Note that any expression can be turned into a sort expression by - calling its `sort` method. + calling its` ``sort`` method. Args: exprs: Sort expressions, applied in order. @@ -208,7 +207,7 @@ def sort(self, *exprs: Expr) -> DataFrame: return DataFrame(self.df.sort(*exprs)) def limit(self, count: int, offset: int = 0) -> DataFrame: - """Return a new `DataFrame` with a limited number of rows. + """Return a new :py:class:`DataFrame` with a limited number of rows. Args: count: Number of rows to limit the DataFrame to. @@ -220,14 +219,14 @@ def limit(self, count: int, offset: int = 0) -> DataFrame: return DataFrame(self.df.limit(count, offset)) def collect(self) -> list[pa.RecordBatch]: - """Execute this `DataFrame` and collect results into memory. + """Execute this :py:class:`DataFrame` and collect results into memory. - Prior to calling `collect`, modifying a DataFrme simply updates a plan - (no actual computation is performed). Calling `collect` triggers the + Prior to calling ``collect``, modifying a DataFrme simply updates a plan + (no actual computation is performed). Calling ``collect`` triggers the computation. Returns: - List of `pyarrow.RecordBatch`es collected from the DataFrame. + List of :py:class:`pyarrow.RecordBatch` collected from the DataFrame. """ return self.df.collect() @@ -242,11 +241,11 @@ def cache(self) -> DataFrame: def collect_partitioned(self) -> list[list[pa.RecordBatch]]: """Execute this DataFrame and collect all partitioned results. - This operation returns ``RecordBatch`` maintaining the input + This operation returns :py:class:`pyarrow.RecordBatch` maintaining the input partitioning. Returns: - List of list of ``RecordBatch`` collected from the + List of list of :py:class:`RecordBatch` collected from the DataFrame. """ return self.df.collect_partitioned() @@ -260,7 +259,7 @@ def show(self, num: int = 20) -> None: self.df.show(num) def distinct(self) -> DataFrame: - """Return a new `DataFrame` with all duplicated rows removed. + """Return a new :py:class:`DataFrame` with all duplicated rows removed. Returns: DataFrame after removing duplicates. @@ -273,7 +272,7 @@ def join( join_keys: tuple[list[str], list[str]], how: str, ) -> DataFrame: - """Join this `DataFrame` with another `DataFrame`. + """Join this :py:class:`DataFrame` with another :py:class:`DataFrame`. Join keys are a pair of lists of column names in the left and right dataframes, respectively. These lists must have the same length. @@ -292,11 +291,11 @@ def join( def explain(self, verbose: bool = False, analyze: bool = False) -> DataFrame: """Return a DataFrame with the explanation of its plan so far. - If `analyze` is specified, runs the plan and reports metrics. + If ``analyze`` is specified, runs the plan and reports metrics. Args: - verbose: If `True`, more details will be included. - analyze: If `True`, the plan will run and metrics reported. + verbose: If ``True``, more details will be included. + analyze: If ``Tru`e``, the plan will run and metrics reported. Returns: DataFrame with the explanation of its plan. @@ -304,7 +303,7 @@ def explain(self, verbose: bool = False, analyze: bool = False) -> DataFrame: return DataFrame(self.df.explain(verbose, analyze)) def logical_plan(self) -> LogicalPlan: - """Return the unoptimized `LogicalPlan` that comprises this `DataFrame`. + """Return the unoptimized ``LogicalPlan``. Returns: Unoptimized logical plan. @@ -312,7 +311,7 @@ def logical_plan(self) -> LogicalPlan: return self.df.logical_plan() def optimized_logical_plan(self) -> LogicalPlan: - """Return the optimized `LogicalPlan` that comprises this `DataFrame`. + """Return the optimized ``LogicalPlan``. Returns: Optimized logical plan. @@ -320,7 +319,7 @@ def optimized_logical_plan(self) -> LogicalPlan: return self.df.optimized_logical_plan() def execution_plan(self) -> ExecutionPlan: - """Return the execution/physical plan that comprises this `DataFrame`. + """Return the execution/physical plan. Returns: Execution plan. @@ -328,7 +327,7 @@ def execution_plan(self) -> ExecutionPlan: return self.df.execution_plan() def repartition(self, num: int) -> DataFrame: - """Repartition a DataFrame into `num` partitions. + """Repartition a DataFrame into ``num`` partitions. The batches allocation uses a round-robin algorithm. @@ -354,13 +353,13 @@ def repartition_by_hash(self, *exprs: Expr, num: int) -> DataFrame: return DataFrame(self.df.repartition_by_hash(*exprs, num=num)) def union(self, other: DataFrame, distinct: bool = False) -> DataFrame: - """Calculate the union of two `DataFrame`s. + """Calculate the union of two :py:class:`DataFrame`. - The two `DataFrame`s must have exactly the same schema. + The two :py:class:`DataFrame` must have exactly the same schema. Args: other: DataFrame to union with. - distinct: If `True`, duplicate rows will be removed. + distinct: If ``True``, duplicate rows will be removed. Returns: DataFrame after union. @@ -368,9 +367,9 @@ def union(self, other: DataFrame, distinct: bool = False) -> DataFrame: return DataFrame(self.df.union(other.df, distinct)) def union_distinct(self, other: DataFrame) -> DataFrame: - """Calculate the distinct union of two `DataFrame`s. + """Calculate the distinct union of two :py:class:`DataFrame`. - The two `DataFrame`s must have exactly the same schema. + The two :py:class:`DataFrame` must have exactly the same schema. Any duplicate rows are discarded. Args: @@ -382,9 +381,9 @@ def union_distinct(self, other: DataFrame) -> DataFrame: return DataFrame(self.df.union_distinct(other.df)) def intersect(self, other: DataFrame) -> DataFrame: - """Calculate the intersection of two `DataFrame`s. + """Calculate the intersection of two :py:class:`DataFrame`. - The two `DataFrame`s must have exactly the same schema. + The two :py:class:`DataFrame` must have exactly the same schema. Args: other: DataFrame to intersect with. @@ -395,9 +394,9 @@ def intersect(self, other: DataFrame) -> DataFrame: return DataFrame(self.df.intersect(other.df)) def except_all(self, other: DataFrame) -> DataFrame: - """Calculate the exception of two `DataFrame`s. + """Calculate the exception of two :py:class:`DataFrame`. - The two `DataFrame`s must have exactly the same schema. + The two :py:class:`DataFrame` must have exactly the same schema. Args: other: DataFrame to calculate exception with. @@ -408,7 +407,7 @@ def except_all(self, other: DataFrame) -> DataFrame: return DataFrame(self.df.except_all(other.df)) def write_csv(self, path: str | pathlib.Path, with_header: bool = False) -> None: - """Execute the `DataFrame` and write the results to a CSV file. + """Execute the :py:class:`DataFrame` and write the results to a CSV file. Args: path: Path of the CSV file to write. @@ -422,7 +421,7 @@ def write_parquet( compression: str = "uncompressed", compression_level: int | None = None, ) -> None: - """Execute the `DataFrame` and write the results to a Parquet file. + """Execute the :py:class:`DataFrame` and write the results to a Parquet file. Args: path: Path of the Parquet file to write. @@ -432,7 +431,7 @@ def write_parquet( self.df.write_parquet(str(path), compression, compression_level) def write_json(self, path: str | pathlib.Path) -> None: - """Execute the `DataFrame` and write the results to a JSON file. + """Execute the :py:class:`DataFrame` and write the results to a JSON file. Args: path: Path of the JSON file to write. @@ -440,7 +439,7 @@ def write_json(self, path: str | pathlib.Path) -> None: self.df.write_json(str(path)) def to_arrow_table(self) -> pa.Table: - """Execute the `DataFrame` and convert it into an Arrow Table. + """Execute the :py:class:`DataFrame` and convert it into an Arrow Table. Returns: Arrow Table. @@ -465,7 +464,7 @@ def execute_stream_partitioned(self) -> list[RecordBatchStream]: return [RecordBatchStream(rbs) for rbs in streams] def to_pandas(self) -> pd.DataFrame: - """Execute the `DataFrame` and convert it into a Pandas DataFrame. + """Execute the :py:class:`DataFrame` and convert it into a Pandas DataFrame. Returns: Pandas DataFrame. @@ -473,7 +472,7 @@ def to_pandas(self) -> pd.DataFrame: return self.df.to_pandas() def to_pylist(self) -> list[dict[str, Any]]: - """Execute the `DataFrame` and convert it into a list of dictionaries. + """Execute the :py:class:`DataFrame` and convert it into a list of dictionaries. Returns: List of dictionaries. @@ -481,7 +480,7 @@ def to_pylist(self) -> list[dict[str, Any]]: return self.df.to_pylist() def to_pydict(self) -> dict[str, list[Any]]: - """Execute the `DataFrame` and convert it into a dictionary of lists. + """Execute the :py:class:`DataFrame` and convert it into a dictionary of lists. Returns: Dictionary of lists. @@ -489,7 +488,7 @@ def to_pydict(self) -> dict[str, list[Any]]: return self.df.to_pydict() def to_polars(self) -> pl.DataFrame: - """Execute the `DataFrame` and convert it into a Polars DataFrame. + """Execute the :py:class:`DataFrame` and convert it into a Polars DataFrame. Returns: Polars DataFrame. @@ -497,7 +496,7 @@ def to_polars(self) -> pl.DataFrame: return self.df.to_polars() def count(self) -> int: - """Return the total number of rows in this `DataFrame`. + """Return the total number of rows in this :py:class:`DataFrame`. Note that this method will actually run a plan to calculate the count, which may be slow for large or complicated DataFrames. @@ -507,9 +506,9 @@ def count(self) -> int: """ return self.df.count() - @deprecated("Use :func:`unnest_columns` instead.") + @deprecated("Use :py:func:`unnest_columns` instead.") def unnest_column(self, column: str, preserve_nulls: bool = True) -> DataFrame: - """See ``unnest_columns``.""" + """See :py:func:`unnest_columns`.""" return DataFrame(self.df.unnest_column(column, preserve_nulls=preserve_nulls)) def unnest_columns(self, *columns: str, preserve_nulls: bool = True) -> DataFrame: diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 955c4e736..71fcf397b 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -17,7 +17,7 @@ """This module supports expressions, one of the core concepts in DataFusion. -See ``Expr`` for more details. +See :ref:`Expressions` in the online documentation for more details. """ from __future__ import annotations @@ -159,8 +159,7 @@ class Expr: """Expression object. Expressions are one of the core concepts in DataFusion. See - https://datafusion.apache.org/python/user-guide/common-operations/expressions.html - for more information. + :ref:`Expressions` in the online documentation for more information. """ def __init__(self, expr: expr_internal.Expr) -> None: @@ -328,7 +327,7 @@ def __lt__(self, rhs: Any) -> Expr: def literal(value: Any) -> Expr: """Creates a new expression representing a scalar value. - `value` must be a valid PyArrow scalar value or easily castable to one. + ``value`` must be a valid PyArrow scalar value or easily castable to one. """ if not isinstance(value, pa.Scalar): value = pa.scalar(value) @@ -336,7 +335,7 @@ def literal(value: Any) -> Expr: @staticmethod def column(value: str) -> Expr: - """Creates a new expression representing a column in a ``DataFrame``.""" + """Creates a new expression representing a column.""" return Expr(expr_internal.Expr.column(value)) def alias(self, name: str) -> Expr: @@ -344,7 +343,7 @@ def alias(self, name: str) -> Expr: return Expr(self.expr.alias(name)) def sort(self, ascending: bool = True, nulls_first: bool = True) -> Expr: - """Creates a sort ``Expr`` from an existing ``Expr``. + """Creates a sort :py:class:`Expr` from an existing :py:class:`Expr`. Args: ascending: If true, sort in ascending order. @@ -365,7 +364,7 @@ def rex_type(self) -> RexType: A Rex (Row Expression) specifies a single row of data.That specification could include user defined functions or types. RexType identifies the - row as one of the possible valid ``RexType``(s). + row as one of the possible valid ``RexType``. """ return self.expr.rex_type() @@ -416,12 +415,12 @@ def __init__( """Construct a window frame using the given parameters. Args: - units: Should be one of `rows`, `range`, or `groups`. + units: Should be one of ``rows``, ``range``, or ``groups``. start_bound: Sets the preceeding bound. Must be >= 0. If none, this - will be set to unbounded. If unit type is `groups`, this + will be set to unbounded. If unit type is ``groups``, this parameter must be set. end_bound: Sets the following bound. Must be >= 0. If none, this - will be set to unbounded. If unit type is `groups`, this + will be set to unbounded. If unit type is ``groups``, this parameter must be set. """ self.window_frame = expr_internal.WindowFrame(units, start_bound, end_bound) @@ -442,7 +441,7 @@ def get_upper_bound(self): class WindowFrameBound: """Defines a single window frame bound. - ``WindowFrame`` typically requires a start and end bound. + :py:class:`WindowFrame` typically requires a start and end bound. """ def __init__(self, frame_bound: expr_internal.WindowFrameBound) -> None: @@ -489,7 +488,7 @@ def __init__(self, case_builder: expr_internal.CaseBuilder) -> None: """Constructs a case builder. This is not typically called by the end user directly. See - ``datafusion.functions.case`` instead. + :py:func:`datafusion.functions.case` instead. """ self.case_builder = case_builder diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index be83d359f..82b5056d7 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""This module contains the user functions for operating on ``Expr``.""" +"""User functions for operating on :py:class:`~datafusion.expr.Expr`.""" from __future__ import annotations @@ -263,12 +263,12 @@ def nullif(expr1: Expr, expr2: Expr) -> Expr: def encode(input: Expr, encoding: Expr) -> Expr: - """Encode the `input`, using the `encoding`. encoding can be base64 or hex.""" + """Encode the ``input``, using the ``encoding``. encoding can be base64 or hex.""" return Expr(f.encode(input.expr, encoding.expr)) def decode(input: Expr, encoding: Expr) -> Expr: - """Decode the `input`, using the `encoding`. encoding can be base64 or hex.""" + """Decode the ``input``, using the ``encoding``. encoding can be base64 or hex.""" return Expr(f.decode(input.expr, encoding.expr)) @@ -280,7 +280,7 @@ def array_to_string(expr: Expr, delimiter: Expr) -> Expr: def array_join(expr: Expr, delimiter: Expr) -> Expr: """Converts each element to its text representation. - This is an alias for :func:`array_to_string`. + This is an alias for :py:func:`array_to_string`. """ return array_to_string(expr, delimiter) @@ -288,7 +288,7 @@ def array_join(expr: Expr, delimiter: Expr) -> Expr: def list_to_string(expr: Expr, delimiter: Expr) -> Expr: """Converts each element to its text representation. - This is an alias for :func:`array_to_string`. + This is an alias for :py:func:`array_to_string`. """ return array_to_string(expr, delimiter) @@ -296,13 +296,13 @@ def list_to_string(expr: Expr, delimiter: Expr) -> Expr: def list_join(expr: Expr, delimiter: Expr) -> Expr: """Converts each element to its text representation. - This is an alias for :func:`array_to_string`. + This is an alias for :py:func:`array_to_string`. """ return array_to_string(expr, delimiter) def in_list(arg: Expr, values: list[Expr], negated: bool = False) -> Expr: - """Returns whether the argument is contained within the list `values`.""" + """Returns whether the argument is contained within the list ``values``.""" values = [v.expr for v in values] return Expr(f.in_list(arg.expr, values, negated)) @@ -326,9 +326,9 @@ def concat(*args: Expr) -> Expr: def concat_ws(separator: str, *args: Expr) -> Expr: - """Concatenates the list `args` with the separator. + """Concatenates the list ``args`` with the separator. - `NULL` arugments are ignored. `separator` should not be `NULL`. + ``NULL`` arugments are ignored. ``separator`` should not be ``NULL``. """ args = [arg.expr for arg in args] return Expr(f.concat_ws(separator, args)) @@ -355,9 +355,11 @@ def count_star() -> Expr: def case(expr: Expr) -> CaseBuilder: - """Create a ``CaseBuilder`` to match cases for the expression ``expr``. + """Create a case expression. - See ``datafusion.expr.CaseBuilder`` for detailed usage of ``CaseBuilder``. + Create a :py:class:`~datafusion.expr.CaseBuilder` to match cases for the + expression ``expr``. See :py:class:`~datafusion.expr.CaseBuilder` for + detailed usage. """ return CaseBuilder(f.case(expr.expr)) @@ -462,12 +464,12 @@ def character_length(arg: Expr) -> Expr: def length(string: Expr) -> Expr: - """The number of characters in the `string`.""" + """The number of characters in the ``string``.""" return Expr(f.length(string.expr)) def char_length(string: Expr) -> Expr: - """The number of characters in the `string`.""" + """The number of characters in the ``string``.""" return Expr(f.char_length(string.expr)) @@ -477,7 +479,7 @@ def chr(arg: Expr) -> Expr: def coalesce(*args: Expr) -> Expr: - """Returns the value of the first expr in `args` which is not NULL.""" + """Returns the value of the first expr in ``args`` which is not NULL.""" args = [arg.expr for arg in args] return Expr(f.coalesce(*args)) @@ -503,7 +505,7 @@ def degrees(arg: Expr) -> Expr: def ends_with(arg: Expr, suffix: Expr) -> Expr: - """Returns true if the `string` ends with the `suffix`, false otherwise.""" + """Returns true if the ``string`` ends with the ``suffix``, false otherwise.""" return Expr(f.ends_with(arg.expr, suffix.expr)) @@ -521,9 +523,9 @@ def find_in_set(string: Expr, string_list: Expr) -> Expr: """Find a string in a list of strings. Returns a value in the range of 1 to N if the string is in the string list - `string_list` consisting of N substrings. + ``string_list`` consisting of N substrings. - The string list is a string composed of substrings separated by `,` characters. + The string list is a string composed of substrings separated by ``,`` characters. """ return Expr(f.find_in_set(string.expr, string_list.expr)) @@ -541,16 +543,16 @@ def gcd(x: Expr, y: Expr) -> Expr: def initcap(string: Expr) -> Expr: """Set the initial letter of each word to capital. - Converts the first letter of each word in `string` to uppercase and the remaining + Converts the first letter of each word in ``string`` to uppercase and the remaining characters to lowercase. """ return Expr(f.initcap(string.expr)) def instr(string: Expr, substring: Expr) -> Expr: - """Finds the position from where the `substring` matches the `string`. + """Finds the position from where the ``substring`` matches the ``string``. - This is an alias for :func:`strpos`. + This is an alias for :py:func:`strpos`. """ return strpos(string, substring) @@ -566,7 +568,7 @@ def lcm(x: Expr, y: Expr) -> Expr: def left(string: Expr, n: Expr) -> Expr: - """Returns the first `n` characters in the `string`.""" + """Returns the first ``n`` characters in the ``string``.""" return Expr(f.left(string.expr, n.expr)) @@ -581,7 +583,7 @@ def ln(arg: Expr) -> Expr: def log(base: Expr, num: Expr) -> Expr: - """Returns the logarithm of a number for a particular `base`.""" + """Returns the logarithm of a number for a particular ``base``.""" return Expr(f.log(base.expr, num.expr)) @@ -622,7 +624,7 @@ def md5(arg: Expr) -> Expr: def nanvl(x: Expr, y: Expr) -> Expr: - """Returns `x` if `x` is not `NaN`. Otherwise returns `y`.""" + """Returns ``x`` if ``x`` is not ``NaN``. Otherwise returns ``y``.""" return Expr(f.nanvl(x.expr, y.expr)) @@ -636,8 +638,8 @@ def overlay( ) -> Expr: """Replace a substring with a new substring. - Replace the substring of string that starts at the `start`'th character and - extends for `length` characters with new substring. + Replace the substring of string that starts at the ``start``'th character and + extends for ``length`` characters with new substring. """ if length is None: return Expr(f.overlay(string.expr, substring.expr, start.expr)) @@ -650,22 +652,22 @@ def pi() -> Expr: def position(string: Expr, substring: Expr) -> Expr: - """Finds the position from where the `substring` matches the `string`. + """Finds the position from where the ``substring`` matches the ``string``. - This is an alias for :func:`strpos`. + This is an alias for :py:func:`strpos`. """ return strpos(string, substring) def power(base: Expr, exponent: Expr) -> Expr: - """Returns `base` raised to the power of `exponent`.""" + """Returns ``base`` raised to the power of ``exponent``.""" return Expr(f.power(base.expr, exponent.expr)) def pow(base: Expr, exponent: Expr) -> Expr: - """Returns `base` raised to the power of `exponent`. + """Returns ``base`` raised to the power of ``exponent``. - This is an alias of `power`. + This is an alias of :py:func:`power`. """ return power(base, exponent) @@ -690,7 +692,7 @@ def regexp_match(string: Expr, regex: Expr, flags: Expr | None = None) -> Expr: """Perform regular expression (regex) matching. Returns an array with each element containing the leftmost-first match of the - corresponding index in `regex` to string in `string`. + corresponding index in ``regex`` to string in ``string``. """ if flags is not None: flags = flags.expr @@ -714,12 +716,12 @@ def regexp_replace( def repeat(string: Expr, n: Expr) -> Expr: - """Repeats the `string` to `n` times.""" + """Repeats the ``string`` to ``n`` times.""" return Expr(f.repeat(string.expr, n.expr)) def replace(string: Expr, from_val: Expr, to_val: Expr) -> Expr: - """Replaces all occurrences of `from` with `to` in the `string`.""" + """Replaces all occurrences of ``from_val`` with ``to_val`` in the ``string``.""" return Expr(f.replace(string.expr, from_val.expr, to_val.expr)) @@ -729,7 +731,7 @@ def reverse(arg: Expr) -> Expr: def right(string: Expr, n: Expr) -> Expr: - """Returns the last `n` characters in the `string`.""" + """Returns the last ``n`` characters in the ``string``.""" return Expr(f.right(string.expr, n.expr)) @@ -738,7 +740,7 @@ def round(value: Expr, decimal_places: Expr = Expr.literal(0)) -> Expr: If the optional ``decimal_places`` is specified, round to the nearest number of decimal places. You can specify a negative number of decimal places. For example - `round(lit(125.2345), lit(-2))` would yield a value of `100.0`. + ``round(lit(125.2345), lit(-2))`` would yield a value of ``100.0``. """ return Expr(f.round(value.expr, decimal_places.expr)) @@ -813,22 +815,26 @@ def starts_with(string: Expr, prefix: Expr) -> Expr: def strpos(string: Expr, substring: Expr) -> Expr: - """Finds the position from where the `substring` matches the `string`.""" + """Finds the position from where the ``substring`` matches the ``string``.""" return Expr(f.strpos(string.expr, substring.expr)) def substr(string: Expr, position: Expr) -> Expr: - """Substring from the `position` to the end.""" + """Substring from the ``position`` to the end.""" return Expr(f.substr(string.expr, position.expr)) def substr_index(string: Expr, delimiter: Expr, count: Expr) -> Expr: - """Returns the substring from `string` before `count` occurrences of `delimiter`.""" + """Returns an indexed substring. + + The return will be the ``string`` from before ``count`` occurrences of + ``delimiter``. + """ return Expr(f.substr_index(string.expr, delimiter.expr, count.expr)) def substring(string: Expr, position: Expr, length: Expr) -> Expr: - """Substring from the `position` with `length` characters.""" + """Substring from the ``position`` with ``length`` characters.""" return Expr(f.substring(string.expr, position.expr, length.expr)) @@ -856,7 +862,7 @@ def now() -> Expr: def to_timestamp(arg: Expr, *formatters: Expr) -> Expr: - """Converts a string and optional formats to a `Timestamp` in nanoseconds. + """Converts a string and optional formats to a ``Timestamp`` in nanoseconds. For usage of ``formatters`` see the rust chrono package ``strftime`` package. @@ -870,33 +876,33 @@ def to_timestamp(arg: Expr, *formatters: Expr) -> Expr: def to_timestamp_millis(arg: Expr, *formatters: Expr) -> Expr: - """Converts a string and optional formats to a `Timestamp` in milliseconds. + """Converts a string and optional formats to a ``Timestamp`` in milliseconds. - See `to_timestamp` for a description on how to use formatters. + See :py:func:`to_timestamp` for a description on how to use formatters. """ return Expr(f.to_timestamp_millis(arg.expr, *formatters)) def to_timestamp_micros(arg: Expr, *formatters: Expr) -> Expr: - """Converts a string and optional formats to a `Timestamp` in microseconds. + """Converts a string and optional formats to a ``Timestamp`` in microseconds. - See `to_timestamp` for a description on how to use formatters. + See :py:func:`to_timestamp` for a description on how to use formatters. """ return Expr(f.to_timestamp_micros(arg.expr, *formatters)) def to_timestamp_nanos(arg: Expr, *formatters: Expr) -> Expr: - """Converts a string and optional formats to a `Timestamp` in nanoseconds. + """Converts a string and optional formats to a ``Timestamp`` in nanoseconds. - See `to_timestamp` for a description on how to use formatters. + See :py:func:`to_timestamp` for a description on how to use formatters. """ return Expr(f.to_timestamp_nanos(arg.expr, *formatters)) def to_timestamp_seconds(arg: Expr, *formatters: Expr) -> Expr: - """Converts a string and optional formats to a `Timestamp` in seconds. + """Converts a string and optional formats to a ``Timestamp`` in seconds. - See `to_timestamp` for a description on how to use formatters. + See :py:func:`to_timestamp` for a description on how to use formatters. """ return Expr(f.to_timestamp_seconds(arg.expr, *formatters)) @@ -920,7 +926,7 @@ def current_time() -> Expr: def datepart(part: Expr, date: Expr) -> Expr: """Return a specified part of a date. - This is an alias for `date_part`. + This is an alias for :py:func:`date_part`. """ return date_part(part, date) @@ -938,7 +944,7 @@ def date_trunc(part: Expr, date: Expr) -> Expr: def datetrunc(part: Expr, date: Expr) -> Expr: """Truncates the date to a specified level of precision. - This is an alias for `date_trunc`. + This is an alias for :py:func:`date_trunc`. """ return date_trunc(part, date) @@ -954,7 +960,7 @@ def make_date(year: Expr, month: Expr, day: Expr) -> Expr: def translate(string: Expr, from_val: Expr, to_val: Expr) -> Expr: - """Replaces the characters in `from_val` with the counterpart in `to_val`.""" + """Replaces the characters in ``from_val`` with the counterpart in ``to_val``.""" return Expr(f.translate(string.expr, from_val.expr, to_val.expr)) @@ -984,7 +990,7 @@ def make_array(*args: Expr) -> Expr: def array(*args: Expr) -> Expr: """Returns an array using the specified input expressions. - This is an alias for `make_array`. + This is an alias for :py:func:`make_array`. """ return make_array(args) @@ -1025,7 +1031,7 @@ def arrow_typeof(arg: Expr) -> Expr: def random() -> Expr: - """Returns a random value in the range `0.0 <= x < 1.0`.""" + """Returns a random value in the range ``0.0 <= x < 1.0``.""" return Expr(f.random()) @@ -1037,7 +1043,7 @@ def array_append(array: Expr, element: Expr) -> Expr: def array_push_back(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. - This is an alias for `array_append`. + This is an alias for :py:func:`array_append`. """ return array_append(array, element) @@ -1045,7 +1051,7 @@ def array_push_back(array: Expr, element: Expr) -> Expr: def list_append(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. - This is an alias for `array_append`. + This is an alias for :py:func:`array_append`. """ return array_append(array, element) @@ -1053,7 +1059,7 @@ def list_append(array: Expr, element: Expr) -> Expr: def list_push_back(array: Expr, element: Expr) -> Expr: """Appends an element to the end of an array. - This is an alias for `array_append`. + This is an alias for :py:func:`array_append`. """ return array_append(array, element) @@ -1067,7 +1073,7 @@ def array_concat(*args: Expr) -> Expr: def array_cat(*args: Expr) -> Expr: """Concatenates the input arrays. - This is an alias for `array_concat`. + This is an alias for :py:func:`array_concat`. """ return array_concat(*args) @@ -1085,7 +1091,7 @@ def array_distinct(array: Expr) -> Expr: def list_distinct(array: Expr) -> Expr: """Returns distinct values from the array after removing duplicates. - This is an alias for `array_distinct`. + This is an alias for :py:func:`array_distinct`. """ return array_distinct(array) @@ -1093,7 +1099,7 @@ def list_distinct(array: Expr) -> Expr: def list_dims(array: Expr) -> Expr: """Returns an array of the array's dimensions. - This is an alias for `array_dims`. + This is an alias for :py:func:`array_dims`. """ return array_dims(array) @@ -1106,7 +1112,7 @@ def array_element(array: Expr, n: Expr) -> Expr: def array_extract(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. - This is an alias for `array_element`. + This is an alias for :py:func:`array_element`. """ return array_element(array, n) @@ -1114,7 +1120,7 @@ def array_extract(array: Expr, n: Expr) -> Expr: def list_element(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. - This is an alias for `array_element`. + This is an alias for :py:func:`array_element`. """ return array_element(array, n) @@ -1122,7 +1128,7 @@ def list_element(array: Expr, n: Expr) -> Expr: def list_extract(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. - This is an alias for `array_element`. + This is an alias for :py:func:`array_element`. """ return array_element(array, n) @@ -1135,7 +1141,7 @@ def array_length(array: Expr) -> Expr: def list_length(array: Expr) -> Expr: """Returns the length of the array. - This is an alias for `array_length`. + This is an alias for :py:func:`array_length`. """ return array_length(array) @@ -1171,7 +1177,7 @@ def array_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: def array_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. - This is an alias for `array_position`. + This is an alias for :py:func:`array_position`. """ return array_position(array, element, index) @@ -1179,7 +1185,7 @@ def array_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: def list_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. - This is an alias for `array_position`. + This is an alias for :py:func:`array_position`. """ return array_position(array, element, index) @@ -1187,7 +1193,7 @@ def list_position(array: Expr, element: Expr, index: int | None = 1) -> Expr: def list_indexof(array: Expr, element: Expr, index: int | None = 1) -> Expr: """Return the position of the first occurrence of ``element`` in ``array``. - This is an alias for `array_position`. + This is an alias for :py:func:`array_position`. """ return array_position(array, element, index) @@ -1200,7 +1206,7 @@ def array_positions(array: Expr, element: Expr) -> Expr: def list_positions(array: Expr, element: Expr) -> Expr: """Searches for an element in the array and returns all occurrences. - This is an alias for `array_positions`. + This is an alias for :py:func:`array_positions`. """ return array_positions(array, element) @@ -1213,7 +1219,7 @@ def array_ndims(array: Expr) -> Expr: def list_ndims(array: Expr) -> Expr: """Returns the number of dimensions of the array. - This is an alias for `array_ndims`. + This is an alias for :py:func:`array_ndims`. """ return array_ndims(array) @@ -1226,7 +1232,7 @@ def array_prepend(element: Expr, array: Expr) -> Expr: def array_push_front(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. - This is an alias for `array_prepend`. + This is an alias for :py:func:`array_prepend`. """ return array_prepend(element, array) @@ -1234,7 +1240,7 @@ def array_push_front(element: Expr, array: Expr) -> Expr: def list_prepend(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. - This is an alias for `array_prepend`. + This is an alias for :py:func:`array_prepend`. """ return array_prepend(element, array) @@ -1242,7 +1248,7 @@ def list_prepend(element: Expr, array: Expr) -> Expr: def list_push_front(element: Expr, array: Expr) -> Expr: """Prepends an element to the beginning of an array. - This is an alias for `array_prepend`. + This is an alias for :py:func:`array_prepend`. """ return array_prepend(element, array) @@ -1265,20 +1271,20 @@ def array_remove(array: Expr, element: Expr) -> Expr: def list_remove(array: Expr, element: Expr) -> Expr: """Removes the first element from the array equal to the given value. - This is an alias for `array_remove`. + This is an alias for :py:func:`array_remove`. """ return array_remove(array, element) def array_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: - """Removes the first `max` elements from the array equal to the given value.""" + """Removes the first ``max`` elements from the array equal to the given value.""" return Expr(f.array_remove_n(array.expr, element.expr, max.expr)) def list_remove_n(array: Expr, element: Expr, max: Expr) -> Expr: - """Removes the first `max` elements from the array equal to the given value. + """Removes the first ``max`` elements from the array equal to the given value. - This is an alias for `array_remove_n`. + This is an alias for :py:func:`array_remove_n`. """ return array_remove_n(array, element, max) @@ -1291,13 +1297,13 @@ def array_remove_all(array: Expr, element: Expr) -> Expr: def list_remove_all(array: Expr, element: Expr) -> Expr: """Removes all elements from the array equal to the given value. - This is an alias for `array_remove_all`. + This is an alias for :py:func:`array_remove_all`. """ return array_remove_all(array, element) def array_repeat(element: Expr, count: Expr) -> Expr: - """Returns an array containing `element` `count` times.""" + """Returns an array containing ``element`` ``count`` times.""" return Expr(f.array_repeat(element.expr, count.expr)) @@ -1309,27 +1315,27 @@ def array_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: def list_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces the first occurrence of ``from_val`` with ``to_val``. - This is an alias for `array_replace`. + This is an alias for :py:func:`array_replace`. """ return array_replace(array, from_val, to_val) def array_replace_n(array: Expr, from_val: Expr, to_val: Expr, max: Expr) -> Expr: - """Replace `n` occurrences of ``from_val`` with ``to_val``. + """Replace ``n`` occurrences of ``from_val`` with ``to_val``. - Replaces the first `max` occurrences of the specified element with another + Replaces the first ``max`` occurrences of the specified element with another specified element. """ return Expr(f.array_replace_n(array.expr, from_val.expr, to_val.expr, max.expr)) def list_replace_n(array: Expr, from_val: Expr, to_val: Expr, max: Expr) -> Expr: - """Replace `n` occurrences of ``from_val`` with ``to_val``. + """Replace ``n`` occurrences of ``from_val`` with ``to_val``. - Replaces the first `max` occurrences of the specified element with another + Replaces the first ``max`` occurrences of the specified element with another specified element. - This is an alias for `array_replace_n`. + This is an alias for :py:func:`array_replace_n`. """ return array_replace_n(array, from_val, to_val, max) @@ -1342,7 +1348,7 @@ def array_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: def list_replace_all(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces all occurrences of ``from_val`` with ``to_val``. - This is an alias for `array_replace_all`. + This is an alias for :py:func:`array_replace_all`. """ return array_replace_all(array, from_val, to_val) @@ -1365,7 +1371,7 @@ def array_sort(array: Expr, descending: bool = False, null_first: bool = False) def list_sort(array: Expr, descending: bool = False, null_first: bool = False) -> Expr: - """This is an alias for ``array_sort``.""" + """This is an alias for :py:func:`array_sort`.""" return array_sort(array, descending=descending, null_first=null_first) @@ -1381,20 +1387,20 @@ def array_slice( def list_slice(array: Expr, begin: Expr, end: Expr, stride: Expr | None = None) -> Expr: """Returns a slice of the array. - This is an alias for `array_slice`. + This is an alias for :py:func:`array_slice`. """ return array_slice(array, begin, end, stride) def array_intersect(array1: Expr, array2: Expr) -> Expr: - """Returns an array of the elements in the intersection of array1 and array2.""" + """Returns the intersection of ``array1`` and ``array2``.""" return Expr(f.array_intersect(array1.expr, array2.expr)) def list_intersect(array1: Expr, array2: Expr) -> Expr: - """Returns an array of the elements in the intersection of `array1` and `array2`. + """Returns an the intersection of ``array1`` and ``array2``. - This is an alias for `array_intersect`. + This is an alias for :py:func:`array_intersect`. """ return array_intersect(array1, array2) @@ -1412,20 +1418,20 @@ def list_union(array1: Expr, array2: Expr) -> Expr: Duplicate rows will not be returned. - This is an alias for `array_union`. + This is an alias for :py:func:`array_union`. """ return array_union(array1, array2) def array_except(array1: Expr, array2: Expr) -> Expr: - """Returns an array of the elements that appear in `array1` but not in `array2`.""" + """Returns the elements that appear in ``array1`` but not in ``array2``.""" return Expr(f.array_except(array1.expr, array2.expr)) def list_except(array1: Expr, array2: Expr) -> Expr: - """Returns an array of the elements that appear in `array1` but not in the `array2`. + """Returns the elements that appear in ``array1`` but not in the ``array2``. - This is an alias for `array_except`. + This is an alias for :py:func:`array_except`. """ return array_except(array1, array2) @@ -1433,8 +1439,8 @@ def list_except(array1: Expr, array2: Expr) -> Expr: def array_resize(array: Expr, size: Expr, value: Expr) -> Expr: """Returns an array with the specified size filled. - If `size` is greater than the `array` length, the additional entries will be filled - with the given `value`. + If ``size`` is greater than the ``array`` length, the additional entries will + be filled with the given ``value``. """ return Expr(f.array_resize(array.expr, size.expr, value.expr)) @@ -1442,8 +1448,8 @@ def array_resize(array: Expr, size: Expr, value: Expr) -> Expr: def list_resize(array: Expr, size: Expr, value: Expr) -> Expr: """Returns an array with the specified size filled. - If `size` is greater than the `array` length, the additional entries will be - filled with the given `value`. This is an alias for `array_resize`. + If ``size`` is greater than the ``array`` length, the additional entries will be + filled with the given ``value``. This is an alias for :py:func:`array_resize`. """ return array_resize(array, size, value) @@ -1489,7 +1495,7 @@ def approx_percentile_cont_with_weight( ) -> Expr: """Returns the value of the approximate percentile. - This function is similar to ``approx_percentile_cont`` except that it uses + This function is similar to :py:func:`approx_percentile_cont` except that it uses the associated associated weights. """ return Expr( @@ -1510,7 +1516,7 @@ def avg(arg: Expr, distinct: bool = False) -> Expr: def corr(value1: Expr, value2: Expr, distinct: bool = False) -> Expr: - """Returns the correlation coefficient between `value1` and `value2`.""" + """Returns the correlation coefficient between ``value1`` and ``value2``.""" return Expr(f.corr(value1.expr, value2.expr, distinct=distinct)) @@ -1528,7 +1534,7 @@ def count(args: Expr | list[Expr] | None = None, distinct: bool = False) -> Expr def covar(y: Expr, x: Expr) -> Expr: """Computes the sample covariance. - This is an alias for `covar_samp`. + This is an alias for :py:func:`covar_samp`. """ return covar_samp(y, x) @@ -1559,7 +1565,7 @@ def max(arg: Expr, distinct: bool = False) -> Expr: def mean(arg: Expr, distinct: bool = False) -> Expr: """Returns the average (mean) value of the argument. - This is an alias for `avg`. + This is an alias for :py:func:`avg`. """ return avg(arg, distinct) @@ -1592,7 +1598,7 @@ def stddev_pop(arg: Expr, distinct: bool = False) -> Expr: def stddev_samp(arg: Expr, distinct: bool = False) -> Expr: """Computes the sample standard deviation of the argument. - This is an alias for `stddev`. + This is an alias for :py:func:`stddev`. """ return stddev(arg, distinct) @@ -1600,7 +1606,7 @@ def stddev_samp(arg: Expr, distinct: bool = False) -> Expr: def var(arg: Expr) -> Expr: """Computes the sample variance of the argument. - This is an alias for `var_samp`. + This is an alias for :py:func:`var_samp`. """ return var_samp(arg) @@ -1616,7 +1622,7 @@ def var_samp(arg: Expr) -> Expr: def regr_avgx(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the average of the independent variable `x`. + """Computes the average of the independent variable ``x``. Only non-null pairs of the inputs are evaluated. """ @@ -1652,7 +1658,7 @@ def regr_slope(y: Expr, x: Expr, distinct: bool = False) -> Expr: def regr_sxx(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the sum of squares of the independent variable `x`.""" + """Computes the sum of squares of the independent variable ``x``.""" return Expr(f.regr_sxx(y.expr, x.expr, distinct)) @@ -1662,7 +1668,7 @@ def regr_sxy(y: Expr, x: Expr, distinct: bool = False) -> Expr: def regr_syy(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the sum of squares of the dependent variable `y`.""" + """Computes the sum of squares of the dependent variable ``y``.""" return Expr(f.regr_syy(y.expr, x.expr, distinct)) diff --git a/python/datafusion/record_batch.py b/python/datafusion/record_batch.py index dcfd55485..44936f7d8 100644 --- a/python/datafusion/record_batch.py +++ b/python/datafusion/record_batch.py @@ -17,7 +17,8 @@ """This module provides the classes for handling record batches. -These are typically the result of dataframe `execute_stream` operations. +These are typically the result of dataframe +:py:func:`datafusion.dataframe.execute_stream` operations. """ from __future__ import annotations @@ -31,24 +32,25 @@ class RecordBatch: - """This class is essentially a wrapper for ``pyarrow.RecordBatch``.""" + """This class is essentially a wrapper for :py:class:`pyarrow.RecordBatch`.""" def __init__(self, record_batch: df_internal.RecordBatch) -> None: """This constructor is generally not called by the end user. - See the ``RecordBatchStream`` iterator for generating this class. + See the :py:class:`RecordBatchStream` iterator for generating this class. """ self.record_batch = record_batch def to_pyarrow(self) -> pyarrow.RecordBatch: - """Convert to pyarrow ``RecordBatch``.""" + """Convert to :py:class:`pyarrow.RecordBatch`.""" return self.record_batch.to_pyarrow() class RecordBatchStream: """This class represents a stream of record batches. - These are typically the result of a ``DataFrame::execute_stream`` operation. + These are typically the result of a + :py:func:`~datafusion.dataframe.DataFrame.execute_stream` operation. """ def __init__(self, record_batch_stream: df_internal.RecordBatchStream) -> None: @@ -56,7 +58,7 @@ def __init__(self, record_batch_stream: df_internal.RecordBatchStream) -> None: self.rbs = record_batch_stream def next(self) -> RecordBatch | None: - """See ``__next__`` for the iterator function.""" + """See :py:func:`__next__` for the iterator function.""" try: next_batch = next(self) except StopIteration: diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index 4b44ad19b..0cdd19a51 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -48,7 +48,8 @@ def __init__(self, plan: substrait_internal.Plan) -> None: """Create a substrait plan. The user should not have to call this constructor directly. Rather, it - should be created via `Serde` or `Producer` classes in this module. + should be created via :py:class:`Serde` or py:class:`Producer` classes + in this module. """ self.plan_internal = plan diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index 4bfbabe69..12563b3d2 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -35,7 +35,7 @@ class Volatility(Enum): """Defines how stable or volatile a function is. When setting the volatility of a function, you can either pass this - enumeration or a `str`. The `str` equivalent is the lower case value of the + enumeration or a ``str``. The ``str`` equivalent is the lower case value of the name (`"immutable"`, `"stable"`, or `"volatile"`). """ @@ -52,9 +52,9 @@ class Volatility(Enum): A stable function may return different values given the same input across different queries but must return the same value for a given input within a - query. An example of this is the `Now` function. DataFusion will attempt to - inline `Stable` functions during planning, when possible. For query - `select col1, now() from t1`, it might take a while to execute but `now()` + query. An example of this is the ``Now`` function. DataFusion will attempt to + inline ``Stable`` functions during planning, when possible. For query + ``select col1, now() from t1``, it might take a while to execute but ``now()`` column will be the same for each output row, which is evaluated during planning. """ @@ -66,7 +66,7 @@ class Volatility(Enum): Multiple invocations of a volatile function may return different results when used in the same query. An example of this is the random() function. DataFusion can not evaluate such functions during planning. In the query - `select col1, random() from t1`, `random()` function will be evaluated + ``select col1, random() from t1``, ``random()`` function will be evaluated for each output row, resulting in a unique random value for each row. """ @@ -78,7 +78,7 @@ def __str__(self): class ScalarUDF: """Class for performing scalar user defined functions (UDF). - Scalar UDFs operate on a row by row basis. See also ``AggregateUDF`` for + Scalar UDFs operate on a row by row basis. See also :py:class:`AggregateUDF` for operating on a group of rows. """ @@ -92,9 +92,9 @@ def __init__( ) -> None: """Instantiate a scalar user defined function (UDF). - See helper method ``udf`` for argument details. + See helper method :py:func:`udf` for argument details. """ - self.udf = df_internal.ScalarUDF( + self._udf = df_internal.ScalarUDF( name, func, input_types, return_type, str(volatility) ) @@ -105,7 +105,7 @@ def __call__(self, *args: Expr) -> Expr: occur during the evaluation of the dataframe. """ args = [arg.expr for arg in args] - return Expr(self.udf.__call__(*args)) + return Expr(self._udf.__call__(*args)) @staticmethod def udf( @@ -119,7 +119,7 @@ def udf( Args: func: A callable python function. - input_types: The data types of the arguments to `func`. This list + input_types: The data types of the arguments to ``func``. This list must be of the same length as the number of arguments. return_type: The data type of the return value from the python function. @@ -144,7 +144,7 @@ def udf( class Accumulator(metaclass=ABCMeta): - """Defines how an `AggregateUDF` accumulates values during an evaluation.""" + """Defines how an :py:class:`AggregateUDF` accumulates values.""" @abstractmethod def state(self) -> List[pyarrow.Scalar]: @@ -175,7 +175,7 @@ class AggregateUDF: """Class for performing scalar user defined functions (UDF). Aggregate UDFs operate on a group of rows and return a single value. See - also ``ScalarUDF`` for operating on a row by row basis. + also :py:class:`ScalarUDF` for operating on a row by row basis. """ def __init__( @@ -189,10 +189,10 @@ def __init__( ) -> None: """Instantiate a user defined aggregate function (UDAF). - See ``Aggregate::udaf`` for a convenience function and arugment + See :py:func:`udaf` for a convenience function and arugment descriptions. """ - self.udf = df_internal.AggregateUDF( + self._udf = df_internal.AggregateUDF( name, accumulator, input_types, return_type, state_type, str(volatility) ) @@ -203,7 +203,7 @@ def __call__(self, *args: Expr) -> Expr: occur during the evaluation of the dataframe. """ args = [arg.expr for arg in args] - return Expr(self.udf.__call__(*args)) + return Expr(self._udf.__call__(*args)) @staticmethod def udaf( @@ -216,14 +216,14 @@ def udaf( ) -> AggregateUDF: """Create a new User Defined Aggregate Function. - The accumulator function must be callable and implement `Accumulator`. + The accumulator function must be callable and implement :py:class:`Accumulator`. Args: accum: The accumulator python function. - input_types: The data types of the arguments to `accum`. + input_types: The data types of the arguments to ``accum``. return_type: The data type of the return value. state_type: The data types of the intermediate accumulation. - volatility: See `Volatility` for allowed values. + volatility: See :py:class:`Volatility` for allowed values. name: A descriptive name for the function. Returns: From 2205e058f01f4f577846833465258bfab12b8d15 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 9 Aug 2024 08:22:45 -0500 Subject: [PATCH 176/413] Re-Enable `num_centroids` to `approx_percentile_cont` (#798) --- Cargo.lock | 164 +++++++++++--------- Cargo.toml | 10 +- python/datafusion/functions.py | 7 +- python/datafusion/tests/test_aggregation.py | 133 ++++++++-------- src/functions.rs | 10 +- 5 files changed, 172 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c41ef771a..e59811210 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,9 +130,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6127ea5e585a12ec9f742232442828ebaf264dfa5eefdd71282376c599562b77" +checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add7f39210b7d726e2a8efc0083e7bf06e8f2d15bdb4896b564dce4410fbf5d" +checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81c16ec702d3898c2f5cfdc148443c6cd7dbe5bac28399859eb0a3d38f072827" +checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae6970bab043c4fbc10aee1660ceb5b306d0c42c8cc5f6ae564efcd9759b663" +checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" dependencies = [ "bytes", "half", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c7ef44f26ef4f8edc392a048324ed5d757ad09135eff6d5509e6450d39e0398" +checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f843490bd258c5182b66e888161bb6f198f49f3792f7c7f98198b924ae0f564" +checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a769666ffac256dd301006faca1ca553d0ae7cffcf4cd07095f73f95eb226514" +checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" dependencies = [ "arrow-buffer", "arrow-schema", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf9c3fb57390a1af0b7bb3b5558c1ee1f63905f3eccf49ae7676a8d1e6e5a72" +checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "654e7f3724176b66ddfacba31af397c48e106fbe4d281c8144e7d237df5acfd7" +checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" dependencies = [ "arrow-array", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8008370e624e8e3c68174faaf793540287106cfda8ad1da862fdc53d8e096b4" +checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5e3a6b7fda8d9fe03f3b18a2d946354ea7f3c8e4076dbdb502ad50d9d44824" +checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" dependencies = [ "ahash", "arrow-array", @@ -307,23 +307,22 @@ dependencies = [ "arrow-data", "arrow-schema", "half", - "hashbrown", ] [[package]] name = "arrow-schema" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab1c12b40e29d9f3b699e0203c2a73ba558444c05e388a4377208f8f9c97eee" +checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e80159088ffe8c48965cb9b1a7c968b2729f29f37363df7eca177fc3281fe7c3" +checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" dependencies = [ "ahash", "arrow-array", @@ -335,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd04a6ea7de183648edbcb7a6dd925bbd04c210895f6384c780e27a9b54afcd" +checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" dependencies = [ "arrow-array", "arrow-buffer", @@ -365,7 +364,7 @@ dependencies = [ "tokio", "xz2", "zstd 0.13.2", - "zstd-safe 7.2.0", + "zstd-safe 7.2.1", ] [[package]] @@ -516,9 +515,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "bzip2" @@ -543,9 +542,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" dependencies = [ "jobserver", "libc", @@ -1163,9 +1162,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" dependencies = [ "crc32fast", "miniz_oxide", @@ -1458,9 +1457,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", @@ -1511,9 +1510,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", "hashbrown", @@ -1995,9 +1994,9 @@ dependencies = [ [[package]] name = "parquet" -version = "52.1.0" +version = "52.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f22ba0d95db56dde8685e3fadcb915cdaadda31ab8abbe3ff7f0ad1ef333267" +checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" dependencies = [ "ahash", "arrow-array", @@ -2181,9 +2180,12 @@ checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "prettyplease" @@ -2347,9 +2349,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ "bytes", "pin-project-lite", @@ -2357,6 +2359,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", + "socket2", "thiserror", "tokio", "tracing", @@ -2364,9 +2367,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ "bytes", "rand", @@ -2388,6 +2391,7 @@ dependencies = [ "libc", "once_cell", "socket2", + "tracing", "windows-sys 0.52.0", ] @@ -2441,9 +2445,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -2558,9 +2562,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" @@ -2613,9 +2617,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ "base64 0.22.1", "rustls-pki-types", @@ -2623,9 +2627,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -2769,11 +2773,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -3020,18 +3025,19 @@ checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" [[package]] name = "target-lexicon" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", "windows-sys 0.52.0", ] @@ -3093,9 +3099,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", @@ -3363,9 +3369,9 @@ dependencies = [ [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "walkdir" @@ -3483,11 +3489,11 @@ dependencies = [ [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3517,6 +3523,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -3663,6 +3678,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -3698,7 +3714,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.2.0", + "zstd-safe 7.2.1", ] [[package]] @@ -3713,18 +3729,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.11+zstd.1.5.6" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index d05a617a3..820118fa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ protoc = [ "datafusion-substrait/protoc" ] substrait = ["dep:datafusion-substrait"] [dependencies] -tokio = { version = "1.35", features = ["macros", "rt", "rt-multi-thread", "sync"] } +tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "52", feature = ["pyarrow"] } @@ -45,17 +45,17 @@ datafusion-functions-array = "40.0.0" datafusion-optimizer = "40.0.0" datafusion-sql = "40.0.0" datafusion-substrait = { version = "40.0.0", optional = true } -prost = "0.12" -prost-types = "0.12" +prost = "0.12" # keep in line with `datafusion-substrait` +prost-types = "0.12" # keep in line with `datafusion-substrait` uuid = { version = "1.9", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" -regex-syntax = "0.8.1" +regex-syntax = "0.8" syn = "2.0.68" -url = "2.2" +url = "2" sqlparser = "0.47.0" [build-dependencies] diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 82b5056d7..2d3d87ee0 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -1473,19 +1473,18 @@ def approx_median(arg: Expr, distinct: bool = False) -> Expr: def approx_percentile_cont( expression: Expr, percentile: Expr, + num_centroids: Expr | None = None, distinct: bool = False, ) -> Expr: """Returns the value that is approximately at a given percentile of ``expr``.""" - # Re-enable num_centroids: https://github.com/apache/datafusion-python/issues/777 - num_centroids = None if num_centroids is None: return Expr( - f.approx_percentile_cont(expression.expr, percentile.expr, distinct=distinct) + f.approx_percentile_cont(expression.expr, percentile.expr, distinct=distinct, num_centroids=None) ) return Expr( f.approx_percentile_cont( - expression.expr, percentile.expr, distinct=distinct + expression.expr, percentile.expr, distinct=distinct, num_centroids=num_centroids.expr ) ) diff --git a/python/datafusion/tests/test_aggregation.py b/python/datafusion/tests/test_aggregation.py index c10e5f36c..03485da4b 100644 --- a/python/datafusion/tests/test_aggregation.py +++ b/python/datafusion/tests/test_aggregation.py @@ -39,79 +39,78 @@ def df(): ) return ctx.create_dataframe([[batch]]) +@pytest.fixture +def df_aggregate_100(): + ctx = SessionContext() + ctx.register_csv("aggregate_test_data", "./testing/data/csv/aggregate_test_100.csv") + return ctx.table("aggregate_test_data") -def test_built_in_aggregation(df): - col_a = column("a") - col_b = column("b") - col_c = column("c") - agg_df = df.aggregate( - [], - [ - f.approx_distinct(col_b), - f.approx_median(col_b), - f.approx_percentile_cont(col_b, lit(0.5)), - f.approx_percentile_cont_with_weight(col_b, lit(0.6), lit(0.5)), - f.array_agg(col_b), - f.avg(col_a), - f.corr(col_a, col_b), - f.count(col_a), - f.covar(col_a, col_b), - f.covar_pop(col_a, col_c), - f.covar_samp(col_b, col_c), - # f.grouping(col_a), # No physical plan implemented yet - f.max(col_a), - f.mean(col_b), - f.median(col_b), - f.min(col_a), - f.sum(col_b), - f.stddev(col_a), - f.stddev_pop(col_b), - f.stddev_samp(col_c), - f.var(col_a), - f.var_pop(col_b), - f.var_samp(col_c), - ], - ) +@pytest.mark.parametrize("agg_expr, calc_expected", [ + (f.avg(column("a")), lambda a, b, c, d: np.array(np.average(a))), + (f.corr(column("a"), column("b")), lambda a, b, c, d: np.array(np.corrcoef(a, b)[0][1])), + (f.count(column("a")), lambda a, b, c, d: pa.array([len(a)])), + # Sample (co)variance -> ddof=1 + # Population (co)variance -> ddof=0 + (f.covar(column("a"), column("b")), lambda a, b, c, d: np.array(np.cov(a, b, ddof=1)[0][1])), + (f.covar_pop(column("a"), column("c")), lambda a, b, c, d: np.array(np.cov(a, c, ddof=0)[0][1])), + (f.covar_samp(column("b"), column("c")), lambda a, b, c, d: np.array(np.cov(b, c, ddof=1)[0][1])), + # f.grouping(col_a), # No physical plan implemented yet + (f.max(column("a")), lambda a, b, c, d: np.array(np.max(a))), + (f.mean(column("b")), lambda a, b, c, d: np.array(np.mean(b))), + (f.median(column("b")), lambda a, b, c, d: np.array(np.median(b))), + (f.min(column("a")), lambda a, b, c, d: np.array(np.min(a))), + (f.sum(column("b")), lambda a, b, c, d: np.array(np.sum(b.to_pylist()))), + # Sample stdev -> ddof=1 + # Population stdev -> ddof=0 + (f.stddev(column("a")), lambda a, b, c, d: np.array(np.std(a, ddof=1))), + (f.stddev_pop(column("b")), lambda a, b, c, d: np.array(np.std(b, ddof=0))), + (f.stddev_samp(column("c")), lambda a, b, c, d: np.array(np.std(c, ddof=1))), + (f.var(column("a")), lambda a, b, c, d: np.array(np.var(a, ddof=1))), + (f.var_pop(column("b")), lambda a, b, c, d: np.array(np.var(b, ddof=0))), + (f.var_samp(column("c")), lambda a, b, c, d: np.array(np.var(c, ddof=1))), +]) +def test_aggregation_stats(df, agg_expr, calc_expected): + + agg_df = df.aggregate([], [agg_expr]) result = agg_df.collect()[0] values_a, values_b, values_c, values_d = df.collect()[0] + expected = calc_expected(values_a, values_b, values_c, values_d) + np.testing.assert_array_almost_equal(result.column(0), expected) + + +@pytest.mark.parametrize("agg_expr, expected", [ + (f.approx_distinct(column("b")), pa.array([2], type=pa.uint64())), + (f.approx_median(column("b")), pa.array([4])), + (f.approx_percentile_cont(column("b"), lit(0.5)), pa.array([4])), + ( + f.approx_percentile_cont_with_weight(column("b"), lit(0.6), lit(0.5)), + pa.array([6], type=pa.float64()) + ), + (f.array_agg(column("b")), pa.array([[4, 4, 6]])), +]) +def test_aggregation(df, agg_expr, expected): + agg_df = df.aggregate([], [agg_expr]) + result = agg_df.collect()[0] + assert result.column(0) == expected - assert result.column(0) == pa.array([2], type=pa.uint64()) - assert result.column(1) == pa.array([4]) - assert result.column(2) == pa.array([4]) - # Ref: https://github.com/apache/datafusion-python/issues/777 - # assert result.column(3) == pa.array([6]) - assert result.column(4) == pa.array([[4, 4, 6]]) - np.testing.assert_array_almost_equal(result.column(5), np.average(values_a)) - np.testing.assert_array_almost_equal( - result.column(6), np.corrcoef(values_a, values_b)[0][1] - ) - assert result.column(7) == pa.array([len(values_a)]) - # Sample (co)variance -> ddof=1 - # Population (co)variance -> ddof=0 - np.testing.assert_array_almost_equal( - result.column(8), np.cov(values_a, values_b, ddof=1)[0][1] - ) - np.testing.assert_array_almost_equal( - result.column(9), np.cov(values_a, values_c, ddof=0)[0][1] - ) - np.testing.assert_array_almost_equal( - result.column(10), np.cov(values_b, values_c, ddof=1)[0][1] - ) - np.testing.assert_array_almost_equal(result.column(11), np.max(values_a)) - np.testing.assert_array_almost_equal(result.column(12), np.mean(values_b)) - np.testing.assert_array_almost_equal(result.column(13), np.median(values_b)) - np.testing.assert_array_almost_equal(result.column(14), np.min(values_a)) - np.testing.assert_array_almost_equal( - result.column(15), np.sum(values_b.to_pylist()) - ) - np.testing.assert_array_almost_equal(result.column(16), np.std(values_a, ddof=1)) - np.testing.assert_array_almost_equal(result.column(17), np.std(values_b, ddof=0)) - np.testing.assert_array_almost_equal(result.column(18), np.std(values_c, ddof=1)) - np.testing.assert_array_almost_equal(result.column(19), np.var(values_a, ddof=1)) - np.testing.assert_array_almost_equal(result.column(20), np.var(values_b, ddof=0)) - np.testing.assert_array_almost_equal(result.column(21), np.var(values_c, ddof=1)) +def test_aggregate_100(df_aggregate_100): + # https://github.com/apache/datafusion/blob/bddb6415a50746d2803dd908d19c3758952d74f9/datafusion/sqllogictest/test_files/aggregate.slt#L1490-L1498 + + result = df_aggregate_100.aggregate( + [ + column("c1") + ], + [ + f.approx_percentile_cont(column("c3"), lit(0.95), lit(200)).alias("c3") + ] + ).sort(column("c1").sort(ascending=True)).collect() + + assert len(result) == 1 + result = result[0] + assert result.column("c1") == pa.array(["a", "b", "c", "d", "e"]) + assert result.column("c3") == pa.array([73, 68, 122, 124, 115]) def test_bit_add_or_xor(df): df = df.aggregate( diff --git a/src/functions.rs b/src/functions.rs index e60c63c8e..f8f478166 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -57,9 +57,15 @@ pub fn approx_percentile_cont( expression: PyExpr, percentile: PyExpr, distinct: bool, + num_centroids: Option, // enforces optional arguments at the end, currently ) -> PyResult { - let expr = - functions_aggregate::expr_fn::approx_percentile_cont(expression.expr, percentile.expr); + let args = if let Some(num_centroids) = num_centroids { + vec![expression.expr, percentile.expr, num_centroids.expr] + } else { + vec![expression.expr, percentile.expr] + }; + let udaf = functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf(); + let expr = udaf.call(args); if distinct { Ok(expr.distinct().build()?.into()) } else { From b6f06f760f8f4048a91538cbde15a4ab813f62ca Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 9 Aug 2024 09:23:09 -0400 Subject: [PATCH 177/413] UDAF process all state variables (#799) --- .../common-operations/udf-and-udfa.rst | 5 +++-- python/datafusion/tests/test_udaf.py | 4 ++-- python/datafusion/udf.py | 2 +- src/udaf.rs | 19 +++++++++++-------- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/docs/source/user-guide/common-operations/udf-and-udfa.rst b/docs/source/user-guide/common-operations/udf-and-udfa.rst index 54c685794..e9c142f0a 100644 --- a/docs/source/user-guide/common-operations/udf-and-udfa.rst +++ b/docs/source/user-guide/common-operations/udf-and-udfa.rst @@ -50,6 +50,7 @@ Additionally the :py:func:`~datafusion.udf.AggregateUDF.udaf` function allows yo import pyarrow.compute import datafusion from datafusion import col, udaf, Accumulator + from typing import List class MyAccumulator(Accumulator): """ @@ -62,9 +63,9 @@ Additionally the :py:func:`~datafusion.udf.AggregateUDF.udaf` function allows yo # not nice since pyarrow scalars can't be summed yet. This breaks on `None` self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(values).as_py()) - def merge(self, states: pyarrow.Array) -> None: + def merge(self, states: List[pyarrow.Array]) -> None: # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(states).as_py()) + self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(states[0]).as_py()) def state(self) -> pyarrow.Array: return pyarrow.array([self._sum.as_py()]) diff --git a/python/datafusion/tests/test_udaf.py b/python/datafusion/tests/test_udaf.py index 81194927c..76488e19b 100644 --- a/python/datafusion/tests/test_udaf.py +++ b/python/datafusion/tests/test_udaf.py @@ -38,10 +38,10 @@ def update(self, values: pa.Array) -> None: # This breaks on `None` self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py()) - def merge(self, states: pa.Array) -> None: + def merge(self, states: List[pa.Array]) -> None: # Not nice since pyarrow scalars can't be summed yet. # This breaks on `None` - self._sum = pa.scalar(self._sum.as_py() + pc.sum(states).as_py()) + self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py()) def evaluate(self) -> pa.Scalar: return self._sum diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index 12563b3d2..bdbad661a 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -157,7 +157,7 @@ def update(self, values: pyarrow.Array) -> None: pass @abstractmethod - def merge(self, states: pyarrow.Array) -> None: + def merge(self, states: List[pyarrow.Array]) -> None: """Merge a set of states.""" pass diff --git a/src/udaf.rs b/src/udaf.rs index 7b5e03668..2041e5a74 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -72,18 +72,21 @@ impl Accumulator for RustAccumulator { fn merge_batch(&mut self, states: &[ArrayRef]) -> Result<()> { Python::with_gil(|py| { - let state = &states[0]; - - // 1. cast states to Pyarrow array - let state = state - .into_data() - .to_pyarrow(py) - .map_err(|e| DataFusionError::Execution(format!("{e}")))?; + // // 1. cast states to Pyarrow arrays + let py_states: Result> = states + .iter() + .map(|state| { + state + .into_data() + .to_pyarrow(py) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) + }) + .collect(); // 2. call merge self.accum .bind(py) - .call_method1("merge", (state,)) + .call_method1("merge", (py_states?,)) .map_err(|e| DataFusionError::Execution(format!("{e}")))?; Ok(()) From e206a0cbfa54e13fa9008d7217c1843ab80cb788 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 9 Aug 2024 08:22:02 -0600 Subject: [PATCH 178/413] chore: Prepare for 40.0.0 release (#801) * use new changelog script from DataFusion * Update changelog --- CHANGELOG.md | 35 ++++++++++ dev/release/README.md | 2 +- dev/release/generate-changelog.py | 111 +++++++++++++++++++++--------- requirements.in | 1 + 4 files changed, 115 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32a74d825..305af5720 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,41 @@ # DataFusion Python Changelog +## [40.0.0](https://github.com/apache/datafusion-python/tree/40.0.0) (2024-08-09) + +This release consists of 18 commits from 4 contributors. See credits at the end of this changelog for more information. + +- Update changelog for 39.0.0 [#742](https://github.com/apache/datafusion-python/pull/742) (andygrove) +- build(deps): bump uuid from 1.8.0 to 1.9.1 [#744](https://github.com/apache/datafusion-python/pull/744) (dependabot[bot]) +- build(deps): bump mimalloc from 0.1.42 to 0.1.43 [#745](https://github.com/apache/datafusion-python/pull/745) (dependabot[bot]) +- build(deps): bump syn from 2.0.67 to 2.0.68 [#746](https://github.com/apache/datafusion-python/pull/746) (dependabot[bot]) +- Tsaucer/find window fn [#747](https://github.com/apache/datafusion-python/pull/747) (timsaucer) +- Python wrapper classes for all user interfaces [#750](https://github.com/apache/datafusion-python/pull/750) (timsaucer) +- Expose array sort [#764](https://github.com/apache/datafusion-python/pull/764) (timsaucer) +- Upgrade protobuf and remove GH Action googletest-installer [#773](https://github.com/apache/datafusion-python/pull/773) (Michael-J-Ward) +- Upgrade Datafusion 40 [#771](https://github.com/apache/datafusion-python/pull/771) (Michael-J-Ward) +- Bugfix: Calling count with None arguments [#768](https://github.com/apache/datafusion-python/pull/768) (timsaucer) +- Add in user example that compares a two different approaches to UDFs [#770](https://github.com/apache/datafusion-python/pull/770) (timsaucer) +- Add missing exports for wrapper modules [#782](https://github.com/apache/datafusion-python/pull/782) (timsaucer) +- Add PyExpr to_variant conversions [#793](https://github.com/apache/datafusion-python/pull/793) (Michael-J-Ward) +- Add missing expressions to wrapper export [#795](https://github.com/apache/datafusion-python/pull/795) (timsaucer) +- Doc/cross reference [#791](https://github.com/apache/datafusion-python/pull/791) (timsaucer) +- Re-Enable `num_centroids` to `approx_percentile_cont` [#798](https://github.com/apache/datafusion-python/pull/798) (Michael-J-Ward) +- UDAF process all state variables [#799](https://github.com/apache/datafusion-python/pull/799) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 9 Tim Saucer + 4 Michael J Ward + 3 dependabot[bot] + 2 Andy Grove +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + ## [39.0.0](https://github.com/apache/datafusion-python/tree/39.0.0) (2024-06-25) **Merged pull requests:** diff --git a/dev/release/README.md b/dev/release/README.md index c4372c832..93c2f97b9 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -63,7 +63,7 @@ We maintain a `CHANGELOG.md` so our users know what has been changed between rel The changelog is generated using a Python script: ```bash -$ GITHUB_TOKEN= ./dev/release/generate-changelog.py apache/datafusion-python 24.0.0 HEAD > dev/changelog/25.0.0.md +$ GITHUB_TOKEN= ./dev/release/generate-changelog.py 24.0.0 HEAD 25.0.0 > dev/changelog/25.0.0.md ``` This script creates a changelog from GitHub PRs based on the labels associated with them as well as looking for diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index af097ce98..5645d2f74 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -20,23 +20,20 @@ from github import Github import os import re - +import subprocess def print_pulls(repo_name, title, pulls): - if len(pulls) > 0: + if len(pulls) > 0: print("**{}:**".format(title)) print() - for pull, commit in pulls: + for (pull, commit) in pulls: url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) - print( - "- {} [#{}]({}) ({})".format( - pull.title, pull.number, url, commit.author.login - ) - ) + print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) print() -def generate_changelog(repo, repo_name, tag1, tag2): +def generate_changelog(repo, repo_name, tag1, tag2, version): + # get a list of commits between two tags print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) comparison = repo.compare(tag1, tag2) @@ -55,45 +52,95 @@ def generate_changelog(repo, repo_name, tag1, tag2): all_pulls.append((pull, commit)) # we split the pulls into categories - # TODO: make categories configurable breaking = [] bugs = [] docs = [] enhancements = [] + performance = [] + other = [] # categorize the pull requests based on GitHub labels print("Categorizing pull requests", file=sys.stderr) - for pull, commit in all_pulls: + for (pull, commit) in all_pulls: + # see if PR title uses Conventional Commits - cc_type = "" - # cc_scope = '' - cc_breaking = "" - parts = re.findall(r"^([a-z]+)(\([a-z]+\))?(!)?:", pull.title) + cc_type = '' + cc_scope = '' + cc_breaking = '' + parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) if len(parts) == 1: parts_tuple = parts[0] - cc_type = parts_tuple[0] # fix, feat, docs, chore - # cc_scope = parts_tuple[1] # component within project - cc_breaking = parts_tuple[2] == "!" + cc_type = parts_tuple[0] # fix, feat, docs, chore + cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == '!' labels = [label.name for label in pull.labels] - # print(pull.number, labels, parts, file=sys.stderr) - if "api change" in labels or cc_breaking: + if 'api change' in labels or cc_breaking: breaking.append((pull, commit)) - elif "bug" in labels or cc_type == "fix": + elif 'bug' in labels or cc_type == 'fix': bugs.append((pull, commit)) - elif "enhancement" in labels or cc_type == "feat": + elif 'performance' in labels or cc_type == 'perf': + performance.append((pull, commit)) + elif 'enhancement' in labels or cc_type == 'feat': enhancements.append((pull, commit)) - elif "documentation" in labels or cc_type == "docs": + elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc': docs.append((pull, commit)) + else: + other.append((pull, commit)) # produce the changelog content print("Generating changelog content", file=sys.stderr) + + # ASF header + print("""\n""") + + print(f"# Apache DataFusion Python {version} Changelog\n") + + # get the number of commits + commit_count = subprocess.check_output(f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + # get number of contributors + contributor_count = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + + print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " + f"See credits at the end of this changelog for more information.\n") + print_pulls(repo_name, "Breaking changes", breaking) + print_pulls(repo_name, "Performance related", performance) print_pulls(repo_name, "Implemented enhancements", enhancements) print_pulls(repo_name, "Fixed bugs", bugs) print_pulls(repo_name, "Documentation updates", docs) - print_pulls(repo_name, "Merged pull requests", all_pulls) + print_pulls(repo_name, "Other", other) + # show code contributions + credits = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True).rstrip() + + print("## Credits\n") + print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " + "per contributor.\n") + print("```") + print(credits) + print("```\n") + + print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " + "PRs, and providing feedback on this release.\n") def cli(args=None): """Process command line arguments.""" @@ -101,19 +148,17 @@ def cli(args=None): args = sys.argv[1:] parser = argparse.ArgumentParser() - parser.add_argument( - "project", help="The project name e.g. apache/datafusion-python" - ) - parser.add_argument("tag1", help="The previous release tag") - parser.add_argument("tag2", help="The current release tag") + parser.add_argument("tag1", help="The previous commit or tag (e.g. 0.1.0)") + parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") + parser.add_argument("version", help="The version number to include in the changelog") args = parser.parse_args() token = os.getenv("GITHUB_TOKEN") + project = "apache/datafusion-python" g = Github(token) - repo = g.get_repo(args.project) - generate_changelog(repo, args.project, args.tag1, args.tag2) - + repo = g.get_repo(project) + generate_changelog(repo, project, args.tag1, args.tag2, args.version) if __name__ == "__main__": - cli() + cli() \ No newline at end of file diff --git a/requirements.in b/requirements.in index b2a1a48df..1b7f62052 100644 --- a/requirements.in +++ b/requirements.in @@ -23,3 +23,4 @@ pytest ruff toml importlib_metadata; python_version < "3.8" +PyGitHub From b2982ecda723f77c2b0b343b030fd3fe589b2062 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 13 Aug 2024 12:11:29 -0400 Subject: [PATCH 179/413] Add typing-extensions dependency to pyproject (#805) * Add typing-extensions dependency to pyproject * Specify versions which require typing extensions. Co-authored-by: Kyle Barron --------- Co-authored-by: Kyle Barron --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a18ef0e5e..4e03ce8db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ classifier = [ ] dependencies = [ "pyarrow>=11.0.0", + "typing-extensions;python_version<'3.13'", ] [project.urls] From 805183bd9ae32b628e641d9552f1d0dded4a7a85 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Thu, 22 Aug 2024 03:53:30 +0200 Subject: [PATCH 180/413] feat: enable list of paths for read_csv (#824) --- python/datafusion/context.py | 7 +++++-- python/datafusion/tests/test_context.py | 16 ++++++++++++++++ src/context.rs | 15 +++++++-------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 922cc87a3..47f2b9cf9 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -883,7 +883,7 @@ def read_json( def read_csv( self, - path: str | pathlib.Path, + path: str | pathlib.Path | list[str] | list[pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ",", @@ -914,9 +914,12 @@ def read_csv( """ if table_partition_cols is None: table_partition_cols = [] + + path = [str(p) for p in path] if isinstance(path, list) else str(path) + return DataFrame( self.ctx.read_csv( - str(path), + path, schema, has_header, delimiter, diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index 8373659b0..1b424db8b 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -484,6 +484,22 @@ def test_read_csv(ctx): csv_df.select(column("c1")).show() +def test_read_csv_list(ctx): + csv_df = ctx.read_csv(path=["testing/data/csv/aggregate_test_100.csv"]) + expected = csv_df.count() * 2 + + double_csv_df = ctx.read_csv( + path=[ + "testing/data/csv/aggregate_test_100.csv", + "testing/data/csv/aggregate_test_100.csv", + ] + ) + actual = double_csv_df.count() + + double_csv_df.select(column("c1")).show() + assert actual == expected + + def test_read_csv_compressed(ctx, tmp_path): test_data_path = "testing/data/csv/aggregate_test_100.csv" diff --git a/src/context.rs b/src/context.rs index 50c4a1994..d7890e3f5 100644 --- a/src/context.rs +++ b/src/context.rs @@ -805,7 +805,7 @@ impl PySessionContext { file_compression_type=None))] pub fn read_csv( &self, - path: PathBuf, + path: &Bound<'_, PyAny>, schema: Option>, has_header: bool, delimiter: &str, @@ -815,10 +815,6 @@ impl PySessionContext { file_compression_type: Option, py: Python, ) -> PyResult { - let path = path - .to_str() - .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; - let delimiter = delimiter.as_bytes(); if delimiter.len() != 1 { return Err(PyValueError::new_err( @@ -833,13 +829,16 @@ impl PySessionContext { .file_extension(file_extension) .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) .file_compression_type(parse_file_compression_type(file_compression_type)?); + options.schema = schema.as_ref().map(|x| &x.0); - if let Some(py_schema) = schema { - options.schema = Some(&py_schema.0); - let result = self.ctx.read_csv(path, options); + if path.is_instance_of::() { + let paths = path.extract::>()?; + let paths = paths.iter().map(|p| p as &str).collect::>(); + let result = self.ctx.read_csv(paths, options); let df = PyDataFrame::new(wait_for_future(py, result).map_err(DataFusionError::from)?); Ok(df) } else { + let path = path.extract::()?; let result = self.ctx.read_csv(path, options); let df = PyDataFrame::new(wait_for_future(py, result).map_err(DataFusionError::from)?); Ok(df) From 3b5085e0c716298af7c8f025b8b9edf601344f02 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 23 Aug 2024 09:43:39 -0500 Subject: [PATCH 181/413] Upgrade deps to datafusion 41 (#802) * update datafusion deps to point to githuc.com/apache/datafusion Datafusion 41 is not yet released on crates.io. * update TableProvider::scan Ref: https://github.com/apache/datafusion/pull/11516 * use SessionStateBuilder The old constructor is deprecated. Ref: https://github.com/apache/datafusion/pull/11403 * update AggregateFunction Upstream Changes: - The field name was switched from `func_name` to func. - AggregateFunctionDefinition was removed Ref: https://github.com/apache/datafusion/pull/11803 * update imports in catalog Catlog API was extracted to a separate crate. Ref: https://github.com/apache/datafusion/pull/11516 * use appropriate path for approx_distinct Ref: https://github.com/apache/datafusion/pull/11644 * migrate AggregateExt to ExprFunctionExt Also removed `sqlparser` dependency since it's re-exported upstream. Ref: https://github.com/apache/datafusion/pull/11550 * update regr_count tests for new return type Ref: https://github.com/apache/datafusion/pull/11731 * migrate from function-array to functions-nested The package was renamed upstream. Ref: https://github.com/apache/datafusion/pull/11602 * cargo fmt * lock datafusion deps to 41 * remove todo from cargo.toml All the datafusion dependencies are re-exported, but I still need to figure out *why*. --- Cargo.lock | 108 ++++++++++++++-------- Cargo.toml | 15 ++- python/datafusion/tests/test_functions.py | 6 +- src/catalog.rs | 2 +- src/common/data_type.rs | 17 ++-- src/context.rs | 11 ++- src/dataset.rs | 4 +- src/expr/aggregate.rs | 10 +- src/expr/aggregate_expr.rs | 4 +- src/functions.rs | 44 ++++----- 10 files changed, 126 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e59811210..bca4bf066 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -680,6 +680,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crunchy" version = "0.2.2" @@ -725,11 +731,12 @@ checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" [[package]] name = "dashmap" -version = "5.5.3" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" dependencies = [ "cfg-if", + "crossbeam-utils", "hashbrown", "lock_api", "once_cell", @@ -738,9 +745,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab9d55a9cd2634818953809f75ebe5248b00dd43c3227efb2a51a2d5feaad54e" +checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb" dependencies = [ "ahash", "apache-avro", @@ -754,16 +761,18 @@ dependencies = [ "bzip2", "chrono", "dashmap", + "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", - "datafusion-functions-array", + "datafusion-functions-nested", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-sql", "flate2", @@ -792,11 +801,25 @@ dependencies = [ "zstd 0.13.2", ] +[[package]] +name = "datafusion-catalog" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", +] + [[package]] name = "datafusion-common" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "def66b642959e7f96f5d2da22e1f43d3bd35598f821e5ce351a0553e0f1b7367" +checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c" dependencies = [ "ahash", "apache-avro", @@ -818,18 +841,18 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f104bb9cb44c06c9badf8a0d7e0855e5f7fa5e395b887d7f835e8a9457dc1352" +checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac0fd8b5d80bbca3fc3b6f40da4e9f6907354824ec3b18bbd83fee8cf5c3c3e" +checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" dependencies = [ "arrow", "chrono", @@ -848,9 +871,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2103d2cc16fb11ef1fa993a6cac57ed5cb028601db4b97566c90e5fa77aa1e68" +checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2" dependencies = [ "ahash", "arrow", @@ -867,11 +890,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a369332afd0ef5bd565f6db2139fb9f1dfdd0afa75a7f70f000b74208d76994f" +checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb" dependencies = [ "arrow", + "arrow-buffer", "base64 0.22.1", "blake2", "blake3", @@ -893,9 +917,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92718db1aff70c47e5abf9fc975768530097059e5db7c7b78cd64b5e9a11fc77" +checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb" dependencies = [ "ahash", "arrow", @@ -910,10 +934,10 @@ dependencies = [ ] [[package]] -name = "datafusion-functions-array" -version = "40.0.0" +name = "datafusion-functions-nested" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bb80f46ff3dcf4bb4510209c2ba9b8ce1b716ac8b7bf70c6bf7dca6260c831" +checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d" dependencies = [ "arrow", "arrow-array", @@ -928,13 +952,14 @@ dependencies = [ "itertools 0.12.1", "log", "paste", + "rand", ] [[package]] name = "datafusion-optimizer" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f34692011bec4fdd6fc18c264bf8037b8625d801e6dd8f5111af15cb6d71d3" +checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906" dependencies = [ "arrow", "async-trait", @@ -952,9 +977,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45538630defedb553771434a437f7ca8f04b9b3e834344aafacecb27dc65d5e5" +checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4" dependencies = [ "ahash", "arrow", @@ -982,9 +1007,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d8a72b0ca908e074aaeca52c14ddf5c28d22361e9cb6bc79bb733cd6661b536" +checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6" dependencies = [ "ahash", "arrow", @@ -994,11 +1019,23 @@ dependencies = [ "rand", ] +[[package]] +name = "datafusion-physical-optimizer" +version = "41.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0" +dependencies = [ + "datafusion-common", + "datafusion-execution", + "datafusion-physical-expr", + "datafusion-physical-plan", +] + [[package]] name = "datafusion-physical-plan" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b504eae6107a342775e22e323e9103f7f42db593ec6103b28605b7b7b1405c4a" +checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe" dependencies = [ "ahash", "arrow", @@ -1037,7 +1074,7 @@ dependencies = [ "datafusion", "datafusion-common", "datafusion-expr", - "datafusion-functions-array", + "datafusion-functions-nested", "datafusion-optimizer", "datafusion-sql", "datafusion-substrait", @@ -1051,7 +1088,6 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "sqlparser", "syn 2.0.72", "tokio", "url", @@ -1060,9 +1096,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5db33f323f41b95ae201318ba654a9bf11113e58a51a1dff977b1a836d3d889" +checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0" dependencies = [ "arrow", "arrow-array", @@ -1077,9 +1113,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "40.0.0" +version = "41.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434e52fbff22e6e04e6c787f603a6aba4961a7e249a29c743c5d4f609ec2dcef" +checksum = "bf0a0055aa98246c79f98f0d03df11f16cb7adc87818d02d4413e3f3cdadbbee" dependencies = [ "arrow-buffer", "async-recursion", @@ -2898,9 +2934,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.47.0" +version = "0.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" +checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" dependencies = [ "log", "sqlparser_derive", diff --git a/Cargo.toml b/Cargo.toml index 820118fa8..8881884b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,13 +38,13 @@ tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync rand = "0.8" pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "52", feature = ["pyarrow"] } -datafusion = { version = "40.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-common = { version = "40.0.0", features = ["pyarrow"] } -datafusion-expr = "40.0.0" -datafusion-functions-array = "40.0.0" -datafusion-optimizer = "40.0.0" -datafusion-sql = "40.0.0" -datafusion-substrait = { version = "40.0.0", optional = true } +datafusion = { version = "41.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-common = { version = "41.0.0", features = ["pyarrow"] } +datafusion-expr = { version = "41.0.0" } +datafusion-functions-nested = { version = "41.0.0" } +datafusion-optimizer = { version = "41.0.0" } +datafusion-sql = { version = "41.0.0" } +datafusion-substrait = { version = "41.0.0", optional = true } prost = "0.12" # keep in line with `datafusion-substrait` prost-types = "0.12" # keep in line with `datafusion-substrait` uuid = { version = "1.9", features = ["v4"] } @@ -56,7 +56,6 @@ parking_lot = "0.12" regex-syntax = "0.8" syn = "2.0.68" url = "2" -sqlparser = "0.47.0" [build-dependencies] pyo3-build-config = "0.21" diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 293912321..b8ad9c0d7 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -808,7 +808,7 @@ def test_regr_funcs_sql(df): assert result[0].column(0) == pa.array([None], type=pa.float64()) assert result[0].column(1) == pa.array([None], type=pa.float64()) - assert result[0].column(2) == pa.array([1], type=pa.float64()) + assert result[0].column(2) == pa.array([1], type=pa.uint64()) assert result[0].column(3) == pa.array([None], type=pa.float64()) assert result[0].column(4) == pa.array([1], type=pa.float64()) assert result[0].column(5) == pa.array([1], type=pa.float64()) @@ -840,7 +840,7 @@ def test_regr_funcs_sql_2(): # Assertions for SQL results assert result_sql[0].column(0) == pa.array([2], type=pa.float64()) assert result_sql[0].column(1) == pa.array([0], type=pa.float64()) - assert result_sql[0].column(2) == pa.array([3], type=pa.float64()) # todo: i would not expect this to be float + assert result_sql[0].column(2) == pa.array([3], type=pa.uint64()) assert result_sql[0].column(3) == pa.array([1], type=pa.float64()) assert result_sql[0].column(4) == pa.array([2], type=pa.float64()) assert result_sql[0].column(5) == pa.array([4], type=pa.float64()) @@ -852,7 +852,7 @@ def test_regr_funcs_sql_2(): @pytest.mark.parametrize("func, expected", [ pytest.param(f.regr_slope, pa.array([2], type=pa.float64()), id="regr_slope"), pytest.param(f.regr_intercept, pa.array([0], type=pa.float64()), id="regr_intercept"), - pytest.param(f.regr_count, pa.array([3], type=pa.float64()), id="regr_count"), # TODO: I would expect this to return an int array + pytest.param(f.regr_count, pa.array([3], type=pa.uint64()), id="regr_count"), pytest.param(f.regr_r2, pa.array([1], type=pa.float64()), id="regr_r2"), pytest.param(f.regr_avgx, pa.array([2], type=pa.float64()), id="regr_avgx"), pytest.param(f.regr_avgy, pa.array([4], type=pa.float64()), id="regr_avgy"), diff --git a/src/catalog.rs b/src/catalog.rs index 49fe14046..1ce66a4dc 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -25,7 +25,7 @@ use crate::errors::DataFusionError; use crate::utils::wait_for_future; use datafusion::{ arrow::pyarrow::ToPyArrow, - catalog::{schema::SchemaProvider, CatalogProvider}, + catalog::{CatalogProvider, SchemaProvider}, datasource::{TableProvider, TableType}, }; diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 469bb789a..21b085c0e 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -18,6 +18,7 @@ use datafusion::arrow::array::Array; use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; @@ -775,20 +776,20 @@ pub enum NullTreatment { RESPECT_NULLS, } -impl From for sqlparser::ast::NullTreatment { - fn from(null_treatment: NullTreatment) -> sqlparser::ast::NullTreatment { +impl From for DFNullTreatment { + fn from(null_treatment: NullTreatment) -> DFNullTreatment { match null_treatment { - NullTreatment::IGNORE_NULLS => sqlparser::ast::NullTreatment::IgnoreNulls, - NullTreatment::RESPECT_NULLS => sqlparser::ast::NullTreatment::RespectNulls, + NullTreatment::IGNORE_NULLS => DFNullTreatment::IgnoreNulls, + NullTreatment::RESPECT_NULLS => DFNullTreatment::RespectNulls, } } } -impl From for NullTreatment { - fn from(null_treatment: sqlparser::ast::NullTreatment) -> NullTreatment { +impl From for NullTreatment { + fn from(null_treatment: DFNullTreatment) -> NullTreatment { match null_treatment { - sqlparser::ast::NullTreatment::IgnoreNulls => NullTreatment::IGNORE_NULLS, - sqlparser::ast::NullTreatment::RespectNulls => NullTreatment::RESPECT_NULLS, + DFNullTreatment::IgnoreNulls => NullTreatment::IGNORE_NULLS, + DFNullTreatment::RespectNulls => NullTreatment::RESPECT_NULLS, } } } diff --git a/src/context.rs b/src/context.rs index d7890e3f5..a43599cf7 100644 --- a/src/context.rs +++ b/src/context.rs @@ -20,6 +20,7 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use datafusion::execution::session_state::SessionStateBuilder; use object_store::ObjectStore; use url::Url; use uuid::Uuid; @@ -49,9 +50,7 @@ use datafusion::datasource::listing::{ }; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; -use datafusion::execution::context::{ - SQLOptions, SessionConfig, SessionContext, SessionState, TaskContext, -}; +use datafusion::execution::context::{SQLOptions, SessionConfig, SessionContext, TaskContext}; use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; @@ -281,7 +280,11 @@ impl PySessionContext { RuntimeConfig::default() }; let runtime = Arc::new(RuntimeEnv::new(runtime_config)?); - let session_state = SessionState::new_with_config_rt(config, runtime); + let session_state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(runtime) + .with_default_features() + .build(); Ok(PySessionContext { ctx: SessionContext::new_with_state(session_state), }) diff --git a/src/dataset.rs b/src/dataset.rs index 724b4af76..b5704164f 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::catalog::Session; use pyo3::exceptions::PyValueError; /// Implements a Datafusion TableProvider that delegates to a PyArrow Dataset /// This allows us to use PyArrow Datasets as Datafusion tables while pushing down projections and filters @@ -30,7 +31,6 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::{DataFusionError, Result as DFResult}; -use datafusion::execution::context::SessionState; use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion::physical_plan::ExecutionPlan; use datafusion_expr::Expr; @@ -98,7 +98,7 @@ impl TableProvider for Dataset { /// parallelized or distributed. async fn scan( &self, - _ctx: &SessionState, + _ctx: &dyn Session, projection: Option<&Vec>, filters: &[Expr], // limit can be used to reduce the amount scanned diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index 626d92c79..e3d1bb136 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -126,9 +126,9 @@ impl PyAggregate { match expr { // TODO: This Alias logic seems to be returning some strange results that we should investigate Expr::Alias(Alias { expr, .. }) => self._aggregation_arguments(expr.as_ref()), - Expr::AggregateFunction(AggregateFunction { - func_def: _, args, .. - }) => Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()), + Expr::AggregateFunction(AggregateFunction { func: _, args, .. }) => { + Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()) + } _ => Err(py_type_err( "Encountered a non Aggregate type in aggregation_arguments", )), @@ -138,9 +138,7 @@ impl PyAggregate { fn _agg_func_name(expr: &Expr) -> PyResult { match expr { Expr::Alias(Alias { expr, .. }) => Self::_agg_func_name(expr.as_ref()), - Expr::AggregateFunction(AggregateFunction { func_def, .. }) => { - Ok(func_def.name().to_owned()) - } + Expr::AggregateFunction(AggregateFunction { func, .. }) => Ok(func.name().to_owned()), _ => Err(py_type_err( "Encountered a non Aggregate type in agg_func_name", )), diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs index 04ec29a15..15097e007 100644 --- a/src/expr/aggregate_expr.rs +++ b/src/expr/aggregate_expr.rs @@ -41,7 +41,7 @@ impl From for PyAggregateFunction { impl Display for PyAggregateFunction { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { let args: Vec = self.aggr.args.iter().map(|expr| expr.to_string()).collect(); - write!(f, "{}({})", self.aggr.func_def.name(), args.join(", ")) + write!(f, "{}({})", self.aggr.func.name(), args.join(", ")) } } @@ -49,7 +49,7 @@ impl Display for PyAggregateFunction { impl PyAggregateFunction { /// Get the aggregate type, such as "MIN", or "MAX" fn aggregate_type(&self) -> String { - self.aggr.func_def.name().to_string() + self.aggr.func.name().to_string() } /// is this a distinct aggregate such as `COUNT(DISTINCT expr)` diff --git a/src/functions.rs b/src/functions.rs index f8f478166..c53d4ad92 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; -use datafusion_expr::AggregateExt; +use datafusion_expr::ExprFunctionExt; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; @@ -30,16 +30,15 @@ use datafusion::functions; use datafusion::functions_aggregate; use datafusion_common::{Column, ScalarValue, TableReference}; use datafusion_expr::expr::Alias; +use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use datafusion_expr::{ - expr::{ - find_df_window_func, AggregateFunction, AggregateFunctionDefinition, Sort, WindowFunction, - }, + expr::{find_df_window_func, AggregateFunction, Sort, WindowFunction}, lit, Expr, WindowFunctionDefinition, }; #[pyfunction] pub fn approx_distinct(expression: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::approx_distinct::approx_distinct(expression.expr).into() + functions_aggregate::expr_fn::approx_distinct(expression.expr).into() } #[pyfunction] @@ -342,9 +341,8 @@ pub fn first_value( builder = builder.filter(filter.expr); } - if let Some(null_treatment) = null_treatment { - builder = builder.null_treatment(null_treatment.into()) - } + // would be nice if all the options builder methods accepted Option ... + builder = builder.null_treatment(null_treatment.map(DFNullTreatment::from)); Ok(builder.build()?.into()) } @@ -373,9 +371,7 @@ pub fn last_value( builder = builder.filter(filter.expr); } - if let Some(null_treatment) = null_treatment { - builder = builder.null_treatment(null_treatment.into()) - } + builder = builder.null_treatment(null_treatment.map(DFNullTreatment::from)); Ok(builder.build()?.into()) } @@ -392,14 +388,14 @@ fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { #[pyfunction] fn make_array(exprs: Vec) -> PyExpr { - datafusion_functions_array::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) + datafusion_functions_nested::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) .into() } #[pyfunction] fn array_concat(exprs: Vec) -> PyExpr { let exprs = exprs.into_iter().map(|x| x.into()).collect(); - datafusion_functions_array::expr_fn::array_concat(exprs).into() + datafusion_functions_nested::expr_fn::array_concat(exprs).into() } #[pyfunction] @@ -411,12 +407,12 @@ fn array_cat(exprs: Vec) -> PyExpr { fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { let index = ScalarValue::Int64(index); let index = Expr::Literal(index); - datafusion_functions_array::expr_fn::array_position(array.into(), element.into(), index).into() + datafusion_functions_nested::expr_fn::array_position(array.into(), element.into(), index).into() } #[pyfunction] fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { - datafusion_functions_array::expr_fn::array_slice( + datafusion_functions_nested::expr_fn::array_slice( array.into(), begin.into(), end.into(), @@ -638,18 +634,16 @@ fn window( } macro_rules! aggregate_function { - ($NAME: ident, $FUNC: ident) => { + ($NAME: ident, $FUNC: path) => { aggregate_function!($NAME, $FUNC, stringify!($NAME)); }; - ($NAME: ident, $FUNC: ident, $DOC: expr) => { + ($NAME: ident, $FUNC: path, $DOC: expr) => { #[doc = $DOC] #[pyfunction] #[pyo3(signature = (*args, distinct=false))] fn $NAME(args: Vec, distinct: bool) -> PyExpr { let expr = datafusion_expr::Expr::AggregateFunction(AggregateFunction { - func_def: AggregateFunctionDefinition::BuiltIn( - datafusion_expr::aggregate_function::AggregateFunction::$FUNC, - ), + func: $FUNC(), args: args.into_iter().map(|e| e.into()).collect(), distinct, filter: None, @@ -701,7 +695,7 @@ macro_rules! expr_fn_vec { }; } -/// Generates a [pyo3] wrapper for [datafusion_functions_array::expr_fn] +/// Generates a [pyo3] wrapper for [datafusion_functions_nested::expr_fn] /// /// These functions have explicit named arguments. macro_rules! array_fn { @@ -718,7 +712,7 @@ macro_rules! array_fn { #[doc = $DOC] #[pyfunction] fn $FUNC($($arg: PyExpr),*) -> PyExpr { - datafusion_functions_array::expr_fn::$FUNC($($arg.into()),*).into() + datafusion_functions_nested::expr_fn::$FUNC($($arg.into()),*).into() } }; } @@ -884,9 +878,9 @@ array_fn!(array_resize, array size value); array_fn!(flatten, array); array_fn!(range, start stop step); -aggregate_function!(array_agg, ArrayAgg); -aggregate_function!(max, Max); -aggregate_function!(min, Min); +aggregate_function!(array_agg, functions_aggregate::array_agg::array_agg_udaf); +aggregate_function!(max, functions_aggregate::min_max::max_udaf); +aggregate_function!(min, functions_aggregate::min_max::min_udaf); pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(abs))?; From 02414d81b2dae6e9a019a079ce7b7958b80ca538 Mon Sep 17 00:00:00 2001 From: Jim Crist-Harif Date: Fri, 23 Aug 2024 14:29:39 -0500 Subject: [PATCH 182/413] Fix SessionContext init with only SessionConfig (#827) Previously creating a `SessionContext` when specifying only a `SessionConfig` would error. --- python/datafusion/context.py | 2 +- python/datafusion/tests/test_context.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 47f2b9cf9..d4e50cfe2 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -445,7 +445,7 @@ def __init__( df = ctx.read_csv("data.csv") """ config = config.config_internal if config is not None else None - runtime = runtime.config_internal if config is not None else None + runtime = runtime.config_internal if runtime is not None else None self.ctx = SessionContextInternal(config, runtime) diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index 1b424db8b..66d7e013a 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -38,6 +38,14 @@ def test_create_context_no_args(): SessionContext() +def test_create_context_session_config_only(): + SessionContext(config=SessionConfig()) + + +def test_create_context_runtime_config_only(): + SessionContext(runtime=RuntimeConfig()) + + @pytest.mark.parametrize("path_to_str", (True, False)) def test_runtime_configs(tmp_path, path_to_str): path1 = tmp_path / "dir1" From 22c70ef55fb56e38d9f4465b974b33b3685a6a40 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 23 Aug 2024 14:30:52 -0500 Subject: [PATCH 183/413] build(deps): upgrade actions/{upload,download}-artifact@v3 to v4 (#829) * update actions/upload-artifact from v3 to v4 * update actions/download-artifact from v3 to v4 * build(dist): each build artifact gets a unique key and merged separately Uploads are now immutable for actions/upload-artifact@v4. This gives each build artifact a unique key, and then merges them together at the end. The [migration guide] suggests this pattern. [migration guide]: https://github.com/actions/upload-artifact/blob/main/docs/MIGRATION.md#merging-multiple-artifacts * use matrix.os to create unique upload key for build-python-mac-win job * use manylinux-x86_64 name for consistency with manylinux-aarch64 --- .github/workflows/build.yml | 59 ++++++++++++++++++++++++------------- .github/workflows/conda.yml | 2 +- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 350be46d5..0b974f2d0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: override: true - name: Generate license file run: python ./dev/create_license.py - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: python-wheel-license path: LICENSE.txt @@ -84,7 +84,7 @@ jobs: - run: rm LICENSE.txt - name: Download LICENSE.txt - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-wheel-license path: . @@ -110,9 +110,9 @@ jobs: run: find target/wheels/ - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: dist-${{ matrix.os }} path: target/wheels/* build-macos-aarch64: @@ -145,7 +145,7 @@ jobs: - run: rm LICENSE.txt - name: Download LICENSE.txt - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-wheel-license path: . @@ -162,12 +162,12 @@ jobs: run: find target/wheels/ - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: dist-macos-aarch64 path: target/wheels/* - build-manylinux: + build-manylinux-x86_64: needs: [generate-license] name: Manylinux runs-on: ubuntu-latest @@ -175,7 +175,7 @@ jobs: - uses: actions/checkout@v4 - run: rm LICENSE.txt - name: Download LICENSE.txt - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-wheel-license path: . @@ -191,9 +191,9 @@ jobs: rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 args: --release --manylinux 2014 --features protoc,substrait - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: dist-manylinux-x86_64 path: target/wheels/* build-manylinux-aarch64: @@ -204,7 +204,7 @@ jobs: - uses: actions/checkout@v4 - run: rm LICENSE.txt - name: Download LICENSE.txt - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-wheel-license path: . @@ -221,9 +221,9 @@ jobs: rustup-components: rust-std rustfmt # Keep them in one line due to https://github.com/PyO3/maturin-action/issues/153 args: --release --features protoc,substrait - name: Archive wheels - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: dist + name: dist-manylinux-aarch64 path: target/wheels/* build-sdist: @@ -234,7 +234,7 @@ jobs: - uses: actions/checkout@v4 - run: rm LICENSE.txt - name: Download LICENSE.txt - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: python-wheel-license path: . @@ -246,19 +246,38 @@ jobs: manylinux: auto rustup-components: rust-std rustfmt args: --release --sdist --out dist --features protoc,substrait - - name: Archive wheels - uses: actions/upload-artifact@v3 + - name: Assert sdist build does not generate wheels + run: | + if [ "$(ls -A target/wheels)" ]; then + echo "Error: Sdist build generated wheels" + exit 1 + else + echo "Directory is clean" + fi + shell: bash + + merge-build-artifacts: + runs-on: ubuntu-latest + needs: + - build-python-mac-win + - build-macos-aarch64 + - build-manylinux-x86_64 + - build-manylinux-aarch64 + - build-sdist + steps: + - name: Merge Build Artifacts + uses: actions/upload-artifact/merge@v4 with: name: dist - path: target/wheels/* - + pattern: dist-* + # NOTE: PyPI publish needs to be done manually for now after release passed the vote # release: # name: Publish in PyPI # needs: [build-manylinux, build-python-mac-win] # runs-on: ubuntu-latest # steps: - # - uses: actions/download-artifact@v3 + # - uses: actions/download-artifact@v4 # - name: Publish to PyPI # uses: pypa/gh-action-pypi-publish@master # with: diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index f25a431c4..52888cb14 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -101,7 +101,7 @@ jobs: run: | conda mambabuild --test packages/${{ matrix.arch }}/*.tar.bz2 - name: Upload conda packages as artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: "conda nightlies (python - ${{ matrix.python }}, arch - ${{ matrix.arch }})" # need to install all conda channel metadata to properly install locally From 766e2edccb766aacd1bc9de6e8ed1c356f69486a Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 27 Aug 2024 12:18:46 -0400 Subject: [PATCH 184/413] Run ruff format in CI (#837) * Run ruff format in CI * Add --check parameter * Apply ruff format --- .github/workflows/build.yml | 4 +- python/datafusion/functions.py | 9 +- python/datafusion/tests/test_aggregation.py | 105 ++++++++------ python/datafusion/tests/test_dataframe.py | 90 ++++++------ python/datafusion/tests/test_expr.py | 14 +- python/datafusion/tests/test_functions.py | 150 +++++++++++++------- python/datafusion/tests/test_sql.py | 32 +++-- 7 files changed, 251 insertions(+), 153 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0b974f2d0..a4f8b2da5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,7 +38,9 @@ jobs: pip install ruff # Update output format to enable automatic inline annotations. - name: Run Ruff - run: ruff check --output-format=github python/ + run: | + ruff check --output-format=github python/ + ruff format --check python/ generate-license: runs-on: ubuntu-latest diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 2d3d87ee0..59a1974fd 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -1479,12 +1479,17 @@ def approx_percentile_cont( """Returns the value that is approximately at a given percentile of ``expr``.""" if num_centroids is None: return Expr( - f.approx_percentile_cont(expression.expr, percentile.expr, distinct=distinct, num_centroids=None) + f.approx_percentile_cont( + expression.expr, percentile.expr, distinct=distinct, num_centroids=None + ) ) return Expr( f.approx_percentile_cont( - expression.expr, percentile.expr, distinct=distinct, num_centroids=num_centroids.expr + expression.expr, + percentile.expr, + distinct=distinct, + num_centroids=num_centroids.expr, ) ) diff --git a/python/datafusion/tests/test_aggregation.py b/python/datafusion/tests/test_aggregation.py index 03485da4b..ab653c403 100644 --- a/python/datafusion/tests/test_aggregation.py +++ b/python/datafusion/tests/test_aggregation.py @@ -39,6 +39,7 @@ def df(): ) return ctx.create_dataframe([[batch]]) + @pytest.fixture def df_aggregate_100(): ctx = SessionContext() @@ -46,32 +47,46 @@ def df_aggregate_100(): return ctx.table("aggregate_test_data") -@pytest.mark.parametrize("agg_expr, calc_expected", [ - (f.avg(column("a")), lambda a, b, c, d: np.array(np.average(a))), - (f.corr(column("a"), column("b")), lambda a, b, c, d: np.array(np.corrcoef(a, b)[0][1])), - (f.count(column("a")), lambda a, b, c, d: pa.array([len(a)])), - # Sample (co)variance -> ddof=1 - # Population (co)variance -> ddof=0 - (f.covar(column("a"), column("b")), lambda a, b, c, d: np.array(np.cov(a, b, ddof=1)[0][1])), - (f.covar_pop(column("a"), column("c")), lambda a, b, c, d: np.array(np.cov(a, c, ddof=0)[0][1])), - (f.covar_samp(column("b"), column("c")), lambda a, b, c, d: np.array(np.cov(b, c, ddof=1)[0][1])), - # f.grouping(col_a), # No physical plan implemented yet - (f.max(column("a")), lambda a, b, c, d: np.array(np.max(a))), - (f.mean(column("b")), lambda a, b, c, d: np.array(np.mean(b))), - (f.median(column("b")), lambda a, b, c, d: np.array(np.median(b))), - (f.min(column("a")), lambda a, b, c, d: np.array(np.min(a))), - (f.sum(column("b")), lambda a, b, c, d: np.array(np.sum(b.to_pylist()))), - # Sample stdev -> ddof=1 - # Population stdev -> ddof=0 - (f.stddev(column("a")), lambda a, b, c, d: np.array(np.std(a, ddof=1))), - (f.stddev_pop(column("b")), lambda a, b, c, d: np.array(np.std(b, ddof=0))), - (f.stddev_samp(column("c")), lambda a, b, c, d: np.array(np.std(c, ddof=1))), - (f.var(column("a")), lambda a, b, c, d: np.array(np.var(a, ddof=1))), - (f.var_pop(column("b")), lambda a, b, c, d: np.array(np.var(b, ddof=0))), - (f.var_samp(column("c")), lambda a, b, c, d: np.array(np.var(c, ddof=1))), -]) +@pytest.mark.parametrize( + "agg_expr, calc_expected", + [ + (f.avg(column("a")), lambda a, b, c, d: np.array(np.average(a))), + ( + f.corr(column("a"), column("b")), + lambda a, b, c, d: np.array(np.corrcoef(a, b)[0][1]), + ), + (f.count(column("a")), lambda a, b, c, d: pa.array([len(a)])), + # Sample (co)variance -> ddof=1 + # Population (co)variance -> ddof=0 + ( + f.covar(column("a"), column("b")), + lambda a, b, c, d: np.array(np.cov(a, b, ddof=1)[0][1]), + ), + ( + f.covar_pop(column("a"), column("c")), + lambda a, b, c, d: np.array(np.cov(a, c, ddof=0)[0][1]), + ), + ( + f.covar_samp(column("b"), column("c")), + lambda a, b, c, d: np.array(np.cov(b, c, ddof=1)[0][1]), + ), + # f.grouping(col_a), # No physical plan implemented yet + (f.max(column("a")), lambda a, b, c, d: np.array(np.max(a))), + (f.mean(column("b")), lambda a, b, c, d: np.array(np.mean(b))), + (f.median(column("b")), lambda a, b, c, d: np.array(np.median(b))), + (f.min(column("a")), lambda a, b, c, d: np.array(np.min(a))), + (f.sum(column("b")), lambda a, b, c, d: np.array(np.sum(b.to_pylist()))), + # Sample stdev -> ddof=1 + # Population stdev -> ddof=0 + (f.stddev(column("a")), lambda a, b, c, d: np.array(np.std(a, ddof=1))), + (f.stddev_pop(column("b")), lambda a, b, c, d: np.array(np.std(b, ddof=0))), + (f.stddev_samp(column("c")), lambda a, b, c, d: np.array(np.std(c, ddof=1))), + (f.var(column("a")), lambda a, b, c, d: np.array(np.var(a, ddof=1))), + (f.var_pop(column("b")), lambda a, b, c, d: np.array(np.var(b, ddof=0))), + (f.var_samp(column("c")), lambda a, b, c, d: np.array(np.var(c, ddof=1))), + ], +) def test_aggregation_stats(df, agg_expr, calc_expected): - agg_df = df.aggregate([], [agg_expr]) result = agg_df.collect()[0] values_a, values_b, values_c, values_d = df.collect()[0] @@ -79,16 +94,19 @@ def test_aggregation_stats(df, agg_expr, calc_expected): np.testing.assert_array_almost_equal(result.column(0), expected) -@pytest.mark.parametrize("agg_expr, expected", [ - (f.approx_distinct(column("b")), pa.array([2], type=pa.uint64())), - (f.approx_median(column("b")), pa.array([4])), - (f.approx_percentile_cont(column("b"), lit(0.5)), pa.array([4])), - ( - f.approx_percentile_cont_with_weight(column("b"), lit(0.6), lit(0.5)), - pa.array([6], type=pa.float64()) - ), - (f.array_agg(column("b")), pa.array([[4, 4, 6]])), -]) +@pytest.mark.parametrize( + "agg_expr, expected", + [ + (f.approx_distinct(column("b")), pa.array([2], type=pa.uint64())), + (f.approx_median(column("b")), pa.array([4])), + (f.approx_percentile_cont(column("b"), lit(0.5)), pa.array([4])), + ( + f.approx_percentile_cont_with_weight(column("b"), lit(0.6), lit(0.5)), + pa.array([6], type=pa.float64()), + ), + (f.array_agg(column("b")), pa.array([[4, 4, 6]])), + ], +) def test_aggregation(df, agg_expr, expected): agg_df = df.aggregate([], [agg_expr]) result = agg_df.collect()[0] @@ -98,20 +116,21 @@ def test_aggregation(df, agg_expr, expected): def test_aggregate_100(df_aggregate_100): # https://github.com/apache/datafusion/blob/bddb6415a50746d2803dd908d19c3758952d74f9/datafusion/sqllogictest/test_files/aggregate.slt#L1490-L1498 - result = df_aggregate_100.aggregate( - [ - column("c1") - ], - [ - f.approx_percentile_cont(column("c3"), lit(0.95), lit(200)).alias("c3") - ] - ).sort(column("c1").sort(ascending=True)).collect() + result = ( + df_aggregate_100.aggregate( + [column("c1")], + [f.approx_percentile_cont(column("c3"), lit(0.95), lit(200)).alias("c3")], + ) + .sort(column("c1").sort(ascending=True)) + .collect() + ) assert len(result) == 1 result = result[0] assert result.column("c1") == pa.array(["a", "b", "c", "d", "e"]) assert result.column("c3") == pa.array([73, 68, 122, 124, 115]) + def test_bit_add_or_xor(df): df = df.aggregate( [], diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index 6444d9321..e5e0c9c8a 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -279,57 +279,67 @@ def test_distinct(): data_test_window_functions = [ - ("row", f.window("row_number", [], order_by=[f.order_by(column("c"))]), [2, 1, 3]), - ("rank", f.window("rank", [], order_by=[f.order_by(column("c"))]), [2, 1, 2]), - ("dense_rank", f.window("dense_rank", [], order_by=[f.order_by(column("c"))]), [2, 1, 2] ), - ("percent_rank", f.window("percent_rank", [], order_by=[f.order_by(column("c"))]), [0.5, 0, 0.5]), - ("cume_dist", f.window("cume_dist", [], order_by=[f.order_by(column("b"))]), [0.3333333333333333, 0.6666666666666666, 1.0]), - ("ntile", f.window("ntile", [literal(2)], order_by=[f.order_by(column("c"))]), [1, 1, 2]), - ("next", f.window("lead", [column("b")], order_by=[f.order_by(column("b"))]), [5, 6, None]), - ("previous", f.window("lag", [column("b")], order_by=[f.order_by(column("b"))]), [None, 4, 5]), - pytest.param( - "first_value", - f.window( + ("row", f.window("row_number", [], order_by=[f.order_by(column("c"))]), [2, 1, 3]), + ("rank", f.window("rank", [], order_by=[f.order_by(column("c"))]), [2, 1, 2]), + ( + "dense_rank", + f.window("dense_rank", [], order_by=[f.order_by(column("c"))]), + [2, 1, 2], + ), + ( + "percent_rank", + f.window("percent_rank", [], order_by=[f.order_by(column("c"))]), + [0.5, 0, 0.5], + ), + ( + "cume_dist", + f.window("cume_dist", [], order_by=[f.order_by(column("b"))]), + [0.3333333333333333, 0.6666666666666666, 1.0], + ), + ( + "ntile", + f.window("ntile", [literal(2)], order_by=[f.order_by(column("c"))]), + [1, 1, 2], + ), + ( + "next", + f.window("lead", [column("b")], order_by=[f.order_by(column("b"))]), + [5, 6, None], + ), + ( + "previous", + f.window("lag", [column("b")], order_by=[f.order_by(column("b"))]), + [None, 4, 5], + ), + pytest.param( "first_value", - [column("a")], - order_by=[f.order_by(column("b"))] + f.window("first_value", [column("a")], order_by=[f.order_by(column("b"))]), + [1, 1, 1], + ), + pytest.param( + "last_value", + f.window("last_value", [column("b")], order_by=[f.order_by(column("b"))]), + [4, 5, 6], ), - [1, 1, 1], - ), - pytest.param( - "last_value", - f.window("last_value", [column("b")], order_by=[f.order_by(column("b"))]), - [4, 5, 6], - ), - pytest.param( - "2nd_value", - f.window( - "nth_value", - [column("b"), literal(2)], - order_by=[f.order_by(column("b"))], + pytest.param( + "2nd_value", + f.window( + "nth_value", + [column("b"), literal(2)], + order_by=[f.order_by(column("b"))], + ), + [None, 5, 5], ), - [None, 5, 5], - ), ] @pytest.mark.parametrize("name,expr,result", data_test_window_functions) def test_window_functions(df, name, expr, result): - df = df.select( - column("a"), - column("b"), - column("c"), - f.alias(expr, name) - ) + df = df.select(column("a"), column("b"), column("c"), f.alias(expr, name)) table = pa.Table.from_batches(df.collect()) - expected = { - "a": [1, 2, 3], - "b": [4, 5, 6], - "c": [8, 5, 8], - name: result - } + expected = {"a": [1, 2, 3], "b": [4, 5, 6], "c": [8, 5, 8], name: result} assert table.sort_by("a").to_pydict() == expected diff --git a/python/datafusion/tests/test_expr.py b/python/datafusion/tests/test_expr.py index 1a41120a5..9071108cb 100644 --- a/python/datafusion/tests/test_expr.py +++ b/python/datafusion/tests/test_expr.py @@ -146,24 +146,26 @@ def test_expr_to_variant(): from datafusion import SessionContext from datafusion.expr import Filter - def traverse_logical_plan(plan): cur_node = plan.to_variant() if isinstance(cur_node, Filter): return cur_node.predicate().to_variant() - if hasattr(plan, 'inputs'): + if hasattr(plan, "inputs"): for input_plan in plan.inputs(): res = traverse_logical_plan(input_plan) if res is not None: return res ctx = SessionContext() - data = {'id': [1, 2, 3], 'name': ['Alice', 'Bob', 'Charlie']} - ctx.from_pydict(data, name='table1') + data = {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]} + ctx.from_pydict(data, name="table1") query = "SELECT * FROM table1 t1 WHERE t1.name IN ('dfa', 'ad', 'dfre', 'vsa')" logical_plan = ctx.sql(query).optimized_logical_plan() variant = traverse_logical_plan(logical_plan) assert variant is not None - assert variant.expr().to_variant().qualified_name() == 'table1.name' - assert str(variant.list()) == '[Expr(Utf8("dfa")), Expr(Utf8("ad")), Expr(Utf8("dfre")), Expr(Utf8("vsa"))]' + assert variant.expr().to_variant().qualified_name() == "table1.name" + assert ( + str(variant.list()) + == '[Expr(Utf8("dfa")), Expr(Utf8("ad")), Expr(Utf8("dfre")), Expr(Utf8("vsa"))]' + ) assert not variant.negated() diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index b8ad9c0d7..732136eaa 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -567,45 +567,86 @@ def test_array_function_obj_tests(stmt, py_expr): assert a == b -@pytest.mark.parametrize("function, expected_result", [ - (f.ascii(column("a")), pa.array([72, 87, 33], type=pa.int32())), # H = 72; W = 87; ! = 33 - (f.bit_length(column("a")), pa.array([40, 40, 8], type=pa.int32())), - (f.btrim(literal(" World ")), pa.array(["World", "World", "World"])), - (f.character_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), - (f.chr(literal(68)), pa.array(["D", "D", "D"])), - (f.concat_ws("-", column("a"), literal("test")), pa.array(["Hello-test", "World-test", "!-test"])), - (f.concat(column("a"), literal("?")), pa.array(["Hello?", "World?", "!?"])), - (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), - (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), - (f.length(column("c")), pa.array([6, 7, 2], type=pa.int32())), - (f.lower(column("a")), pa.array(["hello", "world", "!"])), - (f.lpad(column("a"), literal(7)), pa.array([" Hello", " World", " !"])), - (f.ltrim(column("c")), pa.array(["hello ", "world ", "!"])), - (f.md5(column("a")), pa.array([ - "8b1a9953c4611296a827abf8c47804d7", - "f5a7924e621e84c9280a9a27e1bcb7f6", - "9033e0e305f247c0c3c80d0c7848c8b3", - ])), - (f.octet_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), - (f.repeat(column("a"), literal(2)), pa.array(["HelloHello", "WorldWorld", "!!"])), - (f.replace(column("a"), literal("l"), literal("?")), pa.array(["He??o", "Wor?d", "!"])), - (f.reverse(column("a")), pa.array(["olleH", "dlroW", "!"])), - (f.right(column("a"), literal(4)), pa.array(["ello", "orld", "!"])), - (f.rpad(column("a"), literal(8)), pa.array(["Hello ", "World ", "! "])), - (f.rtrim(column("c")), pa.array(["hello", " world", " !"])), - (f.split_part(column("a"), literal("l"), literal(1)), pa.array(["He", "Wor", "!"])), - (f.starts_with(column("a"), literal("Wor")), pa.array([False, True, False])), - (f.strpos(column("a"), literal("o")), pa.array([5, 2, 0], type=pa.int32())), - (f.substr(column("a"), literal(3)), pa.array(["llo", "rld", ""])), - (f.translate(column("a"), literal("or"), literal("ld")), pa.array(["Helll", "Wldld", "!"])), - (f.trim(column("c")), pa.array(["hello", "world", "!"])), - (f.upper(column("c")), pa.array(["HELLO ", " WORLD ", " !"])), - (f.ends_with(column("a"), literal("llo")), pa.array([True, False, False])), - (f.overlay(column("a"), literal("--"), literal(2)), pa.array(["H--lo", "W--ld", "--"])), - (f.regexp_like(column("a"), literal("(ell|orl)")), pa.array([True, True, False])), - (f.regexp_match(column("a"), literal("(ell|orl)")), pa.array([["ell"], ["orl"], None])), - (f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), pa.array(["H-o", "W-d", "!"])), -]) +@pytest.mark.parametrize( + "function, expected_result", + [ + ( + f.ascii(column("a")), + pa.array([72, 87, 33], type=pa.int32()), + ), # H = 72; W = 87; ! = 33 + (f.bit_length(column("a")), pa.array([40, 40, 8], type=pa.int32())), + (f.btrim(literal(" World ")), pa.array(["World", "World", "World"])), + (f.character_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), + (f.chr(literal(68)), pa.array(["D", "D", "D"])), + ( + f.concat_ws("-", column("a"), literal("test")), + pa.array(["Hello-test", "World-test", "!-test"]), + ), + (f.concat(column("a"), literal("?")), pa.array(["Hello?", "World?", "!?"])), + (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), + (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), + (f.length(column("c")), pa.array([6, 7, 2], type=pa.int32())), + (f.lower(column("a")), pa.array(["hello", "world", "!"])), + (f.lpad(column("a"), literal(7)), pa.array([" Hello", " World", " !"])), + (f.ltrim(column("c")), pa.array(["hello ", "world ", "!"])), + ( + f.md5(column("a")), + pa.array( + [ + "8b1a9953c4611296a827abf8c47804d7", + "f5a7924e621e84c9280a9a27e1bcb7f6", + "9033e0e305f247c0c3c80d0c7848c8b3", + ] + ), + ), + (f.octet_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), + ( + f.repeat(column("a"), literal(2)), + pa.array(["HelloHello", "WorldWorld", "!!"]), + ), + ( + f.replace(column("a"), literal("l"), literal("?")), + pa.array(["He??o", "Wor?d", "!"]), + ), + (f.reverse(column("a")), pa.array(["olleH", "dlroW", "!"])), + (f.right(column("a"), literal(4)), pa.array(["ello", "orld", "!"])), + ( + f.rpad(column("a"), literal(8)), + pa.array(["Hello ", "World ", "! "]), + ), + (f.rtrim(column("c")), pa.array(["hello", " world", " !"])), + ( + f.split_part(column("a"), literal("l"), literal(1)), + pa.array(["He", "Wor", "!"]), + ), + (f.starts_with(column("a"), literal("Wor")), pa.array([False, True, False])), + (f.strpos(column("a"), literal("o")), pa.array([5, 2, 0], type=pa.int32())), + (f.substr(column("a"), literal(3)), pa.array(["llo", "rld", ""])), + ( + f.translate(column("a"), literal("or"), literal("ld")), + pa.array(["Helll", "Wldld", "!"]), + ), + (f.trim(column("c")), pa.array(["hello", "world", "!"])), + (f.upper(column("c")), pa.array(["HELLO ", " WORLD ", " !"])), + (f.ends_with(column("a"), literal("llo")), pa.array([True, False, False])), + ( + f.overlay(column("a"), literal("--"), literal(2)), + pa.array(["H--lo", "W--ld", "--"]), + ), + ( + f.regexp_like(column("a"), literal("(ell|orl)")), + pa.array([True, True, False]), + ), + ( + f.regexp_match(column("a"), literal("(ell|orl)")), + pa.array([["ell"], ["orl"], None]), + ), + ( + f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), + pa.array(["H-o", "W-d", "!"]), + ), + ], +) def test_string_functions(df, function, expected_result): df = df.select(function) result = df.collect() @@ -849,27 +890,30 @@ def test_regr_funcs_sql_2(): assert result_sql[0].column(8) == pa.array([4], type=pa.float64()) -@pytest.mark.parametrize("func, expected", [ - pytest.param(f.regr_slope, pa.array([2], type=pa.float64()), id="regr_slope"), - pytest.param(f.regr_intercept, pa.array([0], type=pa.float64()), id="regr_intercept"), - pytest.param(f.regr_count, pa.array([3], type=pa.uint64()), id="regr_count"), - pytest.param(f.regr_r2, pa.array([1], type=pa.float64()), id="regr_r2"), - pytest.param(f.regr_avgx, pa.array([2], type=pa.float64()), id="regr_avgx"), - pytest.param(f.regr_avgy, pa.array([4], type=pa.float64()), id="regr_avgy"), - pytest.param(f.regr_sxx, pa.array([2], type=pa.float64()), id="regr_sxx"), - pytest.param(f.regr_syy, pa.array([8], type=pa.float64()), id="regr_syy"), - pytest.param(f.regr_sxy, pa.array([4], type=pa.float64()), id="regr_sxy") -]) +@pytest.mark.parametrize( + "func, expected", + [ + pytest.param(f.regr_slope, pa.array([2], type=pa.float64()), id="regr_slope"), + pytest.param( + f.regr_intercept, pa.array([0], type=pa.float64()), id="regr_intercept" + ), + pytest.param(f.regr_count, pa.array([3], type=pa.uint64()), id="regr_count"), + pytest.param(f.regr_r2, pa.array([1], type=pa.float64()), id="regr_r2"), + pytest.param(f.regr_avgx, pa.array([2], type=pa.float64()), id="regr_avgx"), + pytest.param(f.regr_avgy, pa.array([4], type=pa.float64()), id="regr_avgy"), + pytest.param(f.regr_sxx, pa.array([2], type=pa.float64()), id="regr_sxx"), + pytest.param(f.regr_syy, pa.array([8], type=pa.float64()), id="regr_syy"), + pytest.param(f.regr_sxy, pa.array([4], type=pa.float64()), id="regr_sxy"), + ], +) def test_regr_funcs_df(func, expected): - # test case based on `regr_*() basic tests # https://github.com/apache/datafusion/blob/d1361d56b9a9e0c165d3d71a8df6795d2a5f51dd/datafusion/core/tests/sqllogictests/test_files/aggregate.slt#L2358C1-L2374C1 - ctx = SessionContext() # Create a DataFrame - data = {'column1': [1, 2, 3], 'column2': [2, 4, 6]} + data = {"column1": [1, 2, 3], "column2": [2, 4, 6]} df = ctx.from_pydict(data, name="test_table") # Perform the regression function using DataFrame API diff --git a/python/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py index 1505fb1e7..e41d01004 100644 --- a/python/datafusion/tests/test_sql.py +++ b/python/datafusion/tests/test_sql.py @@ -381,15 +381,31 @@ def test_udf( id="binary4", marks=pytest.mark.xfail, ), - pytest.param(helpers.data_datetime("s"), id="datetime_s", marks=pytest.mark.xfail), - pytest.param(helpers.data_datetime("ms"), id="datetime_ms", marks=pytest.mark.xfail), - pytest.param(helpers.data_datetime("us"), id="datetime_us", marks=pytest.mark.xfail), - pytest.param(helpers.data_datetime("ns"), id="datetime_ns", marks=pytest.mark.xfail), + pytest.param( + helpers.data_datetime("s"), id="datetime_s", marks=pytest.mark.xfail + ), + pytest.param( + helpers.data_datetime("ms"), id="datetime_ms", marks=pytest.mark.xfail + ), + pytest.param( + helpers.data_datetime("us"), id="datetime_us", marks=pytest.mark.xfail + ), + pytest.param( + helpers.data_datetime("ns"), id="datetime_ns", marks=pytest.mark.xfail + ), # Not writtable to parquet - pytest.param(helpers.data_timedelta("s"), id="timedelta_s", marks=pytest.mark.xfail), - pytest.param(helpers.data_timedelta("ms"), id="timedelta_ms", marks=pytest.mark.xfail), - pytest.param(helpers.data_timedelta("us"), id="timedelta_us", marks=pytest.mark.xfail), - pytest.param(helpers.data_timedelta("ns"), id="timedelta_ns", marks=pytest.mark.xfail), + pytest.param( + helpers.data_timedelta("s"), id="timedelta_s", marks=pytest.mark.xfail + ), + pytest.param( + helpers.data_timedelta("ms"), id="timedelta_ms", marks=pytest.mark.xfail + ), + pytest.param( + helpers.data_timedelta("us"), id="timedelta_us", marks=pytest.mark.xfail + ), + pytest.param( + helpers.data_timedelta("ns"), id="timedelta_ns", marks=pytest.mark.xfail + ), ], ) def test_simple_select(ctx, tmp_path, arr): From 69ed7fe41cc5524fcf2664af32aa78181e188e0b Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 30 Aug 2024 17:04:38 -0400 Subject: [PATCH 185/413] Add PyCapsule support for Arrow import and export (#825) --- docs/source/user-guide/io/arrow.rst | 73 ++++++++++++++ docs/source/user-guide/io/index.rst | 6 +- examples/import.py | 2 +- python/datafusion/context.py | 24 +++-- python/datafusion/dataframe.py | 16 +++ python/datafusion/tests/test_context.py | 37 ++++++- python/datafusion/tests/test_dataframe.py | 51 ++++++++-- src/context.rs | 46 ++++++--- src/dataframe.rs | 116 +++++++++++++++++++++- 9 files changed, 330 insertions(+), 41 deletions(-) create mode 100644 docs/source/user-guide/io/arrow.rst diff --git a/docs/source/user-guide/io/arrow.rst b/docs/source/user-guide/io/arrow.rst new file mode 100644 index 000000000..d571aa99c --- /dev/null +++ b/docs/source/user-guide/io/arrow.rst @@ -0,0 +1,73 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Arrow +===== + +DataFusion implements the +`Apache Arrow PyCapsule interface `_ +for importing and exporting DataFrames with zero copy. With this feature, any Python +project that implements this interface can share data back and forth with DataFusion +with zero copy. + +We can demonstrate using `pyarrow `_. + +Importing to DataFusion +----------------------- + +Here we will create an Arrow table and import it to DataFusion. + +To import an Arrow table, use :py:func:`datafusion.context.SessionContext.from_arrow`. +This will accept any Python object that implements +`__arrow_c_stream__ `_ +or `__arrow_c_array__ `_ +and returns a ``StructArray``. Common pyarrow sources you can use are: + +- `Array `_ (but it must return a Struct Array) +- `Record Batch `_ +- `Record Batch Reader `_ +- `Table `_ + +.. ipython:: python + + from datafusion import SessionContext + import pyarrow as pa + + data = {"a": [1, 2, 3], "b": [4, 5, 6]} + table = pa.Table.from_pydict(data) + + ctx = SessionContext() + df = ctx.from_arrow(table) + df + +Exporting from DataFusion +------------------------- + +DataFusion DataFrames implement ``__arrow_c_stream__`` PyCapsule interface, so any +Python library that accepts these can import a DataFusion DataFrame directly. + +.. warning:: + It is important to note that this will cause the DataFrame execution to happen, which may be + a time consuming task. That is, you will cause a + :py:func:`datafusion.dataframe.DataFrame.collect` operation call to occur. + + +.. ipython:: python + + df = df.select((col("a") * lit(1.5)).alias("c"), lit("df").alias("d")) + pa.table(df) + diff --git a/docs/source/user-guide/io/index.rst b/docs/source/user-guide/io/index.rst index af08240ff..05411327e 100644 --- a/docs/source/user-guide/io/index.rst +++ b/docs/source/user-guide/io/index.rst @@ -21,8 +21,8 @@ IO .. toctree:: :maxdepth: 2 + arrow + avro csv - parquet json - avro - + parquet diff --git a/examples/import.py b/examples/import.py index a249a1c4e..cd965cb46 100644 --- a/examples/import.py +++ b/examples/import.py @@ -54,5 +54,5 @@ # Convert Arrow Table to datafusion DataFrame arrow_table = pa.Table.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]}) -df = ctx.from_arrow_table(arrow_table) +df = ctx.from_arrow(arrow_table) assert type(df) == datafusion.DataFrame diff --git a/python/datafusion/context.py b/python/datafusion/context.py index d4e50cfe2..283f71e1e 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -586,19 +586,31 @@ def from_pydict( """ return DataFrame(self.ctx.from_pydict(data, name)) - def from_arrow_table( - self, data: pyarrow.Table, name: str | None = None - ) -> DataFrame: - """Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow table. + def from_arrow(self, data: Any, name: str | None = None) -> DataFrame: + """Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow source. + + The Arrow data source can be any object that implements either + ``__arrow_c_stream__`` or ``__arrow_c_array__``. For the latter, it must return + a struct array. Common examples of sources from pyarrow include Args: - data: Arrow table. + data: Arrow data source. name: Name of the DataFrame. Returns: DataFrame representation of the Arrow table. """ - return DataFrame(self.ctx.from_arrow_table(data, name)) + return DataFrame(self.ctx.from_arrow(data, name)) + + @deprecated("Use ``from_arrow`` instead.") + def from_arrow_table( + self, data: pyarrow.Table, name: str | None = None + ) -> DataFrame: + """Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow table. + + This is an alias for :py:func:`from_arrow`. + """ + return self.from_arrow(data, name) def from_pandas(self, data: pandas.DataFrame, name: str | None = None) -> DataFrame: """Create a :py:class:`~datafusion.dataframe.DataFrame` from a Pandas DataFrame. diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index fa7398442..4f1760135 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -524,3 +524,19 @@ def unnest_columns(self, *columns: str, preserve_nulls: bool = True) -> DataFram """ columns = [c for c in columns] return DataFrame(self.df.unnest_columns(columns, preserve_nulls=preserve_nulls)) + + def __arrow_c_stream__(self, requested_schema: pa.Schema) -> Any: + """Export an Arrow PyCapsule Stream. + + This will execute and collect the DataFrame. We will attempt to respect the + requested schema, but only trivial transformations will be applied such as only + returning the fields listed in the requested schema if their data types match + those in the DataFrame. + + Args: + requested_schema: Attempt to provide the DataFrame using this schema. + + Returns: + Arrow PyCapsule object. + """ + return self.df.__arrow_c_stream__(requested_schema) diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index 66d7e013a..0184280c2 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -156,7 +156,7 @@ def test_from_arrow_table(ctx): table = pa.Table.from_pydict(data) # convert to DataFrame - df = ctx.from_arrow_table(table) + df = ctx.from_arrow(table) tables = list(ctx.catalog().database().names()) assert df @@ -166,13 +166,42 @@ def test_from_arrow_table(ctx): assert df.collect()[0].num_rows == 3 +def record_batch_generator(num_batches: int): + schema = pa.schema([("a", pa.int64()), ("b", pa.int64())]) + for i in range(num_batches): + yield pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], schema=schema + ) + + +@pytest.mark.parametrize( + "source", + [ + # __arrow_c_array__ sources + pa.array([{"a": 1, "b": 4}, {"a": 2, "b": 5}, {"a": 3, "b": 6}]), + # __arrow_c_stream__ sources + pa.RecordBatch.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]}), + pa.RecordBatchReader.from_batches( + pa.schema([("a", pa.int64()), ("b", pa.int64())]), record_batch_generator(1) + ), + pa.Table.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]}), + ], +) +def test_from_arrow_sources(ctx, source) -> None: + df = ctx.from_arrow(source) + assert df + assert isinstance(df, DataFrame) + assert df.schema().names == ["a", "b"] + assert df.count() == 3 + + def test_from_arrow_table_with_name(ctx): # create a PyArrow table data = {"a": [1, 2, 3], "b": [4, 5, 6]} table = pa.Table.from_pydict(data) # convert to DataFrame with optional name - df = ctx.from_arrow_table(table, name="tbl") + df = ctx.from_arrow(table, name="tbl") tables = list(ctx.catalog().database().names()) assert df @@ -185,7 +214,7 @@ def test_from_arrow_table_empty(ctx): table = pa.Table.from_pydict(data, schema=schema) # convert to DataFrame - df = ctx.from_arrow_table(table) + df = ctx.from_arrow(table) tables = list(ctx.catalog().database().names()) assert df @@ -200,7 +229,7 @@ def test_from_arrow_table_empty_no_schema(ctx): table = pa.Table.from_pydict(data) # convert to DataFrame - df = ctx.from_arrow_table(table) + df = ctx.from_arrow(table) tables = list(ctx.catalog().database().names()) assert df diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index e5e0c9c8a..477bc0fce 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -47,7 +47,7 @@ def df(): names=["a", "b", "c"], ) - return ctx.create_dataframe([[batch]]) + return ctx.from_arrow(batch) @pytest.fixture @@ -835,13 +835,42 @@ def test_write_compressed_parquet_missing_compression_level(df, tmp_path, compre df.write_parquet(str(path), compression=compression) -# ctx = SessionContext() - -# # create a RecordBatch and a new DataFrame from it -# batch = pa.RecordBatch.from_arrays( -# [pa.array([1, 2, 3]), pa.array([4, 5, 6]), pa.array([8, 5, 8])], -# names=["a", "b", "c"], -# ) - -# df = ctx.create_dataframe([[batch]]) -# test_execute_stream(df) +def test_dataframe_export(df) -> None: + # Guarantees that we have the canonical implementation + # reading our dataframe export + table = pa.table(df) + assert table.num_columns == 3 + assert table.num_rows == 3 + + desired_schema = pa.schema([("a", pa.int64())]) + + # Verify we can request a schema + table = pa.table(df, schema=desired_schema) + assert table.num_columns == 1 + assert table.num_rows == 3 + + # Expect a table of nulls if the schema don't overlap + desired_schema = pa.schema([("g", pa.string())]) + table = pa.table(df, schema=desired_schema) + assert table.num_columns == 1 + assert table.num_rows == 3 + for i in range(0, 3): + assert table[0][i].as_py() is None + + # Expect an error when we cannot convert schema + desired_schema = pa.schema([("a", pa.float32())]) + failed_convert = False + try: + table = pa.table(df, schema=desired_schema) + except Exception: + failed_convert = True + assert failed_convert + + # Expect an error when we have a not set non-nullable + desired_schema = pa.schema([("g", pa.string(), False)]) + failed_convert = False + try: + table = pa.table(df, schema=desired_schema) + except Exception: + failed_convert = True + assert failed_convert diff --git a/src/context.rs b/src/context.rs index a43599cf7..4433d94c2 100644 --- a/src/context.rs +++ b/src/context.rs @@ -20,12 +20,15 @@ use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; +use arrow::array::RecordBatchReader; +use arrow::ffi_stream::ArrowArrayStreamReader; +use arrow::pyarrow::FromPyArrow; use datafusion::execution::session_state::SessionStateBuilder; use object_store::ObjectStore; use url::Url; use uuid::Uuid; -use pyo3::exceptions::{PyKeyError, PyValueError}; +use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; use pyo3::prelude::*; use crate::catalog::{PyCatalog, PyTable}; @@ -444,7 +447,7 @@ impl PySessionContext { let table = table_class.call_method1("from_pylist", args)?; // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; + let df = self.from_arrow(table, name, py)?; Ok(df) } @@ -463,29 +466,42 @@ impl PySessionContext { let table = table_class.call_method1("from_pydict", args)?; // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; + let df = self.from_arrow(table, name, py)?; Ok(df) } /// Construct datafusion dataframe from Arrow Table - pub fn from_arrow_table( + pub fn from_arrow( &mut self, data: Bound<'_, PyAny>, name: Option<&str>, py: Python, ) -> PyResult { - // Instantiate pyarrow Table object & convert to batches - let table = data.call_method0("to_batches")?; + let (schema, batches) = + if let Ok(stream_reader) = ArrowArrayStreamReader::from_pyarrow_bound(&data) { + // Works for any object that implements __arrow_c_stream__ in pycapsule. + + let schema = stream_reader.schema().as_ref().to_owned(); + let batches = stream_reader + .collect::, arrow::error::ArrowError>>() + .map_err(DataFusionError::from)?; + + (schema, batches) + } else if let Ok(array) = RecordBatch::from_pyarrow_bound(&data) { + // While this says RecordBatch, it will work for any object that implements + // __arrow_c_array__ and returns a StructArray. + + (array.schema().as_ref().to_owned(), vec![array]) + } else { + return Err(PyTypeError::new_err( + "Expected either a Arrow Array or Arrow Stream in from_arrow().", + )); + }; - let schema = data.getattr("schema")?; - let schema = schema.extract::>()?; - - // Cast PyAny to RecordBatch type // Because create_dataframe() expects a vector of vectors of record batches // here we need to wrap the vector of record batches in an additional vector - let batches = table.extract::>>()?; - let list_of_batches = PyArrowType::from(vec![batches.0]); - self.create_dataframe(list_of_batches, name, Some(schema), py) + let list_of_batches = PyArrowType::from(vec![batches]); + self.create_dataframe(list_of_batches, name, Some(schema.into()), py) } /// Construct datafusion dataframe from pandas @@ -504,7 +520,7 @@ impl PySessionContext { let table = table_class.call_method1("from_pandas", args)?; // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, py)?; + let df = self.from_arrow(table, name, py)?; Ok(df) } @@ -518,7 +534,7 @@ impl PySessionContext { let table = data.call_method0("to_arrow")?; // Convert Arrow Table to datafusion DataFrame - let df = self.from_arrow_table(table, name, data.py())?; + let df = self.from_arrow(table, name, data.py())?; Ok(df) } diff --git a/src/dataframe.rs b/src/dataframe.rs index 4db59d4fe..22b05226c 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -15,8 +15,14 @@ // specific language governing permissions and limitations // under the License. +use std::ffi::CString; use std::sync::Arc; +use arrow::array::{new_null_array, RecordBatch, RecordBatchIterator, RecordBatchReader}; +use arrow::compute::can_cast_types; +use arrow::error::ArrowError; +use arrow::ffi::FFI_ArrowSchema; +use arrow::ffi_stream::FFI_ArrowArrayStream; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; @@ -29,7 +35,7 @@ use datafusion_common::UnnestOptions; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; -use pyo3::types::PyTuple; +use pyo3::types::{PyCapsule, PyTuple}; use tokio::task::JoinHandle; use crate::errors::py_datafusion_err; @@ -451,6 +457,39 @@ impl PyDataFrame { Ok(table) } + fn __arrow_c_stream__<'py>( + &'py mut self, + py: Python<'py>, + requested_schema: Option>, + ) -> PyResult> { + let mut batches = wait_for_future(py, self.df.as_ref().clone().collect())?; + let mut schema: Schema = self.df.schema().to_owned().into(); + + if let Some(schema_capsule) = requested_schema { + validate_pycapsule(&schema_capsule, "arrow_schema")?; + + let schema_ptr = unsafe { schema_capsule.reference::() }; + let desired_schema = Schema::try_from(schema_ptr).map_err(DataFusionError::from)?; + + schema = project_schema(schema, desired_schema).map_err(DataFusionError::ArrowError)?; + + batches = batches + .into_iter() + .map(|record_batch| record_batch_into_schema(record_batch, &schema)) + .collect::, ArrowError>>() + .map_err(DataFusionError::ArrowError)?; + } + + let batches_wrapped = batches.into_iter().map(Ok); + + let reader = RecordBatchIterator::new(batches_wrapped, Arc::new(schema)); + let reader: Box = Box::new(reader); + + let ffi_stream = FFI_ArrowArrayStream::new(reader); + let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); + PyCapsule::new_bound(py, ffi_stream, Some(stream_capsule_name)) + } + fn execute_stream(&self, py: Python) -> PyResult { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; @@ -539,3 +578,78 @@ fn print_dataframe(py: Python, df: DataFrame) -> PyResult<()> { print.call1((result,))?; Ok(()) } + +fn project_schema(from_schema: Schema, to_schema: Schema) -> Result { + let merged_schema = Schema::try_merge(vec![from_schema, to_schema.clone()])?; + + let project_indices: Vec = to_schema + .fields + .iter() + .map(|field| field.name()) + .filter_map(|field_name| merged_schema.index_of(field_name).ok()) + .collect(); + + merged_schema.project(&project_indices) +} + +fn record_batch_into_schema( + record_batch: RecordBatch, + schema: &Schema, +) -> Result { + let schema = Arc::new(schema.clone()); + let base_schema = record_batch.schema(); + if base_schema.fields().len() == 0 { + // Nothing to project + return Ok(RecordBatch::new_empty(schema)); + } + + let array_size = record_batch.column(0).len(); + let mut data_arrays = Vec::with_capacity(schema.fields().len()); + + for field in schema.fields() { + let desired_data_type = field.data_type(); + if let Some(original_data) = record_batch.column_by_name(field.name()) { + let original_data_type = original_data.data_type(); + + if can_cast_types(original_data_type, desired_data_type) { + data_arrays.push(arrow::compute::kernels::cast( + original_data, + desired_data_type, + )?); + } else if field.is_nullable() { + data_arrays.push(new_null_array(desired_data_type, array_size)); + } else { + return Err(ArrowError::CastError(format!("Attempting to cast to non-nullable and non-castable field {} during schema projection.", field.name()))); + } + } else { + if !field.is_nullable() { + return Err(ArrowError::CastError(format!( + "Attempting to set null to non-nullable field {} during schema projection.", + field.name() + ))); + } + data_arrays.push(new_null_array(desired_data_type, array_size)); + } + } + + RecordBatch::try_new(schema, data_arrays) +} + +fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<()> { + let capsule_name = capsule.name()?; + if capsule_name.is_none() { + return Err(PyValueError::new_err( + "Expected schema PyCapsule to have name set.", + )); + } + + let capsule_name = capsule_name.unwrap().to_str()?; + if capsule_name != name { + return Err(PyValueError::new_err(format!( + "Expected name '{}' in PyCapsule, instead got '{}'", + name, capsule_name + ))); + } + + Ok(()) +} From 003eea8fe75bed9c12200ce296a8c7fa5212b534 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 30 Aug 2024 17:05:00 -0400 Subject: [PATCH 186/413] Feature/expose when function (#836) --- python/datafusion/functions.py | 11 +++++++++++ python/datafusion/tests/test_functions.py | 19 +++++++++++++++++++ src/functions.rs | 9 +++++++++ 3 files changed, 39 insertions(+) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 59a1974fd..ec0c1104d 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -245,6 +245,7 @@ "var", "var_pop", "var_samp", + "when", "window", ] @@ -364,6 +365,16 @@ def case(expr: Expr) -> CaseBuilder: return CaseBuilder(f.case(expr.expr)) +def when(when: Expr, then: Expr) -> CaseBuilder: + """Create a case expression that has no base expression. + + Create a :py:class:`~datafusion.expr.CaseBuilder` to match cases for the + expression ``expr``. See :py:class:`~datafusion.expr.CaseBuilder` for + detailed usage. + """ + return CaseBuilder(f.when(when.expr, then.expr)) + + def window( name: str, args: list[Expr], diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 732136eaa..e5429bd60 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -836,6 +836,25 @@ def test_case(df): assert result.column(2) == pa.array(["Hola", "Mundo", None]) +def test_when_with_no_base(df): + df.show() + df = df.select( + column("b"), + f.when(column("b") > literal(5), literal("too big")) + .when(column("b") < literal(5), literal("too small")) + .otherwise(literal("just right")) + .alias("goldilocks"), + f.when(column("a") == literal("Hello"), column("a")).end().alias("greeting"), + ) + df.show() + + result = df.collect() + result = result[0] + assert result.column(0) == pa.array([4, 5, 6]) + assert result.column(1) == pa.array(["too small", "just right", "too big"]) + assert result.column(2) == pa.array(["Hello", None, None]) + + def test_regr_funcs_sql(df): # test case base on # https://github.com/apache/arrow-datafusion/blob/d1361d56b9a9e0c165d3d71a8df6795d2a5f51dd/datafusion/core/tests/sqllogictests/test_files/aggregate.slt#L2330 diff --git a/src/functions.rs b/src/functions.rs index c53d4ad92..252563621 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -533,6 +533,14 @@ fn case(expr: PyExpr) -> PyResult { }) } +/// Create a CASE WHEN statement with literal WHEN expressions for comparison to the base expression. +#[pyfunction] +fn when(when: PyExpr, then: PyExpr) -> PyResult { + Ok(PyCaseBuilder { + case_builder: datafusion_expr::when(when.expr, then.expr), + }) +} + /// Helper function to find the appropriate window function. /// /// Search procedure: @@ -910,6 +918,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(char_length))?; m.add_wrapped(wrap_pyfunction!(coalesce))?; m.add_wrapped(wrap_pyfunction!(case))?; + m.add_wrapped(wrap_pyfunction!(when))?; m.add_wrapped(wrap_pyfunction!(col))?; m.add_wrapped(wrap_pyfunction!(concat_ws))?; m.add_wrapped(wrap_pyfunction!(concat))?; From 90f5b5b355d79b56ae3607b7b0cdeb09b67e5121 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 2 Sep 2024 10:51:00 -0400 Subject: [PATCH 187/413] Add Window Functions for use with function builder (#808) * Add window function as template for others and function builder * Adding docstrings * Change last_value to use function builder instead of explicitly passing values * Allow any value for lead function default value and add unit test * Add lead window function and unit tests * Temporarily commenting out deprecated functions in documenation so builder will pass * Expose row_number window function * Add rank window function * Add percent rank and dense rank * Add cume_dist * Add ntile window function * Add comment to update when upstream merges * Window frame required calling inner value * Add unit test for avg as window function * Working on documentation for window functions * Add pyo build config file to git ignore since this is user specific * Add examples to docstring * Optionally add window function parameters during function call * Update sort and order_by to apply automatic ordering if any other expression is given * Update unit tests to be cleaner and use default sort on expressions * Ignore vscode folder specific settings * Window frames should only apply to aggregate functions used as window functions. Also pass in scalar pyarrow values so we can set a range other than a uint * Remove deprecated warning until we actually have a way to use all functions without calling window() * Built in window functions do not have any impact by setting null_treatment so remove from user facing * Update user documentation on how to pass parameters for different window functions and what their impacts are * Make first_value and last_value identical in the interface --- .gitignore | 4 + .../common-operations/aggregations.rst | 2 + .../user-guide/common-operations/windows.rst | 187 +++++++-- python/datafusion/dataframe.py | 7 +- python/datafusion/expr.py | 113 ++++- python/datafusion/functions.py | 390 +++++++++++++++++- python/datafusion/tests/test_dataframe.py | 182 ++++++-- python/datafusion/tests/test_functions.py | 1 + src/dataframe.rs | 3 +- src/expr.rs | 110 ++++- src/expr/window.rs | 12 +- src/functions.rs | 176 ++++++-- 12 files changed, 1059 insertions(+), 128 deletions(-) diff --git a/.gitignore b/.gitignore index 0030b907b..aaeaaa5b1 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ target /docs/temp /docs/build .DS_Store +.vscode # Byte-compiled / optimized / DLL files __pycache__/ @@ -31,3 +32,6 @@ apache-rat-*.jar CHANGELOG.md.bak docs/mdbook/book + +.pyo3_build_config + diff --git a/docs/source/user-guide/common-operations/aggregations.rst b/docs/source/user-guide/common-operations/aggregations.rst index b9202129e..7ad402210 100644 --- a/docs/source/user-guide/common-operations/aggregations.rst +++ b/docs/source/user-guide/common-operations/aggregations.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _aggregation: + Aggregation ============ diff --git a/docs/source/user-guide/common-operations/windows.rst b/docs/source/user-guide/common-operations/windows.rst index 5ef3c986c..609176897 100644 --- a/docs/source/user-guide/common-operations/windows.rst +++ b/docs/source/user-guide/common-operations/windows.rst @@ -15,13 +15,16 @@ .. specific language governing permissions and limitations .. under the License. +.. _window_functions: + Window Functions ================ -In this section you will learn about window functions. A window function utilizes values from one or multiple rows to -produce a result for each individual row, unlike an aggregate function that provides a single value for multiple rows. +In this section you will learn about window functions. A window function utilizes values from one or +multiple rows to produce a result for each individual row, unlike an aggregate function that +provides a single value for multiple rows. -The functionality of window functions in DataFusion is supported by the dedicated :py:func:`~datafusion.functions.window` function. +The window functions are availble in the :py:mod:`~datafusion.functions` module. We'll use the pokemon dataset (from Ritchie Vink) in the following examples. @@ -40,20 +43,25 @@ We'll use the pokemon dataset (from Ritchie Vink) in the following examples. ctx = SessionContext() df = ctx.read_csv("pokemon.csv") -Here is an example that shows how to compare each pokemons’s attack power with the average attack power in its ``"Type 1"`` +Here is an example that shows how you can compare each pokemon's speed to the speed of the +previous row in the DataFrame. .. ipython:: python df.select( col('"Name"'), - col('"Attack"'), - f.alias( - f.window("avg", [col('"Attack"')], partition_by=[col('"Type 1"')]), - "Average Attack", - ) + col('"Speed"'), + f.lag(col('"Speed"')).alias("Previous Speed") ) -You can also control the order in which rows are processed by window functions by providing +Setting Parameters +------------------ + + +Ordering +^^^^^^^^ + +You can control the order in which rows are processed by window functions by providing a list of ``order_by`` functions for the ``order_by`` parameter. .. ipython:: python @@ -61,33 +69,150 @@ a list of ``order_by`` functions for the ``order_by`` parameter. df.select( col('"Name"'), col('"Attack"'), - f.alias( - f.window( - "rank", - [], - partition_by=[col('"Type 1"')], - order_by=[f.order_by(col('"Attack"'))], - ), - "rank", - ), + col('"Type 1"'), + f.rank( + partition_by=[col('"Type 1"')], + order_by=[col('"Attack"').sort(ascending=True)], + ).alias("rank"), + ).sort(col('"Type 1"'), col('"Attack"')) + +Partitions +^^^^^^^^^^ + +A window function can take a list of ``partition_by`` columns similar to an +:ref:`Aggregation Function`. This will cause the window values to be evaluated +independently for each of the partitions. In the example above, we found the rank of each +Pokemon per ``Type 1`` partitions. We can see the first couple of each partition if we do +the following: + +.. ipython:: python + + df.select( + col('"Name"'), + col('"Attack"'), + col('"Type 1"'), + f.rank( + partition_by=[col('"Type 1"')], + order_by=[col('"Attack"').sort(ascending=True)], + ).alias("rank"), + ).filter(col("rank") < lit(3)).sort(col('"Type 1"'), col("rank")) + +Window Frame +^^^^^^^^^^^^ + +When using aggregate functions, the Window Frame of defines the rows over which it operates. +If you do not specify a Window Frame, the frame will be set depending on the following +criteria. + +* If an ``order_by`` clause is set, the default window frame is defined as the rows between + unbounded preceeding and the current row. +* If an ``order_by`` is not set, the default frame is defined as the rows betwene unbounded + and unbounded following (the entire partition). + +Window Frames are defined by three parameters: unit type, starting bound, and ending bound. + +The unit types available are: + +* Rows: The starting and ending boundaries are defined by the number of rows relative to the + current row. +* Range: When using Range, the ``order_by`` clause must have exactly one term. The boundaries + are defined bow how close the rows are to the value of the expression in the ``order_by`` + parameter. +* Groups: A "group" is the set of all rows that have equivalent values for all terms in the + ``order_by`` clause. + +In this example we perform a "rolling average" of the speed of the current Pokemon and the +two preceeding rows. + +.. ipython:: python + + from datafusion.expr import WindowFrame + + df.select( + col('"Name"'), + col('"Speed"'), + f.window("avg", + [col('"Speed"')], + order_by=[col('"Speed"')], + window_frame=WindowFrame("rows", 2, 0) + ).alias("Previous Speed") + ) + +Null Treatment +^^^^^^^^^^^^^^ + +When using aggregate functions as window functions, it is often useful to specify how null values +should be treated. In order to do this you need to use the builder function. In future releases +we expect this to be simplified in the interface. + +One common usage for handling nulls is the case where you want to find the last value up to the +current row. In the following example we demonstrate how setting the null treatment to ignore +nulls will fill in with the value of the most recent non-null row. To do this, we also will set +the window frame so that we only process up to the current row. + +In this example, we filter down to one specific type of Pokemon that does have some entries in +it's ``Type 2`` column that are null. + +.. ipython:: python + + from datafusion.common import NullTreatment + + df.filter(col('"Type 1"') == lit("Bug")).select( + '"Name"', + '"Type 2"', + f.window("last_value", [col('"Type 2"')]) + .window_frame(WindowFrame("rows", None, 0)) + .order_by(col('"Speed"')) + .null_treatment(NullTreatment.IGNORE_NULLS) + .build() + .alias("last_wo_null"), + f.window("last_value", [col('"Type 2"')]) + .window_frame(WindowFrame("rows", None, 0)) + .order_by(col('"Speed"')) + .null_treatment(NullTreatment.RESPECT_NULLS) + .build() + .alias("last_with_null") + ) + +Aggregate Functions +------------------- + +You can use any :ref:`Aggregation Function` as a window function. Currently +aggregate functions must use the deprecated +:py:func:`datafusion.functions.window` API but this should be resolved in +DataFusion 42.0 (`Issue Link `_). Here +is an example that shows how to compare each pokemons’s attack power with the average attack +power in its ``"Type 1"`` using the :py:func:`datafusion.functions.avg` function. + +.. ipython:: python + :okwarning: + + df.select( + col('"Name"'), + col('"Attack"'), + col('"Type 1"'), + f.window("avg", [col('"Attack"')]) + .partition_by(col('"Type 1"')) + .build() + .alias("Average Attack"), ) +Available Functions +------------------- + The possible window functions are: 1. Rank Functions - - rank - - dense_rank - - row_number - - ntile + - :py:func:`datafusion.functions.rank` + - :py:func:`datafusion.functions.dense_rank` + - :py:func:`datafusion.functions.ntile` + - :py:func:`datafusion.functions.row_number` 2. Analytical Functions - - cume_dist - - percent_rank - - lag - - lead - - first_value - - last_value - - nth_value + - :py:func:`datafusion.functions.cume_dist` + - :py:func:`datafusion.functions.percent_rank` + - :py:func:`datafusion.functions.lag` + - :py:func:`datafusion.functions.lead` 3. Aggregate Functions - - All aggregate functions can be used as window functions. + - All :ref:`Aggregation Functions` can be used as window functions. diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 4f1760135..0e7d82e29 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -123,11 +123,10 @@ def select(self, *exprs: Expr | str) -> DataFrame: df = df.select("a", col("b"), col("a").alias("alternate_a")) """ - exprs = [ - arg.expr if isinstance(arg, Expr) else Expr.column(arg).expr - for arg in exprs + exprs_internal = [ + Expr.column(arg).expr if isinstance(arg, str) else arg.expr for arg in exprs ] - return DataFrame(self.df.select(*exprs)) + return DataFrame(self.df.select(*exprs_internal)) def filter(self, *predicates: Expr) -> DataFrame: """Return a DataFrame for which ``predicate`` evaluates to ``True``. diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 71fcf397b..c7272bb3b 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -23,8 +23,8 @@ from __future__ import annotations from ._internal import expr as expr_internal, LogicalPlan -from datafusion.common import RexType, DataTypeMap -from typing import Any +from datafusion.common import NullTreatment, RexType, DataTypeMap +from typing import Any, Optional import pyarrow as pa # The following are imported from the internal representation. We may choose to @@ -355,6 +355,10 @@ def is_null(self) -> Expr: """Returns ``True`` if this expression is null.""" return Expr(self.expr.is_null()) + def is_not_null(self) -> Expr: + """Returns ``True`` if this expression is not null.""" + return Expr(self.expr.is_not_null()) + def cast(self, to: pa.DataType[Any]) -> Expr: """Cast to a new data type.""" return Expr(self.expr.cast(to)) @@ -405,12 +409,107 @@ def column_name(self, plan: LogicalPlan) -> str: """Compute the output column name based on the provided logical plan.""" return self.expr.column_name(plan) + def order_by(self, *exprs: Expr) -> ExprFuncBuilder: + """Set the ordering for a window or aggregate function. + + This function will create an :py:class:`ExprFuncBuilder` that can be used to + set parameters for either window or aggregate functions. If used on any other + type of expression, an error will be generated when ``build()`` is called. + """ + return ExprFuncBuilder(self.expr.order_by(list(e.expr for e in exprs))) + + def filter(self, filter: Expr) -> ExprFuncBuilder: + """Filter an aggregate function. + + This function will create an :py:class:`ExprFuncBuilder` that can be used to + set parameters for either window or aggregate functions. If used on any other + type of expression, an error will be generated when ``build()`` is called. + """ + return ExprFuncBuilder(self.expr.filter(filter.expr)) + + def distinct(self) -> ExprFuncBuilder: + """Only evaluate distinct values for an aggregate function. + + This function will create an :py:class:`ExprFuncBuilder` that can be used to + set parameters for either window or aggregate functions. If used on any other + type of expression, an error will be generated when ``build()`` is called. + """ + return ExprFuncBuilder(self.expr.distinct()) + + def null_treatment(self, null_treatment: NullTreatment) -> ExprFuncBuilder: + """Set the treatment for ``null`` values for a window or aggregate function. + + This function will create an :py:class:`ExprFuncBuilder` that can be used to + set parameters for either window or aggregate functions. If used on any other + type of expression, an error will be generated when ``build()`` is called. + """ + return ExprFuncBuilder(self.expr.null_treatment(null_treatment)) + + def partition_by(self, *partition_by: Expr) -> ExprFuncBuilder: + """Set the partitioning for a window function. + + This function will create an :py:class:`ExprFuncBuilder` that can be used to + set parameters for either window or aggregate functions. If used on any other + type of expression, an error will be generated when ``build()`` is called. + """ + return ExprFuncBuilder( + self.expr.partition_by(list(e.expr for e in partition_by)) + ) + + def window_frame(self, window_frame: WindowFrame) -> ExprFuncBuilder: + """Set the frame fora window function. + + This function will create an :py:class:`ExprFuncBuilder` that can be used to + set parameters for either window or aggregate functions. If used on any other + type of expression, an error will be generated when ``build()`` is called. + """ + return ExprFuncBuilder(self.expr.window_frame(window_frame.window_frame)) + + +class ExprFuncBuilder: + def __init__(self, builder: expr_internal.ExprFuncBuilder): + self.builder = builder + + def order_by(self, *exprs: Expr) -> ExprFuncBuilder: + """Set the ordering for a window or aggregate function. + + Values given in ``exprs`` must be sort expressions. You can convert any other + expression to a sort expression using `.sort()`. + """ + return ExprFuncBuilder(self.builder.order_by(list(e.expr for e in exprs))) + + def filter(self, filter: Expr) -> ExprFuncBuilder: + """Filter values during aggregation.""" + return ExprFuncBuilder(self.builder.filter(filter.expr)) + + def distinct(self) -> ExprFuncBuilder: + """Only evaluate distinct values during aggregation.""" + return ExprFuncBuilder(self.builder.distinct()) + + def null_treatment(self, null_treatment: NullTreatment) -> ExprFuncBuilder: + """Set how nulls are treated for either window or aggregate functions.""" + return ExprFuncBuilder(self.builder.null_treatment(null_treatment)) + + def partition_by(self, *partition_by: Expr) -> ExprFuncBuilder: + """Set partitioning for window functions.""" + return ExprFuncBuilder( + self.builder.partition_by(list(e.expr for e in partition_by)) + ) + + def window_frame(self, window_frame: WindowFrame) -> ExprFuncBuilder: + """Set window frame for window functions.""" + return ExprFuncBuilder(self.builder.window_frame(window_frame.window_frame)) + + def build(self) -> Expr: + """Create an expression from a Function Builder.""" + return Expr(self.builder.build()) + class WindowFrame: """Defines a window frame for performing window operations.""" def __init__( - self, units: str, start_bound: int | None, end_bound: int | None + self, units: str, start_bound: Optional[Any], end_bound: Optional[Any] ) -> None: """Construct a window frame using the given parameters. @@ -423,6 +522,14 @@ def __init__( will be set to unbounded. If unit type is ``groups``, this parameter must be set. """ + if not isinstance(start_bound, pa.Scalar) and start_bound is not None: + start_bound = pa.scalar(start_bound) + if units == "rows" or units == "groups": + start_bound = start_bound.cast(pa.uint64()) + if not isinstance(end_bound, pa.Scalar) and end_bound is not None: + end_bound = pa.scalar(end_bound) + if units == "rows" or units == "groups": + end_bound = end_bound.cast(pa.uint64()) self.window_frame = expr_internal.WindowFrame(units, start_bound, end_bound) def get_frame_units(self) -> str: diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index ec0c1104d..28201c1d1 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -27,6 +27,10 @@ from datafusion.expr import CaseBuilder, Expr, WindowFrame from datafusion.context import SessionContext +from typing import Any, Optional + +import pyarrow as pa + __all__ = [ "abs", "acos", @@ -246,7 +250,16 @@ "var_pop", "var_samp", "when", + # Window Functions "window", + "lead", + "lag", + "row_number", + "rank", + "dense_rank", + "percent_rank", + "cume_dist", + "ntile", ] @@ -383,7 +396,14 @@ def window( window_frame: WindowFrame | None = None, ctx: SessionContext | None = None, ) -> Expr: - """Creates a new Window function expression.""" + """Creates a new Window function expression. + + This interface will soon be deprecated. Instead of using this interface, + users should call the window functions directly. For example, to perform a + lag use:: + + df.select(functions.lag(col("a")).partition_by(col("b")).build()) + """ args = [a.expr for a in args] partition_by = [e.expr for e in partition_by] if partition_by is not None else None order_by = [o.expr for o in order_by] if order_by is not None else None @@ -1022,12 +1042,12 @@ def struct(*args: Expr) -> Expr: return Expr(f.struct(*args)) -def named_struct(name_pairs: list[(str, Expr)]) -> Expr: +def named_struct(name_pairs: list[tuple[str, Expr]]) -> Expr: """Returns a struct with the given names and arguments pairs.""" - name_pairs = [[Expr.literal(pair[0]), pair[1]] for pair in name_pairs] + name_pair_exprs = [[Expr.literal(pair[0]), pair[1]] for pair in name_pairs] # flatten - name_pairs = [x.expr for xs in name_pairs for x in xs] + name_pairs = [x.expr for xs in name_pair_exprs for x in xs] return Expr(f.named_struct(*name_pairs)) @@ -1690,17 +1710,19 @@ def regr_syy(y: Expr, x: Expr, distinct: bool = False) -> Expr: def first_value( arg: Expr, distinct: bool = False, - filter: bool = None, - order_by: Expr | None = None, - null_treatment: common.NullTreatment | None = None, + filter: Optional[bool] = None, + order_by: Optional[list[Expr]] = None, + null_treatment: Optional[common.NullTreatment] = None, ) -> Expr: """Returns the first value in a group of values.""" + order_by_cols = [e.expr for e in order_by] if order_by is not None else None + return Expr( f.first_value( arg.expr, distinct=distinct, filter=filter, - order_by=order_by, + order_by=order_by_cols, null_treatment=null_treatment, ) ) @@ -1709,17 +1731,23 @@ def first_value( def last_value( arg: Expr, distinct: bool = False, - filter: bool = None, - order_by: Expr | None = None, - null_treatment: common.NullTreatment | None = None, + filter: Optional[bool] = None, + order_by: Optional[list[Expr]] = None, + null_treatment: Optional[common.NullTreatment] = None, ) -> Expr: - """Returns the last value in a group of values.""" + """Returns the last value in a group of values. + + To set parameters on this expression, use ``.order_by()``, ``.distinct()``, + ``.filter()``, or ``.null_treatment()``. + """ + order_by_cols = [e.expr for e in order_by] if order_by is not None else None + return Expr( f.last_value( arg.expr, distinct=distinct, filter=filter, - order_by=order_by, + order_by=order_by_cols, null_treatment=null_treatment, ) ) @@ -1748,3 +1776,339 @@ def bool_and(arg: Expr, distinct: bool = False) -> Expr: def bool_or(arg: Expr, distinct: bool = False) -> Expr: """Computes the boolean OR of the arguement.""" return Expr(f.bool_or(arg.expr, distinct=distinct)) + + +def lead( + arg: Expr, + shift_offset: int = 1, + default_value: Optional[Any] = None, + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a lead window function. + + Lead operation will return the argument that is in the next shift_offset-th row in + the partition. For example ``lead(col("b"), shift_offset=3, default_value=5)`` will + return the 3rd following value in column ``b``. At the end of the partition, where + no futher values can be returned it will return the default value of 5. + + Here is an example of both the ``lead`` and :py:func:`datafusion.functions.lag` + functions on a simple DataFrame:: + + +--------+------+-----+ + | points | lead | lag | + +--------+------+-----+ + | 100 | 100 | | + | 100 | 50 | 100 | + | 50 | 25 | 100 | + | 25 | | 50 | + +--------+------+-----+ + + To set window function parameters use the window builder approach described in the + ref:`_window_functions` online documentation. + + Args: + arg: Value to return + shift_offset: Number of rows following the current row. + default_value: Value to return if shift_offet row does not exist. + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + if not isinstance(default_value, pa.Scalar) and default_value is not None: + default_value = pa.scalar(default_value) + + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.lead( + arg.expr, + shift_offset, + default_value, + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def lag( + arg: Expr, + shift_offset: int = 1, + default_value: Optional[Any] = None, + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a lag window function. + + Lag operation will return the argument that is in the previous shift_offset-th row + in the partition. For example ``lag(col("b"), shift_offset=3, default_value=5)`` + will return the 3rd previous value in column ``b``. At the beginnig of the + partition, where no values can be returned it will return the default value of 5. + + Here is an example of both the ``lag`` and :py:func:`datafusion.functions.lead` + functions on a simple DataFrame:: + + +--------+------+-----+ + | points | lead | lag | + +--------+------+-----+ + | 100 | 100 | | + | 100 | 50 | 100 | + | 50 | 25 | 100 | + | 25 | | 50 | + +--------+------+-----+ + + Args: + arg: Value to return + shift_offset: Number of rows before the current row. + default_value: Value to return if shift_offet row does not exist. + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + if not isinstance(default_value, pa.Scalar): + default_value = pa.scalar(default_value) + + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.lag( + arg.expr, + shift_offset, + default_value, + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def row_number( + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a row number window function. + + Returns the row number of the window function. + + Here is an example of the ``row_number`` on a simple DataFrame:: + + +--------+------------+ + | points | row number | + +--------+------------+ + | 100 | 1 | + | 100 | 2 | + | 50 | 3 | + | 25 | 4 | + +--------+------------+ + + Args: + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.row_number( + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def rank( + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a rank window function. + + Returns the rank based upon the window order. Consecutive equal values will receive + the same rank, but the next different value will not be consecutive but rather the + number of rows that preceed it plus one. This is similar to Olympic medals. If two + people tie for gold, the next place is bronze. There would be no silver medal. Here + is an example of a dataframe with a window ordered by descending ``points`` and the + associated rank. + + You should set ``order_by`` to produce meaningful results:: + + +--------+------+ + | points | rank | + +--------+------+ + | 100 | 1 | + | 100 | 1 | + | 50 | 3 | + | 25 | 4 | + +--------+------+ + + Args: + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.rank( + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def dense_rank( + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a dense_rank window function. + + This window function is similar to :py:func:`rank` except that the returned values + will be consecutive. Here is an example of a dataframe with a window ordered by + descending ``points`` and the associated dense rank:: + + +--------+------------+ + | points | dense_rank | + +--------+------------+ + | 100 | 1 | + | 100 | 1 | + | 50 | 2 | + | 25 | 3 | + +--------+------------+ + + Args: + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.dense_rank( + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def percent_rank( + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a percent_rank window function. + + This window function is similar to :py:func:`rank` except that the returned values + are the percentage from 0.0 to 1.0 from first to last. Here is an example of a + dataframe with a window ordered by descending ``points`` and the associated percent + rank:: + + +--------+--------------+ + | points | percent_rank | + +--------+--------------+ + | 100 | 0.0 | + | 100 | 0.0 | + | 50 | 0.666667 | + | 25 | 1.0 | + +--------+--------------+ + + Args: + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.percent_rank( + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def cume_dist( + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a cumulative distribution window function. + + This window function is similar to :py:func:`rank` except that the returned values + are the ratio of the row number to the total numebr of rows. Here is an example of a + dataframe with a window ordered by descending ``points`` and the associated + cumulative distribution:: + + +--------+-----------+ + | points | cume_dist | + +--------+-----------+ + | 100 | 0.5 | + | 100 | 0.5 | + | 50 | 0.75 | + | 25 | 1.0 | + +--------+-----------+ + + Args: + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.cume_dist( + partition_by=partition_cols, + order_by=order_cols, + ) + ) + + +def ntile( + groups: int, + partition_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Create a n-tile window function. + + This window function orders the window frame into a give number of groups based on + the ordering criteria. It then returns which group the current row is assigned to. + Here is an example of a dataframe with a window ordered by descending ``points`` + and the associated n-tile function:: + + +--------+-------+ + | points | ntile | + +--------+-------+ + | 120 | 1 | + | 100 | 1 | + | 80 | 2 | + | 60 | 2 | + | 40 | 3 | + | 20 | 3 | + +--------+-------+ + + Args: + groups: Number of groups for the n-tile to be divided into. + partition_by: Expressions to partition the window frame on. + order_by: Set ordering within the window frame. + """ + partition_cols = ( + [col.expr for col in partition_by] if partition_by is not None else None + ) + order_cols = [col.expr for col in order_by] if order_by is not None else None + + return Expr( + f.ntile( + Expr.literal(groups).expr, + partition_by=partition_cols, + order_by=order_cols, + ) + ) diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index 477bc0fce..c2a5f22ba 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -84,6 +84,23 @@ def aggregate_df(): return ctx.sql("select c1, sum(c2) from test group by c1") +@pytest.fixture +def partitioned_df(): + ctx = SessionContext() + + # create a RecordBatch and a new DataFrame from it + batch = pa.RecordBatch.from_arrays( + [ + pa.array([0, 1, 2, 3, 4, 5, 6]), + pa.array([7, None, 7, 8, 9, None, 9]), + pa.array(["A", "A", "A", "A", "B", "B", "B"]), + ], + names=["a", "b", "c"], + ) + + return ctx.create_dataframe([[batch]]) + + def test_select(df): df = df.select( column("a") + column("b"), @@ -249,7 +266,7 @@ def test_join(): df = df.join(df1, join_keys=(["a"], ["a"]), how="inner") df.show() - df = df.sort(column("l.a").sort(ascending=True)) + df = df.sort(column("l.a")) table = pa.Table.from_batches(df.collect()) expected = {"a": [1, 2], "c": [8, 10], "b": [4, 5]} @@ -263,83 +280,162 @@ def test_distinct(): [pa.array([1, 2, 3, 1, 2, 3]), pa.array([4, 5, 6, 4, 5, 6])], names=["a", "b"], ) - df_a = ( - ctx.create_dataframe([[batch]]) - .distinct() - .sort(column("a").sort(ascending=True)) - ) + df_a = ctx.create_dataframe([[batch]]).distinct().sort(column("a")) batch = pa.RecordBatch.from_arrays( [pa.array([1, 2, 3]), pa.array([4, 5, 6])], names=["a", "b"], ) - df_b = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) + df_b = ctx.create_dataframe([[batch]]).sort(column("a")) assert df_a.collect() == df_b.collect() data_test_window_functions = [ - ("row", f.window("row_number", [], order_by=[f.order_by(column("c"))]), [2, 1, 3]), - ("rank", f.window("rank", [], order_by=[f.order_by(column("c"))]), [2, 1, 2]), + ( + "row", + f.row_number(order_by=[column("b"), column("a").sort(ascending=False)]), + [4, 2, 3, 5, 7, 1, 6], + ), + ( + "row_w_params", + f.row_number( + order_by=[column("b"), column("a")], + partition_by=[column("c")], + ), + [2, 1, 3, 4, 2, 1, 3], + ), + ("rank", f.rank(order_by=[column("b")]), [3, 1, 3, 5, 6, 1, 6]), + ( + "rank_w_params", + f.rank(order_by=[column("b"), column("a")], partition_by=[column("c")]), + [2, 1, 3, 4, 2, 1, 3], + ), ( "dense_rank", - f.window("dense_rank", [], order_by=[f.order_by(column("c"))]), - [2, 1, 2], + f.dense_rank(order_by=[column("b")]), + [2, 1, 2, 3, 4, 1, 4], + ), + ( + "dense_rank_w_params", + f.dense_rank(order_by=[column("b"), column("a")], partition_by=[column("c")]), + [2, 1, 3, 4, 2, 1, 3], ), ( "percent_rank", - f.window("percent_rank", [], order_by=[f.order_by(column("c"))]), - [0.5, 0, 0.5], + f.round(f.percent_rank(order_by=[column("b")]), literal(3)), + [0.333, 0.0, 0.333, 0.667, 0.833, 0.0, 0.833], + ), + ( + "percent_rank_w_params", + f.round( + f.percent_rank( + order_by=[column("b"), column("a")], partition_by=[column("c")] + ), + literal(3), + ), + [0.333, 0.0, 0.667, 1.0, 0.5, 0.0, 1.0], ), ( "cume_dist", - f.window("cume_dist", [], order_by=[f.order_by(column("b"))]), - [0.3333333333333333, 0.6666666666666666, 1.0], + f.round(f.cume_dist(order_by=[column("b")]), literal(3)), + [0.571, 0.286, 0.571, 0.714, 1.0, 0.286, 1.0], + ), + ( + "cume_dist_w_params", + f.round( + f.cume_dist( + order_by=[column("b"), column("a")], partition_by=[column("c")] + ), + literal(3), + ), + [0.5, 0.25, 0.75, 1.0, 0.667, 0.333, 1.0], ), ( "ntile", - f.window("ntile", [literal(2)], order_by=[f.order_by(column("c"))]), - [1, 1, 2], + f.ntile(2, order_by=[column("b")]), + [1, 1, 1, 2, 2, 1, 2], ), ( - "next", - f.window("lead", [column("b")], order_by=[f.order_by(column("b"))]), - [5, 6, None], + "ntile_w_params", + f.ntile(2, order_by=[column("b"), column("a")], partition_by=[column("c")]), + [1, 1, 2, 2, 1, 1, 2], ), + ("lead", f.lead(column("b"), order_by=[column("b")]), [7, None, 8, 9, 9, 7, None]), ( - "previous", - f.window("lag", [column("b")], order_by=[f.order_by(column("b"))]), - [None, 4, 5], + "lead_w_params", + f.lead( + column("b"), + shift_offset=2, + default_value=-1, + order_by=[column("b"), column("a")], + partition_by=[column("c")], + ), + [8, 7, -1, -1, -1, 9, -1], ), + ("lag", f.lag(column("b"), order_by=[column("b")]), [None, None, 7, 7, 8, None, 9]), + ( + "lag_w_params", + f.lag( + column("b"), + shift_offset=2, + default_value=-1, + order_by=[column("b"), column("a")], + partition_by=[column("c")], + ), + [-1, -1, None, 7, -1, -1, None], + ), + # TODO update all aggregate functions as windows once upstream merges https://github.com/apache/datafusion-python/issues/833 pytest.param( "first_value", - f.window("first_value", [column("a")], order_by=[f.order_by(column("b"))]), - [1, 1, 1], + f.window( + "first_value", + [column("a")], + order_by=[f.order_by(column("b"))], + partition_by=[column("c")], + ), + [1, 1, 1, 1, 5, 5, 5], ), pytest.param( "last_value", - f.window("last_value", [column("b")], order_by=[f.order_by(column("b"))]), - [4, 5, 6], + f.window("last_value", [column("a")]) + .window_frame(WindowFrame("rows", 0, None)) + .order_by(column("b")) + .partition_by(column("c")) + .build(), + [3, 3, 3, 3, 6, 6, 6], ), pytest.param( - "2nd_value", + "3rd_value", f.window( "nth_value", - [column("b"), literal(2)], - order_by=[f.order_by(column("b"))], + [column("b"), literal(3)], + order_by=[f.order_by(column("a"))], ), - [None, 5, 5], + [None, None, 7, 7, 7, 7, 7], + ), + pytest.param( + "avg", + f.round(f.window("avg", [column("b")], order_by=[column("a")]), literal(3)), + [7.0, 7.0, 7.0, 7.333, 7.75, 7.75, 8.0], ), ] @pytest.mark.parametrize("name,expr,result", data_test_window_functions) -def test_window_functions(df, name, expr, result): - df = df.select(column("a"), column("b"), column("c"), f.alias(expr, name)) - +def test_window_functions(partitioned_df, name, expr, result): + df = partitioned_df.select( + column("a"), column("b"), column("c"), f.alias(expr, name) + ) + df.sort(column("a")).show() table = pa.Table.from_batches(df.collect()) - expected = {"a": [1, 2, 3], "b": [4, 5, 6], "c": [8, 5, 8], name: result} + expected = { + "a": [0, 1, 2, 3, 4, 5, 6], + "b": [7, None, 7, 8, 9, None, 9], + "c": ["A", "A", "A", "A", "B", "B", "B"], + name: result, + } assert table.sort_by("a").to_pydict() == expected @@ -512,9 +608,9 @@ def test_intersect(): [pa.array([3]), pa.array([6])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) + df_c = ctx.create_dataframe([[batch]]).sort(column("a")) - df_a_i_b = df_a.intersect(df_b).sort(column("a").sort(ascending=True)) + df_a_i_b = df_a.intersect(df_b).sort(column("a")) assert df_c.collect() == df_a_i_b.collect() @@ -538,9 +634,9 @@ def test_except_all(): [pa.array([1, 2]), pa.array([4, 5])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) + df_c = ctx.create_dataframe([[batch]]).sort(column("a")) - df_a_e_b = df_a.except_all(df_b).sort(column("a").sort(ascending=True)) + df_a_e_b = df_a.except_all(df_b).sort(column("a")) assert df_c.collect() == df_a_e_b.collect() @@ -573,9 +669,9 @@ def test_union(ctx): [pa.array([1, 2, 3, 3, 4, 5]), pa.array([4, 5, 6, 6, 7, 8])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) + df_c = ctx.create_dataframe([[batch]]).sort(column("a")) - df_a_u_b = df_a.union(df_b).sort(column("a").sort(ascending=True)) + df_a_u_b = df_a.union(df_b).sort(column("a")) assert df_c.collect() == df_a_u_b.collect() @@ -597,9 +693,9 @@ def test_union_distinct(ctx): [pa.array([1, 2, 3, 4, 5]), pa.array([4, 5, 6, 7, 8])], names=["a", "b"], ) - df_c = ctx.create_dataframe([[batch]]).sort(column("a").sort(ascending=True)) + df_c = ctx.create_dataframe([[batch]]).sort(column("a")) - df_a_u_b = df_a.union(df_b, True).sort(column("a").sort(ascending=True)) + df_a_u_b = df_a.union(df_b, True).sort(column("a")) assert df_c.collect() == df_a_u_b.collect() assert df_c.collect() == df_a_u_b.collect() diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index e5429bd60..fe092c456 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -963,6 +963,7 @@ def test_first_last_value(df): assert result.column(3) == pa.array(["!"]) assert result.column(4) == pa.array([6]) assert result.column(5) == pa.array([datetime(2020, 7, 2)]) + df.show() def test_binary_string_functions(df): diff --git a/src/dataframe.rs b/src/dataframe.rs index 22b05226c..d7abab400 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -39,6 +39,7 @@ use pyo3::types::{PyCapsule, PyTuple}; use tokio::task::JoinHandle; use crate::errors::py_datafusion_err; +use crate::expr::to_sort_expressions; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; @@ -150,7 +151,7 @@ impl PyDataFrame { #[pyo3(signature = (*exprs))] fn sort(&self, exprs: Vec) -> PyResult { - let exprs = exprs.into_iter().map(|e| e.into()).collect(); + let exprs = to_sort_expressions(exprs); let df = self.df.as_ref().clone().sort(exprs)?; Ok(Self::new(df)) } diff --git a/src/expr.rs b/src/expr.rs index 04bfc85c2..697682d4c 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -16,10 +16,11 @@ // under the License. use datafusion_expr::utils::exprlist_to_fields; -use datafusion_expr::LogicalPlan; +use datafusion_expr::{ExprFuncBuilder, ExprFunctionExt, LogicalPlan}; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use std::sync::Arc; +use window::PyWindowFrame; use arrow::pyarrow::ToPyArrow; use datafusion::arrow::datatypes::{DataType, Field}; @@ -32,7 +33,7 @@ use datafusion_expr::{ lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast, }; -use crate::common::data_type::{DataTypeMap, RexType}; +use crate::common::data_type::{DataTypeMap, NullTreatment, RexType}; use crate::errors::{py_runtime_err, py_type_err, py_unsupported_variant_err, DataFusionError}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; @@ -281,6 +282,10 @@ impl PyExpr { self.expr.clone().is_null().into() } + pub fn is_not_null(&self) -> PyExpr { + self.expr.clone().is_not_null().into() + } + pub fn cast(&self, to: PyArrowType) -> PyExpr { // self.expr.cast_to() requires DFSchema to validate that the cast // is supported, omit that for now @@ -510,6 +515,107 @@ impl PyExpr { pub fn column_name(&self, plan: PyLogicalPlan) -> PyResult { self._column_name(&plan.plan()).map_err(py_runtime_err) } + + // Expression Function Builder functions + + pub fn order_by(&self, order_by: Vec) -> PyExprFuncBuilder { + self.expr + .clone() + .order_by(to_sort_expressions(order_by)) + .into() + } + + pub fn filter(&self, filter: PyExpr) -> PyExprFuncBuilder { + self.expr.clone().filter(filter.expr.clone()).into() + } + + pub fn distinct(&self) -> PyExprFuncBuilder { + self.expr.clone().distinct().into() + } + + pub fn null_treatment(&self, null_treatment: NullTreatment) -> PyExprFuncBuilder { + self.expr + .clone() + .null_treatment(Some(null_treatment.into())) + .into() + } + + pub fn partition_by(&self, partition_by: Vec) -> PyExprFuncBuilder { + let partition_by = partition_by.iter().map(|e| e.expr.clone()).collect(); + self.expr.clone().partition_by(partition_by).into() + } + + pub fn window_frame(&self, window_frame: PyWindowFrame) -> PyExprFuncBuilder { + self.expr.clone().window_frame(window_frame.into()).into() + } +} + +#[pyclass(name = "ExprFuncBuilder", module = "datafusion.expr", subclass)] +#[derive(Debug, Clone)] +pub struct PyExprFuncBuilder { + pub builder: ExprFuncBuilder, +} + +impl From for PyExprFuncBuilder { + fn from(builder: ExprFuncBuilder) -> Self { + Self { builder } + } +} + +pub fn to_sort_expressions(order_by: Vec) -> Vec { + order_by + .iter() + .map(|e| e.expr.clone()) + .map(|e| match e { + Expr::Sort(_) => e, + _ => e.sort(true, true), + }) + .collect() +} + +#[pymethods] +impl PyExprFuncBuilder { + pub fn order_by(&self, order_by: Vec) -> PyExprFuncBuilder { + self.builder + .clone() + .order_by(to_sort_expressions(order_by)) + .into() + } + + pub fn filter(&self, filter: PyExpr) -> PyExprFuncBuilder { + self.builder.clone().filter(filter.expr.clone()).into() + } + + pub fn distinct(&self) -> PyExprFuncBuilder { + self.builder.clone().distinct().into() + } + + pub fn null_treatment(&self, null_treatment: NullTreatment) -> PyExprFuncBuilder { + self.builder + .clone() + .null_treatment(Some(null_treatment.into())) + .into() + } + + pub fn partition_by(&self, partition_by: Vec) -> PyExprFuncBuilder { + let partition_by = partition_by.iter().map(|e| e.expr.clone()).collect(); + self.builder.clone().partition_by(partition_by).into() + } + + pub fn window_frame(&self, window_frame: PyWindowFrame) -> PyExprFuncBuilder { + self.builder + .clone() + .window_frame(window_frame.into()) + .into() + } + + pub fn build(&self) -> PyResult { + self.builder + .clone() + .build() + .map(|expr| expr.into()) + .map_err(|err| err.into()) + } } impl PyExpr { diff --git a/src/expr/window.rs b/src/expr/window.rs index 786651194..7eb586082 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -168,7 +168,11 @@ fn not_window_function_err(expr: Expr) -> PyErr { impl PyWindowFrame { #[new] #[pyo3(signature=(unit, start_bound, end_bound))] - pub fn new(unit: &str, start_bound: Option, end_bound: Option) -> PyResult { + pub fn new( + unit: &str, + start_bound: Option, + end_bound: Option, + ) -> PyResult { let units = unit.to_ascii_lowercase(); let units = match units.as_str() { "rows" => WindowFrameUnits::Rows, @@ -182,9 +186,7 @@ impl PyWindowFrame { } }; let start_bound = match start_bound { - Some(start_bound) => { - WindowFrameBound::Preceding(ScalarValue::UInt64(Some(start_bound))) - } + Some(start_bound) => WindowFrameBound::Preceding(start_bound), None => match units { WindowFrameUnits::Range => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), @@ -197,7 +199,7 @@ impl PyWindowFrame { }, }; let end_bound = match end_bound { - Some(end_bound) => WindowFrameBound::Following(ScalarValue::UInt64(Some(end_bound))), + Some(end_bound) => WindowFrameBound::Following(end_bound), None => match units { WindowFrameUnits::Rows => WindowFrameBound::Following(ScalarValue::UInt64(None)), WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)), diff --git a/src/functions.rs b/src/functions.rs index 252563621..aed4de474 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,13 +16,16 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; +use datafusion_expr::window_function; use datafusion_expr::ExprFunctionExt; +use datafusion_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; use crate::context::PySessionContext; use crate::errors::DataFusionError; use crate::expr::conditional_expr::PyCaseBuilder; +use crate::expr::to_sort_expressions; use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; use datafusion::execution::FunctionRegistry; @@ -316,18 +319,15 @@ pub fn regr_syy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult, order_by: Option>, null_treatment: Option, ) -> PyResult { - // If we initialize the UDAF with order_by directly, then it gets over-written by the builder - let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); - - // luckily, I can guarantee initializing a builder with an `order_by` default of empty vec + // Since ExprFuncBuilder::new() is private, we can guarantee initializing + // a builder with an `order_by` default of empty vec let order_by = order_by .map(|x| x.into_iter().map(|x| x.expr).collect::>()) .unwrap_or_default(); @@ -348,32 +348,30 @@ pub fn first_value( } #[pyfunction] -pub fn last_value( +pub fn first_value( expr: PyExpr, distinct: bool, filter: Option, order_by: Option>, null_treatment: Option, ) -> PyResult { - let agg_fn = functions_aggregate::expr_fn::last_value(vec![expr.expr]); - - // luckily, I can guarantee initializing a builder with an `order_by` default of empty vec - let order_by = order_by - .map(|x| x.into_iter().map(|x| x.expr).collect::>()) - .unwrap_or_default(); - let mut builder = agg_fn.order_by(order_by); - - if distinct { - builder = builder.distinct(); - } + // If we initialize the UDAF with order_by directly, then it gets over-written by the builder + let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); - if let Some(filter) = filter { - builder = builder.filter(filter.expr); - } + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) +} - builder = builder.null_treatment(null_treatment.map(DFNullTreatment::from)); +#[pyfunction] +pub fn last_value( + expr: PyExpr, + distinct: bool, + filter: Option, + order_by: Option>, + null_treatment: Option, +) -> PyResult { + let agg_fn = functions_aggregate::expr_fn::last_value(vec![expr.expr]); - Ok(builder.build()?.into()) + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } #[pyfunction] @@ -618,9 +616,11 @@ fn window( ctx: Option, ) -> PyResult { let fun = find_window_fn(name, ctx)?; + let window_frame = window_frame - .unwrap_or_else(|| PyWindowFrame::new("rows", None, Some(0)).unwrap()) - .into(); + .map(|w| w.into()) + .unwrap_or(WindowFrame::new(order_by.as_ref().map(|v| !v.is_empty()))); + Ok(PyExpr { expr: datafusion_expr::Expr::WindowFunction(WindowFunction { fun, @@ -634,6 +634,10 @@ fn window( .unwrap_or_default() .into_iter() .map(|x| x.expr) + .map(|e| match e { + Expr::Sort(_) => e, + _ => e.sort(true, true), + }) .collect::>(), window_frame, null_treatment: None, @@ -890,6 +894,116 @@ aggregate_function!(array_agg, functions_aggregate::array_agg::array_agg_udaf); aggregate_function!(max, functions_aggregate::min_max::max_udaf); aggregate_function!(min, functions_aggregate::min_max::min_udaf); +fn add_builder_fns_to_window( + window_fn: Expr, + partition_by: Option>, + order_by: Option>, +) -> PyResult { + // Since ExprFuncBuilder::new() is private, set an empty partition and then + // override later if appropriate. + let mut builder = window_fn.partition_by(vec![]); + + if let Some(partition_cols) = partition_by { + builder = builder.partition_by( + partition_cols + .into_iter() + .map(|col| col.clone().into()) + .collect(), + ); + } + + if let Some(order_by_cols) = order_by { + let order_by_cols = to_sort_expressions(order_by_cols); + builder = builder.order_by(order_by_cols); + } + + builder.build().map(|e| e.into()).map_err(|err| err.into()) +} + +#[pyfunction] +pub fn lead( + arg: PyExpr, + shift_offset: i64, + default_value: Option, + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::lead(arg.expr, Some(shift_offset), default_value); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn lag( + arg: PyExpr, + shift_offset: i64, + default_value: Option, + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::lag(arg.expr, Some(shift_offset), default_value); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn row_number( + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::row_number(); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn rank(partition_by: Option>, order_by: Option>) -> PyResult { + let window_fn = window_function::rank(); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn dense_rank( + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::dense_rank(); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn percent_rank( + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::percent_rank(); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn cume_dist( + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::cume_dist(); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + +#[pyfunction] +pub fn ntile( + arg: PyExpr, + partition_by: Option>, + order_by: Option>, +) -> PyResult { + let window_fn = window_function::ntile(arg.into()); + + add_builder_fns_to_window(window_fn, partition_by, order_by) +} + pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(abs))?; m.add_wrapped(wrap_pyfunction!(acos))?; @@ -1075,5 +1189,15 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(array_slice))?; m.add_wrapped(wrap_pyfunction!(flatten))?; + // Window Functions + m.add_wrapped(wrap_pyfunction!(lead))?; + m.add_wrapped(wrap_pyfunction!(lag))?; + m.add_wrapped(wrap_pyfunction!(row_number))?; + m.add_wrapped(wrap_pyfunction!(rank))?; + m.add_wrapped(wrap_pyfunction!(dense_rank))?; + m.add_wrapped(wrap_pyfunction!(percent_rank))?; + m.add_wrapped(wrap_pyfunction!(cume_dist))?; + m.add_wrapped(wrap_pyfunction!(ntile))?; + Ok(()) } From 57eb9596e0a3d6205bd3e94e83854111145572e6 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Mon, 2 Sep 2024 16:51:52 +0200 Subject: [PATCH 188/413] chore: fix typos (#844) - run [codespell](https://github.com/codespell-project/codespell) on the source code - change name of parameter in db-benchmark.dockerfile based on spelling suggestion and the documentation: https://www.rdocumentation.org/packages/utils/versions/3.6.2/topics/install.packages --- benchmarks/db-benchmark/db-benchmark.dockerfile | 2 +- docs/mdbook/src/index.md | 2 +- docs/source/_static/theme_overrides.css | 2 +- docs/source/conf.py | 2 +- .../user-guide/common-operations/expressions.rst | 2 +- examples/export.py | 2 +- examples/python-udf-comparisons.py | 2 +- examples/tpch/q02_minimum_cost_supplier.py | 2 +- examples/tpch/q04_order_priority_checking.py | 4 ++-- examples/tpch/q06_forecasting_revenue_change.py | 2 +- examples/tpch/q07_volume_shipping.py | 2 +- examples/tpch/q11_important_stock_identification.py | 2 +- examples/tpch/q15_top_supplier.py | 2 +- examples/tpch/q20_potential_part_promotion.py | 2 +- examples/tpch/q21_suppliers_kept_orders_waiting.py | 2 +- examples/tpch/q22_global_sales_opportunity.py | 4 ++-- python/datafusion/context.py | 4 ++-- python/datafusion/expr.py | 2 +- python/datafusion/functions.py | 10 +++++----- python/datafusion/input/location.py | 4 ++-- python/datafusion/udf.py | 4 ++-- src/common/data_type.rs | 2 +- src/expr/table_scan.rs | 2 +- 23 files changed, 32 insertions(+), 32 deletions(-) diff --git a/benchmarks/db-benchmark/db-benchmark.dockerfile b/benchmarks/db-benchmark/db-benchmark.dockerfile index d8842b250..af2edd0f4 100644 --- a/benchmarks/db-benchmark/db-benchmark.dockerfile +++ b/benchmarks/db-benchmark/db-benchmark.dockerfile @@ -58,7 +58,7 @@ RUN cd pandas && \ RUN cd modin && \ virtualenv py-modin --python=/usr/bin/python3.10 -RUN Rscript -e 'install.packages(c("jsonlite","bit64","devtools","rmarkdown"), dependecies=TRUE, repos="https://cloud.r-project.org")' +RUN Rscript -e 'install.packages(c("jsonlite","bit64","devtools","rmarkdown"), dependencies=TRUE, repos="https://cloud.r-project.org")' SHELL ["/bin/bash", "-c"] diff --git a/docs/mdbook/src/index.md b/docs/mdbook/src/index.md index 3cd0fec1d..2c1d217f8 100644 --- a/docs/mdbook/src/index.md +++ b/docs/mdbook/src/index.md @@ -18,7 +18,7 @@ DataFusion is a blazing fast query engine that lets you run data analyses quickly and reliably. -DataFusion is written in Rust, but also exposes Python and SQL bindings, so you can easily query data in your langauge of choice. You don't need to know any Rust to be a happy and productive user of DataFusion. +DataFusion is written in Rust, but also exposes Python and SQL bindings, so you can easily query data in your language of choice. You don't need to know any Rust to be a happy and productive user of DataFusion. DataFusion lets you run queries faster than pandas. Let's compare query runtimes for a 5GB CSV file with 100 million rows of data. diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css index 1e972cc6f..aaa40fba2 100644 --- a/docs/source/_static/theme_overrides.css +++ b/docs/source/_static/theme_overrides.css @@ -56,7 +56,7 @@ a.navbar-brand img { /* This is the bootstrap CSS style for "table-striped". Since the theme does -not yet provide an easy way to configure this globaly, it easier to simply +not yet provide an easy way to configure this globally, it easier to simply include this snippet here than updating each table in all rst files to add ":class: table-striped" */ diff --git a/docs/source/conf.py b/docs/source/conf.py index d5084551e..2e5a41339 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Documenation generation.""" +"""Documentation generation.""" # Configuration file for the Sphinx documentation builder. # diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index c8f8b8f29..e35234c32 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -21,7 +21,7 @@ Expressions =========== In DataFusion an expression is an abstraction that represents a computation. -Expressions are used as the primary inputs and ouputs for most functions within +Expressions are used as the primary inputs and outputs for most functions within DataFusion. As such, expressions can be combined to create expression trees, a concept shared across most compilers and databases. diff --git a/examples/export.py b/examples/export.py index d179bf39d..cc02de52b 100644 --- a/examples/export.py +++ b/examples/export.py @@ -48,6 +48,6 @@ pylist = df.to_pylist() assert pylist == [{"a": 1, "b": 4}, {"a": 2, "b": 5}, {"a": 3, "b": 6}] -# export to Pyton dictionary of columns +# export to Python dictionary of columns pydict = df.to_pydict() assert pydict == {"a": [1, 2, 3], "b": [4, 5, 6]} diff --git a/examples/python-udf-comparisons.py b/examples/python-udf-comparisons.py index 5a6f548fb..9a84dd730 100644 --- a/examples/python-udf-comparisons.py +++ b/examples/python-udf-comparisons.py @@ -28,7 +28,7 @@ # question "return all of the rows that have a specific combination of these # values". We have the combinations we care about provided as a python # list of tuples. There is no built in function that supports this operation, -# but it can be explicilty specified via a single expression or we can +# but it can be explicitly specified via a single expression or we can # use a user defined function. ctx = SessionContext() diff --git a/examples/tpch/q02_minimum_cost_supplier.py b/examples/tpch/q02_minimum_cost_supplier.py index f4020d7bb..2171a2083 100644 --- a/examples/tpch/q02_minimum_cost_supplier.py +++ b/examples/tpch/q02_minimum_cost_supplier.py @@ -96,7 +96,7 @@ # create a column of that value. We can then filter down any rows for which the cost and # minimum do not match. -# The default window frame as of 5/6/2024 is from unbounded preceeding to the current row. +# The default window frame as of 5/6/2024 is from unbounded preceding to the current row. # We want to evaluate the entire data frame, so we specify this. window_frame = datafusion.WindowFrame("rows", None, None) df = df.with_column( diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py index 9dbd81674..77c3bd43e 100644 --- a/examples/tpch/q04_order_priority_checking.py +++ b/examples/tpch/q04_order_priority_checking.py @@ -53,9 +53,9 @@ # Limit results to cases where commitment date before receipt date # Aggregate the results so we only get one row to join with the order table. -# Alterately, and likely more idomatic is instead of `.aggregate` you could +# Alternately, and likely more idiomatic is instead of `.aggregate` you could # do `.select_columns("l_orderkey").distinct()`. The goal here is to show -# mulitple examples of how to use Data Fusion. +# multiple examples of how to use Data Fusion. df_lineitem = df_lineitem.filter(col("l_commitdate") < col("l_receiptdate")).aggregate( [col("l_orderkey")], [] ) diff --git a/examples/tpch/q06_forecasting_revenue_change.py b/examples/tpch/q06_forecasting_revenue_change.py index ec98aaf5e..3beb9eb1f 100644 --- a/examples/tpch/q06_forecasting_revenue_change.py +++ b/examples/tpch/q06_forecasting_revenue_change.py @@ -82,5 +82,5 @@ revenue = df.collect()[0]["revenue"][0].as_py() -# Note: the output value from this query may be dependant on the size of the database generated +# Note: the output value from this query may be dependent on the size of the database generated print(f"Potential lost revenue: {revenue:.2f}") diff --git a/examples/tpch/q07_volume_shipping.py b/examples/tpch/q07_volume_shipping.py index fd7323b79..44c605a9b 100644 --- a/examples/tpch/q07_volume_shipping.py +++ b/examples/tpch/q07_volume_shipping.py @@ -77,7 +77,7 @@ # the two nations of interest. Since there is no `otherwise()` statement, any values that do # not match these will result in a null value and then get filtered out. # -# To do the same using a simle filter would be: +# To do the same using a simple filter would be: # df_nation = df_nation.filter((F.col("n_name") == nation_1) | (F.col("n_name") == nation_2)) df_nation = df_nation.with_column( "n_name", diff --git a/examples/tpch/q11_important_stock_identification.py b/examples/tpch/q11_important_stock_identification.py index 267248707..391eb45b1 100644 --- a/examples/tpch/q11_important_stock_identification.py +++ b/examples/tpch/q11_important_stock_identification.py @@ -63,7 +63,7 @@ # Compute total value of specific parts df = df.aggregate([col("ps_partkey")], [F.sum(col("value")).alias("value")]) -# By default window functions go from unbounded preceeding to current row, but we want +# By default window functions go from unbounded preceding to current row, but we want # to compute this sum across all rows window_frame = WindowFrame("rows", None, None) diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py index 4b9e4c1dd..aa76093ec 100644 --- a/examples/tpch/q15_top_supplier.py +++ b/examples/tpch/q15_top_supplier.py @@ -78,7 +78,7 @@ # from the supplier table df = df.join(df_supplier, (["l_suppkey"], ["s_suppkey"]), "inner") -# Return only the colums requested +# Return only the columns requested df = df.select_columns("s_suppkey", "s_name", "s_address", "s_phone", "total_revenue") # If we have more than one, sort by supplier number (suppkey) diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index 05a267450..4ced7aaa1 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -74,7 +74,7 @@ # This will filter down the line items to the parts of interest df = df.join(df_part, (["l_partkey"], ["p_partkey"]), "inner") -# Compute the total sold and limit ourselves to indivdual supplier/part combinations +# Compute the total sold and limit ourselves to individual supplier/part combinations df = df.aggregate( [col("l_partkey"), col("l_suppkey")], [F.sum(col("l_quantity")).alias("total_sold")] ) diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py index 9f59804e5..6b1679e7d 100644 --- a/examples/tpch/q21_suppliers_kept_orders_waiting.py +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -74,7 +74,7 @@ # only orders where this array is larger than one for multiple supplier orders. The second column # is all of the suppliers who failed to make their commitment. We can filter the second column for # arrays with size one. That combination will give us orders that had multiple suppliers where only -# one failed. Use distinct=True in the blow aggregation so we don't get multipe line items from the +# one failed. Use distinct=True in the blow aggregation so we don't get multiple line items from the # same supplier reported in either array. df = df.aggregate( [col("o_orderkey")], diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py index 622c1429f..41fd5de9e 100644 --- a/examples/tpch/q22_global_sales_opportunity.py +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -45,14 +45,14 @@ # The nation code is a two digit number, but we need to convert it to a string literal nation_codes = F.make_array(*[lit(str(n)) for n in NATION_CODES]) -# Use the substring operation to extract the first two charaters of the phone number +# Use the substring operation to extract the first two characters of the phone number df = df_customer.with_column("cntrycode", F.substring(col("c_phone"), lit(0), lit(3))) # Limit our search to customers with some balance and in the country code above df = df.filter(col("c_acctbal") > lit(0.0)) df = df.filter(~F.array_position(nation_codes, col("cntrycode")).is_null()) -# Compute the average balance. By default, the window frame is from unbounded preceeding to the +# Compute the average balance. By default, the window frame is from unbounded preceding to the # current row. We want our frame to cover the entire data frame. window_frame = WindowFrame("rows", None, None) df = df.with_column( diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 283f71e1e..f6bf1c6ce 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -436,7 +436,7 @@ def __init__( Example usage: - The following example demostrates how to use the context to execute + The following example demonstrates how to use the context to execute a query against a CSV data source using the :py:class:`DataFrame` API:: from datafusion import SessionContext @@ -853,7 +853,7 @@ def empty_table(self) -> DataFrame: return DataFrame(self.ctx.empty_table()) def session_id(self) -> str: - """Retrun an id that uniquely identifies this :py:class:`SessionContext`.""" + """Return an id that uniquely identifies this :py:class:`SessionContext`.""" return self.ctx.session_id() def read_json( diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index c7272bb3b..742f8e43d 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -515,7 +515,7 @@ def __init__( Args: units: Should be one of ``rows``, ``range``, or ``groups``. - start_bound: Sets the preceeding bound. Must be >= 0. If none, this + start_bound: Sets the preceding bound. Must be >= 0. If none, this will be set to unbounded. If unit type is ``groups``, this parameter must be set. end_bound: Sets the following bound. Must be >= 0. If none, this diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 28201c1d1..120fed819 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -342,7 +342,7 @@ def concat(*args: Expr) -> Expr: def concat_ws(separator: str, *args: Expr) -> Expr: """Concatenates the list ``args`` with the separator. - ``NULL`` arugments are ignored. ``separator`` should not be ``NULL``. + ``NULL`` arguments are ignored. ``separator`` should not be ``NULL``. """ args = [arg.expr for arg in args] return Expr(f.concat_ws(separator, args)) @@ -541,7 +541,7 @@ def ends_with(arg: Expr, suffix: Expr) -> Expr: def exp(arg: Expr) -> Expr: - """Returns the exponential of the arugment.""" + """Returns the exponential of the argument.""" return Expr(f.exp(arg.expr)) @@ -1593,7 +1593,7 @@ def grouping(arg: Expr, distinct: bool = False) -> Expr: def max(arg: Expr, distinct: bool = False) -> Expr: - """Returns the maximum value of the arugment.""" + """Returns the maximum value of the argument.""" return Expr(f.max(arg.expr, distinct=distinct)) @@ -1769,12 +1769,12 @@ def bit_xor(arg: Expr, distinct: bool = False) -> Expr: def bool_and(arg: Expr, distinct: bool = False) -> Expr: - """Computes the boolean AND of the arugment.""" + """Computes the boolean AND of the argument.""" return Expr(f.bool_and(arg.expr, distinct=distinct)) def bool_or(arg: Expr, distinct: bool = False) -> Expr: - """Computes the boolean OR of the arguement.""" + """Computes the boolean OR of the argument.""" return Expr(f.bool_or(arg.expr, distinct=distinct)) diff --git a/python/datafusion/input/location.py b/python/datafusion/input/location.py index 566a63da9..b274539fc 100644 --- a/python/datafusion/input/location.py +++ b/python/datafusion/input/location.py @@ -66,7 +66,7 @@ def build_table( # Consume header row and count number of rows for statistics. # TODO: Possibly makes sense to have the eager number of rows # calculated as a configuration since you must read the entire file - # to get that information. However, this should only be occuring + # to get that information. However, this should only be occurring # at table creation time and therefore shouldn't # slow down query performance. with open(input_file, "r") as file: @@ -75,7 +75,7 @@ def build_table( print(header_row) for _ in reader: num_rows += 1 - # TODO: Need to actually consume this row into resonable columns + # TODO: Need to actually consume this row into reasonable columns raise RuntimeError("TODO: Currently unable to support CSV input files.") else: raise RuntimeError( diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index bdbad661a..a3b74bb11 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -153,7 +153,7 @@ def state(self) -> List[pyarrow.Scalar]: @abstractmethod def update(self, values: pyarrow.Array) -> None: - """Evalute an array of values and update state.""" + """Evaluate an array of values and update state.""" pass @abstractmethod @@ -189,7 +189,7 @@ def __init__( ) -> None: """Instantiate a user defined aggregate function (UDAF). - See :py:func:`udaf` for a convenience function and arugment + See :py:func:`udaf` for a convenience function and argument descriptions. """ self._udf = df_internal.AggregateUDF( diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 21b085c0e..bdfc5a6b1 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -40,7 +40,7 @@ pub enum RexType { /// Arrow types which represents the underlying arrow format /// Python types which represent the type in Python /// It is important to keep all of those types in a single -/// and managable location. Therefore this structure exists +/// and manageable location. Therefore this structure exists /// to map those types and provide a simple place for developers /// to map types from one system to another. #[derive(Debug, Clone)] diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index 8fafd12ee..386c43b6a 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -94,7 +94,7 @@ impl PyTableScan { /// The column indexes that should be. Note if this is empty then /// all columns should be read by the `TableProvider`. This function - /// provides a Tuple of the (index, column_name) to make things simplier + /// provides a Tuple of the (index, column_name) to make things simpler /// for the calling code since often times the name is preferred to /// the index which is a lower level abstraction. #[pyo3(name = "projection")] From e8ebc4fab68941b5fb9adadd3a39ec2c8ac120ce Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Mon, 2 Sep 2024 09:52:21 -0500 Subject: [PATCH 189/413] build(ci): use proper mac runners (#841) * build(ci): use proper mac runners GH updated the runner images. macos-14 is now ARM64. macos-13 is last x86_64. Closes https://github.com/apache/datafusion-python/issues/831 * update name for macos x86_64 build job * update manylinux x86_64 job name for consistency --- .github/workflows/build.yml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a4f8b2da5..5fae13f2e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -117,10 +117,10 @@ jobs: name: dist-${{ matrix.os }} path: target/wheels/* - build-macos-aarch64: + build-macos-x86_64: needs: [generate-license] - name: Mac arm64 - runs-on: macos-latest + name: Mac x86_64 + runs-on: macos-13 strategy: fail-fast: false matrix: @@ -136,9 +136,6 @@ jobs: with: toolchain: stable - - name: Set up Rust targets - run: rustup target add aarch64-apple-darwin - - name: Upgrade pip run: python -m pip install --upgrade pip @@ -159,7 +156,7 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package - run: maturin build --release --strip --target aarch64-apple-darwin --features substrait + run: maturin build --release --strip --features substrait - name: List Mac wheels run: find target/wheels/ @@ -171,7 +168,7 @@ jobs: build-manylinux-x86_64: needs: [generate-license] - name: Manylinux + name: Manylinux x86_64 runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -262,7 +259,7 @@ jobs: runs-on: ubuntu-latest needs: - build-python-mac-win - - build-macos-aarch64 + - build-macos-x86_64 - build-manylinux-x86_64 - build-manylinux-aarch64 - build-sdist From 909b809b7a047d470ce5d187c5e9dd734c9d3163 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 2 Sep 2024 11:27:44 -0400 Subject: [PATCH 190/413] Set of small features (#839) * Add repr_html to give nice displays in notebooks when using display(df) * Allow get_item to get index of an array or a field in a struct * add test for getting array elements * Small typo in array * Add DataFrame transform * Update index in unit test * Add dataframe transform unit test * Add unit test for repr_html * Updating documentation * fix typo --------- Co-authored-by: Andy Grove --- .../common-operations/expressions.rst | 37 +++++++++++++++ python/datafusion/dataframe.py | 26 +++++++++++ python/datafusion/expr.py | 19 ++++++-- python/datafusion/functions.py | 2 +- python/datafusion/tests/test_dataframe.py | 32 +++++++++++++ python/datafusion/tests/test_expr.py | 23 ++++++++++ src/dataframe.rs | 46 +++++++++++++++++++ 7 files changed, 181 insertions(+), 4 deletions(-) diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index e35234c32..6014c9d2e 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -60,6 +60,43 @@ examples for the and, or, and not operations. heavy_red_units = (col("color") == lit("red")) & (col("weight") > lit(42)) not_red_units = ~(col("color") == lit("red")) +Arrays +------ + +For columns that contain arrays of values, you can access individual elements of the array by index +using bracket indexing. This is similar to callling the function +:py:func:`datafusion.functions.array_element`, except that array indexing using brackets is 0 based, +similar to Python arrays and ``array_element`` is 1 based indexing to be compatible with other SQL +approaches. + +.. ipython:: python + + from datafusion import SessionContext, col + + ctx = SessionContext() + df = ctx.from_pydict({"a": [[1, 2, 3], [4, 5, 6]]}) + df.select(col("a")[0].alias("a0")) + + +.. warning:: + + Indexing an element of an array via ``[]`` starts at index 0 whereas + :py:func:`~datafusion.functions.array_element` starts at index 1. + +Structs +------- + +Columns that contain struct elements can be accessed using the bracket notation as if they were +Python dictionary style objects. This expects a string key as the parameter passed. + +.. ipython:: python + + ctx = SessionContext() + data = {"a": [{"size": 15, "color": "green"}, {"size": 10, "color": "blue"}]} + df = ctx.from_pydict(data) + df.select(col("a")["size"].alias("a_size")) + + Functions --------- diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 0e7d82e29..46b8fa1bd 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -30,6 +30,7 @@ import pandas as pd import polars as pl import pathlib + from typing import Callable from datafusion._internal import DataFrame as DataFrameInternal from datafusion.expr import Expr @@ -72,6 +73,9 @@ def __repr__(self) -> str: """ return self.df.__repr__() + def _repr_html_(self) -> str: + return self.df._repr_html_() + def describe(self) -> DataFrame: """Return the statistics for this DataFrame. @@ -539,3 +543,25 @@ def __arrow_c_stream__(self, requested_schema: pa.Schema) -> Any: Arrow PyCapsule object. """ return self.df.__arrow_c_stream__(requested_schema) + + def transform(self, func: Callable[..., DataFrame], *args: Any) -> DataFrame: + """Apply a function to the current DataFrame which returns another DataFrame. + + This is useful for chaining together multiple functions. For example:: + + def add_3(df: DataFrame) -> DataFrame: + return df.with_column("modified", lit(3)) + + def within_limit(df: DataFrame, limit: int) -> DataFrame: + return df.filter(col("a") < lit(limit)).distinct() + + df = df.transform(modify_df).transform(within_limit, 4) + + Args: + func: A callable function that takes a DataFrame as it's first argument + args: Zero or more arguments to pass to `func` + + Returns: + DataFrame: After applying func to the original dataframe. + """ + return func(self, *args) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 742f8e43d..7bea0289b 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -22,7 +22,11 @@ from __future__ import annotations -from ._internal import expr as expr_internal, LogicalPlan +from ._internal import ( + expr as expr_internal, + LogicalPlan, + functions as functions_internal, +) from datafusion.common import NullTreatment, RexType, DataTypeMap from typing import Any, Optional import pyarrow as pa @@ -257,8 +261,17 @@ def __invert__(self) -> Expr: """Binary not (~).""" return Expr(self.expr.__invert__()) - def __getitem__(self, key: str) -> Expr: - """For struct data types, return the field indicated by ``key``.""" + def __getitem__(self, key: str | int) -> Expr: + """Retrieve sub-object. + + If ``key`` is a string, returns the subfield of the struct. + If ``key`` is an integer, retrieves the element in the array. Note that the + element index begins at ``0``, unlike `array_element` which begines at ``1``. + """ + if isinstance(key, int): + return Expr( + functions_internal.array_element(self.expr, Expr.literal(key + 1).expr) + ) return Expr(self.expr.__getitem__(key)) def __eq__(self, rhs: Any) -> Expr: diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 120fed819..4c701b24d 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -1023,7 +1023,7 @@ def array(*args: Expr) -> Expr: This is an alias for :py:func:`make_array`. """ - return make_array(args) + return make_array(*args) def range(start: Expr, stop: Expr, step: Expr) -> Expr: diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index c2a5f22ba..90954d09a 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import os +from typing import Any import pyarrow as pa from pyarrow.csv import write_csv @@ -970,3 +971,34 @@ def test_dataframe_export(df) -> None: except Exception: failed_convert = True assert failed_convert + + +def test_dataframe_transform(df): + def add_string_col(df_internal) -> DataFrame: + return df_internal.with_column("string_col", literal("string data")) + + def add_with_parameter(df_internal, value: Any) -> DataFrame: + return df_internal.with_column("new_col", literal(value)) + + df = df.transform(add_string_col).transform(add_with_parameter, 3) + + result = df.to_pydict() + + assert result["a"] == [1, 2, 3] + assert result["string_col"] == ["string data" for _i in range(0, 3)] + assert result["new_col"] == [3 for _i in range(0, 3)] + + +def test_dataframe_repr_html(df) -> None: + output = df._repr_html_() + + ref_html = """ + + + + +
abc
148
255
368
+ """ + + # Ignore whitespace just to make this test look cleaner + assert output.replace(" ", "") == ref_html.replace(" ", "") diff --git a/python/datafusion/tests/test_expr.py b/python/datafusion/tests/test_expr.py index 9071108cb..056d2ea03 100644 --- a/python/datafusion/tests/test_expr.py +++ b/python/datafusion/tests/test_expr.py @@ -169,3 +169,26 @@ def traverse_logical_plan(plan): == '[Expr(Utf8("dfa")), Expr(Utf8("ad")), Expr(Utf8("dfre")), Expr(Utf8("vsa"))]' ) assert not variant.negated() + + +def test_expr_getitem() -> None: + ctx = SessionContext() + data = { + "array_values": [[1, 2, 3], [4, 5], [6], []], + "struct_values": [ + {"name": "Alice", "age": 15}, + {"name": "Bob", "age": 14}, + {"name": "Charlie", "age": 13}, + {"name": None, "age": 12}, + ], + } + df = ctx.from_pydict(data, name="table1") + + names = df.select(col("struct_values")["name"].alias("name")).collect() + names = [r.as_py() for rs in names for r in rs["name"]] + + array_values = df.select(col("array_values")[1].alias("value")).collect() + array_values = [r.as_py() for rs in array_values for r in rs["value"]] + + assert names == ["Alice", "Bob", "Charlie", None] + assert array_values == [2, 5, None, None] diff --git a/src/dataframe.rs b/src/dataframe.rs index d7abab400..3fb8b2292 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -23,6 +23,7 @@ use arrow::compute::can_cast_types; use arrow::error::ArrowError; use arrow::ffi::FFI_ArrowSchema; use arrow::ffi_stream::FFI_ArrowArrayStream; +use arrow::util::display::{ArrayFormatter, FormatOptions}; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; @@ -95,6 +96,51 @@ impl PyDataFrame { } } + fn _repr_html_(&self, py: Python) -> PyResult { + let mut html_str = "\n".to_string(); + + let df = self.df.as_ref().clone().limit(0, Some(10))?; + let batches = wait_for_future(py, df.collect())?; + + if batches.is_empty() { + html_str.push_str("
\n"); + return Ok(html_str); + } + + let schema = batches[0].schema(); + + let mut header = Vec::new(); + for field in schema.fields() { + header.push(format!("{}", field.name())); + } + let header_str = header.join(""); + html_str.push_str(&format!("{}\n", header_str)); + + for batch in batches { + let formatters = batch + .columns() + .iter() + .map(|c| ArrayFormatter::try_new(c.as_ref(), &FormatOptions::default())) + .map(|c| { + c.map_err(|e| PyValueError::new_err(format!("Error: {:?}", e.to_string()))) + }) + .collect::, _>>()?; + + for row in 0..batch.num_rows() { + let mut cells = Vec::new(); + for formatter in &formatters { + cells.push(format!("{}", formatter.value(row))); + } + let row_str = cells.join(""); + html_str.push_str(&format!("{}\n", row_str)); + } + } + + html_str.push_str("\n"); + + Ok(html_str) + } + /// Calculate summary statistics for a DataFrame fn describe(&self, py: Python) -> PyResult { let df = self.df.as_ref().clone(); From fe0738a9c0b536cdf20b0dc0455d14a0d16d2835 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Mon, 2 Sep 2024 17:28:59 +0200 Subject: [PATCH 191/413] feat: better exception and message for table not found (#851) closes #796 --- python/datafusion/tests/test_context.py | 7 +++++++ src/context.rs | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index 0184280c2..4af00a3b4 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -466,6 +466,13 @@ def test_table_exist(ctx): assert ctx.table_exist("t") is True +def test_table_not_found(ctx): + from uuid import uuid4 + + with pytest.raises(KeyError): + ctx.table(f"not-found-{uuid4()}") + + def test_read_json(ctx): path = os.path.dirname(os.path.abspath(__file__)) diff --git a/src/context.rs b/src/context.rs index 4433d94c2..3ab783495 100644 --- a/src/context.rs +++ b/src/context.rs @@ -765,7 +765,8 @@ impl PySessionContext { } pub fn table(&self, name: &str, py: Python) -> PyResult { - let x = wait_for_future(py, self.ctx.table(name)).map_err(DataFusionError::from)?; + let x = wait_for_future(py, self.ctx.table(name)) + .map_err(|e| PyKeyError::new_err(e.to_string()))?; Ok(PyDataFrame::new(x)) } From 859acb4c582eb6ab0147843ace748a93c359264f Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Fri, 6 Sep 2024 17:28:53 +0200 Subject: [PATCH 192/413] feat: make cast accept built-in Python types (#858) --- python/datafusion/expr.py | 21 +++++++++++-- python/datafusion/tests/test_functions.py | 37 +++++++++++++++++------ 2 files changed, 47 insertions(+), 11 deletions(-) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 7bea0289b..a58634b53 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -28,7 +28,7 @@ functions as functions_internal, ) from datafusion.common import NullTreatment, RexType, DataTypeMap -from typing import Any, Optional +from typing import Any, Optional, Type import pyarrow as pa # The following are imported from the internal representation. We may choose to @@ -372,8 +372,25 @@ def is_not_null(self) -> Expr: """Returns ``True`` if this expression is not null.""" return Expr(self.expr.is_not_null()) - def cast(self, to: pa.DataType[Any]) -> Expr: + _to_pyarrow_types = { + float: pa.float64(), + int: pa.int64(), + str: pa.string(), + bool: pa.bool_(), + } + + def cast( + self, to: pa.DataType[Any] | Type[float] | Type[int] | Type[str] | Type[bool] + ) -> Expr: """Cast to a new data type.""" + if not isinstance(to, pa.DataType): + try: + to = self._to_pyarrow_types[to] + except KeyError: + raise TypeError( + "Expected instance of pyarrow.DataType or builtins.type" + ) + return Expr(self.expr.cast(to)) def rex_type(self) -> RexType: diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index fe092c456..e7e6d79e1 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -44,8 +44,9 @@ def df(): datetime(2020, 7, 2), ] ), + pa.array([False, True, True]), ], - names=["a", "b", "c", "d"], + names=["a", "b", "c", "d", "e"], ) return ctx.create_dataframe([[batch]]) @@ -63,15 +64,14 @@ def test_named_struct(df): ) expected = """DataFrame() -+-------+---+---------+------------------------------+ -| a | b | c | d | -+-------+---+---------+------------------------------+ -| Hello | 4 | hello | {a: Hello, b: 4, c: hello } | -| World | 5 | world | {a: World, b: 5, c: world } | -| ! | 6 | ! | {a: !, b: 6, c: !} | -+-------+---+---------+------------------------------+ ++-------+---+---------+------------------------------+-------+ +| a | b | c | d | e | ++-------+---+---------+------------------------------+-------+ +| Hello | 4 | hello | {a: Hello, b: 4, c: hello } | false | +| World | 5 | world | {a: World, b: 5, c: world } | true | +| ! | 6 | ! | {a: !, b: 6, c: !} | true | ++-------+---+---------+------------------------------+-------+ """.strip() - assert str(df) == expected @@ -978,3 +978,22 @@ def test_binary_string_functions(df): assert pa.array(result.column(1)).cast(pa.string()) == pa.array( ["Hello", "World", "!"] ) + + +@pytest.mark.parametrize( + "python_datatype, name, expected", + [ + pytest.param(bool, "e", pa.bool_(), id="bool"), + pytest.param(int, "b", pa.int64(), id="int"), + pytest.param(float, "b", pa.float64(), id="float"), + pytest.param(str, "b", pa.string(), id="str"), + ], +) +def test_cast(df, python_datatype, name: str, expected): + df = df.select( + column(name).cast(python_datatype).alias("actual"), + column(name).cast(expected).alias("expected"), + ) + result = df.collect() + result = result[0] + assert result.column(0) == result.column(1) From a0913c728f5f323c1eb4913e614c9d996083e274 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Fri, 6 Sep 2024 17:30:04 +0200 Subject: [PATCH 193/413] chore: fix docstrings, typos (#852) --- python/datafusion/context.py | 12 +++++++----- python/datafusion/expr.py | 2 +- python/datafusion/functions.py | 5 ----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index f6bf1c6ce..903d4a107 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -73,7 +73,7 @@ def with_create_default_catalog_and_schema( def with_default_catalog_and_schema( self, catalog: str, schema: str ) -> SessionConfig: - """Select a name for the default catalog and shcema. + """Select a name for the default catalog and schema. Args: catalog: Catalog name. @@ -517,7 +517,7 @@ def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame: def sql_with_options(self, query: str, options: SQLOptions) -> DataFrame: """Create a :py:class:`~datafusion.dataframe.DataFrame` from SQL query text. - This function will first validating that the query is allowed by the + This function will first validate that the query is allowed by the provided options. Args: @@ -636,12 +636,14 @@ def from_polars(self, data: polars.DataFrame, name: str | None = None) -> DataFr """ return DataFrame(self.ctx.from_polars(data, name)) - def register_table(self, name: str, table: pyarrow.Table) -> None: - """Register a table with the given name into the session. + def register_table(self, name: str, table: Table) -> None: + """Register a :py:class: `~datafusion.catalog.Table` as a table. + + The registered table can be referenced from SQL statement executed against. Args: name: Name of the resultant table. - table: PyArrow table to add to the session context. + table: DataFusion table to add to the session context. """ self.ctx.register_table(name, table) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index a58634b53..7fa608037 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -266,7 +266,7 @@ def __getitem__(self, key: str | int) -> Expr: If ``key`` is a string, returns the subfield of the struct. If ``key`` is an integer, retrieves the element in the array. Note that the - element index begins at ``0``, unlike `array_element` which begines at ``1``. + element index begins at ``0``, unlike `array_element` which begins at ``1``. """ if isinstance(key, int): return Expr( diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 4c701b24d..97b4fe1d5 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -18,11 +18,6 @@ from __future__ import annotations -# from datafusion._internal.context import SessionContext -# from datafusion._internal.expr import Expr -# from datafusion._internal.expr.conditional_expr import CaseBuilder -# from datafusion._internal.expr.window import WindowFrame - from datafusion._internal import functions as f, common from datafusion.expr import CaseBuilder, Expr, WindowFrame from datafusion.context import SessionContext From 6154ef983faeb01da98e9fcd6755b89fdef3d9ea Mon Sep 17 00:00:00 2001 From: Matt Green Date: Fri, 6 Sep 2024 18:10:10 -0700 Subject: [PATCH 194/413] chore: Use datafusion re-exported dependencies (#856) * Use datafusion re-exported crates * update dependency export syntax --- .gitignore | 1 + Cargo.lock | 5 ---- Cargo.toml | 7 +----- src/common/data_type.rs | 4 ++-- src/common/df_schema.rs | 2 +- src/common/schema.rs | 14 +++++------ src/config.rs | 2 +- src/context.rs | 6 ++--- src/dataframe.rs | 6 ++--- src/dataset.rs | 2 +- src/dataset_exec.rs | 4 ++-- src/expr.rs | 8 +++---- src/expr/aggregate.rs | 8 +++---- src/expr/aggregate_expr.rs | 2 +- src/expr/alias.rs | 2 +- src/expr/analyze.rs | 2 +- src/expr/between.rs | 2 +- src/expr/binary_expr.rs | 2 +- src/expr/bool_expr.rs | 2 +- src/expr/case.rs | 2 +- src/expr/cast.rs | 2 +- src/expr/column.rs | 2 +- src/expr/conditional_expr.rs | 2 +- src/expr/create_memory_table.rs | 2 +- src/expr/create_view.rs | 2 +- src/expr/cross_join.rs | 2 +- src/expr/distinct.rs | 2 +- src/expr/drop_table.rs | 2 +- src/expr/empty_relation.rs | 2 +- src/expr/exists.rs | 2 +- src/expr/explain.rs | 2 +- src/expr/extension.rs | 2 +- src/expr/filter.rs | 2 +- src/expr/grouping_set.rs | 2 +- src/expr/in_list.rs | 2 +- src/expr/in_subquery.rs | 2 +- src/expr/indexed_field.rs | 2 +- src/expr/join.rs | 2 +- src/expr/like.rs | 2 +- src/expr/limit.rs | 2 +- src/expr/literal.rs | 2 +- src/expr/placeholder.rs | 2 +- src/expr/projection.rs | 4 ++-- src/expr/repartition.rs | 2 +- src/expr/scalar_subquery.rs | 2 +- src/expr/signature.rs | 2 +- src/expr/sort.rs | 4 ++-- src/expr/sort_expr.rs | 2 +- src/expr/subquery.rs | 2 +- src/expr/subquery_alias.rs | 2 +- src/expr/table_scan.rs | 4 ++-- src/expr/union.rs | 2 +- src/expr/unnest.rs | 2 +- src/expr/unnest_expr.rs | 2 +- src/expr/window.rs | 6 ++--- src/functions.rs | 41 ++++++++++++++++---------------- src/lib.rs | 9 +++---- src/pyarrow_filter_expression.rs | 4 ++-- src/sql/logical.rs | 2 +- src/udaf.rs | 4 +++- src/udf.rs | 6 ++--- src/utils.rs | 2 +- 62 files changed, 113 insertions(+), 118 deletions(-) diff --git a/.gitignore b/.gitignore index aaeaaa5b1..614d82327 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ dist # intended to run in multiple environments; otherwise, check them in: .python-version venv +.venv apache-rat-*.jar *rat.txt diff --git a/Cargo.lock b/Cargo.lock index bca4bf066..43ee3055d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1072,11 +1072,6 @@ dependencies = [ "arrow", "async-trait", "datafusion", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", - "datafusion-optimizer", - "datafusion-sql", "datafusion-substrait", "futures", "mimalloc", diff --git a/Cargo.toml b/Cargo.toml index 8881884b4..e9a4ababb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,11 +39,6 @@ rand = "0.8" pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "52", feature = ["pyarrow"] } datafusion = { version = "41.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-common = { version = "41.0.0", features = ["pyarrow"] } -datafusion-expr = { version = "41.0.0" } -datafusion-functions-nested = { version = "41.0.0" } -datafusion-optimizer = { version = "41.0.0" } -datafusion-sql = { version = "41.0.0" } datafusion-substrait = { version = "41.0.0", optional = true } prost = "0.12" # keep in line with `datafusion-substrait` prost-types = "0.12" # keep in line with `datafusion-substrait` @@ -67,4 +62,4 @@ crate-type = ["cdylib", "rlib"] [profile.release] lto = true codegen-units = 1 - \ No newline at end of file + diff --git a/src/common/data_type.rs b/src/common/data_type.rs index bdfc5a6b1..a29d1799c 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -17,8 +17,8 @@ use datafusion::arrow::array::Array; use datafusion::arrow::datatypes::{DataType, IntervalUnit, TimeUnit}; -use datafusion_common::{DataFusionError, ScalarValue}; -use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; +use datafusion::common::{DataFusionError, ScalarValue}; +use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; diff --git a/src/common/df_schema.rs b/src/common/df_schema.rs index c16b8eba0..4e1d84060 100644 --- a/src/common/df_schema.rs +++ b/src/common/df_schema.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use datafusion_common::DFSchema; +use datafusion::common::DFSchema; use pyo3::prelude::*; #[derive(Debug, Clone)] diff --git a/src/common/schema.rs b/src/common/schema.rs index 00113a510..5806c90e2 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -18,10 +18,10 @@ use std::any::Any; use datafusion::arrow::datatypes::SchemaRef; -use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource}; +use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource}; use pyo3::prelude::*; -use datafusion_expr::utils::split_conjunction; +use datafusion::logical_expr::utils::split_conjunction; use super::{data_type::DataTypeMap, function::SqlFunction}; @@ -166,7 +166,7 @@ impl TableSource for SqlTableSource { fn supports_filter_pushdown( &self, filter: &Expr, - ) -> datafusion_common::Result { + ) -> datafusion::common::Result { let filters = split_conjunction(filter); if filters.iter().all(|f| is_supported_push_down_expr(f)) { // Push down filters to the tablescan operation if all are supported @@ -180,22 +180,22 @@ impl TableSource for SqlTableSource { } } - fn table_type(&self) -> datafusion_expr::TableType { - datafusion_expr::TableType::Base + fn table_type(&self) -> datafusion::logical_expr::TableType { + datafusion::logical_expr::TableType::Base } #[allow(deprecated)] fn supports_filters_pushdown( &self, filters: &[&Expr], - ) -> datafusion_common::Result> { + ) -> datafusion::common::Result> { filters .iter() .map(|f| self.supports_filter_pushdown(f)) .collect() } - fn get_logical_plan(&self) -> Option<&datafusion_expr::LogicalPlan> { + fn get_logical_plan(&self) -> Option<&datafusion::logical_expr::LogicalPlan> { None } } diff --git a/src/config.rs b/src/config.rs index 82a4f93ab..3f2a05580 100644 --- a/src/config.rs +++ b/src/config.rs @@ -18,8 +18,8 @@ use pyo3::prelude::*; use pyo3::types::*; +use datafusion::common::ScalarValue; use datafusion::config::ConfigOptions; -use datafusion_common::ScalarValue; #[pyclass(name = "Config", module = "datafusion", subclass)] #[derive(Clone)] diff --git a/src/context.rs b/src/context.rs index 3ab783495..11b9fed5f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -46,6 +46,7 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::ScalarValue; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ @@ -61,7 +62,6 @@ use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; -use datafusion_common::ScalarValue; use pyo3::types::{PyDict, PyList, PyTuple}; use tokio::task::JoinHandle; @@ -962,7 +962,7 @@ impl PySessionContext { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let plan = plan.plan.clone(); - let fut: JoinHandle> = + let fut: JoinHandle> = rt.spawn(async move { plan.execute(part, Arc::new(ctx)) }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; Ok(PyRecordBatchStream::new(stream?)) @@ -970,7 +970,7 @@ impl PySessionContext { } impl PySessionContext { - async fn _table(&self, name: &str) -> datafusion_common::Result { + async fn _table(&self, name: &str) -> datafusion::common::Result { self.ctx.table(name).await } } diff --git a/src/dataframe.rs b/src/dataframe.rs index 3fb8b2292..f33622cc0 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -27,12 +27,12 @@ use arrow::util::display::{ArrayFormatter, FormatOptions}; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; +use datafusion::common::UnnestOptions; use datafusion::config::{CsvOptions, TableParquetOptions}; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::prelude::*; -use datafusion_common::UnnestOptions; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; @@ -541,7 +541,7 @@ impl PyDataFrame { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let df = self.df.as_ref().clone(); - let fut: JoinHandle> = + let fut: JoinHandle> = rt.spawn(async move { df.execute_stream().await }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; Ok(PyRecordBatchStream::new(stream?)) @@ -551,7 +551,7 @@ impl PyDataFrame { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime(py).0; let df = self.df.as_ref().clone(); - let fut: JoinHandle>> = + let fut: JoinHandle>> = rt.spawn(async move { df.execute_stream_partitioned().await }); let stream = wait_for_future(py, fut).map_err(py_datafusion_err)?; diff --git a/src/dataset.rs b/src/dataset.rs index b5704164f..de7402fd6 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -31,9 +31,9 @@ use datafusion::arrow::datatypes::SchemaRef; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::datasource::{TableProvider, TableType}; use datafusion::error::{DataFusionError, Result as DFResult}; +use datafusion::logical_expr::Expr; use datafusion::logical_expr::TableProviderFilterPushDown; use datafusion::physical_plan::ExecutionPlan; -use datafusion_expr::Expr; use crate::dataset_exec::DatasetExec; use crate::pyarrow_filter_expression::PyArrowFilterExpression; diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 5fe1f4d1b..a377e2555 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -32,14 +32,14 @@ use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; use datafusion::error::{DataFusionError as InnerDataFusionError, Result as DFResult}; use datafusion::execution::context::TaskContext; +use datafusion::logical_expr::utils::conjunction; +use datafusion::logical_expr::Expr; use datafusion::physical_expr::{EquivalenceProperties, PhysicalSortExpr}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, Partitioning, SendableRecordBatchStream, Statistics, }; -use datafusion_expr::utils::conjunction; -use datafusion_expr::Expr; use crate::errors::DataFusionError; use crate::pyarrow_filter_expression::PyArrowFilterExpression; diff --git a/src/expr.rs b/src/expr.rs index 697682d4c..ab16f2872 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::utils::exprlist_to_fields; -use datafusion_expr::{ExprFuncBuilder, ExprFunctionExt, LogicalPlan}; +use datafusion::logical_expr::utils::exprlist_to_fields; +use datafusion::logical_expr::{ExprFuncBuilder, ExprFunctionExt, LogicalPlan}; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use std::sync::Arc; @@ -26,12 +26,12 @@ use arrow::pyarrow::ToPyArrow; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::functions::core::expr_ext::FieldAccessor; -use datafusion::scalar::ScalarValue; -use datafusion_expr::{ +use datafusion::logical_expr::{ col, expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast, }; +use datafusion::scalar::ScalarValue; use crate::common::data_type::{DataTypeMap, NullTreatment, RexType}; use crate::errors::{py_runtime_err, py_type_err, py_unsupported_variant_err, DataFusionError}; diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index e3d1bb136..389bfb332 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::DataFusionError; -use datafusion_expr::expr::{AggregateFunction, Alias}; -use datafusion_expr::logical_plan::Aggregate; -use datafusion_expr::Expr; +use datafusion::common::DataFusionError; +use datafusion::logical_expr::expr::{AggregateFunction, Alias}; +use datafusion::logical_expr::logical_plan::Aggregate; +use datafusion::logical_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs index 15097e007..09471097f 100644 --- a/src/expr/aggregate_expr.rs +++ b/src/expr/aggregate_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::AggregateFunction; +use datafusion::logical_expr::expr::AggregateFunction; use pyo3::prelude::*; use std::fmt::{Display, Formatter}; diff --git a/src/expr/alias.rs b/src/expr/alias.rs index 3208800ad..e8e03cfad 100644 --- a/src/expr/alias.rs +++ b/src/expr/alias.rs @@ -19,7 +19,7 @@ use crate::expr::PyExpr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; -use datafusion_expr::expr::Alias; +use datafusion::logical_expr::expr::Alias; #[pyclass(name = "Alias", module = "datafusion.expr", subclass)] #[derive(Clone)] diff --git a/src/expr/analyze.rs b/src/expr/analyze.rs index bbec3a808..084513971 100644 --- a/src/expr/analyze.rs +++ b/src/expr/analyze.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Analyze; +use datafusion::logical_expr::logical_plan::Analyze; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/between.rs b/src/expr/between.rs index 9b78b9eeb..a2cac1442 100644 --- a/src/expr/between.rs +++ b/src/expr/between.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::Between; +use datafusion::logical_expr::expr::Between; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/binary_expr.rs b/src/expr/binary_expr.rs index 5f382b770..740299211 100644 --- a/src/expr/binary_expr.rs +++ b/src/expr/binary_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::BinaryExpr; +use datafusion::logical_expr::BinaryExpr; use pyo3::prelude::*; #[pyclass(name = "BinaryExpr", module = "datafusion.expr", subclass)] diff --git a/src/expr/bool_expr.rs b/src/expr/bool_expr.rs index d1502a4eb..e67e25d74 100644 --- a/src/expr/bool_expr.rs +++ b/src/expr/bool_expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Expr; +use datafusion::logical_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/case.rs b/src/expr/case.rs index 605275376..92e28ba56 100644 --- a/src/expr/case.rs +++ b/src/expr/case.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::Case; +use datafusion::logical_expr::Case; use pyo3::prelude::*; #[pyclass(name = "Case", module = "datafusion.expr", subclass)] diff --git a/src/expr/cast.rs b/src/expr/cast.rs index a72199876..b8faea634 100644 --- a/src/expr/cast.rs +++ b/src/expr/cast.rs @@ -16,7 +16,7 @@ // under the License. use crate::{common::data_type::PyDataType, expr::PyExpr}; -use datafusion_expr::{Cast, TryCast}; +use datafusion::logical_expr::{Cast, TryCast}; use pyo3::prelude::*; #[pyclass(name = "Cast", module = "datafusion.expr", subclass)] diff --git a/src/expr/column.rs b/src/expr/column.rs index 68123fb04..365dbc0d2 100644 --- a/src/expr/column.rs +++ b/src/expr/column.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::Column; +use datafusion::common::Column; use pyo3::prelude::*; #[pyclass(name = "Column", module = "datafusion.expr", subclass)] diff --git a/src/expr/conditional_expr.rs b/src/expr/conditional_expr.rs index 96ef58f56..a8a885c54 100644 --- a/src/expr/conditional_expr.rs +++ b/src/expr/conditional_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::conditional_expressions::CaseBuilder; +use datafusion::logical_expr::conditional_expressions::CaseBuilder; use pyo3::prelude::*; #[pyclass(name = "CaseBuilder", module = "datafusion.expr", subclass)] diff --git a/src/expr/create_memory_table.rs b/src/expr/create_memory_table.rs index 509bf2168..01ebb66b0 100644 --- a/src/expr/create_memory_table.rs +++ b/src/expr/create_memory_table.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::CreateMemoryTable; +use datafusion::logical_expr::CreateMemoryTable; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/create_view.rs b/src/expr/create_view.rs index febd723c5..d119f5c21 100644 --- a/src/expr/create_view.rs +++ b/src/expr/create_view.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::{CreateView, DdlStatement, LogicalPlan}; +use datafusion::logical_expr::{CreateView, DdlStatement, LogicalPlan}; use pyo3::prelude::*; use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; diff --git a/src/expr/cross_join.rs b/src/expr/cross_join.rs index 68793f249..5bc202aac 100644 --- a/src/expr/cross_join.rs +++ b/src/expr/cross_join.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::CrossJoin; +use datafusion::logical_expr::logical_plan::CrossJoin; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs index 5d3a0b459..061ab4824 100644 --- a/src/expr/distinct.rs +++ b/src/expr/distinct.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::Distinct; +use datafusion::logical_expr::Distinct; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/drop_table.rs b/src/expr/drop_table.rs index 2a8836db5..330156abe 100644 --- a/src/expr/drop_table.rs +++ b/src/expr/drop_table.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::logical_plan::DropTable; +use datafusion::logical_expr::logical_plan::DropTable; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/empty_relation.rs b/src/expr/empty_relation.rs index 0bc222e59..ce7163466 100644 --- a/src/expr/empty_relation.rs +++ b/src/expr/empty_relation.rs @@ -16,7 +16,7 @@ // under the License. use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; -use datafusion_expr::EmptyRelation; +use datafusion::logical_expr::EmptyRelation; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/exists.rs b/src/expr/exists.rs index fd2aa8c2f..693357836 100644 --- a/src/expr/exists.rs +++ b/src/expr/exists.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Exists; +use datafusion::logical_expr::expr::Exists; use pyo3::prelude::*; use super::subquery::PySubquery; diff --git a/src/expr/explain.rs b/src/expr/explain.rs index d5d6a7bbd..8e7fb8843 100644 --- a/src/expr/explain.rs +++ b/src/expr/explain.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::{logical_plan::Explain, LogicalPlan}; +use datafusion::logical_expr::{logical_plan::Explain, LogicalPlan}; use pyo3::prelude::*; use crate::{common::df_schema::PyDFSchema, errors::py_type_err, sql::logical::PyLogicalPlan}; diff --git a/src/expr/extension.rs b/src/expr/extension.rs index 81a435c23..a29802b0b 100644 --- a/src/expr/extension.rs +++ b/src/expr/extension.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Extension; +use datafusion::logical_expr::Extension; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/filter.rs b/src/expr/filter.rs index 2def2f7d6..a6d8aa7ee 100644 --- a/src/expr/filter.rs +++ b/src/expr/filter.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Filter; +use datafusion::logical_expr::logical_plan::Filter; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/grouping_set.rs b/src/expr/grouping_set.rs index b73932863..63a1c0b50 100644 --- a/src/expr/grouping_set.rs +++ b/src/expr/grouping_set.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::GroupingSet; +use datafusion::logical_expr::GroupingSet; use pyo3::prelude::*; #[pyclass(name = "GroupingSet", module = "datafusion.expr", subclass)] diff --git a/src/expr/in_list.rs b/src/expr/in_list.rs index c1a99a3c8..5dfd8d8eb 100644 --- a/src/expr/in_list.rs +++ b/src/expr/in_list.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::InList; +use datafusion::logical_expr::expr::InList; use pyo3::prelude::*; #[pyclass(name = "InList", module = "datafusion.expr", subclass)] diff --git a/src/expr/in_subquery.rs b/src/expr/in_subquery.rs index 7dfafdbf0..306b68a6e 100644 --- a/src/expr/in_subquery.rs +++ b/src/expr/in_subquery.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::InSubquery; +use datafusion::logical_expr::expr::InSubquery; use pyo3::prelude::*; use super::{subquery::PySubquery, PyExpr}; diff --git a/src/expr/indexed_field.rs b/src/expr/indexed_field.rs index e0dad6a4a..a22dc6b27 100644 --- a/src/expr/indexed_field.rs +++ b/src/expr/indexed_field.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::expr::{GetFieldAccess, GetIndexedField}; +use datafusion::logical_expr::expr::{GetFieldAccess, GetIndexedField}; use pyo3::prelude::*; use std::fmt::{Display, Formatter}; diff --git a/src/expr/join.rs b/src/expr/join.rs index a53ddd3ba..66e677f8a 100644 --- a/src/expr/join.rs +++ b/src/expr/join.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::{Join, JoinConstraint, JoinType}; +use datafusion::logical_expr::logical_plan::{Join, JoinConstraint, JoinType}; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/like.rs b/src/expr/like.rs index 6ed3c2467..2e1f060bd 100644 --- a/src/expr/like.rs +++ b/src/expr/like.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Like; +use datafusion::logical_expr::expr::Like; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/limit.rs b/src/expr/limit.rs index d7b3f4ca5..876e154c1 100644 --- a/src/expr/limit.rs +++ b/src/expr/limit.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Limit; +use datafusion::logical_expr::logical_plan::Limit; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/literal.rs b/src/expr/literal.rs index 0333432f6..43084ba4b 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -16,7 +16,7 @@ // under the License. use crate::errors::DataFusionError; -use datafusion_common::ScalarValue; +use datafusion::common::ScalarValue; use pyo3::prelude::*; #[pyclass(name = "Literal", module = "datafusion.expr", subclass)] diff --git a/src/expr/placeholder.rs b/src/expr/placeholder.rs index ca75ce37e..4ac2c47e3 100644 --- a/src/expr/placeholder.rs +++ b/src/expr/placeholder.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Placeholder; +use datafusion::logical_expr::expr::Placeholder; use pyo3::prelude::*; use crate::common::data_type::PyDataType; diff --git a/src/expr/projection.rs b/src/expr/projection.rs index 8c1423df4..36534fdb2 100644 --- a/src/expr/projection.rs +++ b/src/expr/projection.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Projection; -use datafusion_expr::Expr; +use datafusion::logical_expr::logical_plan::Projection; +use datafusion::logical_expr::Expr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/repartition.rs b/src/expr/repartition.rs index e3e14f878..4e680e181 100644 --- a/src/expr/repartition.rs +++ b/src/expr/repartition.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::{logical_plan::Repartition, Expr, Partitioning}; +use datafusion::logical_expr::{logical_plan::Repartition, Expr, Partitioning}; use pyo3::prelude::*; use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; diff --git a/src/expr/scalar_subquery.rs b/src/expr/scalar_subquery.rs index c71bb9905..9d35f28a9 100644 --- a/src/expr/scalar_subquery.rs +++ b/src/expr/scalar_subquery.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::Subquery; +use datafusion::logical_expr::Subquery; use pyo3::prelude::*; use super::subquery::PySubquery; diff --git a/src/expr/signature.rs b/src/expr/signature.rs index 7882cebed..e85763555 100644 --- a/src/expr/signature.rs +++ b/src/expr/signature.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::{TypeSignature, Volatility}; +use datafusion::logical_expr::{TypeSignature, Volatility}; use pyo3::prelude::*; #[allow(dead_code)] diff --git a/src/expr/sort.rs b/src/expr/sort.rs index f9f9e5899..b31ebfe0b 100644 --- a/src/expr/sort.rs +++ b/src/expr/sort.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::DataFusionError; -use datafusion_expr::logical_plan::Sort; +use datafusion::common::DataFusionError; +use datafusion::logical_expr::logical_plan::Sort; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs index 6a8a0cf0c..4299d1f71 100644 --- a/src/expr/sort_expr.rs +++ b/src/expr/sort_expr.rs @@ -16,7 +16,7 @@ // under the License. use crate::expr::PyExpr; -use datafusion_expr::SortExpr; +use datafusion::logical_expr::SortExpr; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/subquery.rs b/src/expr/subquery.rs index f6f7b7fe5..dac8d0a2b 100644 --- a/src/expr/subquery.rs +++ b/src/expr/subquery.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::Subquery; +use datafusion::logical_expr::Subquery; use pyo3::prelude::*; use crate::sql::logical::PyLogicalPlan; diff --git a/src/expr/subquery_alias.rs b/src/expr/subquery_alias.rs index d3abd2e8c..a83cff96d 100644 --- a/src/expr/subquery_alias.rs +++ b/src/expr/subquery_alias.rs @@ -17,7 +17,7 @@ use std::fmt::{self, Display, Formatter}; -use datafusion_expr::SubqueryAlias; +use datafusion::logical_expr::SubqueryAlias; use pyo3::prelude::*; use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index 386c43b6a..f61be7fe4 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -15,8 +15,8 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::TableReference; -use datafusion_expr::logical_plan::TableScan; +use datafusion::common::TableReference; +use datafusion::logical_expr::logical_plan::TableScan; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/union.rs b/src/expr/union.rs index 98e8eaae6..62488d9a1 100644 --- a/src/expr/union.rs +++ b/src/expr/union.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Union; +use datafusion::logical_expr::logical_plan::Union; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/unnest.rs b/src/expr/unnest.rs index 33fb82feb..adc705035 100644 --- a/src/expr/unnest.rs +++ b/src/expr/unnest.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::logical_plan::Unnest; +use datafusion::logical_expr::logical_plan::Unnest; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/unnest_expr.rs b/src/expr/unnest_expr.rs index a2f8230cc..2234d24b1 100644 --- a/src/expr/unnest_expr.rs +++ b/src/expr/unnest_expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use datafusion_expr::expr::Unnest; +use datafusion::logical_expr::expr::Unnest; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/expr/window.rs b/src/expr/window.rs index 7eb586082..f17a6dd9b 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -15,9 +15,9 @@ // specific language governing permissions and limitations // under the License. -use datafusion_common::{DataFusionError, ScalarValue}; -use datafusion_expr::expr::WindowFunction; -use datafusion_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; +use datafusion::common::{DataFusionError, ScalarValue}; +use datafusion::logical_expr::expr::WindowFunction; +use datafusion::logical_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; diff --git a/src/functions.rs b/src/functions.rs index aed4de474..b5b003dfe 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,9 +16,9 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; -use datafusion_expr::window_function; -use datafusion_expr::ExprFunctionExt; -use datafusion_expr::WindowFrame; +use datafusion::logical_expr::window_function; +use datafusion::logical_expr::ExprFunctionExt; +use datafusion::logical_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; @@ -28,13 +28,13 @@ use crate::expr::conditional_expr::PyCaseBuilder; use crate::expr::to_sort_expressions; use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; +use datafusion::common::{Column, ScalarValue, TableReference}; use datafusion::execution::FunctionRegistry; use datafusion::functions; use datafusion::functions_aggregate; -use datafusion_common::{Column, ScalarValue, TableReference}; -use datafusion_expr::expr::Alias; -use datafusion_expr::sqlparser::ast::NullTreatment as DFNullTreatment; -use datafusion_expr::{ +use datafusion::logical_expr::expr::Alias; +use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; +use datafusion::logical_expr::{ expr::{find_df_window_func, AggregateFunction, Sort, WindowFunction}, lit, Expr, WindowFunctionDefinition, }; @@ -376,7 +376,7 @@ pub fn last_value( #[pyfunction] fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { - datafusion_expr::in_list( + datafusion::logical_expr::in_list( expr.expr, value.into_iter().map(|x| x.expr).collect::>(), negated, @@ -386,14 +386,14 @@ fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { #[pyfunction] fn make_array(exprs: Vec) -> PyExpr { - datafusion_functions_nested::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) + datafusion::functions_nested::expr_fn::make_array(exprs.into_iter().map(|x| x.into()).collect()) .into() } #[pyfunction] fn array_concat(exprs: Vec) -> PyExpr { let exprs = exprs.into_iter().map(|x| x.into()).collect(); - datafusion_functions_nested::expr_fn::array_concat(exprs).into() + datafusion::functions_nested::expr_fn::array_concat(exprs).into() } #[pyfunction] @@ -405,12 +405,13 @@ fn array_cat(exprs: Vec) -> PyExpr { fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { let index = ScalarValue::Int64(index); let index = Expr::Literal(index); - datafusion_functions_nested::expr_fn::array_position(array.into(), element.into(), index).into() + datafusion::functions_nested::expr_fn::array_position(array.into(), element.into(), index) + .into() } #[pyfunction] fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { - datafusion_functions_nested::expr_fn::array_slice( + datafusion::functions_nested::expr_fn::array_slice( array.into(), begin.into(), end.into(), @@ -476,7 +477,7 @@ fn regexp_replace( #[pyfunction] fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { Ok(PyExpr { - expr: datafusion_expr::Expr::Sort(Sort { + expr: datafusion::logical_expr::Expr::Sort(Sort { expr: Box::new(expr.expr), asc, nulls_first, @@ -489,7 +490,7 @@ fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { fn alias(expr: PyExpr, name: &str) -> PyResult { let relation: Option = None; Ok(PyExpr { - expr: datafusion_expr::Expr::Alias(Alias::new(expr.expr, relation, name)), + expr: datafusion::logical_expr::Expr::Alias(Alias::new(expr.expr, relation, name)), }) } @@ -497,7 +498,7 @@ fn alias(expr: PyExpr, name: &str) -> PyResult { #[pyfunction] fn col(name: &str) -> PyResult { Ok(PyExpr { - expr: datafusion_expr::Expr::Column(Column { + expr: datafusion::logical_expr::Expr::Column(Column { relation: None, name: name.to_string(), }), @@ -527,7 +528,7 @@ fn count(expr: PyExpr, distinct: bool) -> PyResult { #[pyfunction] fn case(expr: PyExpr) -> PyResult { Ok(PyCaseBuilder { - case_builder: datafusion_expr::case(expr.expr), + case_builder: datafusion::logical_expr::case(expr.expr), }) } @@ -535,7 +536,7 @@ fn case(expr: PyExpr) -> PyResult { #[pyfunction] fn when(when: PyExpr, then: PyExpr) -> PyResult { Ok(PyCaseBuilder { - case_builder: datafusion_expr::when(when.expr, then.expr), + case_builder: datafusion::logical_expr::when(when.expr, then.expr), }) } @@ -622,7 +623,7 @@ fn window( .unwrap_or(WindowFrame::new(order_by.as_ref().map(|v| !v.is_empty()))); Ok(PyExpr { - expr: datafusion_expr::Expr::WindowFunction(WindowFunction { + expr: datafusion::logical_expr::Expr::WindowFunction(WindowFunction { fun, args: args.into_iter().map(|x| x.expr).collect::>(), partition_by: partition_by @@ -654,7 +655,7 @@ macro_rules! aggregate_function { #[pyfunction] #[pyo3(signature = (*args, distinct=false))] fn $NAME(args: Vec, distinct: bool) -> PyExpr { - let expr = datafusion_expr::Expr::AggregateFunction(AggregateFunction { + let expr = datafusion::logical_expr::Expr::AggregateFunction(AggregateFunction { func: $FUNC(), args: args.into_iter().map(|e| e.into()).collect(), distinct, @@ -724,7 +725,7 @@ macro_rules! array_fn { #[doc = $DOC] #[pyfunction] fn $FUNC($($arg: PyExpr),*) -> PyExpr { - datafusion_functions_nested::expr_fn::$FUNC($($arg.into()),*).into() + datafusion::functions_nested::expr_fn::$FUNC($($arg.into()),*).into() } }; } diff --git a/src/lib.rs b/src/lib.rs index 357eaacd9..e4cc24078 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -21,10 +21,10 @@ use pyo3::prelude::*; // Re-export Apache Arrow DataFusion dependencies pub use datafusion; -pub use datafusion_common; -pub use datafusion_expr; -pub use datafusion_optimizer; -pub use datafusion_sql; +pub use datafusion::common as datafusion_common; +pub use datafusion::logical_expr as datafusion_expr; +pub use datafusion::optimizer; +pub use datafusion::sql as datafusion_sql; #[cfg(feature = "substrait")] pub use datafusion_substrait; @@ -32,6 +32,7 @@ pub use datafusion_substrait; #[allow(clippy::borrow_deref_ref)] pub mod catalog; pub mod common; + #[allow(clippy::borrow_deref_ref)] mod config; #[allow(clippy::borrow_deref_ref)] diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index ff447e1ab..6e2a45e1a 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -22,8 +22,8 @@ use std::convert::TryFrom; use std::result::Result; use arrow::pyarrow::ToPyArrow; -use datafusion_common::{Column, ScalarValue}; -use datafusion_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; +use datafusion::common::{Column, ScalarValue}; +use datafusion::logical_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; use crate::errors::DataFusionError; diff --git a/src/sql/logical.rs b/src/sql/logical.rs index c4471f503..89655ab70 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -35,7 +35,7 @@ use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; use crate::expr::unnest::PyUnnest; use crate::expr::window::PyWindow; -use datafusion_expr::LogicalPlan; +use datafusion::logical_expr::LogicalPlan; use pyo3::prelude::*; use crate::expr::logical_node::LogicalNode; diff --git a/src/udaf.rs b/src/udaf.rs index 2041e5a74..a6aa59ac3 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -24,7 +24,9 @@ use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::common::ScalarValue; use datafusion::error::{DataFusionError, Result}; -use datafusion_expr::{create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF}; +use datafusion::logical_expr::{ + create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF, +}; use crate::expr::PyExpr; use crate::utils::parse_volatility; diff --git a/src/udf.rs b/src/udf.rs index 8f5ca30b1..8bd9021d4 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -23,9 +23,9 @@ use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; -use datafusion_expr::create_udf; -use datafusion_expr::function::ScalarFunctionImplementation; -use datafusion_expr::ScalarUDF; +use datafusion::logical_expr::create_udf; +use datafusion::logical_expr::function::ScalarFunctionImplementation; +use datafusion::logical_expr::ScalarUDF; use crate::expr::PyExpr; use crate::utils::parse_volatility; diff --git a/src/utils.rs b/src/utils.rs index 4334f86cd..0d72eaf75 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -17,7 +17,7 @@ use crate::errors::DataFusionError; use crate::TokioRuntime; -use datafusion_expr::Volatility; +use datafusion::logical_expr::Volatility; use pyo3::prelude::*; use std::future::Future; use tokio::runtime::Runtime; From 4ea0032ff890fb5ab0838e42b8b66ffbd521f4df Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sat, 7 Sep 2024 06:07:27 -0500 Subject: [PATCH 195/413] add section to the contributor guide on separating python and rust code (#860) --- docs/source/contributor-guide/introduction.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/source/contributor-guide/introduction.rst b/docs/source/contributor-guide/introduction.rst index 6de2b87bc..4457a898f 100644 --- a/docs/source/contributor-guide/introduction.rst +++ b/docs/source/contributor-guide/introduction.rst @@ -71,6 +71,17 @@ Our pre-commit hooks can be installed by running :code:`pre-commit install`, whi The pre-commit hooks can also be run adhoc without installing them by simply running :code:`pre-commit run --all-files` +Guidelines for Separating Python and Rust Code +---------------------------------------------- + +Version 40 of ``datafusion-python`` introduced ``python`` wrappers around the ``pyo3`` generated code to vastly improve the user experience. (See the `blog post `_ and `pull request `_ for more details.) + +Mostly, the ``python`` code is limited to pure wrappers with type hints and good docstrings, but there are a few reasons for when the code does more: + +1. Trivial aliases like :py:func:`~datafusion.functions.array_append` and :py:func:`~datafusion.functions.list_append`. +2. Simple type conversion, like from a ``path`` to a ``string`` of the path or from ``number`` to ``lit(number)``. +3. The additional code makes an API **much** more pythonic, like we do for :py:func:`~datafusion.functions.named_struct` (see `source code `_). + Update Dependencies ------------------- From fd8224e1d32282c1664575f32a2eb5d39ffaabba Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 9 Sep 2024 14:13:21 -0400 Subject: [PATCH 196/413] Update Aggregate functions to take builder parameters (#859) * Add NullTreatment enum wrapper and add filter option to approx_distinct * Small usability on aggregate * Adding documentation and additional unit test for approx_median * Update approx_percentil_cont with builder parameters it uses, which is filter but not distinct * Update approx_percentil_cont_with_weight with builder parameters it uses, which is filter but not distinct * Update array_agg to use aggregate options * Update builder options for avg aggregate function * move bit_and bit_or to use macro to generaty python fn * Update builder arguments for bitwise operators * Use macro for bool_and and bool_or * Update python wrapper for arguments appropriate to bool operators * Set corr to use macro for pyfunction * Update unit test to make it easier to debug * Update corr python wrapper to expose only builder parameters used * Update count and count_star to use macro for exposing * Update count and count_star with approprate aggregation options * Move covar_pop and covar_samp to use macro for aggregates * Updateing covar_pop and covar_samp with builder option * Use macro for last_value and move first_value to be near it * Update first_value and last_value with the builder parameters that are relevant * Remove grouping since it is not actually implemented upstream * Move median to use macro * Expose builder options for median * Expose nth value * Updating linear regression functions to use filter and macro * Update stddev and stddev_pop to use filter and macro * Expose string_agg * Add string_agg to python wrappers and add unit test * Switch sum to use macro in rust side and expose correct options in python wrapper * Use macro for exposing var_pop and var_samp * Add unit tests for filtering on var_pop and var_samp * Move approximation functions to use macro when possible * Update user documentation to explain in detail the options for aggregate functions * Update unit test to handle Python 3.10 * Clean up commented code --- .../common-operations/aggregations.rst | 206 ++++- python/datafusion/common.py | 15 +- python/datafusion/dataframe.py | 7 +- python/datafusion/expr.py | 4 +- python/datafusion/functions.py | 867 ++++++++++++++---- python/datafusion/tests/test_aggregation.py | 359 +++++++- python/datafusion/tests/test_functions.py | 99 +- .../datafusion/tests/test_wrapper_coverage.py | 11 + src/functions.rs | 521 ++++------- 9 files changed, 1470 insertions(+), 619 deletions(-) diff --git a/docs/source/user-guide/common-operations/aggregations.rst b/docs/source/user-guide/common-operations/aggregations.rst index 7ad402210..8fee26a15 100644 --- a/docs/source/user-guide/common-operations/aggregations.rst +++ b/docs/source/user-guide/common-operations/aggregations.rst @@ -20,43 +20,205 @@ Aggregation ============ -An aggregate or aggregation is a function where the values of multiple rows are processed together to form a single summary value. -For performing an aggregation, DataFusion provides the :py:func:`~datafusion.dataframe.DataFrame.aggregate` +An aggregate or aggregation is a function where the values of multiple rows are processed together +to form a single summary value. For performing an aggregation, DataFusion provides the +:py:func:`~datafusion.dataframe.DataFrame.aggregate` .. ipython:: python + import urllib.request from datafusion import SessionContext - from datafusion import column, lit + from datafusion import col, lit from datafusion import functions as f - import random - ctx = SessionContext() - df = ctx.from_pydict( - { - "a": ["foo", "bar", "foo", "bar", "foo", "bar", "foo", "foo"], - "b": ["one", "one", "two", "three", "two", "two", "one", "three"], - "c": [random.randint(0, 100) for _ in range(8)], - "d": [random.random() for _ in range(8)], - }, - name="foo_bar" + urllib.request.urlretrieve( + "https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv", + "pokemon.csv", ) - col_a = column("a") - col_b = column("b") - col_c = column("c") - col_d = column("d") + ctx = SessionContext() + df = ctx.read_csv("pokemon.csv") + + col_type_1 = col('"Type 1"') + col_type_2 = col('"Type 2"') + col_speed = col('"Speed"') + col_attack = col('"Attack"') - df.aggregate([], [f.approx_distinct(col_c), f.approx_median(col_d), f.approx_percentile_cont(col_d, lit(0.5))]) + df.aggregate([col_type_1], [ + f.approx_distinct(col_speed).alias("Count"), + f.approx_median(col_speed).alias("Median Speed"), + f.approx_percentile_cont(col_speed, 0.9).alias("90% Speed")]) -When the :code:`group_by` list is empty the aggregation is done over the whole :class:`.DataFrame`. For grouping -the :code:`group_by` list must contain at least one column +When the :code:`group_by` list is empty the aggregation is done over the whole :class:`.DataFrame`. +For grouping the :code:`group_by` list must contain at least one column. .. ipython:: python - df.aggregate([col_a], [f.sum(col_c), f.max(col_d), f.min(col_d)]) + df.aggregate([col_type_1], [ + f.max(col_speed).alias("Max Speed"), + f.avg(col_speed).alias("Avg Speed"), + f.min(col_speed).alias("Min Speed")]) More than one column can be used for grouping .. ipython:: python - df.aggregate([col_a, col_b], [f.sum(col_c), f.max(col_d), f.min(col_d)]) + df.aggregate([col_type_1, col_type_2], [ + f.max(col_speed).alias("Max Speed"), + f.avg(col_speed).alias("Avg Speed"), + f.min(col_speed).alias("Min Speed")]) + + + +Setting Parameters +------------------ + +Each of the built in aggregate functions provides arguments for the parameters that affect their +operation. These can also be overridden using the builder approach to setting any of the following +parameters. When you use the builder, you must call ``build()`` to finish. For example, these two +expressions are equivalent. + +.. ipython:: python + + first_1 = f.first_value(col("a"), order_by=[col("a")]) + first_2 = f.first_value(col("a")).order_by(col("a")).build() + +Ordering +^^^^^^^^ + +You can control the order in which rows are processed by window functions by providing +a list of ``order_by`` functions for the ``order_by`` parameter. In the following example, we +sort the Pokemon by their attack in increasing order and take the first value, which gives us the +Pokemon with the smallest attack value in each ``Type 1``. + +.. ipython:: python + + df.aggregate( + [col('"Type 1"')], + [f.first_value( + col('"Name"'), + order_by=[col('"Attack"').sort(ascending=True)] + ).alias("Smallest Attack") + ]) + +Distinct +^^^^^^^^ + +When you set the parameter ``distinct`` to ``True``, then unique values will only be evaluated one +time each. Suppose we want to create an array of all of the ``Type 2`` for each ``Type 1`` of our +Pokemon set. Since there will be many entries of ``Type 2`` we only one each distinct value. + +.. ipython:: python + + df.aggregate([col_type_1], [f.array_agg(col_type_2, distinct=True).alias("Type 2 List")]) + +In the output of the above we can see that there are some ``Type 1`` for which the ``Type 2`` entry +is ``null``. In reality, we probably want to filter those out. We can do this in two ways. First, +we can filter DataFrame rows that have no ``Type 2``. If we do this, we might have some ``Type 1`` +entries entirely removed. The second is we can use the ``filter`` argument described below. + +.. ipython:: python + + df.filter(col_type_2.is_not_null()).aggregate([col_type_1], [f.array_agg(col_type_2, distinct=True).alias("Type 2 List")]) + + df.aggregate([col_type_1], [f.array_agg(col_type_2, distinct=True, filter=col_type_2.is_not_null()).alias("Type 2 List")]) + +Which approach you take should depend on your use case. + +Null Treatment +^^^^^^^^^^^^^^ + +This option allows you to either respect or ignore null values. + +One common usage for handling nulls is the case where you want to find the first value within a +partition. By setting the null treatment to ignore nulls, we can find the first non-null value +in our partition. + + +.. ipython:: python + + from datafusion.common import NullTreatment + + df.aggregate([col_type_1], [ + f.first_value( + col_type_2, + order_by=[col_attack], + null_treatment=NullTreatment.RESPECT_NULLS + ).alias("Lowest Attack Type 2")]) + + df.aggregate([col_type_1], [ + f.first_value( + col_type_2, + order_by=[col_attack], + null_treatment=NullTreatment.IGNORE_NULLS + ).alias("Lowest Attack Type 2")]) + +Filter +^^^^^^ + +Using the filter option is useful for filtering results to include in the aggregate function. It can +be seen in the example above on how this can be useful to only filter rows evaluated by the +aggregate function without filtering rows from the entire DataFrame. + +Filter takes a single expression. + +Suppose we want to find the speed values for only Pokemon that have low Attack values. + +.. ipython:: python + + df.aggregate([col_type_1], [ + f.avg(col_speed).alias("Avg Speed All"), + f.avg(col_speed, filter=col_attack < lit(50)).alias("Avg Speed Low Attack")]) + + +Aggregate Functions +------------------- + +The available aggregate functions are: + +1. Comparison Functions + - :py:func:`datafusion.functions.min` + - :py:func:`datafusion.functions.max` +2. Math Functions + - :py:func:`datafusion.functions.sum` + - :py:func:`datafusion.functions.avg` + - :py:func:`datafusion.functions.median` +3. Array Functions + - :py:func:`datafusion.functions.array_agg` +4. Logical Functions + - :py:func:`datafusion.functions.bit_and` + - :py:func:`datafusion.functions.bit_or` + - :py:func:`datafusion.functions.bit_xor` + - :py:func:`datafusion.functions.bool_and` + - :py:func:`datafusion.functions.bool_or` +5. Statistical Functions + - :py:func:`datafusion.functions.count` + - :py:func:`datafusion.functions.corr` + - :py:func:`datafusion.functions.covar_samp` + - :py:func:`datafusion.functions.covar_pop` + - :py:func:`datafusion.functions.stddev` + - :py:func:`datafusion.functions.stddev_pop` + - :py:func:`datafusion.functions.var_samp` + - :py:func:`datafusion.functions.var_pop` +6. Linear Regression Functions + - :py:func:`datafusion.functions.regr_count` + - :py:func:`datafusion.functions.regr_slope` + - :py:func:`datafusion.functions.regr_intercept` + - :py:func:`datafusion.functions.regr_r2` + - :py:func:`datafusion.functions.regr_avgx` + - :py:func:`datafusion.functions.regr_avgy` + - :py:func:`datafusion.functions.regr_sxx` + - :py:func:`datafusion.functions.regr_syy` + - :py:func:`datafusion.functions.regr_slope` +7. Positional Functions + - :py:func:`datafusion.functions.first_value` + - :py:func:`datafusion.functions.last_value` + - :py:func:`datafusion.functions.nth_value` +8. String Functions + - :py:func:`datafusion.functions.string_agg` +9. Approximation Functions + - :py:func:`datafusion.functions.approx_distinct` + - :py:func:`datafusion.functions.approx_median` + - :py:func:`datafusion.functions.approx_percentile_cont` + - :py:func:`datafusion.functions.approx_percentile_cont_with_weight` + diff --git a/python/datafusion/common.py b/python/datafusion/common.py index 225e33304..7db8333f2 100644 --- a/python/datafusion/common.py +++ b/python/datafusion/common.py @@ -17,13 +17,13 @@ """Common data types used throughout the DataFusion project.""" from ._internal import common as common_internal +from enum import Enum # TODO these should all have proper wrapper classes DFSchema = common_internal.DFSchema DataType = common_internal.DataType DataTypeMap = common_internal.DataTypeMap -NullTreatment = common_internal.NullTreatment PythonType = common_internal.PythonType RexType = common_internal.RexType SqlFunction = common_internal.SqlFunction @@ -47,3 +47,16 @@ "SqlStatistics", "SqlFunction", ] + + +class NullTreatment(Enum): + """Describe how null values are to be treated by functions. + + This is used primarily by aggregate and window functions. It can be set on + these functions using the builder approach described in + ref:`_window_functions` and ref:`_aggregation` in the online documentation. + + """ + + RESPECT_NULLS = common_internal.NullTreatment.RESPECT_NULLS + IGNORE_NULLS = common_internal.NullTreatment.IGNORE_NULLS diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 46b8fa1bd..56dff22a4 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -180,7 +180,9 @@ def with_column_renamed(self, old_name: str, new_name: str) -> DataFrame: """ return DataFrame(self.df.with_column_renamed(old_name, new_name)) - def aggregate(self, group_by: list[Expr], aggs: list[Expr]) -> DataFrame: + def aggregate( + self, group_by: list[Expr] | Expr, aggs: list[Expr] | Expr + ) -> DataFrame: """Aggregates the rows of the current DataFrame. Args: @@ -190,6 +192,9 @@ def aggregate(self, group_by: list[Expr], aggs: list[Expr]) -> DataFrame: Returns: DataFrame after aggregation. """ + group_by = group_by if isinstance(group_by, list) else [group_by] + aggs = aggs if isinstance(aggs, list) else [aggs] + group_by = [e.expr for e in group_by] aggs = [e.expr for e in aggs] return DataFrame(self.df.aggregate(group_by, aggs)) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 7fa608037..bd6a86fb8 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -473,7 +473,7 @@ def null_treatment(self, null_treatment: NullTreatment) -> ExprFuncBuilder: set parameters for either window or aggregate functions. If used on any other type of expression, an error will be generated when ``build()`` is called. """ - return ExprFuncBuilder(self.expr.null_treatment(null_treatment)) + return ExprFuncBuilder(self.expr.null_treatment(null_treatment.value)) def partition_by(self, *partition_by: Expr) -> ExprFuncBuilder: """Set the partitioning for a window function. @@ -518,7 +518,7 @@ def distinct(self) -> ExprFuncBuilder: def null_treatment(self, null_treatment: NullTreatment) -> ExprFuncBuilder: """Set how nulls are treated for either window or aggregate functions.""" - return ExprFuncBuilder(self.builder.null_treatment(null_treatment)) + return ExprFuncBuilder(self.builder.null_treatment(null_treatment.value)) def partition_by(self, *partition_by: Expr) -> ExprFuncBuilder: """Set partitioning for window functions.""" diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 97b4fe1d5..163ff04e4 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -18,9 +18,10 @@ from __future__ import annotations -from datafusion._internal import functions as f, common +from datafusion._internal import functions as f, expr as expr_internal from datafusion.expr import CaseBuilder, Expr, WindowFrame from datafusion.context import SessionContext +from datafusion.common import NullTreatment from typing import Any, Optional @@ -126,7 +127,6 @@ "floor", "from_unixtime", "gcd", - "grouping", "in_list", "initcap", "isnan", @@ -180,6 +180,7 @@ "named_struct", "nanvl", "now", + "nth_value", "nullif", "octet_length", "order_by", @@ -222,6 +223,7 @@ "stddev", "stddev_pop", "stddev_samp", + "string_agg", "strpos", "struct", "substr", @@ -244,6 +246,7 @@ "var", "var_pop", "var_samp", + "var_sample", "when", # Window Functions "window", @@ -258,6 +261,12 @@ ] +def expr_list_to_raw_expr_list( + expr_list: Optional[list[Expr]], +) -> Optional[list[expr_internal.Expr]]: + return [e.expr for e in expr_list] if expr_list is not None else None + + def isnan(expr: Expr) -> Expr: """Returns true if a given number is +NaN or -NaN otherwise returns false.""" return Expr(f.isnan(expr.expr)) @@ -358,9 +367,18 @@ def col(name: str) -> Expr: return Expr(f.col(name)) -def count_star() -> Expr: - """Create a COUNT(1) aggregate expression.""" - return Expr(f.count_star()) +def count_star(filter: Optional[Expr] = None) -> Expr: + """Create a COUNT(1) aggregate expression. + + This aggregate function will count all of the rows in the partition. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``distinct``, and ``null_treatment``. + + Args: + filter: If provided, only count rows for which the filter is True + """ + return count(Expr.literal(1), filter=filter) def case(expr: Expr) -> CaseBuilder: @@ -400,8 +418,8 @@ def window( df.select(functions.lag(col("a")).partition_by(col("b")).build()) """ args = [a.expr for a in args] - partition_by = [e.expr for e in partition_by] if partition_by is not None else None - order_by = [o.expr for o in order_by] if order_by is not None else None + partition_by = expr_list_to_raw_expr_list(partition_by) + order_by = expr_list_to_raw_expr_list(order_by) window_frame = window_frame.window_frame if window_frame is not None else None return Expr(f.window(name, args, partition_by, order_by, window_frame, ctx)) @@ -1486,291 +1504,788 @@ def flatten(array: Expr) -> Expr: # aggregate functions -def approx_distinct(expression: Expr) -> Expr: - """Returns the approximate number of distinct values.""" - return Expr(f.approx_distinct(expression.expr)) +def approx_distinct( + expression: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Returns the approximate number of distinct values. + + This aggregate function is similar to :py:func:`count` with distinct set, but it + will approximate the number of distinct entries. It may return significantly faster + than :py:func:`count` for some DataFrames. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: Values to check for distinct entries + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + + return Expr(f.approx_distinct(expression.expr, filter=filter_raw)) + +def approx_median(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Returns the approximate median value. -def approx_median(arg: Expr, distinct: bool = False) -> Expr: - """Returns the approximate median value.""" - return Expr(f.approx_median(arg.expr, distinct=distinct)) + This aggregate function is similar to :py:func:`median`, but it will only + approximate the median. It may return significantly faster for some DataFrames. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by`` and ``null_treatment``, and ``distinct``. + + Args: + expression: Values to find the median for + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.approx_median(expression.expr, filter=filter_raw)) def approx_percentile_cont( expression: Expr, - percentile: Expr, - num_centroids: Expr | None = None, - distinct: bool = False, + percentile: float, + num_centroids: Optional[int] = None, + filter: Optional[Expr] = None, ) -> Expr: - """Returns the value that is approximately at a given percentile of ``expr``.""" - if num_centroids is None: - return Expr( - f.approx_percentile_cont( - expression.expr, percentile.expr, distinct=distinct, num_centroids=None - ) - ) + """Returns the value that is approximately at a given percentile of ``expr``. + + This aggregate function assumes the input values form a continuous distribution. + Suppose you have a DataFrame which consists of 100 different test scores. If you + called this function with a percentile of 0.9, it would return the value of the + test score that is above 90% of the other test scores. The returned value may be + between two of the values. + + This function uses the [t-digest](https://arxiv.org/abs/1902.04023) algorithm to + compute the percentil. You can limit the number of bins used in this algorithm by + setting the ``num_centroids`` parameter. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: Values for which to find the approximate percentile + percentile: This must be between 0.0 and 1.0, inclusive + num_centroids: Max bin size for the t-digest algorithm + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None return Expr( f.approx_percentile_cont( - expression.expr, - percentile.expr, - distinct=distinct, - num_centroids=num_centroids.expr, + expression.expr, percentile, num_centroids=num_centroids, filter=filter_raw ) ) def approx_percentile_cont_with_weight( - arg: Expr, weight: Expr, percentile: Expr, distinct: bool = False + expression: Expr, weight: Expr, percentile: float, filter: Optional[Expr] = None ) -> Expr: - """Returns the value of the approximate percentile. + """Returns the value of the weighted approximate percentile. + + This aggregate function is similar to :py:func:`approx_percentile_cont` except that + it uses the associated associated weights. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: Values for which to find the approximate percentile + weight: Relative weight for each of the values in ``expression`` + percentile: This must be between 0.0 and 1.0, inclusive + filter: If provided, only compute against rows for which the filter is True - This function is similar to :py:func:`approx_percentile_cont` except that it uses - the associated associated weights. """ + filter_raw = filter.expr if filter is not None else None return Expr( f.approx_percentile_cont_with_weight( - arg.expr, weight.expr, percentile.expr, distinct=distinct + expression.expr, weight.expr, percentile, filter=filter_raw ) ) -def array_agg(arg: Expr, distinct: bool = False) -> Expr: - """Aggregate values into an array.""" - return Expr(f.array_agg(arg.expr, distinct=distinct)) +def array_agg( + expression: Expr, + distinct: bool = False, + filter: Optional[Expr] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Aggregate values into an array. + Currently ``distinct`` and ``order_by`` cannot be used together. As a work around, + consider :py:func:`array_sort` after aggregation. + [Issue Tracker](https://github.com/apache/datafusion/issues/12371) -def avg(arg: Expr, distinct: bool = False) -> Expr: - """Returns the average value.""" - return Expr(f.avg(arg.expr, distinct=distinct)) + If using the builder functions described in ref:`_aggregation` this function ignores + the option ``null_treatment``. + Args: + expression: Values to combine into an array + distinct: If True, a single entry for each distinct value will be in the result + filter: If provided, only compute against rows for which the filter is True + order_by: Order the resultant array values + """ + order_by_raw = expr_list_to_raw_expr_list(order_by) + filter_raw = filter.expr if filter is not None else None + + return Expr( + f.array_agg( + expression.expr, distinct=distinct, filter=filter_raw, order_by=order_by_raw + ) + ) -def corr(value1: Expr, value2: Expr, distinct: bool = False) -> Expr: - """Returns the correlation coefficient between ``value1`` and ``value2``.""" - return Expr(f.corr(value1.expr, value2.expr, distinct=distinct)) +def avg( + expression: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Returns the average value. -def count(args: Expr | list[Expr] | None = None, distinct: bool = False) -> Expr: - """Returns the number of rows that match the given arguments.""" - if args is None: - return count(Expr.literal(1), distinct=distinct) - if isinstance(args, list): - args = [arg.expr for arg in args] - elif isinstance(args, Expr): - args = [args.expr] - return Expr(f.count(*args, distinct=distinct)) + This aggregate function expects a numeric expression and will return a float. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. -def covar(y: Expr, x: Expr) -> Expr: - """Computes the sample covariance. + Args: + expression: Values to combine into an array + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.avg(expression.expr, filter=filter_raw)) - This is an alias for :py:func:`covar_samp`. + +def corr(value_y: Expr, value_x: Expr, filter: Optional[Expr] = None) -> Expr: + """Returns the correlation coefficient between ``value1`` and ``value2``. + + This aggregate function expects both values to be numeric and will return a float. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + value_y: The dependent variable for correlation + value_x: The independent variable for correlation + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.corr(value_y.expr, value_x.expr, filter=filter_raw)) + + +def count( + expressions: Expr | list[Expr] | None = None, + distinct: bool = False, + filter: Optional[Expr] = None, +) -> Expr: + """Returns the number of rows that match the given arguments. + + This aggregate function will count the non-null rows provided in the expression. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by`` and ``null_treatment``. + + Args: + expressions: Argument to perform bitwise calculation on + distinct: If True, a single entry for each distinct value will be in the result + filter: If provided, only compute against rows for which the filter is True """ - return covar_samp(y, x) + filter_raw = filter.expr if filter is not None else None + if expressions is None: + args = [Expr.literal(1).expr] + elif isinstance(expressions, list): + args = [arg.expr for arg in expressions] + else: + args = [expressions.expr] -def covar_pop(y: Expr, x: Expr) -> Expr: - """Computes the population covariance.""" - return Expr(f.covar_pop(y.expr, x.expr)) + return Expr(f.count(*args, distinct=distinct, filter=filter_raw)) -def covar_samp(y: Expr, x: Expr) -> Expr: - """Computes the sample covariance.""" - return Expr(f.covar_samp(y.expr, x.expr)) +def covar_pop(value_y: Expr, value_x: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the population covariance. + This aggregate function expects both values to be numeric and will return a float. -def grouping(arg: Expr, distinct: bool = False) -> Expr: - """Indicates if the expression is aggregated or not. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. - Returns 1 if the value of the argument is aggregated, 0 if not. + Args: + value_y: The dependent variable for covariance + value_x: The independent variable for covariance + filter: If provided, only compute against rows for which the filter is True """ - return Expr(f.grouping(arg.expr, distinct=distinct)) + filter_raw = filter.expr if filter is not None else None + return Expr(f.covar_pop(value_y.expr, value_x.expr, filter=filter_raw)) -def max(arg: Expr, distinct: bool = False) -> Expr: - """Returns the maximum value of the argument.""" - return Expr(f.max(arg.expr, distinct=distinct)) +def covar_samp(value_y: Expr, value_x: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the sample covariance. + This aggregate function expects both values to be numeric and will return a float. -def mean(arg: Expr, distinct: bool = False) -> Expr: + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + value_y: The dependent variable for covariance + value_x: The independent variable for covariance + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.covar_samp(value_y.expr, value_x.expr, filter=filter_raw)) + + +def covar(value_y: Expr, value_x: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the sample covariance. + + This is an alias for :py:func:`covar_samp`. + """ + return covar_samp(value_y, value_x, filter) + + +def max(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Aggregate function that returns the maximum value of the argument. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: The value to find the maximum of + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.max(expression.expr, filter=filter_raw)) + + +def mean(expression: Expr, filter: Optional[Expr] = None) -> Expr: """Returns the average (mean) value of the argument. This is an alias for :py:func:`avg`. """ - return avg(arg, distinct) + return avg(expression, filter) + + +def median( + expression: Expr, distinct: bool = False, filter: Optional[Expr] = None +) -> Expr: + """Computes the median of a set of numbers. + + This aggregate function returns the median value of the expression for the given + aggregate function. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by`` and ``null_treatment``. + + Args: + expression: The value to compute the median of + distinct: If True, a single entry for each distinct value will be in the result + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.median(expression.expr, distinct=distinct, filter=filter_raw)) + +def min(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Returns the minimum value of the argument. -def median(arg: Expr) -> Expr: - """Computes the median of a set of numbers.""" - return Expr(f.median(arg.expr)) + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + Args: + expression: The value to find the minimum of + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.min(expression.expr, filter=filter_raw)) -def min(arg: Expr, distinct: bool = False) -> Expr: - """Returns the minimum value of the argument.""" - return Expr(f.min(arg.expr, distinct=distinct)) +def sum( + expression: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the sum of a set of numbers. -def sum(arg: Expr) -> Expr: - """Computes the sum of a set of numbers.""" - return Expr(f.sum(arg.expr)) + This aggregate function expects a numeric expression. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. -def stddev(arg: Expr, distinct: bool = False) -> Expr: - """Computes the standard deviation of the argument.""" - return Expr(f.stddev(arg.expr, distinct=distinct)) + Args: + expression: Values to combine into an array + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.sum(expression.expr, filter=filter_raw)) -def stddev_pop(arg: Expr, distinct: bool = False) -> Expr: - """Computes the population standard deviation of the argument.""" - return Expr(f.stddev_pop(arg.expr, distinct=distinct)) +def stddev(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the standard deviation of the argument. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. -def stddev_samp(arg: Expr, distinct: bool = False) -> Expr: + Args: + expression: The value to find the minimum of + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.stddev(expression.expr, filter=filter_raw)) + + +def stddev_pop(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the population standard deviation of the argument. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: The value to find the minimum of + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.stddev_pop(expression.expr, filter=filter_raw)) + + +def stddev_samp(arg: Expr, filter: Optional[Expr] = None) -> Expr: """Computes the sample standard deviation of the argument. This is an alias for :py:func:`stddev`. """ - return stddev(arg, distinct) + return stddev(arg, filter=filter) -def var(arg: Expr) -> Expr: +def var(expression: Expr, filter: Optional[Expr] = None) -> Expr: """Computes the sample variance of the argument. This is an alias for :py:func:`var_samp`. """ - return var_samp(arg) + return var_samp(expression, filter) + + +def var_pop(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the population variance of the argument. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. -def var_pop(arg: Expr, distinct: bool = False) -> Expr: - """Computes the population variance of the argument.""" - return Expr(f.var_pop(arg.expr, distinct=distinct)) + Args: + expression: The variable to compute the variance for + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.var_pop(expression.expr, filter=filter_raw)) -def var_samp(arg: Expr) -> Expr: - """Computes the sample variance of the argument.""" - return Expr(f.var_samp(arg.expr)) +def var_samp(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the sample variance of the argument. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. -def regr_avgx(y: Expr, x: Expr, distinct: bool = False) -> Expr: + Args: + expression: The variable to compute the variance for + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.var_sample(expression.expr, filter=filter_raw)) + + +def var_sample(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the sample variance of the argument. + + This is an alias for :py:func:`var_samp`. + """ + return var_samp(expression, filter) + + +def regr_avgx( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: """Computes the average of the independent variable ``x``. - Only non-null pairs of the inputs are evaluated. + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True """ - return Expr(f.regr_avgx(y.expr, x.expr, distinct)) + filter_raw = filter.expr if filter is not None else None + + return Expr(f.regr_avgx(y.expr, x.expr, filter=filter_raw)) -def regr_avgy(y: Expr, x: Expr, distinct: bool = False) -> Expr: +def regr_avgy( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: """Computes the average of the dependent variable ``y``. - Only non-null pairs of the inputs are evaluated. + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + + return Expr(f.regr_avgy(y.expr, x.expr, filter=filter_raw)) + + +def regr_count( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Counts the number of rows in which both expressions are not null. + + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + + return Expr(f.regr_count(y.expr, x.expr, filter=filter_raw)) + + +def regr_intercept( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the intercept from the linear regression. + + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + + return Expr(f.regr_intercept(y.expr, x.expr, filter=filter_raw)) + + +def regr_r2( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the R-squared value from linear regression. + + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + + return Expr(f.regr_r2(y.expr, x.expr, filter=filter_raw)) + + +def regr_slope( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the slope from linear regression. + + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True """ - return Expr(f.regr_avgy(y.expr, x.expr, distinct)) + filter_raw = filter.expr if filter is not None else None + + return Expr(f.regr_slope(y.expr, x.expr, filter=filter_raw)) -def regr_count(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Counts the number of rows in which both expressions are not null.""" - return Expr(f.regr_count(y.expr, x.expr, distinct)) +def regr_sxx( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the sum of squares of the independent variable ``x``. + + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.regr_sxx(y.expr, x.expr, filter=filter_raw)) -def regr_intercept(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the intercept from the linear regression.""" - return Expr(f.regr_intercept(y.expr, x.expr, distinct)) +def regr_sxy( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the sum of products of pairs of numbers. + + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. -def regr_r2(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the R-squared value from linear regression.""" - return Expr(f.regr_r2(y.expr, x.expr, distinct)) + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None -def regr_slope(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the slope from linear regression.""" - return Expr(f.regr_slope(y.expr, x.expr, distinct)) + return Expr(f.regr_sxy(y.expr, x.expr, filter=filter_raw)) -def regr_sxx(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the sum of squares of the independent variable ``x``.""" - return Expr(f.regr_sxx(y.expr, x.expr, distinct)) +def regr_syy( + y: Expr, + x: Expr, + filter: Optional[Expr] = None, +) -> Expr: + """Computes the sum of squares of the dependent variable ``y``. + This is a linear regression aggregate function. Only non-null pairs of the inputs + are evaluated. -def regr_sxy(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the sum of products of pairs of numbers.""" - return Expr(f.regr_sxy(y.expr, x.expr, distinct)) + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + Args: + y: The linear regression dependent variable + x: The linear regression independent variable + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None -def regr_syy(y: Expr, x: Expr, distinct: bool = False) -> Expr: - """Computes the sum of squares of the dependent variable ``y``.""" - return Expr(f.regr_syy(y.expr, x.expr, distinct)) + return Expr(f.regr_syy(y.expr, x.expr, filter=filter_raw)) def first_value( - arg: Expr, - distinct: bool = False, - filter: Optional[bool] = None, + expression: Expr, + filter: Optional[Expr] = None, order_by: Optional[list[Expr]] = None, - null_treatment: Optional[common.NullTreatment] = None, + null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS, ) -> Expr: - """Returns the first value in a group of values.""" - order_by_cols = [e.expr for e in order_by] if order_by is not None else None + """Returns the first value in a group of values. + + This aggregate function will return the first value in the partition. + + If using the builder functions described in ref:`_aggregation` this function ignores + the option ``distinct``. + + Args: + expression: Argument to perform bitwise calculation on + filter: If provided, only compute against rows for which the filter is True + order_by: Set the ordering of the expression to evaluate + null_treatment: Assign whether to respect or ignull null values. + """ + order_by_raw = expr_list_to_raw_expr_list(order_by) + filter_raw = filter.expr if filter is not None else None return Expr( f.first_value( - arg.expr, - distinct=distinct, - filter=filter, - order_by=order_by_cols, - null_treatment=null_treatment, + expression.expr, + filter=filter_raw, + order_by=order_by_raw, + null_treatment=null_treatment.value, ) ) def last_value( - arg: Expr, - distinct: bool = False, - filter: Optional[bool] = None, + expression: Expr, + filter: Optional[Expr] = None, order_by: Optional[list[Expr]] = None, - null_treatment: Optional[common.NullTreatment] = None, + null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS, ) -> Expr: """Returns the last value in a group of values. - To set parameters on this expression, use ``.order_by()``, ``.distinct()``, - ``.filter()``, or ``.null_treatment()``. + This aggregate function will return the last value in the partition. + + If using the builder functions described in ref:`_aggregation` this function ignores + the option ``distinct``. + + Args: + expression: Argument to perform bitwise calculation on + filter: If provided, only compute against rows for which the filter is True + order_by: Set the ordering of the expression to evaluate + null_treatment: Assign whether to respect or ignull null values. """ - order_by_cols = [e.expr for e in order_by] if order_by is not None else None + order_by_raw = expr_list_to_raw_expr_list(order_by) + filter_raw = filter.expr if filter is not None else None return Expr( f.last_value( - arg.expr, - distinct=distinct, - filter=filter, - order_by=order_by_cols, - null_treatment=null_treatment, + expression.expr, + filter=filter_raw, + order_by=order_by_raw, + null_treatment=null_treatment.value, + ) + ) + + +def nth_value( + expression: Expr, + n: int, + filter: Optional[Expr] = None, + order_by: Optional[list[Expr]] = None, + null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS, +) -> Expr: + """Returns the n-th value in a group of values. + + This aggregate function will return the n-th value in the partition. + + If using the builder functions described in ref:`_aggregation` this function ignores + the option ``distinct``. + + Args: + expression: Argument to perform bitwise calculation on + n: Index of value to return. Starts at 1. + filter: If provided, only compute against rows for which the filter is True + order_by: Set the ordering of the expression to evaluate + null_treatment: Assign whether to respect or ignull null values. + """ + order_by_raw = expr_list_to_raw_expr_list(order_by) + filter_raw = filter.expr if filter is not None else None + + return Expr( + f.nth_value( + expression.expr, + n, + filter=filter_raw, + order_by=order_by_raw, + null_treatment=null_treatment.value, ) ) -def bit_and(arg: Expr, distinct: bool = False) -> Expr: - """Computes the bitwise AND of the argument.""" - return Expr(f.bit_and(arg.expr, distinct=distinct)) +def bit_and(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the bitwise AND of the argument. + + This aggregate function will bitwise compare every value in the input partition. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + Args: + expression: Argument to perform bitwise calculation on + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.bit_and(expression.expr, filter=filter_raw)) -def bit_or(arg: Expr, distinct: bool = False) -> Expr: - """Computes the bitwise OR of the argument.""" - return Expr(f.bit_or(arg.expr, distinct=distinct)) +def bit_or(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the bitwise OR of the argument. -def bit_xor(arg: Expr, distinct: bool = False) -> Expr: - """Computes the bitwise XOR of the argument.""" - return Expr(f.bit_xor(arg.expr, distinct=distinct)) + This aggregate function will bitwise compare every value in the input partition. + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: Argument to perform bitwise calculation on + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.bit_or(expression.expr, filter=filter_raw)) -def bool_and(arg: Expr, distinct: bool = False) -> Expr: - """Computes the boolean AND of the argument.""" - return Expr(f.bool_and(arg.expr, distinct=distinct)) +def bit_xor( + expression: Expr, distinct: bool = False, filter: Optional[Expr] = None +) -> Expr: + """Computes the bitwise XOR of the argument. -def bool_or(arg: Expr, distinct: bool = False) -> Expr: - """Computes the boolean OR of the argument.""" - return Expr(f.bool_or(arg.expr, distinct=distinct)) + This aggregate function will bitwise compare every value in the input partition. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by`` and ``null_treatment``. + + Args: + expression: Argument to perform bitwise calculation on + distinct: If True, evaluate each unique value of expression only once + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.bit_xor(expression.expr, distinct=distinct, filter=filter_raw)) + + +def bool_and(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the boolean AND of the argument. + + This aggregate function will compare every value in the input partition. These are + expected to be boolean values. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: Argument to perform calculation on + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.bool_and(expression.expr, filter=filter_raw)) + + +def bool_or(expression: Expr, filter: Optional[Expr] = None) -> Expr: + """Computes the boolean OR of the argument. + + This aggregate function will compare every value in the input partition. These are + expected to be boolean values. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``order_by``, ``null_treatment``, and ``distinct``. + + Args: + expression: Argument to perform calculation on + filter: If provided, only compute against rows for which the filter is True + """ + filter_raw = filter.expr if filter is not None else None + return Expr(f.bool_or(expression.expr, filter=filter_raw)) def lead( @@ -2107,3 +2622,37 @@ def ntile( order_by=order_cols, ) ) + + +def string_agg( + expression: Expr, + delimiter: str, + filter: Optional[Expr] = None, + order_by: Optional[list[Expr]] = None, +) -> Expr: + """Concatenates the input strings. + + This aggregate function will concatenate input strings, ignoring null values, and + seperating them with the specified delimiter. Non-string values will be converted to + their string equivalents. + + If using the builder functions described in ref:`_aggregation` this function ignores + the options ``distinct`` and ``null_treatment``. + + Args: + expression: Argument to perform bitwise calculation on + delimiter: Text to place between each value of expression + filter: If provided, only compute against rows for which the filter is True + order_by: Set the ordering of the expression to evaluate + """ + order_by_raw = expr_list_to_raw_expr_list(order_by) + filter_raw = filter.expr if filter is not None else None + + return Expr( + f.string_agg( + expression.expr, + delimiter, + filter=filter_raw, + order_by=order_by_raw, + ) + ) diff --git a/python/datafusion/tests/test_aggregation.py b/python/datafusion/tests/test_aggregation.py index ab653c403..243a8c3c9 100644 --- a/python/datafusion/tests/test_aggregation.py +++ b/python/datafusion/tests/test_aggregation.py @@ -21,6 +21,7 @@ from datafusion import SessionContext, column, lit from datafusion import functions as f +from datafusion.common import NullTreatment @pytest.fixture @@ -34,12 +35,30 @@ def df(): pa.array([4, 4, 6]), pa.array([9, 8, 5]), pa.array([True, True, False]), + pa.array([1, 2, None]), ], - names=["a", "b", "c", "d"], + names=["a", "b", "c", "d", "e"], ) return ctx.create_dataframe([[batch]]) +@pytest.fixture +def df_partitioned(): + ctx = SessionContext() + + # create a RecordBatch and a new DataFrame from it + batch = pa.RecordBatch.from_arrays( + [ + pa.array([0, 1, 2, 3, 4, 5, 6]), + pa.array([7, None, 7, 8, 9, None, 9]), + pa.array(["A", "A", "A", "A", "B", "B", "B"]), + ], + names=["a", "b", "c"], + ) + + return ctx.create_dataframe([[batch]]) + + @pytest.fixture def df_aggregate_100(): ctx = SessionContext() @@ -87,6 +106,7 @@ def df_aggregate_100(): ], ) def test_aggregation_stats(df, agg_expr, calc_expected): + df = df.select("a", "b", "c", "d") agg_df = df.aggregate([], [agg_expr]) result = agg_df.collect()[0] values_a, values_b, values_c, values_d = df.collect()[0] @@ -95,68 +115,323 @@ def test_aggregation_stats(df, agg_expr, calc_expected): @pytest.mark.parametrize( - "agg_expr, expected", + "agg_expr, expected, array_sort", [ - (f.approx_distinct(column("b")), pa.array([2], type=pa.uint64())), - (f.approx_median(column("b")), pa.array([4])), - (f.approx_percentile_cont(column("b"), lit(0.5)), pa.array([4])), + (f.approx_distinct(column("b")), pa.array([2], type=pa.uint64()), False), + ( + f.approx_distinct( + column("b"), + filter=column("a") != lit(3), + ), + pa.array([1], type=pa.uint64()), + False, + ), + (f.approx_median(column("b")), pa.array([4]), False), + (f.median(column("b"), distinct=True), pa.array([5]), False), + (f.median(column("b"), filter=column("a") != 2), pa.array([5]), False), + (f.approx_median(column("b"), filter=column("a") != 2), pa.array([5]), False), + (f.approx_percentile_cont(column("b"), 0.5), pa.array([4]), False), ( - f.approx_percentile_cont_with_weight(column("b"), lit(0.6), lit(0.5)), + f.approx_percentile_cont_with_weight(column("b"), lit(0.6), 0.5), pa.array([6], type=pa.float64()), + False, + ), + ( + f.approx_percentile_cont_with_weight( + column("b"), lit(0.6), 0.5, filter=column("a") != lit(3) + ), + pa.array([4], type=pa.float64()), + False, + ), + (f.array_agg(column("b")), pa.array([[4, 4, 6]]), False), + (f.array_agg(column("b"), distinct=True), pa.array([[4, 6]]), True), + ( + f.array_agg(column("e"), filter=column("e").is_not_null()), + pa.array([[1, 2]]), + False, + ), + ( + f.array_agg(column("b"), order_by=[column("c")]), + pa.array([[6, 4, 4]]), + False, + ), + (f.avg(column("b"), filter=column("a") != lit(1)), pa.array([5.0]), False), + (f.sum(column("b"), filter=column("a") != lit(1)), pa.array([10]), False), + (f.count(column("b"), distinct=True), pa.array([2]), False), + (f.count(column("b"), filter=column("a") != 3), pa.array([2]), False), + (f.count(), pa.array([3]), False), + (f.count(column("e")), pa.array([2]), False), + (f.count_star(filter=column("a") != 3), pa.array([2]), False), + (f.max(column("a"), filter=column("a") != lit(3)), pa.array([2]), False), + (f.min(column("a"), filter=column("a") != lit(1)), pa.array([2]), False), + ( + f.stddev(column("a"), filter=column("a") != lit(2)), + pa.array([np.sqrt(2)]), + False, + ), + ( + f.stddev_pop(column("a"), filter=column("a") != lit(2)), + pa.array([1.0]), + False, ), - (f.array_agg(column("b")), pa.array([[4, 4, 6]])), ], ) -def test_aggregation(df, agg_expr, expected): - agg_df = df.aggregate([], [agg_expr]) +def test_aggregation(df, agg_expr, expected, array_sort): + agg_df = df.aggregate([], [agg_expr.alias("agg_expr")]) + if array_sort: + agg_df = agg_df.select(f.array_sort(column("agg_expr"))) + agg_df.show() result = agg_df.collect()[0] + + print(result) assert result.column(0) == expected -def test_aggregate_100(df_aggregate_100): +@pytest.mark.parametrize( + "name,expr,expected", + [ + ( + "approx_percentile_cont", + f.approx_percentile_cont(column("c3"), 0.95, num_centroids=200), + [73, 68, 122, 124, 115], + ), + ( + "approx_perc_cont_few_centroids", + f.approx_percentile_cont(column("c3"), 0.95, num_centroids=5), + [72, 68, 119, 124, 115], + ), + ( + "approx_perc_cont_filtered", + f.approx_percentile_cont( + column("c3"), 0.95, num_centroids=200, filter=column("c3") > lit(0) + ), + [83, 68, 122, 124, 117], + ), + ( + "corr", + f.corr(column("c3"), column("c2")), + [-0.1056, -0.2808, 0.0023, 0.0022, -0.2473], + ), + ( + "corr_w_filter", + f.corr(column("c3"), column("c2"), filter=column("c3") > lit(0)), + [-0.3298, 0.2925, 0.2467, -0.2269, 0.0358], + ), + ( + "covar_pop", + f.covar_pop(column("c3"), column("c2")), + [-7.2857, -25.6731, 0.2222, 0.2469, -20.2857], + ), + ( + "covar_pop_w_filter", + f.covar_pop(column("c3"), column("c2"), filter=column("c3") > lit(0)), + [-9.25, 9.0579, 13.7521, -9.9669, 1.1641], + ), + ( + "covar_samp", + f.covar_samp(column("c3"), column("c2")), + [-7.65, -27.0994, 0.2333, 0.2614, -21.3], + ), + ( + "covar_samp_w_filter", + f.covar_samp(column("c3"), column("c2"), filter=column("c3") > lit(0)), + [-10.5714, 9.9636, 15.1273, -10.9636, 1.2417], + ), + ( + "var_samp", + f.var_samp(column("c2")), + [1.9286, 2.2047, 1.6333, 2.1438, 1.6], + ), + ( + "var_samp_w_filter", + f.var_samp(column("c2"), filter=column("c3") > lit(0)), + [1.4286, 2.4182, 1.8545, 1.4727, 1.6292], + ), + ( + "var_pop", + f.var_pop(column("c2")), + [1.8367, 2.0886, 1.5556, 2.0247, 1.5238], + ), + ( + "var_pop_w_filter", + f.var_pop(column("c2"), filter=column("c3") > lit(0)), + [1.25, 2.1983, 1.686, 1.3388, 1.5273], + ), + ], +) +def test_aggregate_100(df_aggregate_100, name, expr, expected): # https://github.com/apache/datafusion/blob/bddb6415a50746d2803dd908d19c3758952d74f9/datafusion/sqllogictest/test_files/aggregate.slt#L1490-L1498 - result = ( + df = ( df_aggregate_100.aggregate( [column("c1")], - [f.approx_percentile_cont(column("c3"), lit(0.95), lit(200)).alias("c3")], + [expr.alias(name)], ) + .select("c1", f.round(column(name), lit(4)).alias(name)) .sort(column("c1").sort(ascending=True)) - .collect() ) + df.show() - assert len(result) == 1 - result = result[0] - assert result.column("c1") == pa.array(["a", "b", "c", "d", "e"]) - assert result.column("c3") == pa.array([73, 68, 122, 124, 115]) + expected_dict = { + "c1": ["a", "b", "c", "d", "e"], + name: expected, + } + assert df.collect()[0].to_pydict() == expected_dict -def test_bit_add_or_xor(df): - df = df.aggregate( - [], - [ - f.bit_and(column("a")), - f.bit_or(column("b")), - f.bit_xor(column("c")), - ], - ) - result = df.collect() - result = result[0] - assert result.column(0) == pa.array([0]) - assert result.column(1) == pa.array([6]) - assert result.column(2) == pa.array([4]) +data_test_bitwise_and_boolean_functions = [ + ("bit_and", f.bit_and(column("a")), [0]), + ("bit_and_filter", f.bit_and(column("a"), filter=column("a") != lit(2)), [1]), + ("bit_or", f.bit_or(column("b")), [6]), + ("bit_or_filter", f.bit_or(column("b"), filter=column("a") != lit(3)), [4]), + ("bit_xor", f.bit_xor(column("c")), [4]), + ("bit_xor_distinct", f.bit_xor(column("b"), distinct=True), [2]), + ("bit_xor_filter", f.bit_xor(column("b"), filter=column("a") != lit(3)), [0]), + ( + "bit_xor_filter_distinct", + f.bit_xor(column("b"), distinct=True, filter=column("a") != lit(3)), + [4], + ), + ("bool_and", f.bool_and(column("d")), [False]), + ("bool_and_filter", f.bool_and(column("d"), filter=column("a") != lit(3)), [True]), + ("bool_or", f.bool_or(column("d")), [True]), + ("bool_or_filter", f.bool_or(column("d"), filter=column("a") == lit(3)), [False]), +] -def test_bool_and_or(df): - df = df.aggregate( - [], - [ - f.bool_and(column("d")), - f.bool_or(column("d")), - ], +@pytest.mark.parametrize("name,expr,result", data_test_bitwise_and_boolean_functions) +def test_bit_and_bool_fns(df, name, expr, result): + df = df.aggregate([], [expr.alias(name)]) + + expected = { + name: result, + } + + assert df.collect()[0].to_pydict() == expected + + +@pytest.mark.parametrize( + "name,expr,result", + [ + ("first_value", f.first_value(column("a")), [0, 4]), + ( + "first_value_ordered", + f.first_value(column("a"), order_by=[column("a").sort(ascending=False)]), + [3, 6], + ), + ( + "first_value_with_null", + f.first_value( + column("b"), + order_by=[column("b").sort(ascending=True)], + null_treatment=NullTreatment.RESPECT_NULLS, + ), + [None, None], + ), + ( + "first_value_ignore_null", + f.first_value( + column("b"), + order_by=[column("b").sort(ascending=True)], + null_treatment=NullTreatment.IGNORE_NULLS, + ), + [7, 9], + ), + ("last_value", f.last_value(column("a")), [3, 6]), + ( + "last_value_ordered", + f.last_value(column("a"), order_by=[column("a").sort(ascending=False)]), + [0, 4], + ), + ( + "last_value_with_null", + f.last_value( + column("b"), + order_by=[column("b").sort(ascending=True, nulls_first=False)], + null_treatment=NullTreatment.RESPECT_NULLS, + ), + [None, None], + ), + ( + "last_value_ignore_null", + f.last_value( + column("b"), + order_by=[column("b").sort(ascending=True)], + null_treatment=NullTreatment.IGNORE_NULLS, + ), + [8, 9], + ), + ("first_value", f.first_value(column("a")), [0, 4]), + ( + "nth_value_ordered", + f.nth_value(column("a"), 2, order_by=[column("a").sort(ascending=False)]), + [2, 5], + ), + ( + "nth_value_with_null", + f.nth_value( + column("b"), + 3, + order_by=[column("b").sort(ascending=True, nulls_first=False)], + null_treatment=NullTreatment.RESPECT_NULLS, + ), + [8, None], + ), + ( + "nth_value_ignore_null", + f.nth_value( + column("b"), + 2, + order_by=[column("b").sort(ascending=True)], + null_treatment=NullTreatment.IGNORE_NULLS, + ), + [7, 9], + ), + ], +) +def test_first_last_value(df_partitioned, name, expr, result) -> None: + df = df_partitioned.aggregate([column("c")], [expr.alias(name)]).sort(column("c")) + + expected = { + "c": ["A", "B"], + name: result, + } + + assert df.collect()[0].to_pydict() == expected + + +@pytest.mark.parametrize( + "name,expr,result", + [ + ("string_agg", f.string_agg(column("a"), ","), "one,two,three,two"), + ("string_agg", f.string_agg(column("b"), ""), "03124"), + ( + "string_agg", + f.string_agg(column("a"), ",", filter=column("b") != lit(3)), + "one,three,two", + ), + ( + "string_agg", + f.string_agg(column("a"), ",", order_by=[column("b")]), + "one,three,two,two", + ), + ], +) +def test_string_agg(name, expr, result) -> None: + ctx = SessionContext() + + df = ctx.from_pydict( + { + "a": ["one", "two", None, "three", "two"], + "b": [0, 3, 1, 2, 4], + } ) - result = df.collect() - result = result[0] - assert result.column(0) == pa.array([False]) - assert result.column(1) == pa.array([True]) + + df = df.aggregate([], [expr.alias(name)]) + + expected = { + name: [result], + } + df.show() + assert df.collect()[0].to_pydict() == expected diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index e7e6d79e1..8e3c51397 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -912,17 +912,64 @@ def test_regr_funcs_sql_2(): @pytest.mark.parametrize( "func, expected", [ - pytest.param(f.regr_slope, pa.array([2], type=pa.float64()), id="regr_slope"), + pytest.param(f.regr_slope(column("c2"), column("c1")), [4.6], id="regr_slope"), pytest.param( - f.regr_intercept, pa.array([0], type=pa.float64()), id="regr_intercept" + f.regr_slope(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [8], + id="regr_slope_filter", + ), + pytest.param( + f.regr_intercept(column("c2"), column("c1")), [-4], id="regr_intercept" + ), + pytest.param( + f.regr_intercept( + column("c2"), column("c1"), filter=column("c1") > literal(2) + ), + [-16], + id="regr_intercept_filter", + ), + pytest.param(f.regr_count(column("c2"), column("c1")), [4], id="regr_count"), + pytest.param( + f.regr_count(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [2], + id="regr_count_filter", + ), + pytest.param(f.regr_r2(column("c2"), column("c1")), [0.92], id="regr_r2"), + pytest.param( + f.regr_r2(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [1.0], + id="regr_r2_filter", + ), + pytest.param(f.regr_avgx(column("c2"), column("c1")), [2.5], id="regr_avgx"), + pytest.param( + f.regr_avgx(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [3.5], + id="regr_avgx_filter", + ), + pytest.param(f.regr_avgy(column("c2"), column("c1")), [7.5], id="regr_avgy"), + pytest.param( + f.regr_avgy(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [12.0], + id="regr_avgy_filter", + ), + pytest.param(f.regr_sxx(column("c2"), column("c1")), [5.0], id="regr_sxx"), + pytest.param( + f.regr_sxx(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [0.5], + id="regr_sxx_filter", + ), + pytest.param(f.regr_syy(column("c2"), column("c1")), [115.0], id="regr_syy"), + pytest.param( + f.regr_syy(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [32.0], + id="regr_syy_filter", + ), + pytest.param(f.regr_sxy(column("c2"), column("c1")), [23.0], id="regr_sxy"), + pytest.param( + f.regr_sxy(column("c2"), column("c1"), filter=column("c1") > literal(2)), + [4.0], + id="regr_sxy_filter", ), - pytest.param(f.regr_count, pa.array([3], type=pa.uint64()), id="regr_count"), - pytest.param(f.regr_r2, pa.array([1], type=pa.float64()), id="regr_r2"), - pytest.param(f.regr_avgx, pa.array([2], type=pa.float64()), id="regr_avgx"), - pytest.param(f.regr_avgy, pa.array([4], type=pa.float64()), id="regr_avgy"), - pytest.param(f.regr_sxx, pa.array([2], type=pa.float64()), id="regr_sxx"), - pytest.param(f.regr_syy, pa.array([8], type=pa.float64()), id="regr_syy"), - pytest.param(f.regr_sxy, pa.array([4], type=pa.float64()), id="regr_sxy"), ], ) def test_regr_funcs_df(func, expected): @@ -932,38 +979,18 @@ def test_regr_funcs_df(func, expected): ctx = SessionContext() # Create a DataFrame - data = {"column1": [1, 2, 3], "column2": [2, 4, 6]} + data = {"c1": [1, 2, 3, 4, 5, None], "c2": [2, 4, 8, 16, None, 64]} df = ctx.from_pydict(data, name="test_table") # Perform the regression function using DataFrame API - result_df = df.aggregate([], [func(f.col("column2"), f.col("column1"))]).collect() - - # Assertion for DataFrame API result - assert result_df[0].column(0) == expected - + df = df.aggregate([], [func.alias("result")]) + df.show() -def test_first_last_value(df): - df = df.aggregate( - [], - [ - f.first_value(column("a")), - f.first_value(column("b")), - f.first_value(column("d")), - f.last_value(column("a")), - f.last_value(column("b")), - f.last_value(column("d")), - ], - ) + expected_dict = { + "result": expected, + } - result = df.collect() - result = result[0] - assert result.column(0) == pa.array(["Hello"]) - assert result.column(1) == pa.array([4]) - assert result.column(2) == pa.array([datetime(2022, 12, 31)]) - assert result.column(3) == pa.array(["!"]) - assert result.column(4) == pa.array([6]) - assert result.column(5) == pa.array([datetime(2020, 7, 2)]) - df.show() + assert df.collect()[0].to_pydict() == expected_dict def test_binary_string_functions(df): diff --git a/python/datafusion/tests/test_wrapper_coverage.py b/python/datafusion/tests/test_wrapper_coverage.py index 44b9ca831..4a47de2e1 100644 --- a/python/datafusion/tests/test_wrapper_coverage.py +++ b/python/datafusion/tests/test_wrapper_coverage.py @@ -20,8 +20,19 @@ import datafusion.object_store import datafusion.substrait +# EnumType introduced in 3.11. 3.10 and prior it was called EnumMeta. +try: + from enum import EnumType +except ImportError: + from enum import EnumMeta as EnumType + def missing_exports(internal_obj, wrapped_obj) -> None: + # Special case enums - just make sure they exist since dir() + # and other functions get overridden. + if isinstance(wrapped_obj, EnumType): + return + for attr in dir(internal_obj): assert attr in dir(wrapped_obj) diff --git a/src/functions.rs b/src/functions.rs index b5b003dfe..b9ca6301a 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -35,305 +35,27 @@ use datafusion::functions_aggregate; use datafusion::logical_expr::expr::Alias; use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use datafusion::logical_expr::{ - expr::{find_df_window_func, AggregateFunction, Sort, WindowFunction}, + expr::{find_df_window_func, Sort, WindowFunction}, lit, Expr, WindowFunctionDefinition, }; -#[pyfunction] -pub fn approx_distinct(expression: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::approx_distinct(expression.expr).into() -} - -#[pyfunction] -pub fn approx_median(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::approx_median(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn approx_percentile_cont( - expression: PyExpr, - percentile: PyExpr, - distinct: bool, - num_centroids: Option, // enforces optional arguments at the end, currently -) -> PyResult { - let args = if let Some(num_centroids) = num_centroids { - vec![expression.expr, percentile.expr, num_centroids.expr] - } else { - vec![expression.expr, percentile.expr] - }; - let udaf = functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf(); - let expr = udaf.call(args); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn approx_percentile_cont_with_weight( - expression: PyExpr, - weight: PyExpr, - percentile: PyExpr, - distinct: bool, -) -> PyResult { - let expr = functions_aggregate::expr_fn::approx_percentile_cont_with_weight( - expression.expr, - weight.expr, - percentile.expr, - ); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn avg(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::avg(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn bit_and(expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::bit_and(expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn bit_or(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::bit_or(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn bit_xor(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::bit_xor(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn bool_and(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::bool_and(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn bool_or(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::bool_or(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn corr(y: PyExpr, x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::corr(y.expr, x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn grouping(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::grouping(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn sum(args: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::sum(args.expr).into() -} - -#[pyfunction] -pub fn covar_samp(y: PyExpr, x: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::covar_samp(y.expr, x.expr).into() -} - -#[pyfunction] -pub fn covar_pop(y: PyExpr, x: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::covar_pop(y.expr, x.expr).into() -} - -#[pyfunction] -pub fn median(arg: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::median(arg.expr).into() -} - -#[pyfunction] -pub fn stddev(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::stddev(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn stddev_pop(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::stddev_pop(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn var_samp(expression: PyExpr) -> PyExpr { - functions_aggregate::expr_fn::var_sample(expression.expr).into() -} - -#[pyfunction] -pub fn var_pop(expression: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::var_pop(expression.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_avgx(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_avgx(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_avgy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_avgy(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_count(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_count(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_intercept(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_intercept(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_r2(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_r2(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_slope(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_slope(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_sxx(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_sxx(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_sxy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_sxy(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - -#[pyfunction] -pub fn regr_syy(expr_y: PyExpr, expr_x: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::regr_syy(expr_y.expr, expr_x.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - fn add_builder_fns_to_aggregate( agg_fn: Expr, - distinct: bool, + distinct: Option, filter: Option, order_by: Option>, null_treatment: Option, ) -> PyResult { // Since ExprFuncBuilder::new() is private, we can guarantee initializing - // a builder with an `order_by` default of empty vec - let order_by = order_by - .map(|x| x.into_iter().map(|x| x.expr).collect::>()) - .unwrap_or_default(); - let mut builder = agg_fn.order_by(order_by); + // a builder with an `null_treatment` with option None + let mut builder = agg_fn.null_treatment(None); - if distinct { + if let Some(order_by_cols) = order_by { + let order_by_cols = to_sort_expressions(order_by_cols); + builder = builder.order_by(order_by_cols); + } + + if let Some(true) = distinct { builder = builder.distinct(); } @@ -341,39 +63,11 @@ fn add_builder_fns_to_aggregate( builder = builder.filter(filter.expr); } - // would be nice if all the options builder methods accepted Option ... builder = builder.null_treatment(null_treatment.map(DFNullTreatment::from)); Ok(builder.build()?.into()) } -#[pyfunction] -pub fn first_value( - expr: PyExpr, - distinct: bool, - filter: Option, - order_by: Option>, - null_treatment: Option, -) -> PyResult { - // If we initialize the UDAF with order_by directly, then it gets over-written by the builder - let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); - - add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) -} - -#[pyfunction] -pub fn last_value( - expr: PyExpr, - distinct: bool, - filter: Option, - order_by: Option>, - null_treatment: Option, -) -> PyResult { - let agg_fn = functions_aggregate::expr_fn::last_value(vec![expr.expr]); - - add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) -} - #[pyfunction] fn in_list(expr: PyExpr, value: Vec, negated: bool) -> PyExpr { datafusion::logical_expr::in_list( @@ -505,25 +199,6 @@ fn col(name: &str) -> PyResult { }) } -// TODO: should we just expose this in python? -/// Create a COUNT(1) aggregate expression -#[pyfunction] -fn count_star() -> PyExpr { - functions_aggregate::expr_fn::count(lit(1)).into() -} - -/// Wrapper for [`functions_aggregate::expr_fn::count`] -/// Count the number of non-null values in the column -#[pyfunction] -fn count(expr: PyExpr, distinct: bool) -> PyResult { - let expr = functions_aggregate::expr_fn::count(expr.expr); - if distinct { - Ok(expr.distinct().build()?.into()) - } else { - Ok(expr.into()) - } -} - /// Create a CASE WHEN statement with literal WHEN expressions for comparison to the base expression. #[pyfunction] fn case(expr: PyExpr) -> PyResult { @@ -646,24 +321,46 @@ fn window( }) } +// Generates a [pyo3] wrapper for associated aggregate functions. +// All of the builder options are exposed to the python internal +// function and we rely on the wrappers to only use those that +// are appropriate. macro_rules! aggregate_function { - ($NAME: ident, $FUNC: path) => { - aggregate_function!($NAME, $FUNC, stringify!($NAME)); + ($NAME: ident) => { + aggregate_function!($NAME, expr); }; - ($NAME: ident, $FUNC: path, $DOC: expr) => { - #[doc = $DOC] + ($NAME: ident, $($arg:ident)*) => { #[pyfunction] - #[pyo3(signature = (*args, distinct=false))] - fn $NAME(args: Vec, distinct: bool) -> PyExpr { - let expr = datafusion::logical_expr::Expr::AggregateFunction(AggregateFunction { - func: $FUNC(), - args: args.into_iter().map(|e| e.into()).collect(), - distinct, - filter: None, - order_by: None, - null_treatment: None, - }); - expr.into() + fn $NAME( + $($arg: PyExpr),*, + distinct: Option, + filter: Option, + order_by: Option>, + null_treatment: Option + ) -> PyResult { + let agg_fn = functions_aggregate::expr_fn::$NAME($($arg.into()),*); + + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) + } + }; +} + +macro_rules! aggregate_function_vec_args { + ($NAME: ident) => { + aggregate_function_vec_args!($NAME, expr); + }; + ($NAME: ident, $($arg:ident)*) => { + #[pyfunction] + fn $NAME( + $($arg: PyExpr),*, + distinct: Option, + filter: Option, + order_by: Option>, + null_treatment: Option + ) -> PyResult { + let agg_fn = functions_aggregate::expr_fn::$NAME(vec![$($arg.into()),*]); + + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } }; } @@ -891,9 +588,120 @@ array_fn!(array_resize, array size value); array_fn!(flatten, array); array_fn!(range, start stop step); -aggregate_function!(array_agg, functions_aggregate::array_agg::array_agg_udaf); -aggregate_function!(max, functions_aggregate::min_max::max_udaf); -aggregate_function!(min, functions_aggregate::min_max::min_udaf); +aggregate_function!(array_agg); +aggregate_function!(max); +aggregate_function!(min); +aggregate_function!(avg); +aggregate_function!(sum); +aggregate_function!(bit_and); +aggregate_function!(bit_or); +aggregate_function!(bit_xor); +aggregate_function!(bool_and); +aggregate_function!(bool_or); +aggregate_function!(corr, y x); +aggregate_function!(count); +aggregate_function!(covar_samp, y x); +aggregate_function!(covar_pop, y x); +aggregate_function!(median); +aggregate_function!(regr_slope, y x); +aggregate_function!(regr_intercept, y x); +aggregate_function!(regr_count, y x); +aggregate_function!(regr_r2, y x); +aggregate_function!(regr_avgx, y x); +aggregate_function!(regr_avgy, y x); +aggregate_function!(regr_sxx, y x); +aggregate_function!(regr_syy, y x); +aggregate_function!(regr_sxy, y x); +aggregate_function!(stddev); +aggregate_function!(stddev_pop); +aggregate_function!(var_sample); +aggregate_function!(var_pop); +aggregate_function!(approx_distinct); +aggregate_function!(approx_median); + +// Code is commented out since grouping is not yet implemented +// https://github.com/apache/datafusion-python/issues/861 +// aggregate_function!(grouping); + +#[pyfunction] +pub fn approx_percentile_cont( + expression: PyExpr, + percentile: f64, + num_centroids: Option, // enforces optional arguments at the end, currently + filter: Option, +) -> PyResult { + let args = if let Some(num_centroids) = num_centroids { + vec![expression.expr, lit(percentile), lit(num_centroids)] + } else { + vec![expression.expr, lit(percentile)] + }; + let udaf = functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf(); + let agg_fn = udaf.call(args); + + add_builder_fns_to_aggregate(agg_fn, None, filter, None, None) +} + +#[pyfunction] +pub fn approx_percentile_cont_with_weight( + expression: PyExpr, + weight: PyExpr, + percentile: f64, + filter: Option, +) -> PyResult { + let agg_fn = functions_aggregate::expr_fn::approx_percentile_cont_with_weight( + expression.expr, + weight.expr, + lit(percentile), + ); + + add_builder_fns_to_aggregate(agg_fn, None, filter, None, None) +} + +aggregate_function_vec_args!(last_value); + +// We handle first_value explicitly because the signature expects an order_by +// https://github.com/apache/datafusion/issues/12376 +#[pyfunction] +pub fn first_value( + expr: PyExpr, + distinct: Option, + filter: Option, + order_by: Option>, + null_treatment: Option, +) -> PyResult { + // If we initialize the UDAF with order_by directly, then it gets over-written by the builder + let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); + + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) +} + +// nth_value requires a non-expr argument +#[pyfunction] +pub fn nth_value( + expr: PyExpr, + n: i64, + distinct: Option, + filter: Option, + order_by: Option>, + null_treatment: Option, +) -> PyResult { + let agg_fn = datafusion::functions_aggregate::nth_value::nth_value(vec![expr.expr, lit(n)]); + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) +} + +// string_agg requires a non-expr argument +#[pyfunction] +pub fn string_agg( + expr: PyExpr, + delimiter: String, + distinct: Option, + filter: Option, + order_by: Option>, + null_treatment: Option, +) -> PyResult { + let agg_fn = datafusion::functions_aggregate::string_agg::string_agg(expr.expr, lit(delimiter)); + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) +} fn add_builder_fns_to_window( window_fn: Expr, @@ -1042,7 +850,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(cosh))?; m.add_wrapped(wrap_pyfunction!(cot))?; m.add_wrapped(wrap_pyfunction!(count))?; - m.add_wrapped(wrap_pyfunction!(count_star))?; m.add_wrapped(wrap_pyfunction!(covar_pop))?; m.add_wrapped(wrap_pyfunction!(covar_samp))?; m.add_wrapped(wrap_pyfunction!(current_date))?; @@ -1059,7 +866,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(floor))?; m.add_wrapped(wrap_pyfunction!(from_unixtime))?; m.add_wrapped(wrap_pyfunction!(gcd))?; - m.add_wrapped(wrap_pyfunction!(grouping))?; + // m.add_wrapped(wrap_pyfunction!(grouping))?; m.add_wrapped(wrap_pyfunction!(in_list))?; m.add_wrapped(wrap_pyfunction!(initcap))?; m.add_wrapped(wrap_pyfunction!(isnan))?; @@ -1113,6 +920,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(starts_with))?; m.add_wrapped(wrap_pyfunction!(stddev))?; m.add_wrapped(wrap_pyfunction!(stddev_pop))?; + m.add_wrapped(wrap_pyfunction!(string_agg))?; m.add_wrapped(wrap_pyfunction!(strpos))?; m.add_wrapped(wrap_pyfunction!(r#struct))?; // Use raw identifier since struct is a keyword m.add_wrapped(wrap_pyfunction!(substr))?; @@ -1134,7 +942,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(upper))?; m.add_wrapped(wrap_pyfunction!(self::uuid))?; // Use self to avoid name collision m.add_wrapped(wrap_pyfunction!(var_pop))?; - m.add_wrapped(wrap_pyfunction!(var_samp))?; + m.add_wrapped(wrap_pyfunction!(var_sample))?; m.add_wrapped(wrap_pyfunction!(window))?; m.add_wrapped(wrap_pyfunction!(regr_avgx))?; m.add_wrapped(wrap_pyfunction!(regr_avgy))?; @@ -1147,6 +955,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(regr_syy))?; m.add_wrapped(wrap_pyfunction!(first_value))?; m.add_wrapped(wrap_pyfunction!(last_value))?; + m.add_wrapped(wrap_pyfunction!(nth_value))?; m.add_wrapped(wrap_pyfunction!(bit_and))?; m.add_wrapped(wrap_pyfunction!(bit_or))?; m.add_wrapped(wrap_pyfunction!(bit_xor))?; From 89b77abf14e8e2e7ee2a83ab23717cb7c1491a02 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Mon, 9 Sep 2024 17:44:32 -0500 Subject: [PATCH 197/413] prepare release of `datafusion-python` 41.0.0 (#866) * bump datafusion-python version * generate changelog I used the following command, then pasted the output into CHANGELOG.md. ```console ./dev/release/generate-changelog.py 40.1.0-rc1 HEAD 41.0.0 ``` --- CHANGELOG.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 305af5720..56a2bfc79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,51 @@ # DataFusion Python Changelog +## [41.0.0](https://github.com/apache/datafusion-python/tree/41.0.0) (2024-09-09) + +This release consists of 19 commits from 6 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: enable list of paths for read_csv [#824](https://github.com/apache/datafusion-python/pull/824) (mesejo) +- feat: better exception and message for table not found [#851](https://github.com/apache/datafusion-python/pull/851) (mesejo) +- feat: make cast accept built-in Python types [#858](https://github.com/apache/datafusion-python/pull/858) (mesejo) + +**Other:** + +- chore: Prepare for 40.0.0 release [#801](https://github.com/apache/datafusion-python/pull/801) (andygrove) +- Add typing-extensions dependency to pyproject [#805](https://github.com/apache/datafusion-python/pull/805) (timsaucer) +- Upgrade deps to datafusion 41 [#802](https://github.com/apache/datafusion-python/pull/802) (Michael-J-Ward) +- Fix SessionContext init with only SessionConfig [#827](https://github.com/apache/datafusion-python/pull/827) (jcrist) +- build(deps): upgrade actions/{upload,download}-artifact@v3 to v4 [#829](https://github.com/apache/datafusion-python/pull/829) (Michael-J-Ward) +- Run ruff format in CI [#837](https://github.com/apache/datafusion-python/pull/837) (timsaucer) +- Add PyCapsule support for Arrow import and export [#825](https://github.com/apache/datafusion-python/pull/825) (timsaucer) +- Feature/expose when function [#836](https://github.com/apache/datafusion-python/pull/836) (timsaucer) +- Add Window Functions for use with function builder [#808](https://github.com/apache/datafusion-python/pull/808) (timsaucer) +- chore: fix typos [#844](https://github.com/apache/datafusion-python/pull/844) (mesejo) +- build(ci): use proper mac runners [#841](https://github.com/apache/datafusion-python/pull/841) (Michael-J-Ward) +- Set of small features [#839](https://github.com/apache/datafusion-python/pull/839) (timsaucer) +- chore: fix docstrings, typos [#852](https://github.com/apache/datafusion-python/pull/852) (mesejo) +- chore: Use datafusion re-exported dependencies [#856](https://github.com/apache/datafusion-python/pull/856) (emgeee) +- add guidelines on separating python and rust code [#860](https://github.com/apache/datafusion-python/pull/860) (Michael-J-Ward) +- Update Aggregate functions to take builder parameters [#859](https://github.com/apache/datafusion-python/pull/859) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 7 Tim Saucer + 5 Daniel Mesejo + 4 Michael J Ward + 1 Andy Grove + 1 Jim Crist-Harif + 1 Matt Green +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + + ## [40.0.0](https://github.com/apache/datafusion-python/tree/40.0.0) (2024-08-09) This release consists of 18 commits from 4 contributors. See credits at the end of this changelog for more information. diff --git a/Cargo.lock b/Cargo.lock index 43ee3055d..d507321db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,7 +1067,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "40.0.0" +version = "41.0.0" dependencies = [ "arrow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index e9a4ababb..8635776e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "40.0.0" +version = "41.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] From 02d4453af1c18d7e31c21d60574456226b71793e Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Wed, 11 Sep 2024 19:28:10 +0200 Subject: [PATCH 198/413] feat: expose between (#868) closes #809 --- python/datafusion/expr.py | 16 ++++++++++++ python/datafusion/tests/test_functions.py | 31 +++++++++++++++++++++++ src/expr.rs | 11 ++++++++ 3 files changed, 58 insertions(+) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index bd6a86fb8..f6a51ce2a 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -393,6 +393,22 @@ def cast( return Expr(self.expr.cast(to)) + def between(self, low: Any, high: Any, negated: bool = False) -> Expr: + """Returns ``True`` if this expression is between a given range. + + Args: + low: lower bound of the range (inclusive). + high: higher bound of the range (inclusive). + negated: negates whether the expression is between a given range + """ + if not isinstance(low, Expr): + low = Expr.literal(low) + + if not isinstance(high, Expr): + high = Expr.literal(high) + + return Expr(self.expr.between(low.expr, high.expr, negated=negated)) + def rex_type(self) -> RexType: """Return the Rex Type of this expression. diff --git a/python/datafusion/tests/test_functions.py b/python/datafusion/tests/test_functions.py index 8e3c51397..9353f872d 100644 --- a/python/datafusion/tests/test_functions.py +++ b/python/datafusion/tests/test_functions.py @@ -1024,3 +1024,34 @@ def test_cast(df, python_datatype, name: str, expected): result = df.collect() result = result[0] assert result.column(0) == result.column(1) + + +@pytest.mark.parametrize( + "negated, low, high, expected", + [ + pytest.param(False, 3, 5, {"filtered": [4, 5]}), + pytest.param(False, 4, 5, {"filtered": [4, 5]}), + pytest.param(True, 3, 5, {"filtered": [6]}), + pytest.param(True, 4, 6, []), + ], +) +def test_between(df, negated, low, high, expected): + df = df.filter(column("b").between(low, high, negated=negated)).select( + column("b").alias("filtered") + ) + + actual = df.collect() + + if expected: + actual = actual[0].to_pydict() + assert actual == expected + else: + assert len(actual) == 0 # the rows are empty + + +def test_between_default(df): + df = df.filter(column("b").between(3, 5)).select(column("b").alias("filtered")) + expected = {"filtered": [4, 5]} + + actual = df.collect()[0].to_pydict() + assert actual == expected diff --git a/src/expr.rs b/src/expr.rs index ab16f2872..0e1a193f3 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -293,6 +293,17 @@ impl PyExpr { expr.into() } + #[pyo3(signature = (low, high, negated=false))] + pub fn between(&self, low: PyExpr, high: PyExpr, negated: bool) -> PyExpr { + let expr = Expr::Between(Between::new( + Box::new(self.expr.clone()), + negated, + Box::new(low.into()), + Box::new(high.into()), + )); + expr.into() + } + /// A Rex (Row Expression) specifies a single row of data. That specification /// could include user defined functions or types. RexType identifies the row /// as one of the possible valid `RexTypes`. From 6c8bf5f8aadaf5a3431da272ed6b3afa06efa8ec Mon Sep 17 00:00:00 2001 From: Matt Green Date: Tue, 17 Sep 2024 13:58:11 -0700 Subject: [PATCH 199/413] Upgrade datafusion (#867) * update dependencies * update get_logical_plan signature * remove row_number() function row_number was converted to a UDF in datafusion v42 https://github.com/apache/datafusion/pull/12030 This specific functionality needs to be added back in. * remove unneeded dependency * fix pyo3 warnings Implicit defaults for trailing optional arguments have been deprecated in pyo3 v0.22.0 https://github.com/PyO3/pyo3/pull/4078 * update object_store dependency * change PyExpr -> PySortExpr * comment out key.extract::<&PyTuple>() condition statement * change more instances of PyExpr > PySortExpr * update function signatures to use _bound versions * remove clone * Working through some of the sort requirement changes * remove unused import * expr.display_name is deprecated, used format!() + schema_name() instead * expr.canonical_name() is deprecated, use format!() expr instead * remove comment * fix tuple extraction in dataframe.__getitem__() * remove unneeded import * Add docstring comments to SortExpr python class * change extract() to downcast() Co-authored-by: Michael J Ward * deprecate Expr::display_name Ref: https://github.com/apache/datafusion/pull/11797 * fix lint errors * update datafusion commit hash * fix type in cargo file for arrow features * upgrade to datafusion 42 * cleanup --------- Co-authored-by: Tim Saucer Co-authored-by: Michael J Ward Co-authored-by: Michael-J-Ward --- Cargo.lock | 784 +++++++++--------- Cargo.toml | 17 +- python/datafusion/context.py | 13 +- python/datafusion/dataframe.py | 8 +- python/datafusion/expr.py | 83 +- python/datafusion/functions.py | 95 +-- python/datafusion/tests/test_expr.py | 34 +- python/datafusion/tests/test_sql.py | 11 +- .../datafusion/tests/test_wrapper_coverage.py | 5 +- src/common/data_type.rs | 8 +- src/common/schema.rs | 37 +- src/context.rs | 18 +- src/dataframe.rs | 14 +- src/dataset.rs | 2 +- src/dataset_exec.rs | 4 +- src/expr.rs | 41 +- src/expr/sort.rs | 6 +- src/expr/sort_expr.rs | 21 +- src/expr/window.rs | 5 +- src/functions.rs | 87 +- src/pyarrow_filter_expression.rs | 2 +- src/udf.rs | 10 +- 22 files changed, 710 insertions(+), 595 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d507321db..d06073b6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,18 +4,18 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" dependencies = [ "gimli", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "adler32" @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "apache-avro" @@ -118,21 +118,21 @@ dependencies = [ [[package]] name = "arrayref" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05048a8932648b63f21c37d88b552ccc8a65afb6dfe9fc9f30ce79174c2e7a85" +checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d8a57966e43bfe9a3277984a14c24ec617ad874e4c0e1d2a1b083a39cfbf22c" +checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f4a9468c882dc66862cef4e1fd8423d47e67972377d85d80e022786427768c" +checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c975484888fc95ec4a632cdc98be39c085b1bb518531b0c80c5d462063e5daa1" +checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d" dependencies = [ "bytes", "half", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da26719e76b81d8bc3faad1d4dbdc1bcc10d14704e63dc17fc9f3e7e1e567c8e" +checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c13c36dc5ddf8c128df19bab27898eea64bf9da2b555ec1cd17a8ff57fba9ec2" +checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd9d6f18c65ef7a2573ab498c374d8ae364b4a4edf67105357491c031f716ca5" +checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65" dependencies = [ "arrow-buffer", "arrow-schema", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e786e1cdd952205d9a8afc69397b317cfbb6e0095e445c69cda7e8da5c1eeb0f" +checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb22284c5a2a01d73cebfd88a33511a3234ab45d66086b2ca2d1228c3498e445" +checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6" dependencies = [ "arrow-array", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42745f86b1ab99ef96d1c0bcf49180848a64fe2c7a7a0d945bc64fa2b21ba9bc" +checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd09a518c602a55bd406bcc291a967b284cfa7a63edfbf8b897ea4748aad23c" +checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb" dependencies = [ "ahash", "arrow-array", @@ -311,18 +311,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e972cd1ff4a4ccd22f86d3e53e835c2ed92e0eea6a3e8eadb72b4f1ac802cf8" +checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "600bae05d43483d216fb3494f8c32fdbefd8aa4e1de237e790dbb3d9f44690a3" +checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba" dependencies = [ "ahash", "arrow-array", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc1985b67cb45f6606a248ac2b4a288849f196bab8c657ea5589f47cdd55e6" +checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230" dependencies = [ "arrow-array", "arrow-buffer", @@ -375,18 +375,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -412,17 +412,17 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", "miniz_oxide", "object", "rustc-demangle", + "windows-targets", ] [[package]] @@ -460,9 +460,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", "arrayvec", @@ -542,12 +542,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.7" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -566,7 +567,7 @@ dependencies = [ "iana-time-zone", "num-traits", "serde", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -593,9 +594,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] @@ -633,9 +634,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" @@ -649,9 +650,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -664,9 +665,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -731,9 +732,9 @@ checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" [[package]] name = "dashmap" -version = "6.0.1" +version = "6.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804c8821570c3f8b70230c2ba75ffa5c0f9a4189b9a432b6656c536712acae28" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", @@ -745,9 +746,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4fd4a99fc70d40ef7e52b243b4a399c3f8d353a40d5ecb200deee05e49c61bb" +checksum = "ee907b081e45e1d14e1f327e89ef134f91fcebad0bfc2dc229fa9f6044379682" dependencies = [ "ahash", "apache-avro", @@ -769,6 +770,7 @@ dependencies = [ "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", + "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -781,7 +783,7 @@ dependencies = [ "half", "hashbrown", "indexmap", - "itertools 0.12.1", + "itertools", "log", "num-traits", "num_cpus", @@ -803,9 +805,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b3cfbd84c6003594ae1972314e3df303a27ce8ce755fcea3240c90f4c0529" +checksum = "6c2b914f6e33c429af7d8696c72a47ed9225d7e2b82c747ebdfa2408ed53579f" dependencies = [ "arrow-schema", "async-trait", @@ -813,13 +815,14 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "parking_lot", ] [[package]] name = "datafusion-common" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44fdbc877e3e40dcf88cc8f283d9f5c8851f0a3aa07fee657b1b75ac1ad49b9c" +checksum = "3a84f8e76330c582a6b8ada0b2c599ca46cfe46b7585e458fc3f4092bc722a18" dependencies = [ "ahash", "apache-avro", @@ -835,24 +838,27 @@ dependencies = [ "num_cpus", "object_store", "parquet", + "paste", "pyo3", "sqlparser", + "tokio", ] [[package]] name = "datafusion-common-runtime" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7496d1f664179f6ce3a5cbef6566056ccaf3ea4aa72cc455f80e62c1dd86b1" +checksum = "cf08cc30d92720d557df13bd5a5696213bd5ea0f38a866d8d85055d866fba774" dependencies = [ + "log", "tokio", ] [[package]] name = "datafusion-execution" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799e70968c815b611116951e3dd876aef04bf217da31b72eec01ee6a959336a1" +checksum = "86bc4183d5c45b9f068a6f351678a0d1eb1225181424542bb75db18ec280b822" dependencies = [ "arrow", "chrono", @@ -871,9 +877,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1841c409d9518c17971d15c9bae62e629eb937e6fb6c68cd32e9186f8b30d2" +checksum = "202119ce58e4d103e37ae64aab40d4e574c97bdd2bea994bf307b175fcbfa74d" dependencies = [ "ahash", "arrow", @@ -881,6 +887,9 @@ dependencies = [ "arrow-buffer", "chrono", "datafusion-common", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "paste", "serde_json", "sqlparser", @@ -888,11 +897,22 @@ dependencies = [ "strum_macros 0.26.4", ] +[[package]] +name = "datafusion-expr-common" +version = "42.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b181ce8569216abb01ef3294aa16c0a40d7d39350c2ff01ede00f167a535f2" +dependencies = [ + "arrow", + "datafusion-common", + "paste", +] + [[package]] name = "datafusion-functions" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e481cf34d2a444bd8fa09b65945f0ce83dc92df8665b761505b3d9f351bebb" +checksum = "6e4124b8066444e05a24472f852e94cf56546c0f4d92d00f018f207216902712" dependencies = [ "arrow", "arrow-buffer", @@ -905,7 +925,7 @@ dependencies = [ "datafusion-expr", "hashbrown", "hex", - "itertools 0.12.1", + "itertools", "log", "md-5", "rand", @@ -917,9 +937,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b4ece19f73c02727e5e8654d79cd5652de371352c1df3c4ac3e419ecd6943fb" +checksum = "b94acdac235ea21810150a89751617ef2db7e32eba27f54be48a81bde2bfe119" dependencies = [ "ahash", "arrow", @@ -927,17 +947,34 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", "datafusion-physical-expr-common", + "half", "log", "paste", "sqlparser", ] +[[package]] +name = "datafusion-functions-aggregate-common" +version = "42.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9ea085bbf900bf16e2ca0f56fc56236b2e4f2e1a2cccb67bcd83c5ab4ad0ef" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", + "rand", +] + [[package]] name = "datafusion-functions-nested" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1474552cc824e8c9c88177d454db5781d4b66757d4aca75719306b8343a5e8d" +checksum = "6c882e61665ed60c5ce9b061c1e587aeb8ae5ae4bcb5e5f2465139ab25328e0f" dependencies = [ "arrow", "arrow-array", @@ -949,17 +986,30 @@ dependencies = [ "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", - "itertools 0.12.1", + "datafusion-physical-expr-common", + "itertools", "log", "paste", "rand", ] +[[package]] +name = "datafusion-functions-window" +version = "42.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98a354ce96df3ca6d025093adac9fd55ca09931c9b6f2630140721a95873fde4" +dependencies = [ + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "log", +] + [[package]] name = "datafusion-optimizer" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791ff56f55608bc542d1ea7a68a64bdc86a9413f5a381d06a39fd49c2a3ab906" +checksum = "baf677c74fb7b5a1899ef52709e4a70fff3ed80bdfb4bbe495909810e83d5f39" dependencies = [ "arrow", "async-trait", @@ -969,7 +1019,7 @@ dependencies = [ "datafusion-physical-expr", "hashbrown", "indexmap", - "itertools 0.12.1", + "itertools", "log", "paste", "regex-syntax", @@ -977,9 +1027,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a223962b3041304a3e20ed07a21d5de3d88d7e4e71ca192135db6d24e3365a4" +checksum = "30b077999f6eb6c43d6b25bc66332a3be2f693c382840f008dd763b8540f9530" dependencies = [ "ahash", "arrow", @@ -993,12 +1043,14 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", "hashbrown", "hex", "indexmap", - "itertools 0.12.1", + "itertools", "log", "paste", "petgraph", @@ -1007,35 +1059,37 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db5e7d8532a1601cd916881db87a70b0a599900d23f3db2897d389032da53bc6" +checksum = "dce847f885c2b13bbe29f5c8b7948797131aa470af6e16d2a94f4428b4f4f1bd" dependencies = [ "ahash", "arrow", "datafusion-common", - "datafusion-expr", + "datafusion-expr-common", "hashbrown", "rand", ] [[package]] name = "datafusion-physical-optimizer" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb9c78f308e050f5004671039786a925c3fee83b90004e9fcfd328d7febdcc0" +checksum = "d13238e3b9fdd62a4c18760bfef714bb990d1e1d3430e9f416aae4b3cfaa71af" dependencies = [ + "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-physical-expr", "datafusion-physical-plan", + "itertools", ] [[package]] name = "datafusion-physical-plan" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d1116949432eb2d30f6362707e2846d942e491052a206f2ddcb42d08aea1ffe" +checksum = "faba6f55a7eaf0241d07d12c2640de52742646b10f754485d5192bdfe2c9ceae" dependencies = [ "ahash", "arrow", @@ -1050,13 +1104,14 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", "half", "hashbrown", "indexmap", - "itertools 0.12.1", + "itertools", "log", "once_cell", "parking_lot", @@ -1083,7 +1138,7 @@ dependencies = [ "pyo3-build-config", "rand", "regex-syntax", - "syn 2.0.72", + "syn", "tokio", "url", "uuid", @@ -1091,9 +1146,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b45d0180711165fe94015d7c4123eb3e1cf5fb60b1506453200b8d1ce666bef0" +checksum = "dad8d96a9b52e1aa24f9373696a815be828193efce7cb0bbd2140b6bb67d1819" dependencies = [ "arrow", "arrow-array", @@ -1108,15 +1163,15 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf0a0055aa98246c79f98f0d03df11f16cb7adc87818d02d4413e3f3cdadbbee" +checksum = "f92b1b80e98bf5a9921bf118816e0e766d18527e343153321fcccfe4d68c5c45" dependencies = [ "arrow-buffer", "async-recursion", "chrono", "datafusion", - "itertools 0.12.1", + "itertools", "object_store", "pbjson-types", "prost", @@ -1135,12 +1190,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dyn-clone" version = "1.0.17" @@ -1171,9 +1220,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fixedbitset" @@ -1193,9 +1242,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.31" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "miniz_oxide", @@ -1272,7 +1321,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -1328,9 +1377,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" [[package]] name = "glob" @@ -1340,9 +1389,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1470,16 +1519,16 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http", "hyper", "hyper-util", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1488,9 +1537,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -1508,9 +1557,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1541,9 +1590,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.3.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown", @@ -1575,27 +1624,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - -[[package]] -name = "itertools" -version = "0.12.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "itertools" @@ -1623,9 +1654,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -1702,9 +1733,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libflate" @@ -1830,18 +1861,18 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi", "libc", @@ -1941,18 +1972,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.2" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" +checksum = "25a0c4b3a0e31f8b66f71ad8064521efa773910196e2cde791436f13409f3b45" dependencies = [ "async-trait", "base64 0.22.1", @@ -1961,7 +1992,7 @@ dependencies = [ "futures", "humantime", "hyper", - "itertools 0.13.0", + "itertools", "md-5", "parking_lot", "percent-encoding", @@ -2020,14 +2051,14 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] name = "parquet" -version = "52.2.0" +version = "53.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e977b9066b4d3b03555c22bdc442f3fadebd96a39111249113087d0edb2691cd" +checksum = "f0fbf928021131daaa57d334ca8e3904fe9ae22f73c56244fc7db9b04eedc3d8" dependencies = [ "ahash", "arrow-array", @@ -2076,9 +2107,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "pbjson" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +checksum = "c7e6349fa080353f4a597daffd05cb81572a9c031a6d4fff7e504947496fcc68" dependencies = [ "base64 0.21.7", "serde", @@ -2086,21 +2117,21 @@ dependencies = [ [[package]] name = "pbjson-build" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" +checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" dependencies = [ - "heck 0.4.1", - "itertools 0.11.0", + "heck 0.5.0", + "itertools", "prost", "prost-types", ] [[package]] name = "pbjson-types" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +checksum = "e54e5e7bfb1652f95bc361d76f3c780d8e526b134b85417e774166ee941f0887" dependencies = [ "bytes", "chrono", @@ -2182,7 +2213,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -2220,12 +2251,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.72", + "syn", ] [[package]] @@ -2239,9 +2270,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" dependencies = [ "bytes", "prost-derive", @@ -2249,13 +2280,13 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302" dependencies = [ "bytes", "heck 0.5.0", - "itertools 0.12.1", + "itertools", "log", "multimap", "once_cell", @@ -2264,28 +2295,28 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.72", + "syn", "tempfile", ] [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools", "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" dependencies = [ "prost", ] @@ -2301,15 +2332,15 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -2319,9 +2350,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883df5835fafdad87c0d888b266c8ec0f4c9ca48a5bed6bbb592e8dedee1b50" +checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3" dependencies = [ "once_cell", "target-lexicon", @@ -2329,9 +2360,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c" dependencies = [ "libc", "pyo3-build-config", @@ -2339,34 +2370,34 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.72", + "syn", ] [[package]] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.72", + "syn", ] [[package]] name = "quad-rand" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88" +checksum = "b76f1009795ca44bb5aaae8fd3f18953e209259c33d9b059b1f53d58ab7511db" [[package]] name = "quick-xml" @@ -2380,9 +2411,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", @@ -2398,9 +2429,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", @@ -2415,22 +2446,22 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -2467,9 +2498,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" dependencies = [ "bitflags 2.6.0", ] @@ -2521,9 +2552,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "base64 0.22.1", "bytes", @@ -2545,7 +2576,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", + "rustls-native-certs 0.7.3", "rustls-pemfile", "rustls-pki-types", "serde", @@ -2561,7 +2592,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "windows-registry", ] [[package]] @@ -2599,18 +2630,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -2621,9 +2652,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" dependencies = [ "once_cell", "ring", @@ -2635,9 +2666,22 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -2664,9 +2708,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring", "rustls-pki-types", @@ -2696,11 +2740,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2724,7 +2768,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.72", + "syn", ] [[package]] @@ -2773,22 +2817,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -2799,14 +2843,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] name = "serde_json" -version = "1.0.122" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -2816,14 +2860,14 @@ dependencies = [ [[package]] name = "serde_tokenstream" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8790a7c3fe883e443eaa2af6f705952bc5d6e8671a220b9335c8cae92c037e74" +checksum = "64060d864397305347a78851c51588fd283767e7e7589829e8121d65512340f1" dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.72", + "syn", ] [[package]] @@ -2862,6 +2906,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "siphasher" version = "0.3.11" @@ -2885,24 +2935,23 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "snafu" -version = "0.7.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" +checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d" dependencies = [ - "doc-comment", "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.7.5" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" +checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -2929,9 +2978,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.49.0" +version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a404d0e14905361b918cb8afdb73605e25c1d5029312bd9785142dcb3aa49e" +checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac" dependencies = [ "log", "sqlparser_derive", @@ -2945,7 +2994,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -2979,7 +3028,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn", ] [[package]] @@ -2992,14 +3041,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn", ] [[package]] name = "substrait" -version = "0.36.0" +version = "0.41.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1ee6e584c8bf37104b7eb51c25eae07a9321b0e01379bec3b7c462d2f42afbf" +checksum = "2a3bf05f1d7a3fd7a97790d410f6e859b3a98dcde05e7a3fc00b31b0f60fe7cb" dependencies = [ "heck 0.5.0", "pbjson", @@ -3015,7 +3064,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.72", + "syn", "typify", "walkdir", ] @@ -3028,20 +3077,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.72" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -3053,6 +3091,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "target-lexicon" @@ -3062,15 +3103,15 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.11.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3090,7 +3131,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -3130,9 +3171,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -3152,7 +3193,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -3168,9 +3209,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -3196,15 +3237,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -3225,7 +3266,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -3270,7 +3311,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -3304,7 +3345,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.72", + "syn", "thiserror", "unicode-ident", ] @@ -3322,7 +3363,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.72", + "syn", "typify-impl", ] @@ -3334,24 +3375,24 @@ checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" @@ -3431,34 +3472,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -3468,9 +3510,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3478,22 +3520,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" @@ -3510,9 +3552,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -3533,49 +3575,55 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "windows-registry" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-targets 0.48.5", + "windows-result", + "windows-strings", + "windows-targets", ] [[package]] -name = "windows-sys" -version = "0.52.0" +name = "windows-result" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-targets 0.52.6", + "windows-result", + "windows-targets", ] [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] -name = "windows-targets" -version = "0.48.5" +name = "windows-sys" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] @@ -3584,46 +3632,28 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -3636,64 +3666,30 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "xz2" version = "0.1.7" @@ -3721,7 +3717,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn", ] [[package]] @@ -3769,9 +3765,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 8635776e2..11dccc4fb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,24 +36,24 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync"] } rand = "0.8" -pyo3 = { version = "0.21", features = ["extension-module", "abi3", "abi3-py38"] } -arrow = { version = "52", feature = ["pyarrow"] } -datafusion = { version = "41.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-substrait = { version = "41.0.0", optional = true } -prost = "0.12" # keep in line with `datafusion-substrait` -prost-types = "0.12" # keep in line with `datafusion-substrait` +pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } +arrow = { version = "53", features = ["pyarrow"] } +datafusion = { version = "42.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-substrait = { version = "42.0.0", optional = true } +prost = "0.13" # keep in line with `datafusion-substrait` +prost-types = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.9", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.10.1", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.11.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8" syn = "2.0.68" url = "2" [build-dependencies] -pyo3-build-config = "0.21" +pyo3-build-config = "0.22" [lib] name = "datafusion_python" @@ -62,4 +62,3 @@ crate-type = ["cdylib", "rlib"] [profile.release] lto = true codegen-units = 1 - diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 903d4a107..35a40ccd4 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -28,7 +28,7 @@ from datafusion._internal import AggregateUDF from datafusion.catalog import Catalog, Table from datafusion.dataframe import DataFrame -from datafusion.expr import Expr +from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list from datafusion.record_batch import RecordBatchStream from datafusion.udf import ScalarUDF @@ -466,7 +466,7 @@ def register_listing_table( table_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = ".parquet", schema: pyarrow.Schema | None = None, - file_sort_order: list[list[Expr]] | None = None, + file_sort_order: list[list[Expr | SortExpr]] | None = None, ) -> None: """Register multiple files as a single table. @@ -484,15 +484,18 @@ def register_listing_table( """ if table_partition_cols is None: table_partition_cols = [] - if file_sort_order is not None: - file_sort_order = [[x.expr for x in xs] for xs in file_sort_order] + file_sort_order_raw = ( + [sort_list_to_raw_sort_list(f) for f in file_sort_order] + if file_sort_order is not None + else None + ) self.ctx.register_listing_table( name, str(path), table_partition_cols, file_extension, schema, - file_sort_order, + file_sort_order_raw, ) def sql(self, query: str, options: SQLOptions | None = None) -> DataFrame: diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 56dff22a4..2328ef8fa 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -33,7 +33,7 @@ from typing import Callable from datafusion._internal import DataFrame as DataFrameInternal -from datafusion.expr import Expr +from datafusion.expr import Expr, SortExpr, sort_or_default from datafusion._internal import ( LogicalPlan, ExecutionPlan, @@ -199,7 +199,7 @@ def aggregate( aggs = [e.expr for e in aggs] return DataFrame(self.df.aggregate(group_by, aggs)) - def sort(self, *exprs: Expr) -> DataFrame: + def sort(self, *exprs: Expr | SortExpr) -> DataFrame: """Sort the DataFrame by the specified sorting expressions. Note that any expression can be turned into a sort expression by @@ -211,8 +211,8 @@ def sort(self, *exprs: Expr) -> DataFrame: Returns: DataFrame after sorting. """ - exprs = [expr.expr for expr in exprs] - return DataFrame(self.df.sort(*exprs)) + exprs_raw = [sort_or_default(expr) for expr in exprs] + return DataFrame(self.df.sort(*exprs_raw)) def limit(self, count: int, offset: int = 0) -> DataFrame: """Return a new :py:class:`DataFrame` with a limited number of rows. diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index f6a51ce2a..fd5e6f04a 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -22,14 +22,15 @@ from __future__ import annotations -from ._internal import ( - expr as expr_internal, - LogicalPlan, - functions as functions_internal, -) -from datafusion.common import NullTreatment, RexType, DataTypeMap from typing import Any, Optional, Type + import pyarrow as pa +from datafusion.common import DataTypeMap, NullTreatment, RexType +from typing_extensions import deprecated + +from ._internal import LogicalPlan +from ._internal import expr as expr_internal +from ._internal import functions as functions_internal # The following are imported from the internal representation. We may choose to # give these all proper wrappers, or to simply leave as is. These were added @@ -84,7 +85,6 @@ ScalarVariable = expr_internal.ScalarVariable SimilarTo = expr_internal.SimilarTo Sort = expr_internal.Sort -SortExpr = expr_internal.SortExpr Subquery = expr_internal.Subquery SubqueryAlias = expr_internal.SubqueryAlias TableScan = expr_internal.TableScan @@ -159,6 +159,27 @@ ] +def expr_list_to_raw_expr_list( + expr_list: Optional[list[Expr]], +) -> Optional[list[expr_internal.Expr]]: + """Helper function to convert an optional list to raw expressions.""" + return [e.expr for e in expr_list] if expr_list is not None else None + + +def sort_or_default(e: Expr | SortExpr) -> expr_internal.SortExpr: + """Helper function to return a default Sort if an Expr is provided.""" + if isinstance(e, SortExpr): + return e.raw_sort + return SortExpr(e.expr, True, True).raw_sort + + +def sort_list_to_raw_sort_list( + sort_list: Optional[list[Expr | SortExpr]], +) -> Optional[list[expr_internal.SortExpr]]: + """Helper function to return an optional sort list to raw variant.""" + return [sort_or_default(e) for e in sort_list] if sort_list is not None else None + + class Expr: """Expression object. @@ -174,12 +195,22 @@ def to_variant(self) -> Any: """Convert this expression into a python object if possible.""" return self.expr.to_variant() + @deprecated( + "display_name() is deprecated. Use :py:meth:`~Expr.schema_name` instead" + ) def display_name(self) -> str: """Returns the name of this expression as it should appear in a schema. This name will not include any CAST expressions. """ - return self.expr.display_name() + return self.schema_name() + + def schema_name(self) -> str: + """Returns the name of this expression as it should appear in a schema. + + This name will not include any CAST expressions. + """ + return self.expr.schema_name() def canonical_name(self) -> str: """Returns a complete string representation of this expression.""" @@ -355,14 +386,14 @@ def alias(self, name: str) -> Expr: """Assign a name to the expression.""" return Expr(self.expr.alias(name)) - def sort(self, ascending: bool = True, nulls_first: bool = True) -> Expr: + def sort(self, ascending: bool = True, nulls_first: bool = True) -> SortExpr: """Creates a sort :py:class:`Expr` from an existing :py:class:`Expr`. Args: ascending: If true, sort in ascending order. nulls_first: Return null values first. """ - return Expr(self.expr.sort(ascending=ascending, nulls_first=nulls_first)) + return SortExpr(self.expr, ascending=ascending, nulls_first=nulls_first) def is_null(self) -> Expr: """Returns ``True`` if this expression is null.""" @@ -455,14 +486,14 @@ def column_name(self, plan: LogicalPlan) -> str: """Compute the output column name based on the provided logical plan.""" return self.expr.column_name(plan) - def order_by(self, *exprs: Expr) -> ExprFuncBuilder: + def order_by(self, *exprs: Expr | SortExpr) -> ExprFuncBuilder: """Set the ordering for a window or aggregate function. This function will create an :py:class:`ExprFuncBuilder` that can be used to set parameters for either window or aggregate functions. If used on any other type of expression, an error will be generated when ``build()`` is called. """ - return ExprFuncBuilder(self.expr.order_by(list(e.expr for e in exprs))) + return ExprFuncBuilder(self.expr.order_by([sort_or_default(e) for e in exprs])) def filter(self, filter: Expr) -> ExprFuncBuilder: """Filter an aggregate function. @@ -522,7 +553,9 @@ def order_by(self, *exprs: Expr) -> ExprFuncBuilder: Values given in ``exprs`` must be sort expressions. You can convert any other expression to a sort expression using `.sort()`. """ - return ExprFuncBuilder(self.builder.order_by(list(e.expr for e in exprs))) + return ExprFuncBuilder( + self.builder.order_by([sort_or_default(e) for e in exprs]) + ) def filter(self, filter: Expr) -> ExprFuncBuilder: """Filter values during aggregation.""" @@ -659,3 +692,27 @@ def end(self) -> Expr: Any non-matching cases will end in a `null` value. """ return Expr(self.case_builder.end()) + + +class SortExpr: + """Used to specify sorting on either a DataFrame or function.""" + + def __init__(self, expr: Expr, ascending: bool, nulls_first: bool) -> None: + """This constructor should not be called by the end user.""" + self.raw_sort = expr_internal.SortExpr(expr, ascending, nulls_first) + + def expr(self) -> Expr: + """Return the raw expr backing the SortExpr.""" + return Expr(self.raw_sort.expr()) + + def ascending(self) -> bool: + """Return ascending property.""" + return self.raw_sort.ascending() + + def nulls_first(self) -> bool: + """Return nulls_first property.""" + return self.raw_sort.nulls_first() + + def __repr__(self) -> str: + """Generate a string representation of this expression.""" + return self.raw_sort.__repr__() diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 163ff04e4..0401afbc4 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -18,8 +18,15 @@ from __future__ import annotations -from datafusion._internal import functions as f, expr as expr_internal -from datafusion.expr import CaseBuilder, Expr, WindowFrame +from datafusion._internal import functions as f +from datafusion.expr import ( + CaseBuilder, + Expr, + WindowFrame, + SortExpr, + sort_list_to_raw_sort_list, + expr_list_to_raw_expr_list, +) from datafusion.context import SessionContext from datafusion.common import NullTreatment @@ -261,12 +268,6 @@ ] -def expr_list_to_raw_expr_list( - expr_list: Optional[list[Expr]], -) -> Optional[list[expr_internal.Expr]]: - return [e.expr for e in expr_list] if expr_list is not None else None - - def isnan(expr: Expr) -> Expr: """Returns true if a given number is +NaN or -NaN otherwise returns false.""" return Expr(f.isnan(expr.expr)) @@ -352,9 +353,9 @@ def concat_ws(separator: str, *args: Expr) -> Expr: return Expr(f.concat_ws(separator, args)) -def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> Expr: +def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> SortExpr: """Creates a new sort expression.""" - return Expr(f.order_by(expr.expr, ascending, nulls_first)) + return SortExpr(expr.expr, ascending=ascending, nulls_first=nulls_first) def alias(expr: Expr, name: str) -> Expr: @@ -405,7 +406,7 @@ def window( name: str, args: list[Expr], partition_by: list[Expr] | None = None, - order_by: list[Expr] | None = None, + order_by: list[Expr | SortExpr] | None = None, window_frame: WindowFrame | None = None, ctx: SessionContext | None = None, ) -> Expr: @@ -419,9 +420,9 @@ def window( """ args = [a.expr for a in args] partition_by = expr_list_to_raw_expr_list(partition_by) - order_by = expr_list_to_raw_expr_list(order_by) + order_by_raw = sort_list_to_raw_sort_list(order_by) window_frame = window_frame.window_frame if window_frame is not None else None - return Expr(f.window(name, args, partition_by, order_by, window_frame, ctx)) + return Expr(f.window(name, args, partition_by, order_by_raw, window_frame, ctx)) # scalar functions @@ -1608,7 +1609,7 @@ def array_agg( expression: Expr, distinct: bool = False, filter: Optional[Expr] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Aggregate values into an array. @@ -1625,7 +1626,7 @@ def array_agg( filter: If provided, only compute against rows for which the filter is True order_by: Order the resultant array values """ - order_by_raw = expr_list_to_raw_expr_list(order_by) + order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None return Expr( @@ -2107,7 +2108,7 @@ def regr_syy( def first_value( expression: Expr, filter: Optional[Expr] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS, ) -> Expr: """Returns the first value in a group of values. @@ -2123,7 +2124,7 @@ def first_value( order_by: Set the ordering of the expression to evaluate null_treatment: Assign whether to respect or ignull null values. """ - order_by_raw = expr_list_to_raw_expr_list(order_by) + order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None return Expr( @@ -2139,7 +2140,7 @@ def first_value( def last_value( expression: Expr, filter: Optional[Expr] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS, ) -> Expr: """Returns the last value in a group of values. @@ -2155,7 +2156,7 @@ def last_value( order_by: Set the ordering of the expression to evaluate null_treatment: Assign whether to respect or ignull null values. """ - order_by_raw = expr_list_to_raw_expr_list(order_by) + order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None return Expr( @@ -2172,7 +2173,7 @@ def nth_value( expression: Expr, n: int, filter: Optional[Expr] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, null_treatment: NullTreatment = NullTreatment.RESPECT_NULLS, ) -> Expr: """Returns the n-th value in a group of values. @@ -2189,7 +2190,7 @@ def nth_value( order_by: Set the ordering of the expression to evaluate null_treatment: Assign whether to respect or ignull null values. """ - order_by_raw = expr_list_to_raw_expr_list(order_by) + order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None return Expr( @@ -2293,7 +2294,7 @@ def lead( shift_offset: int = 1, default_value: Optional[Any] = None, partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a lead window function. @@ -2330,7 +2331,7 @@ def lead( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.lead( @@ -2338,7 +2339,7 @@ def lead( shift_offset, default_value, partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) @@ -2348,7 +2349,7 @@ def lag( shift_offset: int = 1, default_value: Optional[Any] = None, partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a lag window function. @@ -2382,7 +2383,7 @@ def lag( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.lag( @@ -2390,14 +2391,14 @@ def lag( shift_offset, default_value, partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) def row_number( partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a row number window function. @@ -2421,19 +2422,19 @@ def row_number( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.row_number( partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) def rank( partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a rank window function. @@ -2462,19 +2463,19 @@ def rank( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.rank( partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) def dense_rank( partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a dense_rank window function. @@ -2498,19 +2499,19 @@ def dense_rank( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.dense_rank( partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) def percent_rank( partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a percent_rank window function. @@ -2535,19 +2536,19 @@ def percent_rank( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.percent_rank( partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) def cume_dist( partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a cumulative distribution window function. @@ -2572,12 +2573,12 @@ def cume_dist( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.cume_dist( partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) @@ -2585,7 +2586,7 @@ def cume_dist( def ntile( groups: int, partition_by: Optional[list[Expr]] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Create a n-tile window function. @@ -2613,13 +2614,13 @@ def ntile( partition_cols = ( [col.expr for col in partition_by] if partition_by is not None else None ) - order_cols = [col.expr for col in order_by] if order_by is not None else None + order_by_raw = sort_list_to_raw_sort_list(order_by) return Expr( f.ntile( Expr.literal(groups).expr, partition_by=partition_cols, - order_by=order_cols, + order_by=order_by_raw, ) ) @@ -2628,7 +2629,7 @@ def string_agg( expression: Expr, delimiter: str, filter: Optional[Expr] = None, - order_by: Optional[list[Expr]] = None, + order_by: Optional[list[Expr | SortExpr]] = None, ) -> Expr: """Concatenates the input strings. @@ -2645,7 +2646,7 @@ def string_agg( filter: If provided, only compute against rows for which the filter is True order_by: Set the ordering of the expression to evaluate """ - order_by_raw = expr_list_to_raw_expr_list(order_by) + order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None return Expr( diff --git a/python/datafusion/tests/test_expr.py b/python/datafusion/tests/test_expr.py index 056d2ea03..b58177f16 100644 --- a/python/datafusion/tests/test_expr.py +++ b/python/datafusion/tests/test_expr.py @@ -15,18 +15,21 @@ # specific language governing permissions and limitations # under the License. +import pyarrow +import pytest from datafusion import SessionContext, col -from datafusion.expr import Column, Literal, BinaryExpr, AggregateFunction from datafusion.expr import ( - Projection, - Filter, Aggregate, + AggregateFunction, + BinaryExpr, + Column, + Filter, Limit, + Literal, + Projection, Sort, TableScan, ) -import pyarrow -import pytest @pytest.fixture @@ -192,3 +195,24 @@ def test_expr_getitem() -> None: assert names == ["Alice", "Bob", "Charlie", None] assert array_values == [2, 5, None, None] + + +def test_display_name_deprecation(): + import warnings + + expr = col("foo") + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered + warnings.simplefilter("always") + + # should trigger warning + name = expr.display_name() + + # Verify some things + assert len(w) == 1 + assert issubclass(w[-1].category, DeprecationWarning) + assert "deprecated" in str(w[-1].message) + + # returns appropriate result + assert name == expr.schema_name() + assert name == "foo" diff --git a/python/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py index e41d01004..cbb2e9f57 100644 --- a/python/datafusion/tests/test_sql.py +++ b/python/datafusion/tests/test_sql.py @@ -264,14 +264,17 @@ def test_execute(ctx, tmp_path): # count result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").collect() + ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").show() + + expected_schema = pa.schema([("cnt", pa.int64(), False)]) + expected_values = pa.array([7], type=pa.int64()) + expected = [pa.RecordBatch.from_arrays([expected_values], schema=expected_schema)] - expected = pa.array([7], pa.int64()) - expected = [pa.RecordBatch.from_arrays([expected], ["cnt"])] assert result == expected # where - expected = pa.array([2], pa.int64()) - expected = [pa.RecordBatch.from_arrays([expected], ["cnt"])] + expected_values = pa.array([2], type=pa.int64()) + expected = [pa.RecordBatch.from_arrays([expected_values], schema=expected_schema)] result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a > 10").collect() assert result == expected diff --git a/python/datafusion/tests/test_wrapper_coverage.py b/python/datafusion/tests/test_wrapper_coverage.py index 4a47de2e1..c53a89c59 100644 --- a/python/datafusion/tests/test_wrapper_coverage.py +++ b/python/datafusion/tests/test_wrapper_coverage.py @@ -39,7 +39,10 @@ def missing_exports(internal_obj, wrapped_obj) -> None: internal_attr = getattr(internal_obj, attr) wrapped_attr = getattr(wrapped_obj, attr) - assert wrapped_attr is not None if internal_attr is not None else True + if internal_attr is not None: + if wrapped_attr is None: + print("Missing attribute: ", attr) + assert False if attr in ["__self__", "__class__"]: continue diff --git a/src/common/data_type.rs b/src/common/data_type.rs index a29d1799c..7f9c75bfd 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -24,7 +24,7 @@ use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -#[pyclass(name = "RexType", module = "datafusion.common")] +#[pyclass(eq, eq_int, name = "RexType", module = "datafusion.common")] pub enum RexType { Alias, Literal, @@ -692,7 +692,7 @@ impl From for PyDataType { /// Represents the possible Python types that can be mapped to the SQL types #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -#[pyclass(name = "PythonType", module = "datafusion.common")] +#[pyclass(eq, eq_int, name = "PythonType", module = "datafusion.common")] pub enum PythonType { Array, Bool, @@ -712,7 +712,7 @@ pub enum PythonType { #[allow(non_camel_case_types)] #[allow(clippy::upper_case_acronyms)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -#[pyclass(name = "SqlType", module = "datafusion.common")] +#[pyclass(eq, eq_int, name = "SqlType", module = "datafusion.common")] pub enum SqlType { ANY, ARRAY, @@ -770,7 +770,7 @@ pub enum SqlType { #[allow(non_camel_case_types)] #[allow(clippy::upper_case_acronyms)] #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -#[pyclass(name = "NullTreatment", module = "datafusion.common")] +#[pyclass(eq, eq_int, name = "NullTreatment", module = "datafusion.common")] pub enum NullTreatment { IGNORE_NULLS, RESPECT_NULLS, diff --git a/src/common/schema.rs b/src/common/schema.rs index 5806c90e2..66ce925ae 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::any::Any; +use std::{any::Any, borrow::Cow}; use datafusion::arrow::datatypes::SchemaRef; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource}; @@ -62,6 +62,7 @@ pub struct SqlTable { #[pymethods] impl SqlTable { #[new] + #[pyo3(signature = (table_name, columns, row_count, filepaths=None))] pub fn new( table_name: String, columns: Vec<(String, DataTypeMap)>, @@ -163,39 +164,33 @@ impl TableSource for SqlTableSource { self.schema.clone() } - fn supports_filter_pushdown( - &self, - filter: &Expr, - ) -> datafusion::common::Result { - let filters = split_conjunction(filter); - if filters.iter().all(|f| is_supported_push_down_expr(f)) { - // Push down filters to the tablescan operation if all are supported - Ok(TableProviderFilterPushDown::Exact) - } else if filters.iter().any(|f| is_supported_push_down_expr(f)) { - // Partially apply the filter in the TableScan but retain - // the Filter operator in the plan as well - Ok(TableProviderFilterPushDown::Inexact) - } else { - Ok(TableProviderFilterPushDown::Unsupported) - } - } - fn table_type(&self) -> datafusion::logical_expr::TableType { datafusion::logical_expr::TableType::Base } - #[allow(deprecated)] fn supports_filters_pushdown( &self, filters: &[&Expr], ) -> datafusion::common::Result> { filters .iter() - .map(|f| self.supports_filter_pushdown(f)) + .map(|f| { + let filters = split_conjunction(f); + if filters.iter().all(|f| is_supported_push_down_expr(f)) { + // Push down filters to the tablescan operation if all are supported + Ok(TableProviderFilterPushDown::Exact) + } else if filters.iter().any(|f| is_supported_push_down_expr(f)) { + // Partially apply the filter in the TableScan but retain + // the Filter operator in the plan as well + Ok(TableProviderFilterPushDown::Inexact) + } else { + Ok(TableProviderFilterPushDown::Unsupported) + } + }) .collect() } - fn get_logical_plan(&self) -> Option<&datafusion::logical_expr::LogicalPlan> { + fn get_logical_plan(&self) -> Option> { None } } diff --git a/src/context.rs b/src/context.rs index 11b9fed5f..79db2e65c 100644 --- a/src/context.rs +++ b/src/context.rs @@ -35,7 +35,7 @@ use crate::catalog::{PyCatalog, PyTable}; use crate::dataframe::PyDataFrame; use crate::dataset::Dataset; use crate::errors::{py_datafusion_err, DataFusionError}; -use crate::expr::PyExpr; +use crate::expr::sort_expr::PySortExpr; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; @@ -86,7 +86,7 @@ impl PySessionConfig { let mut config = SessionConfig::new(); if let Some(hash_map) = config_options { for (k, v) in &hash_map { - config = config.set(k, ScalarValue::Utf8(Some(v.clone()))); + config = config.set(k, &ScalarValue::Utf8(Some(v.clone()))); } } @@ -294,6 +294,7 @@ impl PySessionContext { } /// Register an object store with the given name + #[pyo3(signature = (scheme, store, host=None))] pub fn register_object_store( &mut self, scheme: &str, @@ -332,7 +333,7 @@ impl PySessionContext { table_partition_cols: Vec<(String, String)>, file_extension: &str, schema: Option>, - file_sort_order: Option>>, + file_sort_order: Option>>, py: Python, ) -> PyResult<()> { let options = ListingOptions::new(Arc::new(ParquetFormat::new())) @@ -374,6 +375,7 @@ impl PySessionContext { Ok(PyDataFrame::new(df)) } + #[pyo3(signature = (query, options=None))] pub fn sql_with_options( &mut self, query: &str, @@ -390,6 +392,7 @@ impl PySessionContext { Ok(PyDataFrame::new(df)) } + #[pyo3(signature = (partitions, name=None, schema=None))] pub fn create_dataframe( &mut self, partitions: PyArrowType>>, @@ -433,6 +436,7 @@ impl PySessionContext { } /// Construct datafusion dataframe from Python list + #[pyo3(signature = (data, name=None))] pub fn from_pylist( &mut self, data: Bound<'_, PyList>, @@ -452,6 +456,7 @@ impl PySessionContext { } /// Construct datafusion dataframe from Python dictionary + #[pyo3(signature = (data, name=None))] pub fn from_pydict( &mut self, data: Bound<'_, PyDict>, @@ -471,6 +476,7 @@ impl PySessionContext { } /// Construct datafusion dataframe from Arrow Table + #[pyo3(signature = (data, name=None))] pub fn from_arrow( &mut self, data: Bound<'_, PyAny>, @@ -506,6 +512,7 @@ impl PySessionContext { /// Construct datafusion dataframe from pandas #[allow(clippy::wrong_self_convention)] + #[pyo3(signature = (data, name=None))] pub fn from_pandas( &mut self, data: Bound<'_, PyAny>, @@ -525,6 +532,7 @@ impl PySessionContext { } /// Construct datafusion dataframe from polars + #[pyo3(signature = (data, name=None))] pub fn from_polars( &mut self, data: Bound<'_, PyAny>, @@ -581,7 +589,7 @@ impl PySessionContext { file_extension: &str, skip_metadata: bool, schema: Option>, - file_sort_order: Option>>, + file_sort_order: Option>>, py: Python, ) -> PyResult<()> { let mut options = ParquetReadOptions::default() @@ -882,7 +890,7 @@ impl PySessionContext { file_extension: &str, skip_metadata: bool, schema: Option>, - file_sort_order: Option>>, + file_sort_order: Option>>, py: Python, ) -> PyResult { let mut options = ParquetReadOptions::default() diff --git a/src/dataframe.rs b/src/dataframe.rs index f33622cc0..1f7f2e643 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -36,16 +36,19 @@ use datafusion::prelude::*; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; -use pyo3::types::{PyCapsule, PyTuple}; +use pyo3::types::{PyCapsule, PyTuple, PyTupleMethods}; use tokio::task::JoinHandle; use crate::errors::py_datafusion_err; -use crate::expr::to_sort_expressions; +use crate::expr::sort_expr::to_sort_expressions; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; use crate::utils::{get_tokio_runtime, wait_for_future}; -use crate::{errors::DataFusionError, expr::PyExpr}; +use crate::{ + errors::DataFusionError, + expr::{sort_expr::PySortExpr, PyExpr}, +}; /// A PyDataFrame is a representation of a logical plan and an API to compose statements. /// Use it to build a plan and `.collect()` to execute the plan and collect the result. @@ -70,7 +73,7 @@ impl PyDataFrame { if let Ok(key) = key.extract::() { // df[col] self.select_columns(vec![key]) - } else if let Ok(tuple) = key.extract::<&PyTuple>() { + } else if let Ok(tuple) = key.downcast::() { // df[col1, col2, col3] let keys = tuple .iter() @@ -196,7 +199,7 @@ impl PyDataFrame { } #[pyo3(signature = (*exprs))] - fn sort(&self, exprs: Vec) -> PyResult { + fn sort(&self, exprs: Vec) -> PyResult { let exprs = to_sort_expressions(exprs); let df = self.df.as_ref().clone().sort(exprs)?; Ok(Self::new(df)) @@ -504,6 +507,7 @@ impl PyDataFrame { Ok(table) } + #[pyo3(signature = (requested_schema=None))] fn __arrow_c_stream__<'py>( &'py mut self, py: Python<'py>, diff --git a/src/dataset.rs b/src/dataset.rs index de7402fd6..a8fa21ec5 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -39,7 +39,7 @@ use crate::dataset_exec::DatasetExec; use crate::pyarrow_filter_expression::PyArrowFilterExpression; // Wraps a pyarrow.dataset.Dataset class and implements a Datafusion TableProvider around it -#[derive(Debug, Clone)] +#[derive(Debug)] pub(crate) struct Dataset { dataset: PyObject, } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index a377e2555..2759aa678 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -53,7 +53,7 @@ impl Iterator for PyArrowBatchesAdapter { fn next(&mut self) -> Option { Python::with_gil(|py| { - let mut batches = self.batches.clone().into_bound(py); + let mut batches = self.batches.clone_ref(py).into_bound(py); Some( batches .next()? @@ -65,7 +65,7 @@ impl Iterator for PyArrowBatchesAdapter { } // Wraps a pyarrow.dataset.Dataset class and implements a Datafusion ExecutionPlan around it -#[derive(Debug, Clone)] +#[derive(Debug)] pub(crate) struct DatasetExec { dataset: PyObject, schema: SchemaRef, diff --git a/src/expr.rs b/src/expr.rs index 0e1a193f3..304d147c9 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -28,7 +28,7 @@ use datafusion::arrow::pyarrow::PyArrowType; use datafusion::functions::core::expr_ext::FieldAccessor; use datafusion::logical_expr::{ col, - expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction}, + expr::{AggregateFunction, InList, InSubquery, ScalarFunction, WindowFunction}, lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast, }; use datafusion::scalar::ScalarValue; @@ -94,6 +94,8 @@ pub mod unnest; pub mod unnest_expr; pub mod window; +use sort_expr::{to_sort_expressions, PySortExpr}; + /// A PyExpr that can be used on a DataFrame #[pyclass(name = "Expr", module = "datafusion.expr", subclass)] #[derive(Debug, Clone)] @@ -150,7 +152,6 @@ impl PyExpr { Expr::Case(value) => Ok(case::PyCase::from(value.clone()).into_py(py)), Expr::Cast(value) => Ok(cast::PyCast::from(value.clone()).into_py(py)), Expr::TryCast(value) => Ok(cast::PyTryCast::from(value.clone()).into_py(py)), - Expr::Sort(value) => Ok(sort_expr::PySortExpr::from(value.clone()).into_py(py)), Expr::ScalarFunction(value) => Err(py_unsupported_variant_err(format!( "Converting Expr::ScalarFunction to a Python object is not implemented: {:?}", value @@ -167,9 +168,9 @@ impl PyExpr { Expr::ScalarSubquery(value) => { Ok(scalar_subquery::PyScalarSubquery::from(value.clone()).into_py(py)) } - Expr::Wildcard { qualifier } => Err(py_unsupported_variant_err(format!( - "Converting Expr::Wildcard to a Python object is not implemented : {:?}", - qualifier + Expr::Wildcard { qualifier, options } => Err(py_unsupported_variant_err(format!( + "Converting Expr::Wildcard to a Python object is not implemented : {:?} {:?}", + qualifier, options ))), Expr::GroupingSet(value) => { Ok(grouping_set::PyGroupingSet::from(value.clone()).into_py(py)) @@ -188,13 +189,13 @@ impl PyExpr { /// Returns the name of this expression as it should appear in a schema. This name /// will not include any CAST expressions. - fn display_name(&self) -> PyResult { - Ok(self.expr.display_name()?) + fn schema_name(&self) -> PyResult { + Ok(format!("{}", self.expr.schema_name())) } /// Returns a full and complete string representation of this expression. fn canonical_name(&self) -> PyResult { - Ok(self.expr.canonical_name()) + Ok(format!("{}", self.expr)) } /// Returns the name of the Expr variant. @@ -274,7 +275,7 @@ impl PyExpr { /// Create a sort PyExpr from an existing PyExpr. #[pyo3(signature = (ascending=true, nulls_first=true))] - pub fn sort(&self, ascending: bool, nulls_first: bool) -> PyExpr { + pub fn sort(&self, ascending: bool, nulls_first: bool) -> PySortExpr { self.expr.clone().sort(ascending, nulls_first).into() } @@ -323,7 +324,6 @@ impl PyExpr { | Expr::Case { .. } | Expr::Cast { .. } | Expr::TryCast { .. } - | Expr::Sort { .. } | Expr::ScalarFunction { .. } | Expr::AggregateFunction { .. } | Expr::WindowFunction { .. } @@ -387,7 +387,6 @@ impl PyExpr { | Expr::Negative(expr) | Expr::Cast(Cast { expr, .. }) | Expr::TryCast(TryCast { expr, .. }) - | Expr::Sort(Sort { expr, .. }) | Expr::InSubquery(InSubquery { expr, .. }) => Ok(vec![PyExpr::from(*expr.clone())]), // Expr variants containing a collection of Expr(s) for operands @@ -529,7 +528,7 @@ impl PyExpr { // Expression Function Builder functions - pub fn order_by(&self, order_by: Vec) -> PyExprFuncBuilder { + pub fn order_by(&self, order_by: Vec) -> PyExprFuncBuilder { self.expr .clone() .order_by(to_sort_expressions(order_by)) @@ -573,20 +572,9 @@ impl From for PyExprFuncBuilder { } } -pub fn to_sort_expressions(order_by: Vec) -> Vec { - order_by - .iter() - .map(|e| e.expr.clone()) - .map(|e| match e { - Expr::Sort(_) => e, - _ => e.sort(true, true), - }) - .collect() -} - #[pymethods] impl PyExprFuncBuilder { - pub fn order_by(&self, order_by: Vec) -> PyExprFuncBuilder { + pub fn order_by(&self, order_by: Vec) -> PyExprFuncBuilder { self.builder .clone() .order_by(to_sort_expressions(order_by)) @@ -641,11 +629,6 @@ impl PyExpr { input_plan: &LogicalPlan, ) -> Result, DataFusionError> { match expr { - Expr::Sort(Sort { expr, .. }) => { - // DataFusion does not support create_name for sort expressions (since they never - // appear in projections) so we just delegate to the contained expression instead - Self::expr_to_field(expr, input_plan) - } Expr::Wildcard { .. } => { // Since * could be any of the valid column names just return the first one Ok(Arc::new(input_plan.schema().field(0).clone())) diff --git a/src/expr/sort.rs b/src/expr/sort.rs index b31ebfe0b..a1803ccaf 100644 --- a/src/expr/sort.rs +++ b/src/expr/sort.rs @@ -22,7 +22,7 @@ use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; use crate::expr::logical_node::LogicalNode; -use crate::expr::PyExpr; +use crate::expr::sort_expr::PySortExpr; use crate::sql::logical::PyLogicalPlan; #[pyclass(name = "Sort", module = "datafusion.expr", subclass)] @@ -63,12 +63,12 @@ impl Display for PySort { #[pymethods] impl PySort { /// Retrieves the sort expressions for this `Sort` - fn sort_exprs(&self) -> PyResult> { + fn sort_exprs(&self) -> PyResult> { Ok(self .sort .expr .iter() - .map(|e| PyExpr::from(e.clone())) + .map(|e| PySortExpr::from(e.clone())) .collect()) } diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs index 4299d1f71..12f74e4d8 100644 --- a/src/expr/sort_expr.rs +++ b/src/expr/sort_expr.rs @@ -51,10 +51,29 @@ impl Display for PySortExpr { } } +pub fn to_sort_expressions(order_by: Vec) -> Vec { + order_by.iter().map(|e| e.sort.clone()).collect() +} + +pub fn py_sort_expr_list(expr: &[SortExpr]) -> PyResult> { + Ok(expr.iter().map(|e| PySortExpr::from(e.clone())).collect()) +} + #[pymethods] impl PySortExpr { + #[new] + fn new(expr: PyExpr, asc: bool, nulls_first: bool) -> Self { + Self { + sort: SortExpr { + expr: expr.into(), + asc, + nulls_first, + }, + } + } + fn expr(&self) -> PyResult { - Ok((*self.sort.expr).clone().into()) + Ok(self.sort.expr.clone().into()) } fn ascending(&self) -> PyResult { diff --git a/src/expr/window.rs b/src/expr/window.rs index f17a6dd9b..950db12ae 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -24,6 +24,7 @@ use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; use crate::errors::py_type_err; use crate::expr::logical_node::LogicalNode; +use crate::expr::sort_expr::{py_sort_expr_list, PySortExpr}; use crate::expr::PyExpr; use crate::sql::logical::PyLogicalPlan; @@ -114,9 +115,9 @@ impl PyWindow { } /// Returns order by columns in a window function expression - pub fn get_sort_exprs(&self, expr: PyExpr) -> PyResult> { + pub fn get_sort_exprs(&self, expr: PyExpr) -> PyResult> { match expr.expr.unalias() { - Expr::WindowFunction(WindowFunction { order_by, .. }) => py_expr_list(&order_by), + Expr::WindowFunction(WindowFunction { order_by, .. }) => py_sort_expr_list(&order_by), other => Err(not_window_function_err(other)), } } diff --git a/src/functions.rs b/src/functions.rs index b9ca6301a..32f6519f7 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -25,7 +25,8 @@ use crate::common::data_type::NullTreatment; use crate::context::PySessionContext; use crate::errors::DataFusionError; use crate::expr::conditional_expr::PyCaseBuilder; -use crate::expr::to_sort_expressions; +use crate::expr::sort_expr::to_sort_expressions; +use crate::expr::sort_expr::PySortExpr; use crate::expr::window::PyWindowFrame; use crate::expr::PyExpr; use datafusion::common::{Column, ScalarValue, TableReference}; @@ -35,7 +36,7 @@ use datafusion::functions_aggregate; use datafusion::logical_expr::expr::Alias; use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use datafusion::logical_expr::{ - expr::{find_df_window_func, Sort, WindowFunction}, + expr::{find_df_window_func, WindowFunction}, lit, Expr, WindowFunctionDefinition, }; @@ -43,7 +44,7 @@ fn add_builder_fns_to_aggregate( agg_fn: Expr, distinct: Option, filter: Option, - order_by: Option>, + order_by: Option>, null_treatment: Option, ) -> PyResult { // Since ExprFuncBuilder::new() is private, we can guarantee initializing @@ -96,6 +97,7 @@ fn array_cat(exprs: Vec) -> PyExpr { } #[pyfunction] +#[pyo3(signature = (array, element, index=None))] fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr { let index = ScalarValue::Int64(index); let index = Expr::Literal(index); @@ -104,6 +106,7 @@ fn array_position(array: PyExpr, element: PyExpr, index: Option) -> PyExpr } #[pyfunction] +#[pyo3(signature = (array, begin, end, stride=None))] fn array_slice(array: PyExpr, begin: PyExpr, end: PyExpr, stride: Option) -> PyExpr { datafusion::functions_nested::expr_fn::array_slice( array.into(), @@ -142,16 +145,19 @@ fn concat_ws(sep: String, args: Vec) -> PyResult { } #[pyfunction] +#[pyo3(signature = (values, regex, flags=None))] fn regexp_like(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { Ok(functions::expr_fn::regexp_like(values.expr, regex.expr, flags.map(|x| x.expr)).into()) } #[pyfunction] +#[pyo3(signature = (values, regex, flags=None))] fn regexp_match(values: PyExpr, regex: PyExpr, flags: Option) -> PyResult { Ok(functions::expr_fn::regexp_match(values.expr, regex.expr, flags.map(|x| x.expr)).into()) } #[pyfunction] +#[pyo3(signature = (string, pattern, replacement, flags=None))] /// Replaces substring(s) matching a POSIX regular expression. fn regexp_replace( string: PyExpr, @@ -169,14 +175,12 @@ fn regexp_replace( } /// Creates a new Sort Expr #[pyfunction] -fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { - Ok(PyExpr { - expr: datafusion::logical_expr::Expr::Sort(Sort { - expr: Box::new(expr.expr), - asc, - nulls_first, - }), - }) +fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { + Ok(PySortExpr::from(datafusion::logical_expr::expr::Sort { + expr: expr.expr, + asc, + nulls_first, + })) } /// Creates a new Alias Expr @@ -283,11 +287,12 @@ fn find_window_fn(name: &str, ctx: Option) -> PyResult, partition_by: Option>, - order_by: Option>, + order_by: Option>, window_frame: Option, ctx: Option, ) -> PyResult { @@ -309,11 +314,7 @@ fn window( order_by: order_by .unwrap_or_default() .into_iter() - .map(|x| x.expr) - .map(|e| match e { - Expr::Sort(_) => e, - _ => e.sort(true, true), - }) + .map(|x| x.into()) .collect::>(), window_frame, null_treatment: None, @@ -331,11 +332,12 @@ macro_rules! aggregate_function { }; ($NAME: ident, $($arg:ident)*) => { #[pyfunction] + #[pyo3(signature = ($($arg),*, distinct=None, filter=None, order_by=None, null_treatment=None))] fn $NAME( $($arg: PyExpr),*, distinct: Option, filter: Option, - order_by: Option>, + order_by: Option>, null_treatment: Option ) -> PyResult { let agg_fn = functions_aggregate::expr_fn::$NAME($($arg.into()),*); @@ -351,11 +353,12 @@ macro_rules! aggregate_function_vec_args { }; ($NAME: ident, $($arg:ident)*) => { #[pyfunction] + #[pyo3(signature = ($($arg),*, distinct=None, filter=None, order_by=None, null_treatment=None))] fn $NAME( $($arg: PyExpr),*, distinct: Option, filter: Option, - order_by: Option>, + order_by: Option>, null_treatment: Option ) -> PyResult { let agg_fn = functions_aggregate::expr_fn::$NAME(vec![$($arg.into()),*]); @@ -624,6 +627,7 @@ aggregate_function!(approx_median); // aggregate_function!(grouping); #[pyfunction] +#[pyo3(signature = (expression, percentile, num_centroids=None, filter=None))] pub fn approx_percentile_cont( expression: PyExpr, percentile: f64, @@ -642,6 +646,7 @@ pub fn approx_percentile_cont( } #[pyfunction] +#[pyo3(signature = (expression, weight, percentile, filter=None))] pub fn approx_percentile_cont_with_weight( expression: PyExpr, weight: PyExpr, @@ -662,11 +667,12 @@ aggregate_function_vec_args!(last_value); // We handle first_value explicitly because the signature expects an order_by // https://github.com/apache/datafusion/issues/12376 #[pyfunction] +#[pyo3(signature = (expr, distinct=None, filter=None, order_by=None, null_treatment=None))] pub fn first_value( expr: PyExpr, distinct: Option, filter: Option, - order_by: Option>, + order_by: Option>, null_treatment: Option, ) -> PyResult { // If we initialize the UDAF with order_by directly, then it gets over-written by the builder @@ -677,26 +683,28 @@ pub fn first_value( // nth_value requires a non-expr argument #[pyfunction] +#[pyo3(signature = (expr, n, distinct=None, filter=None, order_by=None, null_treatment=None))] pub fn nth_value( expr: PyExpr, n: i64, distinct: Option, filter: Option, - order_by: Option>, + order_by: Option>, null_treatment: Option, ) -> PyResult { - let agg_fn = datafusion::functions_aggregate::nth_value::nth_value(vec![expr.expr, lit(n)]); + let agg_fn = datafusion::functions_aggregate::nth_value::nth_value(expr.expr, n, vec![]); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } // string_agg requires a non-expr argument #[pyfunction] +#[pyo3(signature = (expr, delimiter, distinct=None, filter=None, order_by=None, null_treatment=None))] pub fn string_agg( expr: PyExpr, delimiter: String, distinct: Option, filter: Option, - order_by: Option>, + order_by: Option>, null_treatment: Option, ) -> PyResult { let agg_fn = datafusion::functions_aggregate::string_agg::string_agg(expr.expr, lit(delimiter)); @@ -706,7 +714,7 @@ pub fn string_agg( fn add_builder_fns_to_window( window_fn: Expr, partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { // Since ExprFuncBuilder::new() is private, set an empty partition and then // override later if appropriate. @@ -730,12 +738,13 @@ fn add_builder_fns_to_window( } #[pyfunction] +#[pyo3(signature = (arg, shift_offset, default_value=None, partition_by=None, order_by=None))] pub fn lead( arg: PyExpr, shift_offset: i64, default_value: Option, partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { let window_fn = window_function::lead(arg.expr, Some(shift_offset), default_value); @@ -743,12 +752,13 @@ pub fn lead( } #[pyfunction] +#[pyo3(signature = (arg, shift_offset, default_value=None, partition_by=None, order_by=None))] pub fn lag( arg: PyExpr, shift_offset: i64, default_value: Option, partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { let window_fn = window_function::lag(arg.expr, Some(shift_offset), default_value); @@ -756,26 +766,32 @@ pub fn lag( } #[pyfunction] +#[pyo3(signature = (partition_by=None, order_by=None))] pub fn row_number( partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { - let window_fn = window_function::row_number(); + let window_fn = datafusion::functions_window::expr_fn::row_number(); add_builder_fns_to_window(window_fn, partition_by, order_by) } #[pyfunction] -pub fn rank(partition_by: Option>, order_by: Option>) -> PyResult { +#[pyo3(signature = (partition_by=None, order_by=None))] +pub fn rank( + partition_by: Option>, + order_by: Option>, +) -> PyResult { let window_fn = window_function::rank(); add_builder_fns_to_window(window_fn, partition_by, order_by) } #[pyfunction] +#[pyo3(signature = (partition_by=None, order_by=None))] pub fn dense_rank( partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { let window_fn = window_function::dense_rank(); @@ -783,9 +799,10 @@ pub fn dense_rank( } #[pyfunction] +#[pyo3(signature = (partition_by=None, order_by=None))] pub fn percent_rank( partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { let window_fn = window_function::percent_rank(); @@ -793,9 +810,10 @@ pub fn percent_rank( } #[pyfunction] +#[pyo3(signature = (partition_by=None, order_by=None))] pub fn cume_dist( partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { let window_fn = window_function::cume_dist(); @@ -803,10 +821,11 @@ pub fn cume_dist( } #[pyfunction] +#[pyo3(signature = (arg, partition_by=None, order_by=None))] pub fn ntile( arg: PyExpr, partition_by: Option>, - order_by: Option>, + order_by: Option>, ) -> PyResult { let window_fn = window_function::ntile(arg.into()); @@ -1002,8 +1021,8 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { // Window Functions m.add_wrapped(wrap_pyfunction!(lead))?; m.add_wrapped(wrap_pyfunction!(lag))?; - m.add_wrapped(wrap_pyfunction!(row_number))?; m.add_wrapped(wrap_pyfunction!(rank))?; + m.add_wrapped(wrap_pyfunction!(row_number))?; m.add_wrapped(wrap_pyfunction!(dense_rank))?; m.add_wrapped(wrap_pyfunction!(percent_rank))?; m.add_wrapped(wrap_pyfunction!(cume_dist))?; diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index 6e2a45e1a..0f97ea442 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -27,7 +27,7 @@ use datafusion::logical_expr::{expr::InList, Between, BinaryExpr, Expr, Operator use crate::errors::DataFusionError; -#[derive(Debug, Clone)] +#[derive(Debug)] #[repr(transparent)] pub(crate) struct PyArrowFilterExpression(PyObject); diff --git a/src/udf.rs b/src/udf.rs index 8bd9021d4..7d5db2f96 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -21,7 +21,8 @@ use pyo3::{prelude::*, types::PyTuple}; use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::arrow::datatypes::DataType; -use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; +use datafusion::arrow::pyarrow::FromPyArrow; +use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; use datafusion::logical_expr::create_udf; use datafusion::logical_expr::function::ScalarFunctionImplementation; @@ -43,16 +44,15 @@ fn to_rust_function(func: PyObject) -> ScalarFunctionImplementation { .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); - let py_args = PyTuple::new(py, py_args); + let py_args = PyTuple::new_bound(py, py_args); // 2. call function let value = func - .as_ref(py) - .call(py_args, None) + .call_bound(py, py_args, None) .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; // 3. cast to arrow::array::Array - let array_data = ArrayData::from_pyarrow(value).unwrap(); + let array_data = ArrayData::from_pyarrow_bound(value.bind(py)).unwrap(); Ok(make_array(array_data)) }) }, From a00cfbfdbf3143c8c56d9ea043be3fb69da008ee Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 18 Sep 2024 16:56:43 -0400 Subject: [PATCH 200/413] feat: aggregates as windows (#871) * Add to turn any aggregate function into a window function * Rename Window to WindowExpr so we can define Window to mean a window definition to be reused * Add unit test to cover default frames * Improve error report --- python/datafusion/expr.py | 57 ++++++++++++++++- python/datafusion/tests/test_dataframe.py | 75 ++++++++++++++++------- src/expr.rs | 46 +++++++++++++- src/expr/window.rs | 20 +++--- src/functions.rs | 29 +++++---- src/sql/logical.rs | 4 +- 6 files changed, 183 insertions(+), 48 deletions(-) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index fd5e6f04a..152aa38d3 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -92,7 +92,7 @@ Union = expr_internal.Union Unnest = expr_internal.Unnest UnnestExpr = expr_internal.UnnestExpr -Window = expr_internal.Window +WindowExpr = expr_internal.WindowExpr __all__ = [ "Expr", @@ -154,6 +154,7 @@ "Partitioning", "Repartition", "Window", + "WindowExpr", "WindowFrame", "WindowFrameBound", ] @@ -542,6 +543,36 @@ def window_frame(self, window_frame: WindowFrame) -> ExprFuncBuilder: """ return ExprFuncBuilder(self.expr.window_frame(window_frame.window_frame)) + def over(self, window: Window) -> Expr: + """Turn an aggregate function into a window function. + + This function turns any aggregate function into a window function. With the + exception of ``partition_by``, how each of the parameters is used is determined + by the underlying aggregate function. + + Args: + window: Window definition + """ + partition_by_raw = expr_list_to_raw_expr_list(window._partition_by) + order_by_raw = sort_list_to_raw_sort_list(window._order_by) + window_frame_raw = ( + window._window_frame.window_frame + if window._window_frame is not None + else None + ) + null_treatment_raw = ( + window._null_treatment.value if window._null_treatment is not None else None + ) + + return Expr( + self.expr.over( + partition_by=partition_by_raw, + order_by=order_by_raw, + window_frame=window_frame_raw, + null_treatment=null_treatment_raw, + ) + ) + class ExprFuncBuilder: def __init__(self, builder: expr_internal.ExprFuncBuilder): @@ -584,6 +615,30 @@ def build(self) -> Expr: return Expr(self.builder.build()) +class Window: + """Define reusable window parameters.""" + + def __init__( + self, + partition_by: Optional[list[Expr]] = None, + window_frame: Optional[WindowFrame] = None, + order_by: Optional[list[SortExpr | Expr]] = None, + null_treatment: Optional[NullTreatment] = None, + ) -> None: + """Construct a window definition. + + Args: + partition_by: Partitions for window operation + window_frame: Define the start and end bounds of the window frame + order_by: Set ordering + null_treatment: Indicate how nulls are to be treated + """ + self._partition_by = partition_by + self._window_frame = window_frame + self._order_by = order_by + self._null_treatment = null_treatment + + class WindowFrame: """Defines a window frame for performing window operations.""" diff --git a/python/datafusion/tests/test_dataframe.py b/python/datafusion/tests/test_dataframe.py index 90954d09a..ad7f728b4 100644 --- a/python/datafusion/tests/test_dataframe.py +++ b/python/datafusion/tests/test_dataframe.py @@ -31,6 +31,7 @@ literal, udf, ) +from datafusion.expr import Window @pytest.fixture @@ -386,38 +387,32 @@ def test_distinct(): ), [-1, -1, None, 7, -1, -1, None], ), - # TODO update all aggregate functions as windows once upstream merges https://github.com/apache/datafusion-python/issues/833 - pytest.param( + ( "first_value", - f.window( - "first_value", - [column("a")], - order_by=[f.order_by(column("b"))], - partition_by=[column("c")], + f.first_value(column("a")).over( + Window(partition_by=[column("c")], order_by=[column("b")]) ), [1, 1, 1, 1, 5, 5, 5], ), - pytest.param( + ( "last_value", - f.window("last_value", [column("a")]) - .window_frame(WindowFrame("rows", 0, None)) - .order_by(column("b")) - .partition_by(column("c")) - .build(), + f.last_value(column("a")).over( + Window( + partition_by=[column("c")], + order_by=[column("b")], + window_frame=WindowFrame("rows", None, None), + ) + ), [3, 3, 3, 3, 6, 6, 6], ), - pytest.param( + ( "3rd_value", - f.window( - "nth_value", - [column("b"), literal(3)], - order_by=[f.order_by(column("a"))], - ), + f.nth_value(column("b"), 3).over(Window(order_by=[column("a")])), [None, None, 7, 7, 7, 7, 7], ), - pytest.param( + ( "avg", - f.round(f.window("avg", [column("b")], order_by=[column("a")]), literal(3)), + f.round(f.avg(column("b")).over(Window(order_by=[column("a")])), literal(3)), [7.0, 7.0, 7.0, 7.333, 7.75, 7.75, 8.0], ), ] @@ -473,6 +468,44 @@ def test_invalid_window_frame(units, start_bound, end_bound): WindowFrame(units, start_bound, end_bound) +def test_window_frame_defaults_match_postgres(partitioned_df): + # ref: https://github.com/apache/datafusion-python/issues/688 + + window_frame = WindowFrame("rows", None, None) + + col_a = column("a") + + # Using `f.window` with or without an unbounded window_frame produces the same + # results. These tests are included as a regression check but can be removed when + # f.window() is deprecated in favor of using the .over() approach. + no_frame = f.window("avg", [col_a]).alias("no_frame") + with_frame = f.window("avg", [col_a], window_frame=window_frame).alias("with_frame") + df_1 = partitioned_df.select(col_a, no_frame, with_frame) + + expected = { + "a": [0, 1, 2, 3, 4, 5, 6], + "no_frame": [3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0], + "with_frame": [3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0], + } + + assert df_1.sort(col_a).to_pydict() == expected + + # When order is not set, the default frame should be unounded preceeding to + # unbounded following. When order is set, the default frame is unbounded preceeding + # to current row. + no_order = f.avg(col_a).over(Window()).alias("over_no_order") + with_order = f.avg(col_a).over(Window(order_by=[col_a])).alias("over_with_order") + df_2 = partitioned_df.select(col_a, no_order, with_order) + + expected = { + "a": [0, 1, 2, 3, 4, 5, 6], + "over_no_order": [3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0], + "over_with_order": [0.0, 0.5, 1.0, 1.5, 2.0, 2.5, 3.0], + } + + assert df_2.sort(col_a).to_pydict() == expected + + def test_get_dataframe(tmp_path): ctx = SessionContext() diff --git a/src/expr.rs b/src/expr.rs index 304d147c9..49fa4b845 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -16,7 +16,9 @@ // under the License. use datafusion::logical_expr::utils::exprlist_to_fields; -use datafusion::logical_expr::{ExprFuncBuilder, ExprFunctionExt, LogicalPlan}; +use datafusion::logical_expr::{ + ExprFuncBuilder, ExprFunctionExt, LogicalPlan, WindowFunctionDefinition, +}; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use std::sync::Arc; @@ -39,6 +41,7 @@ use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; +use crate::functions::add_builder_fns_to_window; use crate::sql::logical::PyLogicalPlan; use self::alias::PyAlias; @@ -558,6 +561,45 @@ impl PyExpr { pub fn window_frame(&self, window_frame: PyWindowFrame) -> PyExprFuncBuilder { self.expr.clone().window_frame(window_frame.into()).into() } + + #[pyo3(signature = (partition_by=None, window_frame=None, order_by=None, null_treatment=None))] + pub fn over( + &self, + partition_by: Option>, + window_frame: Option, + order_by: Option>, + null_treatment: Option, + ) -> PyResult { + match &self.expr { + Expr::AggregateFunction(agg_fn) => { + let window_fn = Expr::WindowFunction(WindowFunction::new( + WindowFunctionDefinition::AggregateUDF(agg_fn.func.clone()), + agg_fn.args.clone(), + )); + + add_builder_fns_to_window( + window_fn, + partition_by, + window_frame, + order_by, + null_treatment, + ) + } + Expr::WindowFunction(_) => add_builder_fns_to_window( + self.expr.clone(), + partition_by, + window_frame, + order_by, + null_treatment, + ), + _ => Err( + DataFusionError::ExecutionError(datafusion::error::DataFusionError::Plan( + format!("Using {} with `over` is not allowed. Must use an aggregate or window function.", self.expr.variant_name()), + )) + .into(), + ), + } + } } #[pyclass(name = "ExprFuncBuilder", module = "datafusion.expr", subclass)] @@ -749,7 +791,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; Ok(()) diff --git a/src/expr/window.rs b/src/expr/window.rs index 950db12ae..6486dbb32 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -32,9 +32,9 @@ use super::py_expr_list; use crate::errors::py_datafusion_err; -#[pyclass(name = "Window", module = "datafusion.expr", subclass)] +#[pyclass(name = "WindowExpr", module = "datafusion.expr", subclass)] #[derive(Clone)] -pub struct PyWindow { +pub struct PyWindowExpr { window: Window, } @@ -62,15 +62,15 @@ pub struct PyWindowFrameBound { frame_bound: WindowFrameBound, } -impl From for Window { - fn from(window: PyWindow) -> Window { +impl From for Window { + fn from(window: PyWindowExpr) -> Window { window.window } } -impl From for PyWindow { - fn from(window: Window) -> PyWindow { - PyWindow { window } +impl From for PyWindowExpr { + fn from(window: Window) -> PyWindowExpr { + PyWindowExpr { window } } } @@ -80,7 +80,7 @@ impl From for PyWindowFrameBound { } } -impl Display for PyWindow { +impl Display for PyWindowExpr { fn fmt(&self, f: &mut Formatter) -> fmt::Result { write!( f, @@ -103,7 +103,7 @@ impl Display for PyWindowFrame { } #[pymethods] -impl PyWindow { +impl PyWindowExpr { /// Returns the schema of the Window pub fn schema(&self) -> PyResult { Ok(self.window.schema.as_ref().clone().into()) @@ -283,7 +283,7 @@ impl PyWindowFrameBound { } } -impl LogicalNode for PyWindow { +impl LogicalNode for PyWindowExpr { fn inputs(&self) -> Vec { vec![self.window.input.as_ref().clone().into()] } diff --git a/src/functions.rs b/src/functions.rs index 32f6519f7..6f8dd7ada 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -711,14 +711,15 @@ pub fn string_agg( add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } -fn add_builder_fns_to_window( +pub(crate) fn add_builder_fns_to_window( window_fn: Expr, partition_by: Option>, + window_frame: Option, order_by: Option>, + null_treatment: Option, ) -> PyResult { - // Since ExprFuncBuilder::new() is private, set an empty partition and then - // override later if appropriate. - let mut builder = window_fn.partition_by(vec![]); + let null_treatment = null_treatment.map(|n| n.into()); + let mut builder = window_fn.null_treatment(null_treatment); if let Some(partition_cols) = partition_by { builder = builder.partition_by( @@ -734,6 +735,10 @@ fn add_builder_fns_to_window( builder = builder.order_by(order_by_cols); } + if let Some(window_frame) = window_frame { + builder = builder.window_frame(window_frame.into()); + } + builder.build().map(|e| e.into()).map_err(|err| err.into()) } @@ -748,7 +753,7 @@ pub fn lead( ) -> PyResult { let window_fn = window_function::lead(arg.expr, Some(shift_offset), default_value); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -762,7 +767,7 @@ pub fn lag( ) -> PyResult { let window_fn = window_function::lag(arg.expr, Some(shift_offset), default_value); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -773,7 +778,7 @@ pub fn row_number( ) -> PyResult { let window_fn = datafusion::functions_window::expr_fn::row_number(); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -784,7 +789,7 @@ pub fn rank( ) -> PyResult { let window_fn = window_function::rank(); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -795,7 +800,7 @@ pub fn dense_rank( ) -> PyResult { let window_fn = window_function::dense_rank(); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -806,7 +811,7 @@ pub fn percent_rank( ) -> PyResult { let window_fn = window_function::percent_rank(); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -817,7 +822,7 @@ pub fn cume_dist( ) -> PyResult { let window_fn = window_function::cume_dist(); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } #[pyfunction] @@ -829,7 +834,7 @@ pub fn ntile( ) -> PyResult { let window_fn = window_function::ntile(arg.into()); - add_builder_fns_to_window(window_fn, partition_by, order_by) + add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 89655ab70..d00f0af3f 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -34,7 +34,7 @@ use crate::expr::subquery::PySubquery; use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; use crate::expr::unnest::PyUnnest; -use crate::expr::window::PyWindow; +use crate::expr::window::PyWindowExpr; use datafusion::logical_expr::LogicalPlan; use pyo3::prelude::*; @@ -80,7 +80,7 @@ impl PyLogicalPlan { LogicalPlan::Subquery(plan) => PySubquery::from(plan.clone()).to_variant(py), LogicalPlan::SubqueryAlias(plan) => PySubqueryAlias::from(plan.clone()).to_variant(py), LogicalPlan::Unnest(plan) => PyUnnest::from(plan.clone()).to_variant(py), - LogicalPlan::Window(plan) => PyWindow::from(plan.clone()).to_variant(py), + LogicalPlan::Window(plan) => PyWindowExpr::from(plan.clone()).to_variant(py), LogicalPlan::Repartition(_) | LogicalPlan::Union(_) | LogicalPlan::Statement(_) From 044bbe2f32a208f3e745971468d8769ca82e5d70 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 21 Sep 2024 10:33:51 -0400 Subject: [PATCH 201/413] Fix regression on register_udaf (#878) * Test no longer hangs, and updated error string to match latest * Add unit tests for registering udf and udaf * Resolve error on registering udaf #874 * remove stale comment * Update unit test text to match in multiple versions of python * Regex for exception that is compatible with python 3.10 and 3.12 --- python/datafusion/tests/test_udaf.py | 47 +++++++++++++++++++++++----- python/datafusion/udf.py | 4 +-- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/python/datafusion/tests/test_udaf.py b/python/datafusion/tests/test_udaf.py index 76488e19b..6f2525b0f 100644 --- a/python/datafusion/tests/test_udaf.py +++ b/python/datafusion/tests/test_udaf.py @@ -21,7 +21,7 @@ import pyarrow.compute as pc import pytest -from datafusion import Accumulator, SessionContext, column, udaf +from datafusion import Accumulator, column, udaf, udf class Summarize(Accumulator): @@ -60,18 +60,15 @@ def state(self) -> List[pa.Scalar]: @pytest.fixture -def df(): - ctx = SessionContext() - +def df(ctx): # create a RecordBatch and a new DataFrame from it batch = pa.RecordBatch.from_arrays( [pa.array([1, 2, 3]), pa.array([4, 4, 6])], names=["a", "b"], ) - return ctx.create_dataframe([[batch]]) + return ctx.create_dataframe([[batch]], name="test_table") -@pytest.mark.skip(reason="df.collect() will hang, need more investigations") def test_errors(df): with pytest.raises(TypeError): udaf( @@ -92,8 +89,9 @@ def test_errors(df): df = df.aggregate([], [accum(column("a"))]) msg = ( - "Can't instantiate abstract class MissingMethods with abstract " - "methods evaluate, merge, update" + "Can't instantiate abstract class MissingMethods (without an implementation " + "for abstract methods 'evaluate', 'merge', 'update'|with abstract methods " + "evaluate, merge, update)" ) with pytest.raises(Exception, match=msg): df.collect() @@ -132,3 +130,36 @@ def test_group_by(df): arrays = [batch.column(1) for batch in batches] joined = pa.concat_arrays(arrays) assert joined == pa.array([1.0 + 2.0, 3.0]) + + +def test_register_udaf(ctx, df) -> None: + summarize = udaf( + Summarize, + pa.float64(), + pa.float64(), + [pa.float64()], + volatility="immutable", + ) + + ctx.register_udaf(summarize) + + df_result = ctx.sql("select summarize(b) from test_table") + + assert df_result.collect()[0][0][0].as_py() == 14.0 + + +def test_register_udf(ctx, df) -> None: + is_null = udf( + lambda x: x.is_null(), + [pa.float64()], + pa.bool_(), + volatility="immutable", + name="is_null", + ) + + ctx.register_udf(is_null) + + df_result = ctx.sql("select is_null(a) from test_table") + result = df_result.collect()[0].column(0) + + assert result == pa.array([False, False, False]) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index a3b74bb11..f74d675e3 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -192,7 +192,7 @@ def __init__( See :py:func:`udaf` for a convenience function and argument descriptions. """ - self._udf = df_internal.AggregateUDF( + self._udaf = df_internal.AggregateUDF( name, accumulator, input_types, return_type, state_type, str(volatility) ) @@ -203,7 +203,7 @@ def __call__(self, *args: Expr) -> Expr: occur during the evaluation of the dataframe. """ args = [arg.expr for arg in args] - return Expr(self._udf.__call__(*args)) + return Expr(self._udaf.__call__(*args)) @staticmethod def udaf( From a18ece8bfb5f5814cb58a731c7856c630882a210 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sun, 22 Sep 2024 12:41:28 -0500 Subject: [PATCH 202/413] build(deps): upgrade setup-protoc action and protoc version number (#873) `arduino/setup-protoc@v3` could not find our old version number of `3.20.2`. I chose protoc version 27.4 because it's the last release from the prior version. --- .github/workflows/build.yml | 8 ++++---- .github/workflows/docs.yaml | 4 ++-- .github/workflows/test.yaml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5fae13f2e..f52913ce8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -92,9 +92,9 @@ jobs: path: . - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v3 with: - version: "3.20.2" + version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package @@ -150,9 +150,9 @@ jobs: path: . - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v3 with: - version: "3.20.2" + version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 60dc15927..e47497b2a 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -52,9 +52,9 @@ jobs: python-version: "3.11" - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v3 with: - version: '3.20.2' + version: '27.4' repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install dependencies diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4f47dc984..f9383db5f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,9 +50,9 @@ jobs: override: true - name: Install Protoc - uses: arduino/setup-protoc@v1 + uses: arduino/setup-protoc@v3 with: - version: '3.20.2' + version: '27.4' repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python From f6261b05ce0d68489403fe966a464cdd7d24b3b5 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sun, 22 Sep 2024 12:42:22 -0500 Subject: [PATCH 203/413] add a test to capture the bug (#843) --- python/datafusion/tests/test_context.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/python/datafusion/tests/test_context.py b/python/datafusion/tests/test_context.py index 4af00a3b4..ab86faa9d 100644 --- a/python/datafusion/tests/test_context.py +++ b/python/datafusion/tests/test_context.py @@ -372,6 +372,25 @@ def test_dataset_filter(ctx, capfd): assert result[0].column(1) == pa.array([-3]) +def test_dataset_count(ctx): + # `datafusion-python` issue: https://github.com/apache/datafusion-python/issues/800 + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], + names=["a", "b"], + ) + dataset = ds.dataset([batch]) + ctx.register_dataset("t", dataset) + + # Testing the dataframe API + df = ctx.table("t") + assert df.count() == 3 + + # Testing the SQL API + count = ctx.sql("SELECT COUNT(*) FROM t") + count = count.collect() + assert count[0].column(0) == pa.array([3]) + + def test_pyarrow_predicate_pushdown_is_null(ctx, capfd): """Ensure that pyarrow filter gets pushed down for `IsNull`""" # create a RecordBatch and register it as a pyarrow.dataset.Dataset From 1952e67d7e10aa18b63b6916be68ae851bee9b14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:52:48 -0500 Subject: [PATCH 204/413] build(deps): bump prost-types from 0.13.2 to 0.13.3 (#881) Bumps [prost-types](https://github.com/tokio-rs/prost) from 0.13.2 to 0.13.3. - [Release notes](https://github.com/tokio-rs/prost/releases) - [Changelog](https://github.com/tokio-rs/prost/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/prost/compare/v0.13.2...v0.13.3) --- updated-dependencies: - dependency-name: prost-types dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d06073b6f..e48339e98 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2270,9 +2270,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ "bytes", "prost-derive", @@ -2301,9 +2301,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", "itertools", @@ -2314,9 +2314,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" dependencies = [ "prost", ] From 8b36aac1cd9eb16189e6464fd4df2a5213a7be7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Sep 2024 15:53:03 -0500 Subject: [PATCH 205/413] build(deps): bump prost from 0.13.2 to 0.13.3 (#882) Bumps [prost](https://github.com/tokio-rs/prost) from 0.13.2 to 0.13.3. - [Release notes](https://github.com/tokio-rs/prost/releases) - [Changelog](https://github.com/tokio-rs/prost/blob/master/CHANGELOG.md) - [Commits](https://github.com/tokio-rs/prost/compare/v0.13.2...v0.13.3) --- updated-dependencies: - dependency-name: prost dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From 2df33d33fcf3f01e83884f4e8943330cb6458d95 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sat, 28 Sep 2024 17:08:36 -0500 Subject: [PATCH 206/413] chore: remove XFAIL from passing tests (#884) * remove xfail from passing tests * update xfail cmoment for datetime_s * document upstream issue for timestamp[s] xfail test --- python/datafusion/tests/test_sql.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/python/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py index cbb2e9f57..bd2ae58d7 100644 --- a/python/datafusion/tests/test_sql.py +++ b/python/datafusion/tests/test_sql.py @@ -378,23 +378,29 @@ def test_udf( helpers.data_binary_other(), helpers.data_date32(), helpers.data_with_nans(), - # C data interface missing pytest.param( pa.array([b"1111", b"2222", b"3333"], pa.binary(4), _null_mask), id="binary4", - marks=pytest.mark.xfail, ), + # `timestamp[s]` does not roundtrip for pyarrow.parquet: https://github.com/apache/arrow/issues/41382 pytest.param( - helpers.data_datetime("s"), id="datetime_s", marks=pytest.mark.xfail + helpers.data_datetime("s"), + id="datetime_s", + marks=pytest.mark.xfail( + reason="pyarrow.parquet does not support timestamp[s] roundtrips" + ), ), pytest.param( - helpers.data_datetime("ms"), id="datetime_ms", marks=pytest.mark.xfail + helpers.data_datetime("ms"), + id="datetime_ms", ), pytest.param( - helpers.data_datetime("us"), id="datetime_us", marks=pytest.mark.xfail + helpers.data_datetime("us"), + id="datetime_us", ), pytest.param( - helpers.data_datetime("ns"), id="datetime_ns", marks=pytest.mark.xfail + helpers.data_datetime("ns"), + id="datetime_ns", ), # Not writtable to parquet pytest.param( From f8224953b281bf157b3ddc18d88a7085852a6bdc Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Sun, 29 Sep 2024 14:19:55 +0200 Subject: [PATCH 207/413] feat: make register_csv accept a list of paths (#883) --- python/datafusion/context.py | 11 +++-- python/datafusion/tests/test_sql.py | 35 ++++++++++++++++ src/context.rs | 64 +++++++++++++++++++++++++---- 3 files changed, 99 insertions(+), 11 deletions(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 35a40ccd4..2c41faba6 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -714,7 +714,7 @@ def register_parquet( def register_csv( self, name: str, - path: str | pathlib.Path, + path: str | pathlib.Path | list[str | pathlib.Path], schema: pyarrow.Schema | None = None, has_header: bool = True, delimiter: str = ",", @@ -728,7 +728,7 @@ def register_csv( Args: name: Name of the table to register. - path: Path to the CSV file. + path: Path to the CSV file. It also accepts a list of Paths. schema: An optional schema representing the CSV file. If None, the CSV reader will try to infer it based on data in file. has_header: Whether the CSV file have a header. If schema inference @@ -741,9 +741,14 @@ def register_csv( selected for data input. file_compression_type: File compression type. """ + if isinstance(path, list): + path = [str(p) for p in path] + else: + path = str(path) + self.ctx.register_csv( name, - str(path), + path, schema, has_header, delimiter, diff --git a/python/datafusion/tests/test_sql.py b/python/datafusion/tests/test_sql.py index bd2ae58d7..e39a9f5c7 100644 --- a/python/datafusion/tests/test_sql.py +++ b/python/datafusion/tests/test_sql.py @@ -104,6 +104,41 @@ def test_register_csv(ctx, tmp_path): ctx.register_csv("csv4", path, file_compression_type="rar") +def test_register_csv_list(ctx, tmp_path): + path = tmp_path / "test.csv" + + int_values = [1, 2, 3, 4] + table = pa.Table.from_arrays( + [ + int_values, + ["a", "b", "c", "d"], + [1.1, 2.2, 3.3, 4.4], + ], + names=["int", "str", "float"], + ) + write_csv(table, path) + ctx.register_csv("csv", path) + + csv_df = ctx.table("csv") + expected_count = csv_df.count() * 2 + ctx.register_csv( + "double_csv", + path=[ + path, + path, + ], + ) + + double_csv_df = ctx.table("double_csv") + actual_count = double_csv_df.count() + assert actual_count == expected_count + + int_sum = ctx.sql("select sum(int) from double_csv").to_pydict()[ + "sum(double_csv.int)" + ][0] + assert int_sum == 2 * sum(int_values) + + def test_register_parquet(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) ctx.register_parquet("t", path) diff --git a/src/context.rs b/src/context.rs index 79db2e65c..7ad12ceb0 100644 --- a/src/context.rs +++ b/src/context.rs @@ -46,7 +46,8 @@ use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::ScalarValue; +use datafusion::catalog_common::TableReference; +use datafusion::common::{exec_err, ScalarValue}; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::file_format::parquet::ParquetFormat; use datafusion::datasource::listing::{ @@ -54,9 +55,12 @@ use datafusion::datasource::listing::{ }; use datafusion::datasource::MemTable; use datafusion::datasource::TableProvider; -use datafusion::execution::context::{SQLOptions, SessionConfig, SessionContext, TaskContext}; +use datafusion::execution::context::{ + DataFilePaths, SQLOptions, SessionConfig, SessionContext, TaskContext, +}; use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; +use datafusion::execution::options::ReadOptions; use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ @@ -621,7 +625,7 @@ impl PySessionContext { pub fn register_csv( &mut self, name: &str, - path: PathBuf, + path: &Bound<'_, PyAny>, schema: Option>, has_header: bool, delimiter: &str, @@ -630,9 +634,6 @@ impl PySessionContext { file_compression_type: Option, py: Python, ) -> PyResult<()> { - let path = path - .to_str() - .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; let delimiter = delimiter.as_bytes(); if delimiter.len() != 1 { return Err(PyValueError::new_err( @@ -648,8 +649,15 @@ impl PySessionContext { .file_compression_type(parse_file_compression_type(file_compression_type)?); options.schema = schema.as_ref().map(|x| &x.0); - let result = self.ctx.register_csv(name, path, options); - wait_for_future(py, result).map_err(DataFusionError::from)?; + if path.is_instance_of::() { + let paths = path.extract::>()?; + let result = self.register_csv_from_multiple_paths(name, paths, options); + wait_for_future(py, result).map_err(DataFusionError::from)?; + } else { + let path = path.extract::()?; + let result = self.ctx.register_csv(name, &path, options); + wait_for_future(py, result).map_err(DataFusionError::from)?; + } Ok(()) } @@ -981,6 +989,46 @@ impl PySessionContext { async fn _table(&self, name: &str) -> datafusion::common::Result { self.ctx.table(name).await } + + async fn register_csv_from_multiple_paths( + &self, + name: &str, + table_paths: Vec, + options: CsvReadOptions<'_>, + ) -> datafusion::common::Result<()> { + let table_paths = table_paths.to_urls()?; + let session_config = self.ctx.copied_config(); + let listing_options = + options.to_listing_options(&session_config, self.ctx.copied_table_options()); + + let option_extension = listing_options.file_extension.clone(); + + if table_paths.is_empty() { + return exec_err!("No table paths were provided"); + } + + // check if the file extension matches the expected extension + for path in &table_paths { + let file_path = path.as_str(); + if !file_path.ends_with(option_extension.clone().as_str()) && !path.is_collection() { + return exec_err!( + "File path '{file_path}' does not match the expected extension '{option_extension}'" + ); + } + } + + let resolved_schema = options + .get_resolved_schema(&session_config, self.ctx.state(), table_paths[0].clone()) + .await?; + + let config = ListingTableConfig::new_with_multi_paths(table_paths) + .with_listing_options(listing_options) + .with_schema(resolved_schema); + let table = ListingTable::try_new(config)?; + self.ctx + .register_table(TableReference::Bare { table: name.into() }, Arc::new(table))?; + Ok(()) + } } pub fn convert_table_partition_cols( From 022e4b323c20dd306eac8c9b4ea65a9f23d9bcac Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 30 Sep 2024 07:43:23 -0400 Subject: [PATCH 208/413] Add user defined window function support (#880) * Adding PyWindowUDF and implementing PartitionEvaluator for it. Still requires python side work. * Add python wrappers for UDWF * adding unit tests for user defined window functions * Change udwf() to take an instance rather than a class so we can parameterize it * Pass multiple arrays for udwf evaluate so we can capture the order_by and also multiple columns * Update udwf to take multiple input columns * Add user exampe for UDWF * Update template for how values are passed to update * Add user documentation for UDWF * Updating documentation per PR review --- .../common-operations/udf-and-udfa.rst | 194 +++++++++-- examples/python-udwf.py | 270 ++++++++++++++++ python/datafusion/__init__.py | 5 +- python/datafusion/context.py | 7 +- python/datafusion/tests/test_udwf.py | 294 +++++++++++++++++ python/datafusion/udf.py | 254 ++++++++++++++- src/context.rs | 6 + src/lib.rs | 2 + src/udwf.rs | 305 ++++++++++++++++++ 9 files changed, 1306 insertions(+), 31 deletions(-) create mode 100644 examples/python-udwf.py create mode 100644 python/datafusion/tests/test_udwf.py create mode 100644 src/udwf.rs diff --git a/docs/source/user-guide/common-operations/udf-and-udfa.rst b/docs/source/user-guide/common-operations/udf-and-udfa.rst index e9c142f0a..ffd7a05cb 100644 --- a/docs/source/user-guide/common-operations/udf-and-udfa.rst +++ b/docs/source/user-guide/common-operations/udf-and-udfa.rst @@ -15,11 +15,24 @@ .. specific language governing permissions and limitations .. under the License. -User Defined Functions +User-Defined Functions ====================== -DataFusion provides powerful expressions and functions, reducing the need for custom Python functions. -However you can still incorporate your own functions, i.e. User-Defined Functions (UDFs), with the :py:func:`~datafusion.udf.ScalarUDF.udf` function. +DataFusion provides powerful expressions and functions, reducing the need for custom Python +functions. However you can still incorporate your own functions, i.e. User-Defined Functions (UDFs). + +Scalar Functions +---------------- + +When writing a user-defined function that can operate on a row by row basis, these are called Scalar +Functions. You can define your own scalar function by calling +:py:func:`~datafusion.udf.ScalarUDF.udf` . + +The basic definition of a scalar UDF is a python function that takes one or more +`pyarrow `_ arrays and returns a single array as +output. DataFusion scalar UDFs operate on an entire batch of records at a time, though the +evaluation of those records should be on a row by row basis. In the following example, we compute +if the input array contains null values. .. ipython:: python @@ -35,14 +48,67 @@ However you can still incorporate your own functions, i.e. User-Defined Function ctx = datafusion.SessionContext() batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], + [pyarrow.array([1, None, 3]), pyarrow.array([4, 5, 6])], names=["a", "b"], ) df = ctx.create_dataframe([[batch]], name="batch_array") - df.select(is_null_arr(col("a"))).to_pandas() + df.select(col("a"), is_null_arr(col("a")).alias("is_null")).show() + +In the previous example, we used the fact that pyarrow provides a variety of built in array +functions such as ``is_null()``. There are additional pyarrow +`compute functions `_ available. When possible, +it is highly recommended to use these functions because they can perform computations without doing +any copy operations from the original arrays. This leads to greatly improved performance. + +If you need to perform an operation in python that is not available with the pyarrow compute +functions, you will need to convert the record batch into python values, perform your operation, +and construct an array. This operation of converting the built in data type of the array into a +python object can be one of the slowest operations in DataFusion, so it should be done sparingly. + +The following example performs the same operation as before with ``is_null`` but demonstrates +converting to Python objects to do the evaluation. + +.. ipython:: python + + import pyarrow + import datafusion + from datafusion import udf, col + + def is_null(array: pyarrow.Array) -> pyarrow.Array: + return pyarrow.array([value.as_py() is None for value in array]) + + is_null_arr = udf(is_null, [pyarrow.int64()], pyarrow.bool_(), 'stable') + + ctx = datafusion.SessionContext() + + batch = pyarrow.RecordBatch.from_arrays( + [pyarrow.array([1, None, 3]), pyarrow.array([4, 5, 6])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]], name="batch_array") -Additionally the :py:func:`~datafusion.udf.AggregateUDF.udaf` function allows you to define User-Defined Aggregate Functions (UDAFs) + df.select(col("a"), is_null_arr(col("a")).alias("is_null")).show() + +Aggregate Functions +------------------- + +The :py:func:`~datafusion.udf.AggregateUDF.udaf` function allows you to define User-Defined +Aggregate Functions (UDAFs). To use this you must implement an +:py:class:`~datafusion.udf.Accumulator` that determines how the aggregation is performed. + +When defining a UDAF there are four methods you need to implement. The ``update`` function takes the +array(s) of input and updates the internal state of the accumulator. You should define this function +to have as many input arguments as you will pass when calling the UDAF. Since aggregation may be +split into multiple batches, we must have a method to combine multiple batches. For this, we have +two functions, ``state`` and ``merge``. ``state`` will return an array of scalar values that contain +the current state of a single batch accumulation. Then we must ``merge`` the results of these +different states. Finally ``evaluate`` is the call that will return the final result after the +``merge`` is complete. + +In the following example we want to define a custom aggregate function that will return the +difference between the sum of two columns. The state can be represented by a single value and we can +also see how the inputs to ``update`` and ``merge`` differ. .. code-block:: python @@ -57,30 +123,122 @@ Additionally the :py:func:`~datafusion.udf.AggregateUDF.udaf` function allows yo Interface of a user-defined accumulation. """ def __init__(self): - self._sum = pyarrow.scalar(0.0) + self._sum = 0.0 - def update(self, values: pyarrow.Array) -> None: - # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(values).as_py()) + def update(self, values_a: pyarrow.Array, values_b: pyarrow.Array) -> None: + self._sum = self._sum + pyarrow.compute.sum(values_a).as_py() - pyarrow.compute.sum(values_b).as_py() def merge(self, states: List[pyarrow.Array]) -> None: - # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar(self._sum.as_py() + pyarrow.compute.sum(states[0]).as_py()) + self._sum = self._sum + pyarrow.compute.sum(states[0]).as_py() def state(self) -> pyarrow.Array: - return pyarrow.array([self._sum.as_py()]) + return pyarrow.array([self._sum]) def evaluate(self) -> pyarrow.Scalar: - return self._sum + return pyarrow.scalar(self._sum) ctx = datafusion.SessionContext() df = ctx.from_pydict( { - "a": [1, 2, 3], - "b": [4, 5, 6], + "a": [4, 5, 6], + "b": [1, 2, 3], } ) - my_udaf = udaf(MyAccumulator, pyarrow.float64(), pyarrow.float64(), [pyarrow.float64()], 'stable') + my_udaf = udaf(MyAccumulator, [pyarrow.float64(), pyarrow.float64()], pyarrow.float64(), [pyarrow.float64()], 'stable') + + df.aggregate([], [my_udaf(col("a"), col("b")).alias("col_diff")]) + +Window Functions +---------------- + +To implement a User-Defined Window Function (UDWF) you must call the +:py:func:`~datafusion.udf.WindowUDF.udwf` function using a class that implements the abstract +class :py:class:`~datafusion.udf.WindowEvaluator`. + +There are three methods of evaluation of UDWFs. + +- ``evaluate`` is the simplest case, where you are given an array and are expected to calculate the + value for a single row of that array. This is the simplest case, but also the least performant. +- ``evaluate_all`` computes the values for all rows for an input array at a single time. +- ``evaluate_all_with_rank`` computes the values for all rows, but you only have the rank + information for the rows. + +Which methods you implement are based upon which of these options are set. + +.. list-table:: + :header-rows: 1 + + * - ``uses_window_frame`` + - ``supports_bounded_execution`` + - ``include_rank`` + - function_to_implement + * - False (default) + - False (default) + - False (default) + - ``evaluate_all`` + * - False + - True + - False + - ``evaluate`` + * - False + - True + - False + - ``evaluate_all_with_rank`` + * - True + - True/False + - True/False + - ``evaluate`` + +UDWF options +^^^^^^^^^^^^ + +When you define your UDWF you can override the functions that return these values. They will +determine which evaluate functions are called. + +- ``uses_window_frame`` is set for functions that compute based on the specified window frame. If + your function depends upon the specified frame, set this to ``True``. +- ``supports_bounded_execution`` specifies if your function can be incrementally computed. +- ``include_rank`` is set to ``True`` for window functions that can be computed only using the rank + information. + + +.. code-block:: python + + import pyarrow as pa + from datafusion import udwf, col, SessionContext + from datafusion.udf import WindowEvaluator + + class ExponentialSmooth(WindowEvaluator): + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + results = [] + curr_value = 0.0 + values = values[0] + for idx in range(num_rows): + if idx == 0: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + results.append(curr_value) + + return pa.array(results) + + exp_smooth = udwf( + ExponentialSmooth(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", + ) + + ctx = SessionContext() + + df = ctx.from_pydict({ + "a": [1.0, 2.1, 2.9, 4.0, 5.1, 6.0, 6.9, 8.0] + }) - df.aggregate([],[my_udaf(col("a"))]) + df.select("a", exp_smooth(col("a")).alias("smooth_a")).show() diff --git a/examples/python-udwf.py b/examples/python-udwf.py new file mode 100644 index 000000000..05b3021d8 --- /dev/null +++ b/examples/python-udwf.py @@ -0,0 +1,270 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pyarrow as pa +import datafusion +from datafusion import udwf, functions as f, col, lit +from datafusion.udf import WindowEvaluator +from datafusion.expr import WindowFrame + +# This example creates five different examples of user defined window functions in order +# to demonstrate the variety of ways a user may need to implement. + + +class ExponentialSmoothDefault(WindowEvaluator): + """Create a running smooth operation across an entire partition at once.""" + + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + results = [] + curr_value = 0.0 + values = values[0] + for idx in range(num_rows): + if idx == 0: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + results.append(curr_value) + + return pa.array(results) + + +class SmoothBoundedFromPreviousRow(WindowEvaluator): + """Smooth over from the previous to current row only.""" + + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def supports_bounded_execution(self) -> bool: + return True + + def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: + # Override the default range of current row since uses_window_frame is False + # So for the purpose of this test we just smooth from the previous row to + # current. + if idx == 0: + return (0, 0) + return (idx - 1, idx) + + def evaluate( + self, values: list[pa.Array], eval_range: tuple[int, int] + ) -> pa.Scalar: + (start, stop) = eval_range + curr_value = 0.0 + values = values[0] + for idx in range(start, stop + 1): + if idx == start: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + return pa.scalar(curr_value).cast(pa.float64()) + + +class SmoothAcrossRank(WindowEvaluator): + """Smooth over the rank from the previous rank to current.""" + + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def include_rank(self) -> bool: + return True + + def evaluate_all_with_rank( + self, num_rows: int, ranks_in_partition: list[tuple[int, int]] + ) -> pa.Array: + results = [] + for idx in range(num_rows): + if idx == 0: + prior_value = 1.0 + matching_row = [ + i + for i in range(len(ranks_in_partition)) + if ranks_in_partition[i][0] <= idx and ranks_in_partition[i][1] > idx + ][0] + 1 + curr_value = matching_row * self.alpha + prior_value * (1.0 - self.alpha) + results.append(curr_value) + prior_value = matching_row + + return pa.array(results) + + +class ExponentialSmoothFrame(WindowEvaluator): + "Find the value across an entire frame using exponential smoothing" + + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def uses_window_frame(self) -> bool: + return True + + def evaluate( + self, values: list[pa.Array], eval_range: tuple[int, int] + ) -> pa.Scalar: + (start, stop) = eval_range + curr_value = 0.0 + if len(values) > 1: + order_by = values[1] # noqa: F841 + values = values[0] + else: + values = values[0] + for idx in range(start, stop): + if idx == start: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + return pa.scalar(curr_value).cast(pa.float64()) + + +class SmoothTwoColumn(WindowEvaluator): + """Smooth once column based on a condition of another column. + + If the second column is above a threshold, then smooth over the first column from + the previous and next rows. + """ + + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + results = [] + values_a = values[0] + values_b = values[1] + for idx in range(num_rows): + if not values_b[idx].is_valid: + if idx == 0: + results.append(values_a[1].cast(pa.float64())) + elif idx == num_rows - 1: + results.append(values_a[num_rows - 2].cast(pa.float64())) + else: + results.append( + pa.scalar( + values_a[idx - 1].as_py() * self.alpha + + values_a[idx + 1].as_py() * (1.0 - self.alpha) + ) + ) + else: + results.append(values_a[idx].cast(pa.float64())) + + return pa.array(results) + + +# create a context +ctx = datafusion.SessionContext() + +# create a RecordBatch and a new DataFrame from it +batch = pa.RecordBatch.from_arrays( + [ + pa.array([1.0, 2.1, 2.9, 4.0, 5.1, 6.0, 6.9, 8.0]), + pa.array([1, 2, None, 4, 5, 6, None, 8]), + pa.array(["A", "A", "A", "A", "A", "B", "B", "B"]), + ], + names=["a", "b", "c"], +) +df = ctx.create_dataframe([[batch]]) + +exp_smooth = udwf( + ExponentialSmoothDefault(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_two_row = udwf( + SmoothBoundedFromPreviousRow(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_rank = udwf( + SmoothAcrossRank(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_frame = udwf( + ExponentialSmoothFrame(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_two_col = udwf( + SmoothTwoColumn(0.9), + [pa.float64(), pa.int64()], + pa.float64(), + volatility="immutable", +) + +# These are done with separate statements instead of one large `select` because that will +# attempt to combine the window operations and our defined UDFs do not all support that. +( + df.with_column("exp_smooth", exp_smooth(col("a"))) + .with_column("smooth_prior_row", smooth_two_row(col("a"))) + .with_column("smooth_rank", smooth_rank(col("a")).order_by(col("c")).build()) + .with_column("smooth_two_col", smooth_two_col(col("a"), col("b"))) + .with_column( + "smooth_frame", + smooth_frame(col("a")).window_frame(WindowFrame("rows", None, 0)).build(), + ) + .select( + "a", + "b", + "c", + "exp_smooth", + "smooth_prior_row", + "smooth_rank", + "smooth_two_col", + "smooth_frame", + ) +).show() + +assert df.select(f.round(exp_smooth(col("a")), lit(3))).collect()[0].column( + 0 +) == pa.array([1, 1.99, 2.809, 3.881, 4.978, 5.898, 6.8, 7.88]) + + +assert df.select(f.round(smooth_two_row(col("a")), lit(3))).collect()[0].column( + 0 +) == pa.array([1.0, 1.99, 2.82, 3.89, 4.99, 5.91, 6.81, 7.89]) + + +assert df.select(smooth_rank(col("a")).order_by(col("c")).build()).collect()[0].column( + 0 +) == pa.array([1, 1, 1, 1, 1, 1.9, 2.0, 2.0]) + + +assert df.select(smooth_two_col(col("a"), col("b"))).collect()[0].column(0) == pa.array( + [1, 2.1, 2.29, 4, 5.1, 6, 6.2, 8.0] +) + + +assert df.select( + f.round( + smooth_frame(col("a")).window_frame(WindowFrame("rows", None, 0)).build(), + lit(3), + ) +).collect()[0].column(0) == pa.array([1, 1.99, 2.809, 3.881, 4.978, 5.898, 6.8, 7.88]) diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 08ca3fe02..4f40b2088 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -40,7 +40,7 @@ from .record_batch import RecordBatchStream, RecordBatch -from .udf import ScalarUDF, AggregateUDF, Accumulator +from .udf import ScalarUDF, AggregateUDF, Accumulator, WindowUDF from .common import ( DFSchema, @@ -78,6 +78,7 @@ "Database", "Table", "AggregateUDF", + "WindowUDF", "LogicalPlan", "ExecutionPlan", "RecordBatch", @@ -113,3 +114,5 @@ def lit(value): udf = ScalarUDF.udf udaf = AggregateUDF.udaf + +udwf = WindowUDF.udwf diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 2c41faba6..5b52d397b 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -25,12 +25,11 @@ from ._internal import SessionContext as SessionContextInternal from ._internal import LogicalPlan, ExecutionPlan -from datafusion._internal import AggregateUDF from datafusion.catalog import Catalog, Table from datafusion.dataframe import DataFrame from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list from datafusion.record_batch import RecordBatchStream -from datafusion.udf import ScalarUDF +from datafusion.udf import ScalarUDF, AggregateUDF, WindowUDF from typing import Any, TYPE_CHECKING from typing_extensions import deprecated @@ -838,6 +837,10 @@ def register_udaf(self, udaf: AggregateUDF) -> None: """Register a user-defined aggregation function (UDAF) with the context.""" self.ctx.register_udaf(udaf._udaf) + def register_udwf(self, udwf: WindowUDF) -> None: + """Register a user-defined window function (UDWF) with the context.""" + self.ctx.register_udwf(udwf._udwf) + def catalog(self, name: str = "datafusion") -> Catalog: """Retrieve a catalog by name.""" return self.ctx.catalog(name) diff --git a/python/datafusion/tests/test_udwf.py b/python/datafusion/tests/test_udwf.py new file mode 100644 index 000000000..67c0979fe --- /dev/null +++ b/python/datafusion/tests/test_udwf.py @@ -0,0 +1,294 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pyarrow as pa +import pytest + +from datafusion import SessionContext, column, udwf, lit, functions as f +from datafusion.udf import WindowEvaluator +from datafusion.expr import WindowFrame + + +class ExponentialSmoothDefault(WindowEvaluator): + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + results = [] + curr_value = 0.0 + values = values[0] + for idx in range(num_rows): + if idx == 0: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + results.append(curr_value) + + return pa.array(results) + + +class ExponentialSmoothBounded(WindowEvaluator): + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def supports_bounded_execution(self) -> bool: + return True + + def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: + # Override the default range of current row since uses_window_frame is False + # So for the purpose of this test we just smooth from the previous row to + # current. + if idx == 0: + return (0, 0) + return (idx - 1, idx) + + def evaluate( + self, values: list[pa.Array], eval_range: tuple[int, int] + ) -> pa.Scalar: + (start, stop) = eval_range + curr_value = 0.0 + values = values[0] + for idx in range(start, stop + 1): + if idx == start: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + return pa.scalar(curr_value).cast(pa.float64()) + + +class ExponentialSmoothRank(WindowEvaluator): + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def include_rank(self) -> bool: + return True + + def evaluate_all_with_rank( + self, num_rows: int, ranks_in_partition: list[tuple[int, int]] + ) -> pa.Array: + results = [] + for idx in range(num_rows): + if idx == 0: + prior_value = 1.0 + matching_row = [ + i + for i in range(len(ranks_in_partition)) + if ranks_in_partition[i][0] <= idx and ranks_in_partition[i][1] > idx + ][0] + 1 + curr_value = matching_row * self.alpha + prior_value * (1.0 - self.alpha) + results.append(curr_value) + prior_value = matching_row + + return pa.array(results) + + +class ExponentialSmoothFrame(WindowEvaluator): + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def uses_window_frame(self) -> bool: + return True + + def evaluate( + self, values: list[pa.Array], eval_range: tuple[int, int] + ) -> pa.Scalar: + (start, stop) = eval_range + curr_value = 0.0 + if len(values) > 1: + order_by = values[1] # noqa: F841 + values = values[0] + else: + values = values[0] + for idx in range(start, stop): + if idx == start: + curr_value = values[idx].as_py() + else: + curr_value = values[idx].as_py() * self.alpha + curr_value * ( + 1.0 - self.alpha + ) + return pa.scalar(curr_value).cast(pa.float64()) + + +class SmoothTwoColumn(WindowEvaluator): + """This class demonstrates using two columns. + + If the second column is above a threshold, then smooth over the first column from + the previous and next rows. + """ + + def __init__(self, alpha: float) -> None: + self.alpha = alpha + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + results = [] + values_a = values[0] + values_b = values[1] + for idx in range(num_rows): + if values_b[idx].as_py() > 7: + if idx == 0: + results.append(values_a[1].cast(pa.float64())) + elif idx == num_rows - 1: + results.append(values_a[num_rows - 2].cast(pa.float64())) + else: + results.append( + pa.scalar( + values_a[idx - 1].as_py() * self.alpha + + values_a[idx + 1].as_py() * (1.0 - self.alpha) + ) + ) + else: + results.append(values_a[idx].cast(pa.float64())) + + return pa.array(results) + + +class NotSubclassOfWindowEvaluator: + pass + + +@pytest.fixture +def df(): + ctx = SessionContext() + + # create a RecordBatch and a new DataFrame from it + batch = pa.RecordBatch.from_arrays( + [ + pa.array([0, 1, 2, 3, 4, 5, 6]), + pa.array([7, 4, 3, 8, 9, 1, 6]), + pa.array(["A", "A", "A", "A", "B", "B", "B"]), + ], + names=["a", "b", "c"], + ) + return ctx.create_dataframe([[batch]]) + + +def test_udwf_errors(df): + with pytest.raises(TypeError): + udwf( + NotSubclassOfWindowEvaluator(), + pa.float64(), + pa.float64(), + volatility="immutable", + ) + + +smooth_default = udwf( + ExponentialSmoothDefault(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_bounded = udwf( + ExponentialSmoothBounded(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_rank = udwf( + ExponentialSmoothRank(0.9), + pa.utf8(), + pa.float64(), + volatility="immutable", +) + +smooth_frame = udwf( + ExponentialSmoothFrame(0.9), + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_two_col = udwf( + SmoothTwoColumn(0.9), + [pa.int64(), pa.int64()], + pa.float64(), + volatility="immutable", +) + +data_test_udwf_functions = [ + ( + "default_udwf", + smooth_default(column("a")), + [0, 0.9, 1.89, 2.889, 3.889, 4.889, 5.889], + ), + ( + "default_udwf_partitioned", + smooth_default(column("a")).partition_by(column("c")).build(), + [0, 0.9, 1.89, 2.889, 4.0, 4.9, 5.89], + ), + ( + "default_udwf_ordered", + smooth_default(column("a")).order_by(column("b")).build(), + [0.551, 1.13, 2.3, 2.755, 3.876, 5.0, 5.513], + ), + ( + "bounded_udwf", + smooth_bounded(column("a")), + [0, 0.9, 1.9, 2.9, 3.9, 4.9, 5.9], + ), + ( + "bounded_udwf_ignores_frame", + smooth_bounded(column("a")) + .window_frame(WindowFrame("rows", None, None)) + .build(), + [0, 0.9, 1.9, 2.9, 3.9, 4.9, 5.9], + ), + ( + "rank_udwf", + smooth_rank(column("c")).order_by(column("c")).build(), + [1, 1, 1, 1, 1.9, 2, 2], + ), + ( + "frame_unbounded_udwf", + smooth_frame(column("a")).window_frame(WindowFrame("rows", None, None)).build(), + [5.889, 5.889, 5.889, 5.889, 5.889, 5.889, 5.889], + ), + ( + "frame_bounded_udwf", + smooth_frame(column("a")).window_frame(WindowFrame("rows", None, 0)).build(), + [0.0, 0.9, 1.89, 2.889, 3.889, 4.889, 5.889], + ), + ( + "frame_bounded_udwf", + smooth_frame(column("a")) + .window_frame(WindowFrame("rows", None, 0)) + .order_by(column("b")) + .build(), + [0.551, 1.13, 2.3, 2.755, 3.876, 5.0, 5.513], + ), + ( + "two_column_udwf", + smooth_two_col(column("a"), column("b")), + [0.0, 1.0, 2.0, 2.2, 3.2, 5.0, 6.0], + ), +] + + +@pytest.mark.parametrize("name,expr,expected", data_test_udwf_functions) +def test_udwf_functions(df, name, expr, expected): + df = df.select("a", "b", f.round(expr, lit(3)).alias(name)) + + # execute and collect the first (and only) batch + result = df.sort(column("a")).select(column(name)).collect()[0] + + assert result.column(0) == pa.array(expected) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index f74d675e3..bb7a90866 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Provides the user defined functions for evaluation of dataframes.""" +"""Provides the user-defined functions for evaluation of dataframes.""" from __future__ import annotations @@ -76,7 +76,7 @@ def __str__(self): class ScalarUDF: - """Class for performing scalar user defined functions (UDF). + """Class for performing scalar user-defined functions (UDF). Scalar UDFs operate on a row by row basis. See also :py:class:`AggregateUDF` for operating on a group of rows. @@ -90,7 +90,7 @@ def __init__( return_type: _R, volatility: Volatility | str, ) -> None: - """Instantiate a scalar user defined function (UDF). + """Instantiate a scalar user-defined function (UDF). See helper method :py:func:`udf` for argument details. """ @@ -115,7 +115,7 @@ def udf( volatility: Volatility | str, name: str | None = None, ) -> ScalarUDF: - """Create a new User Defined Function. + """Create a new User-Defined Function. Args: func: A callable python function. @@ -127,7 +127,7 @@ def udf( name: A descriptive name for the function. Returns: - A user defined aggregate function, which can be used in either data + A user-defined aggregate function, which can be used in either data aggregation or window function calls. """ if not callable(func): @@ -152,7 +152,7 @@ def state(self) -> List[pyarrow.Scalar]: pass @abstractmethod - def update(self, values: pyarrow.Array) -> None: + def update(self, *values: pyarrow.Array) -> None: """Evaluate an array of values and update state.""" pass @@ -172,7 +172,7 @@ def evaluate(self) -> pyarrow.Scalar: class AggregateUDF: - """Class for performing scalar user defined functions (UDF). + """Class for performing scalar user-defined functions (UDF). Aggregate UDFs operate on a group of rows and return a single value. See also :py:class:`ScalarUDF` for operating on a row by row basis. @@ -187,7 +187,7 @@ def __init__( state_type: list[pyarrow.DataType], volatility: Volatility | str, ) -> None: - """Instantiate a user defined aggregate function (UDAF). + """Instantiate a user-defined aggregate function (UDAF). See :py:func:`udaf` for a convenience function and argument descriptions. @@ -214,7 +214,7 @@ def udaf( volatility: Volatility | str, name: str | None = None, ) -> AggregateUDF: - """Create a new User Defined Aggregate Function. + """Create a new User-Defined Aggregate Function. The accumulator function must be callable and implement :py:class:`Accumulator`. @@ -227,7 +227,7 @@ def udaf( name: A descriptive name for the function. Returns: - A user defined aggregate function, which can be used in either data + A user-defined aggregate function, which can be used in either data aggregation or window function calls. """ if not issubclass(accum, Accumulator): @@ -246,3 +246,237 @@ def udaf( state_type=state_type, volatility=volatility, ) + + +class WindowEvaluator(metaclass=ABCMeta): + """Evaluator class for user-defined window functions (UDWF). + + It is up to the user to decide which evaluate function is appropriate. + + +------------------------+--------------------------------+------------------+---------------------------+ + | ``uses_window_frame`` | ``supports_bounded_execution`` | ``include_rank`` | function_to_implement | + +========================+================================+==================+===========================+ + | False (default) | False (default) | False (default) | ``evaluate_all`` | + +------------------------+--------------------------------+------------------+---------------------------+ + | False | True | False | ``evaluate`` | + +------------------------+--------------------------------+------------------+---------------------------+ + | False | True/False | True | ``evaluate_all_with_rank``| + +------------------------+--------------------------------+------------------+---------------------------+ + | True | True/False | True/False | ``evaluate`` | + +------------------------+--------------------------------+------------------+---------------------------+ + """ # noqa: W505 + + def memoize(self) -> None: + """Perform a memoize operation to improve performance. + + When the window frame has a fixed beginning (e.g UNBOUNDED + PRECEDING), some functions such as FIRST_VALUE and + NTH_VALUE do not need the (unbounded) input once they have + seen a certain amount of input. + + `memoize` is called after each input batch is processed, and + such functions can save whatever they need + """ + pass + + def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: + """Return the range for the window fuction. + + If `uses_window_frame` flag is `false`. This method is used to + calculate required range for the window function during + stateful execution. + + Generally there is no required range, hence by default this + returns smallest range(current row). e.g seeing current row is + enough to calculate window result (such as row_number, rank, + etc) + + Args: + idx:: Current index + num_rows: Number of rows. + """ + return (idx, idx + 1) + + def is_causal(self) -> bool: + """Get whether evaluator needs future data for its result.""" + return False + + def evaluate_all(self, values: list[pyarrow.Array], num_rows: int) -> pyarrow.Array: + """Evaluate a window function on an entire input partition. + + This function is called once per input *partition* for window functions that + *do not use* values from the window frame, such as + :py:func:`~datafusion.functions.row_number`, :py:func:`~datafusion.functions.rank`, + :py:func:`~datafusion.functions.dense_rank`, :py:func:`~datafusion.functions.percent_rank`, + :py:func:`~datafusion.functions.cume_dist`, :py:func:`~datafusion.functions.lead`, + and :py:func:`~datafusion.functions.lag`. + + It produces the result of all rows in a single pass. It + expects to receive the entire partition as the ``value`` and + must produce an output column with one output row for every + input row. + + ``num_rows`` is required to correctly compute the output in case + ``len(values) == 0`` + + Implementing this function is an optimization. Certain window + functions are not affected by the window frame definition or + the query doesn't have a frame, and ``evaluate`` skips the + (costly) window frame boundary calculation and the overhead of + calling ``evaluate`` for each output row. + + For example, the `LAG` built in window function does not use + the values of its window frame (it can be computed in one shot + on the entire partition with ``Self::evaluate_all`` regardless of the + window defined in the ``OVER`` clause) + + .. code-block:: text + + lag(x, 1) OVER (ORDER BY z ROWS BETWEEN 2 PRECEDING AND 3 FOLLOWING) + + However, ``avg()`` computes the average in the window and thus + does use its window frame. + + .. code-block:: text + + avg(x) OVER (PARTITION BY y ORDER BY z ROWS BETWEEN 2 PRECEDING AND 3 FOLLOWING) + """ # noqa: W505 + pass + + def evaluate( + self, values: list[pyarrow.Array], eval_range: tuple[int, int] + ) -> pyarrow.Scalar: + """Evaluate window function on a range of rows in an input partition. + + This is the simplest and most general function to implement + but also the least performant as it creates output one row at + a time. It is typically much faster to implement stateful + evaluation using one of the other specialized methods on this + trait. + + Returns a [`ScalarValue`] that is the value of the window + function within `range` for the entire partition. Argument + `values` contains the evaluation result of function arguments + and evaluation results of ORDER BY expressions. If function has a + single argument, `values[1..]` will contain ORDER BY expression results. + """ + pass + + def evaluate_all_with_rank( + self, num_rows: int, ranks_in_partition: list[tuple[int, int]] + ) -> pyarrow.Array: + """Called for window functions that only need the rank of a row. + + Evaluate the partition evaluator against the partition using + the row ranks. For example, ``rank(col("a"))`` produces + + .. code-block:: text + + a | rank + - + ---- + A | 1 + A | 1 + C | 3 + D | 4 + D | 4 + + For this case, `num_rows` would be `5` and the + `ranks_in_partition` would be called with + + .. code-block:: text + + [ + (0,1), + (2,2), + (3,4), + ] + + The user must implement this method if ``include_rank`` returns True. + """ + pass + + def supports_bounded_execution(self) -> bool: + """Can the window function be incrementally computed using bounded memory?""" + return False + + def uses_window_frame(self) -> bool: + """Does the window function use the values from the window frame?""" + return False + + def include_rank(self) -> bool: + """Can this function be evaluated with (only) rank?""" + return False + + +if TYPE_CHECKING: + _W = TypeVar("_W", bound=WindowEvaluator) + + +class WindowUDF: + """Class for performing window user-defined functions (UDF). + + Window UDFs operate on a partition of rows. See + also :py:class:`ScalarUDF` for operating on a row by row basis. + """ + + def __init__( + self, + name: str | None, + func: WindowEvaluator, + input_types: list[pyarrow.DataType], + return_type: pyarrow.DataType, + volatility: Volatility | str, + ) -> None: + """Instantiate a user-defined window function (UDWF). + + See :py:func:`udwf` for a convenience function and argument + descriptions. + """ + self._udwf = df_internal.WindowUDF( + name, func, input_types, return_type, str(volatility) + ) + + def __call__(self, *args: Expr) -> Expr: + """Execute the UDWF. + + This function is not typically called by an end user. These calls will + occur during the evaluation of the dataframe. + """ + args_raw = [arg.expr for arg in args] + return Expr(self._udwf.__call__(*args_raw)) + + @staticmethod + def udwf( + func: WindowEvaluator, + input_types: pyarrow.DataType | list[pyarrow.DataType], + return_type: pyarrow.DataType, + volatility: Volatility | str, + name: str | None = None, + ) -> WindowUDF: + """Create a new User-Defined Window Function. + + Args: + func: The python function. + input_types: The data types of the arguments to ``func``. + return_type: The data type of the return value. + volatility: See :py:class:`Volatility` for allowed values. + name: A descriptive name for the function. + + Returns: + A user-defined window function. + """ + if not isinstance(func, WindowEvaluator): + raise TypeError( + "`func` must implement the abstract base class WindowEvaluator" + ) + if name is None: + name = func.__class__.__qualname__.lower() + if isinstance(input_types, pyarrow.DataType): + input_types = [input_types] + return WindowUDF( + name=name, + func=func, + input_types=input_types, + return_type=return_type, + volatility=volatility, + ) diff --git a/src/context.rs b/src/context.rs index 7ad12ceb0..fde442ce4 100644 --- a/src/context.rs +++ b/src/context.rs @@ -42,6 +42,7 @@ use crate::sql::logical::PyLogicalPlan; use crate::store::StorageContexts; use crate::udaf::PyAggregateUDF; use crate::udf::PyScalarUDF; +use crate::udwf::PyWindowUDF; use crate::utils::{get_tokio_runtime, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; @@ -754,6 +755,11 @@ impl PySessionContext { Ok(()) } + pub fn register_udwf(&mut self, udwf: PyWindowUDF) -> PyResult<()> { + self.ctx.register_udwf(udwf.function); + Ok(()) + } + #[pyo3(signature = (name="datafusion"))] pub fn catalog(&self, name: &str) -> PyResult { match self.ctx.catalog(name) { diff --git a/src/lib.rs b/src/lib.rs index e4cc24078..98821833d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -58,6 +58,7 @@ pub mod substrait; mod udaf; #[allow(clippy::borrow_deref_ref)] mod udf; +mod udwf; pub mod utils; #[cfg(feature = "mimalloc")] @@ -90,6 +91,7 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/udwf.rs b/src/udwf.rs new file mode 100644 index 000000000..31cc5e60e --- /dev/null +++ b/src/udwf.rs @@ -0,0 +1,305 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::any::Any; +use std::ops::Range; +use std::sync::Arc; + +use arrow::array::{make_array, Array, ArrayData, ArrayRef}; +use datafusion::logical_expr::window_state::WindowAggState; +use datafusion::scalar::ScalarValue; +use pyo3::exceptions::PyValueError; +use pyo3::prelude::*; + +use datafusion::arrow::datatypes::DataType; +use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; +use datafusion::error::{DataFusionError, Result}; +use datafusion::logical_expr::{ + PartitionEvaluator, PartitionEvaluatorFactory, Signature, Volatility, WindowUDF, WindowUDFImpl, +}; +use pyo3::types::{PyList, PyTuple}; + +use crate::expr::PyExpr; +use crate::utils::parse_volatility; + +#[derive(Debug)] +struct RustPartitionEvaluator { + evaluator: PyObject, +} + +impl RustPartitionEvaluator { + fn new(evaluator: PyObject) -> Self { + Self { evaluator } + } +} + +impl PartitionEvaluator for RustPartitionEvaluator { + fn memoize(&mut self, _state: &mut WindowAggState) -> Result<()> { + Python::with_gil(|py| self.evaluator.bind(py).call_method0("memoize").map(|_| ())) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) + } + + fn get_range(&self, idx: usize, n_rows: usize) -> Result> { + Python::with_gil(|py| { + let py_args = vec![idx.to_object(py), n_rows.to_object(py)]; + let py_args = PyTuple::new_bound(py, py_args); + + self.evaluator + .bind(py) + .call_method1("get_range", py_args) + .and_then(|v| { + let tuple: Bound<'_, PyTuple> = v.extract()?; + if tuple.len() != 2 { + return Err(PyValueError::new_err(format!( + "Expected get_range to return tuple of length 2. Received length {}", + tuple.len() + ))); + } + + let start: usize = tuple.get_item(0).unwrap().extract()?; + let end: usize = tuple.get_item(1).unwrap().extract()?; + + Ok(Range { start, end }) + }) + }) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) + } + + fn is_causal(&self) -> bool { + Python::with_gil(|py| { + self.evaluator + .bind(py) + .call_method0("is_causal") + .and_then(|v| v.extract()) + .unwrap_or(false) + }) + } + + fn evaluate_all(&mut self, values: &[ArrayRef], num_rows: usize) -> Result { + Python::with_gil(|py| { + let py_values = PyList::new_bound( + py, + values + .iter() + .map(|arg| arg.into_data().to_pyarrow(py).unwrap()), + ); + let py_num_rows = num_rows.to_object(py).into_bound(py); + let py_args = PyTuple::new_bound( + py, + PyTuple::new_bound(py, vec![py_values.as_any(), &py_num_rows]), + ); + + self.evaluator + .bind(py) + .call_method1("evaluate_all", py_args) + .map(|v| { + let array_data = ArrayData::from_pyarrow_bound(&v).unwrap(); + make_array(array_data) + }) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) + }) + } + + fn evaluate(&mut self, values: &[ArrayRef], range: &Range) -> Result { + Python::with_gil(|py| { + let py_values = PyList::new_bound( + py, + values + .iter() + .map(|arg| arg.into_data().to_pyarrow(py).unwrap()), + ); + let range_tuple = + PyTuple::new_bound(py, vec![range.start.to_object(py), range.end.to_object(py)]); + let py_args = PyTuple::new_bound( + py, + PyTuple::new_bound(py, vec![py_values.as_any(), range_tuple.as_any()]), + ); + + self.evaluator + .bind(py) + .call_method1("evaluate", py_args) + .and_then(|v| v.extract()) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) + }) + } + + fn evaluate_all_with_rank( + &self, + num_rows: usize, + ranks_in_partition: &[Range], + ) -> Result { + Python::with_gil(|py| { + let ranks = ranks_in_partition + .iter() + .map(|r| PyTuple::new_bound(py, vec![r.start, r.end])); + + // 1. cast args to Pyarrow array + let py_args = vec![num_rows.to_object(py), PyList::new_bound(py, ranks).into()]; + + let py_args = PyTuple::new_bound(py, py_args); + + // 2. call function + self.evaluator + .bind(py) + .call_method1("evaluate_all_with_rank", py_args) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) + .map(|v| { + let array_data = ArrayData::from_pyarrow_bound(&v).unwrap(); + make_array(array_data) + }) + }) + } + + fn supports_bounded_execution(&self) -> bool { + Python::with_gil(|py| { + self.evaluator + .bind(py) + .call_method0("supports_bounded_execution") + .and_then(|v| v.extract()) + .unwrap_or(false) + }) + } + + fn uses_window_frame(&self) -> bool { + Python::with_gil(|py| { + self.evaluator + .bind(py) + .call_method0("uses_window_frame") + .and_then(|v| v.extract()) + .unwrap_or(false) + }) + } + + fn include_rank(&self) -> bool { + Python::with_gil(|py| { + self.evaluator + .bind(py) + .call_method0("include_rank") + .and_then(|v| v.extract()) + .unwrap_or(false) + }) + } +} + +pub fn to_rust_partition_evaluator(evaluator: PyObject) -> PartitionEvaluatorFactory { + Arc::new(move || -> Result> { + let evaluator = Python::with_gil(|py| evaluator.clone_ref(py)); + Ok(Box::new(RustPartitionEvaluator::new(evaluator))) + }) +} + +/// Represents an WindowUDF +#[pyclass(name = "WindowUDF", module = "datafusion", subclass)] +#[derive(Debug, Clone)] +pub struct PyWindowUDF { + pub(crate) function: WindowUDF, +} + +#[pymethods] +impl PyWindowUDF { + #[new] + #[pyo3(signature=(name, evaluator, input_types, return_type, volatility))] + fn new( + name: &str, + evaluator: PyObject, + input_types: Vec>, + return_type: PyArrowType, + volatility: &str, + ) -> PyResult { + let return_type = return_type.0; + let input_types = input_types.into_iter().map(|t| t.0).collect(); + + let function = WindowUDF::from(MultiColumnWindowUDF::new( + name, + input_types, + return_type, + parse_volatility(volatility)?, + to_rust_partition_evaluator(evaluator), + )); + Ok(Self { function }) + } + + /// creates a new PyExpr with the call of the udf + #[pyo3(signature = (*args))] + fn __call__(&self, args: Vec) -> PyResult { + let args = args.iter().map(|e| e.expr.clone()).collect(); + Ok(self.function.call(args).into()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("WindowUDF({})", self.function.name())) + } +} + +pub struct MultiColumnWindowUDF { + name: String, + signature: Signature, + return_type: DataType, + partition_evaluator_factory: PartitionEvaluatorFactory, +} + +impl std::fmt::Debug for MultiColumnWindowUDF { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.debug_struct("WindowUDF") + .field("name", &self.name) + .field("signature", &self.signature) + .field("return_type", &"") + .field("partition_evaluator_factory", &"") + .finish() + } +} + +impl MultiColumnWindowUDF { + pub fn new( + name: impl Into, + input_types: Vec, + return_type: DataType, + volatility: Volatility, + partition_evaluator_factory: PartitionEvaluatorFactory, + ) -> Self { + let name = name.into(); + let signature = Signature::exact(input_types, volatility); + Self { + name, + signature, + return_type, + partition_evaluator_factory, + } + } +} + +impl WindowUDFImpl for MultiColumnWindowUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + &self.name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + Ok(self.return_type.clone()) + } + + fn partition_evaluator(&self) -> Result> { + (self.partition_evaluator_factory)() + } +} From c49ca9ae299ec14048b5f44c81ff5ccd11ea11ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 13:39:43 -0500 Subject: [PATCH 209/413] build(deps): bump syn from 2.0.77 to 2.0.79 (#886) Bumps [syn](https://github.com/dtolnay/syn) from 2.0.77 to 2.0.79. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/2.0.77...2.0.79) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e48339e98..da5d19a93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3077,9 +3077,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 11dccc4fb..54e53f4b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ futures = "0.3" object_store = { version = "0.11.0", features = ["aws", "gcp", "azure"] } parking_lot = "0.12" regex-syntax = "0.8" -syn = "2.0.68" +syn = "2.0.79" url = "2" [build-dependencies] From 97e330b9469f948da00e777d9e089e0181c1d5bf Mon Sep 17 00:00:00 2001 From: Sergey Fedoseev Date: Fri, 4 Oct 2024 14:37:07 +0200 Subject: [PATCH 210/413] fix example of reading parquet from s3 (#896) --- examples/sql-parquet-s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/sql-parquet-s3.py b/examples/sql-parquet-s3.py index bd7da5e20..61f1e0c50 100644 --- a/examples/sql-parquet-s3.py +++ b/examples/sql-parquet-s3.py @@ -31,7 +31,7 @@ ctx = datafusion.SessionContext() path = f"s3://{bucket_name}/" -ctx.register_object_store(path, s3) +ctx.register_object_store("s3://", s3, None) ctx.register_parquet("trips", path) From 5b9e528b1120676eb224c0581ece928f58f60d14 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 4 Oct 2024 09:36:57 -0500 Subject: [PATCH 211/413] release-testing (#889) * make script shebangs more flexible * move location of python tests This change allows us to run the test suite against a release candidate hosted on test.pypi.org. ``` pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0 pytest --import-mode=importlib python/tests ``` * add documentation for testing against release candidate * add dev script for cleaning maturin build artifacts * update ruff lint config for new test location --- dev/clean.sh | 62 +++++++++++++++++++ dev/python_lint.sh | 2 +- dev/release/README.md | 23 ++++++- dev/release/verify-release-candidate.sh | 2 +- dev/rust_lint.sh | 2 +- docs/build.sh | 2 +- pyproject.toml | 2 +- python/{datafusion => }/tests/__init__.py | 0 python/{datafusion => }/tests/conftest.py | 0 .../tests/data_test_context/data.json | 0 python/{datafusion => }/tests/generic.py | 0 .../tests/test_aggregation.py | 0 python/{datafusion => }/tests/test_catalog.py | 0 python/{datafusion => }/tests/test_config.py | 0 python/{datafusion => }/tests/test_context.py | 0 .../{datafusion => }/tests/test_dataframe.py | 0 python/{datafusion => }/tests/test_expr.py | 0 .../{datafusion => }/tests/test_functions.py | 0 python/{datafusion => }/tests/test_imports.py | 0 .../{datafusion => }/tests/test_indexing.py | 0 python/{datafusion => }/tests/test_input.py | 0 python/{datafusion => }/tests/test_sql.py | 0 python/{datafusion => }/tests/test_store.py | 0 .../{datafusion => }/tests/test_substrait.py | 0 python/{datafusion => }/tests/test_udaf.py | 0 python/{datafusion => }/tests/test_udwf.py | 0 .../tests/test_wrapper_coverage.py | 0 27 files changed, 88 insertions(+), 7 deletions(-) create mode 100755 dev/clean.sh rename python/{datafusion => }/tests/__init__.py (100%) rename python/{datafusion => }/tests/conftest.py (100%) rename python/{datafusion => }/tests/data_test_context/data.json (100%) rename python/{datafusion => }/tests/generic.py (100%) rename python/{datafusion => }/tests/test_aggregation.py (100%) rename python/{datafusion => }/tests/test_catalog.py (100%) rename python/{datafusion => }/tests/test_config.py (100%) rename python/{datafusion => }/tests/test_context.py (100%) rename python/{datafusion => }/tests/test_dataframe.py (100%) rename python/{datafusion => }/tests/test_expr.py (100%) rename python/{datafusion => }/tests/test_functions.py (100%) rename python/{datafusion => }/tests/test_imports.py (100%) rename python/{datafusion => }/tests/test_indexing.py (100%) rename python/{datafusion => }/tests/test_input.py (100%) rename python/{datafusion => }/tests/test_sql.py (100%) rename python/{datafusion => }/tests/test_store.py (100%) rename python/{datafusion => }/tests/test_substrait.py (100%) rename python/{datafusion => }/tests/test_udaf.py (100%) rename python/{datafusion => }/tests/test_udwf.py (100%) rename python/{datafusion => }/tests/test_wrapper_coverage.py (100%) diff --git a/dev/clean.sh b/dev/clean.sh new file mode 100755 index 000000000..0d86680e8 --- /dev/null +++ b/dev/clean.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# This cleans up the project by removing build artifacts and other generated files. + +# Function to remove a directory and print the action +remove_dir() { + if [ -d "$1" ]; then + echo "Removing directory: $1" + rm -rf "$1" + fi +} + +# Function to remove a file and print the action +remove_file() { + if [ -f "$1" ]; then + echo "Removing file: $1" + rm -f "$1" + fi +} + +# Remove .pytest_cache directory +remove_dir .pytest_cache/ + +# Remove target directory +remove_dir target/ + +# Remove any __pycache__ directories +find python/ -type d -name "__pycache__" -print | while read -r dir; do + remove_dir "$dir" +done + +# Remove pytest-coverage.lcov file +# remove_file .coverage +# remove_file pytest-coverage.lcov + +# Remove rust-coverage.lcov file +# remove_file rust-coverage.lcov + +# Remove pyo3 files +find python/ -type f -name '_internal.*.so' -print | while read -r file; do + remove_file "$file" +done + +echo "Cleanup complete." \ No newline at end of file diff --git a/dev/python_lint.sh b/dev/python_lint.sh index 3bc67fb12..29f0d4833 100755 --- a/dev/python_lint.sh +++ b/dev/python_lint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/dev/release/README.md b/dev/release/README.md index 93c2f97b9..49fd9de2d 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -159,10 +159,29 @@ Send the email to start the vote. ## Verifying a Release -Install the release from testpypi: +Running the unit tests against a testpypi release candidate: ```bash -pip install --extra-index-url https://test.pypi.org/simple/ datafusion==0.7.0 +# clone a fresh repo +git clone https://github.com/apache/datafusion-python.git +cd datafusion-python + +# checkout the release commit +git fetch --tags +git checkout 40.0.0-rc1 + +# create the env +python3 -m venv venv +source venv/bin/activate + +# install release candidate +pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0 + +# only dep needed to run tests is pytest +pip install pytest + +# run the tests +pytest --import-mode=importlib python/tests ``` Try running one of the examples from the top-level README, or write some custom Python code to query some available diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 14c0baee8..3879a267f 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/dev/rust_lint.sh b/dev/rust_lint.sh index b1285cbc3..eeb9e2302 100755 --- a/dev/rust_lint.sh +++ b/dev/rust_lint.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/docs/build.sh b/docs/build.sh index 7e8bb0b54..5afe85812 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file diff --git a/pyproject.toml b/pyproject.toml index 4e03ce8db..6e10333a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,7 +78,7 @@ max-doc-length = 88 # Disable docstring checking for these directories [tool.ruff.lint.per-file-ignores] -"python/datafusion/tests/*" = ["D"] +"python/tests/*" = ["D"] "examples/*" = ["D", "W505"] "dev/*" = ["D"] "benchmarks/*" = ["D", "F"] diff --git a/python/datafusion/tests/__init__.py b/python/tests/__init__.py similarity index 100% rename from python/datafusion/tests/__init__.py rename to python/tests/__init__.py diff --git a/python/datafusion/tests/conftest.py b/python/tests/conftest.py similarity index 100% rename from python/datafusion/tests/conftest.py rename to python/tests/conftest.py diff --git a/python/datafusion/tests/data_test_context/data.json b/python/tests/data_test_context/data.json similarity index 100% rename from python/datafusion/tests/data_test_context/data.json rename to python/tests/data_test_context/data.json diff --git a/python/datafusion/tests/generic.py b/python/tests/generic.py similarity index 100% rename from python/datafusion/tests/generic.py rename to python/tests/generic.py diff --git a/python/datafusion/tests/test_aggregation.py b/python/tests/test_aggregation.py similarity index 100% rename from python/datafusion/tests/test_aggregation.py rename to python/tests/test_aggregation.py diff --git a/python/datafusion/tests/test_catalog.py b/python/tests/test_catalog.py similarity index 100% rename from python/datafusion/tests/test_catalog.py rename to python/tests/test_catalog.py diff --git a/python/datafusion/tests/test_config.py b/python/tests/test_config.py similarity index 100% rename from python/datafusion/tests/test_config.py rename to python/tests/test_config.py diff --git a/python/datafusion/tests/test_context.py b/python/tests/test_context.py similarity index 100% rename from python/datafusion/tests/test_context.py rename to python/tests/test_context.py diff --git a/python/datafusion/tests/test_dataframe.py b/python/tests/test_dataframe.py similarity index 100% rename from python/datafusion/tests/test_dataframe.py rename to python/tests/test_dataframe.py diff --git a/python/datafusion/tests/test_expr.py b/python/tests/test_expr.py similarity index 100% rename from python/datafusion/tests/test_expr.py rename to python/tests/test_expr.py diff --git a/python/datafusion/tests/test_functions.py b/python/tests/test_functions.py similarity index 100% rename from python/datafusion/tests/test_functions.py rename to python/tests/test_functions.py diff --git a/python/datafusion/tests/test_imports.py b/python/tests/test_imports.py similarity index 100% rename from python/datafusion/tests/test_imports.py rename to python/tests/test_imports.py diff --git a/python/datafusion/tests/test_indexing.py b/python/tests/test_indexing.py similarity index 100% rename from python/datafusion/tests/test_indexing.py rename to python/tests/test_indexing.py diff --git a/python/datafusion/tests/test_input.py b/python/tests/test_input.py similarity index 100% rename from python/datafusion/tests/test_input.py rename to python/tests/test_input.py diff --git a/python/datafusion/tests/test_sql.py b/python/tests/test_sql.py similarity index 100% rename from python/datafusion/tests/test_sql.py rename to python/tests/test_sql.py diff --git a/python/datafusion/tests/test_store.py b/python/tests/test_store.py similarity index 100% rename from python/datafusion/tests/test_store.py rename to python/tests/test_store.py diff --git a/python/datafusion/tests/test_substrait.py b/python/tests/test_substrait.py similarity index 100% rename from python/datafusion/tests/test_substrait.py rename to python/tests/test_substrait.py diff --git a/python/datafusion/tests/test_udaf.py b/python/tests/test_udaf.py similarity index 100% rename from python/datafusion/tests/test_udaf.py rename to python/tests/test_udaf.py diff --git a/python/datafusion/tests/test_udwf.py b/python/tests/test_udwf.py similarity index 100% rename from python/datafusion/tests/test_udwf.py rename to python/tests/test_udwf.py diff --git a/python/datafusion/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py similarity index 100% rename from python/datafusion/tests/test_wrapper_coverage.py rename to python/tests/test_wrapper_coverage.py From 976b70062cba0d81ca0dd71e0cd6fa7074be3de6 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 4 Oct 2024 09:38:16 -0500 Subject: [PATCH 212/413] chore(bench): fix create_tables.sql for tpch benchmark (#897) The `WITH HEADER ROW` and `DELIMETER` clauses are no longer in use, and upstream datafusion errors out with a message saying so. --- benchmarks/tpch/create_tables.sql | 40 ++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/benchmarks/tpch/create_tables.sql b/benchmarks/tpch/create_tables.sql index 4b2209c4b..9f3aeea20 100644 --- a/benchmarks/tpch/create_tables.sql +++ b/benchmarks/tpch/create_tables.sql @@ -13,7 +13,10 @@ CREATE EXTERNAL TABLE customer ( c_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/customer.csv'; CREATE EXTERNAL TABLE lineitem ( @@ -36,7 +39,10 @@ CREATE EXTERNAL TABLE lineitem ( l_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/lineitem.csv'; CREATE EXTERNAL TABLE nation ( @@ -47,7 +53,10 @@ CREATE EXTERNAL TABLE nation ( n_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/nation.csv'; CREATE EXTERNAL TABLE orders ( @@ -63,7 +72,10 @@ CREATE EXTERNAL TABLE orders ( o_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/orders.csv'; CREATE EXTERNAL TABLE part ( @@ -79,7 +91,10 @@ CREATE EXTERNAL TABLE part ( p_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/part.csv'; CREATE EXTERNAL TABLE partsupp ( @@ -91,7 +106,10 @@ CREATE EXTERNAL TABLE partsupp ( ps_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/partsupp.csv'; CREATE EXTERNAL TABLE region ( @@ -101,7 +119,10 @@ CREATE EXTERNAL TABLE region ( r_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/region.csv'; CREATE EXTERNAL TABLE supplier ( @@ -115,5 +136,8 @@ CREATE EXTERNAL TABLE supplier ( s_extra VARCHAR NOT NULL, ) STORED AS CSV -WITH HEADER ROW DELIMITER '|' +OPTIONS ( + format.delimiter '|', + format.has_header true +) LOCATION '$PATH/supplier.csv'; \ No newline at end of file From d181a3039c3b5059b08016ea3f18a5e961611aef Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 4 Oct 2024 11:29:54 -0400 Subject: [PATCH 213/413] Add physical and logical plan conversion to and from protobuf (#892) * Add physical and logical plan conversion to and from protobuf * Add wrappers for LogicalPlan and ExecutionPlan * Add unit tests for to_proto and from_proto for logical and physical plans --- Cargo.lock | 251 ++++++++++++-------------- Cargo.toml | 1 + python/datafusion/__init__.py | 4 +- python/datafusion/context.py | 12 +- python/datafusion/dataframe.py | 11 +- python/datafusion/expr.py | 8 +- python/datafusion/plan.py | 147 +++++++++++++++ python/datafusion/substrait.py | 10 +- python/datafusion/tests/test_plans.py | 42 +++++ src/physical_plan.rs | 35 +++- src/sql/logical.rs | 35 +++- 11 files changed, 398 insertions(+), 158 deletions(-) create mode 100644 python/datafusion/plan.py create mode 100644 python/datafusion/tests/test_plans.py diff --git a/Cargo.lock b/Cargo.lock index da5d19a93..0935ea9e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,7 +178,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown", + "hashbrown 0.14.5", "num", ] @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" dependencies = [ "bzip2", "flate2", @@ -380,9 +380,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", @@ -406,9 +406,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" @@ -515,9 +515,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bzip2" @@ -542,9 +542,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.20" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45bcde016d64c21da4be18b655631e5ab6d3107607e71a73a9f53eb48aae23fb" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", @@ -738,7 +738,7 @@ checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" dependencies = [ "cfg-if", "crossbeam-utils", - "hashbrown", + "hashbrown 0.14.5", "lock_api", "once_cell", "parking_lot_core", @@ -781,7 +781,7 @@ dependencies = [ "futures", "glob", "half", - "hashbrown", + "hashbrown 0.14.5", "indexmap", "itertools", "log", @@ -832,7 +832,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown", + "hashbrown 0.14.5", "instant", "libc", "num_cpus", @@ -866,7 +866,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "futures", - "hashbrown", + "hashbrown 0.14.5", "log", "object_store", "parking_lot", @@ -923,7 +923,7 @@ dependencies = [ "datafusion-common", "datafusion-execution", "datafusion-expr", - "hashbrown", + "hashbrown 0.14.5", "hex", "itertools", "log", @@ -1017,7 +1017,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown", + "hashbrown 0.14.5", "indexmap", "itertools", "log", @@ -1047,7 +1047,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown", + "hashbrown 0.14.5", "hex", "indexmap", "itertools", @@ -1067,7 +1067,7 @@ dependencies = [ "arrow", "datafusion-common", "datafusion-expr-common", - "hashbrown", + "hashbrown 0.14.5", "rand", ] @@ -1109,7 +1109,7 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown", + "hashbrown 0.14.5", "indexmap", "itertools", "log", @@ -1120,6 +1120,35 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-proto" +version = "42.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585357d621fa03ea85a7fefca79ebc5ef0ee13a7f82be0762a414879a4d190a7" +dependencies = [ + "arrow", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-proto-common", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "42.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4db6534382f92f528bdb5d925b4214c31ffd84fa7fe1eff3ed0d2f1286851ab8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "object_store", + "prost", +] + [[package]] name = "datafusion-python" version = "41.0.0" @@ -1127,6 +1156,7 @@ dependencies = [ "arrow", "async-trait", "datafusion", + "datafusion-proto", "datafusion-substrait", "futures", "mimalloc", @@ -1242,9 +1272,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "miniz_oxide", @@ -1427,6 +1457,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "heck" version = "0.4.1" @@ -1487,9 +1523,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "humantime" @@ -1528,7 +1564,7 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs 0.8.0", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1537,9 +1573,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" dependencies = [ "bytes", "futures-channel", @@ -1550,7 +1586,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", "tower-service", "tracing", ] @@ -1590,12 +1625,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.15.0", ] [[package]] @@ -1733,9 +1768,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libflate" @@ -1757,7 +1792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" dependencies = [ "core2", - "hashbrown", + "hashbrown 0.14.5", "rle-decode-fast", ] @@ -2012,9 +2047,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "openssl-probe" @@ -2075,7 +2113,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown", + "hashbrown 0.14.5", "lz4_flex", "num", "num-bigint", @@ -2196,26 +2234,6 @@ dependencies = [ "siphasher", ] -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2230,15 +2248,15 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "ppv-lite86" @@ -2280,9 +2298,9 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8650aabb6c35b860610e9cff5dc1af886c9e25073b7b1712a68972af4281302" +checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" dependencies = [ "bytes", "heck 0.5.0", @@ -2401,9 +2419,9 @@ checksum = "b76f1009795ca44bb5aaae8fd3f18953e209259c33d9b059b1f53d58ab7511db" [[package]] name = "quick-xml" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" dependencies = [ "memchr", "serde", @@ -2498,18 +2516,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.4" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0884ad60e090bf1345b93da0a5de8923c93884cd03f40dfcfddd3b4bee661853" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", @@ -2519,9 +2537,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", @@ -2536,9 +2554,9 @@ checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" @@ -2546,15 +2564,15 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eae2a1ebfecc58aff952ef8ccd364329abe627762f5bf09ff42eb9d98522479" dependencies = [ - "hashbrown", + "hashbrown 0.14.5", "memchr", ] [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", @@ -2576,7 +2594,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs 0.7.3", + "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", "serde", @@ -2664,19 +2682,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-native-certs" version = "0.8.0" @@ -2692,19 +2697,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -2792,9 +2796,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -2935,18 +2939,18 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "snafu" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b835cb902660db3415a672d862905e791e54d306c6e8189168c7f3d9ae1c79d" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" dependencies = [ "snafu-derive", ] [[package]] name = "snafu-derive" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d1e02fca405f6280643174a50c942219f0bbf4dbf7d480f1dd864d6f211ae5" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -3103,9 +3107,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", @@ -3116,18 +3120,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -3220,27 +3224,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -3369,9 +3352,9 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" @@ -3396,9 +3379,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unindent" @@ -3539,9 +3522,9 @@ checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" dependencies = [ "futures-util", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index 54e53f4b0..a0723984f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] arrow = { version = "53", features = ["pyarrow"] } datafusion = { version = "42.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } datafusion-substrait = { version = "42.0.0", optional = true } +datafusion-proto = { version = "42.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` prost-types = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.9", features = ["v4"] } diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 4f40b2088..63c19b3e1 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -36,7 +36,7 @@ from .catalog import Catalog, Database, Table # The following imports are okay to remain as opaque to the user. -from ._internal import Config, LogicalPlan, ExecutionPlan, runtime +from ._internal import Config, runtime from .record_batch import RecordBatchStream, RecordBatch @@ -53,6 +53,8 @@ WindowFrame, ) +from .plan import LogicalPlan, ExecutionPlan + from . import functions, object_store, substrait __version__ = importlib_metadata.version(__name__) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 5b52d397b..b08e62d77 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -23,7 +23,6 @@ from ._internal import RuntimeConfig as RuntimeConfigInternal from ._internal import SQLOptions as SQLOptionsInternal from ._internal import SessionContext as SessionContextInternal -from ._internal import LogicalPlan, ExecutionPlan from datafusion.catalog import Catalog, Table from datafusion.dataframe import DataFrame @@ -39,6 +38,7 @@ import pandas import polars import pathlib + from datafusion.plan import LogicalPlan, ExecutionPlan class SessionConfig: @@ -268,8 +268,10 @@ def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConf Returns: A new :py:class:`RuntimeConfig` object with the updated setting. """ - paths = [str(p) for p in paths] - self.config_internal = self.config_internal.with_disk_manager_specified(paths) + paths_list = [str(p) for p in paths] + self.config_internal = self.config_internal.with_disk_manager_specified( + paths_list + ) return self def with_unbounded_memory_pool(self) -> RuntimeConfig: @@ -558,7 +560,7 @@ def create_dataframe_from_logical_plan(self, plan: LogicalPlan) -> DataFrame: Returns: DataFrame representation of the logical plan. """ - return DataFrame(self.ctx.create_dataframe_from_logical_plan(plan)) + return DataFrame(self.ctx.create_dataframe_from_logical_plan(plan._raw_plan)) def from_pylist( self, data: list[dict[str, Any]], name: str | None = None @@ -1034,4 +1036,4 @@ def read_table(self, table: Table) -> DataFrame: def execute(self, plan: ExecutionPlan, partitions: int) -> RecordBatchStream: """Execute the ``plan`` and return the results.""" - return RecordBatchStream(self.ctx.execute(plan, partitions)) + return RecordBatchStream(self.ctx.execute(plan._raw_plan, partitions)) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 2328ef8fa..c5ac0bb89 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -24,6 +24,7 @@ from typing import Any, List, TYPE_CHECKING from datafusion.record_batch import RecordBatchStream from typing_extensions import deprecated +from datafusion.plan import LogicalPlan, ExecutionPlan if TYPE_CHECKING: import pyarrow as pa @@ -34,10 +35,6 @@ from datafusion._internal import DataFrame as DataFrameInternal from datafusion.expr import Expr, SortExpr, sort_or_default -from datafusion._internal import ( - LogicalPlan, - ExecutionPlan, -) class DataFrame: @@ -316,7 +313,7 @@ def logical_plan(self) -> LogicalPlan: Returns: Unoptimized logical plan. """ - return self.df.logical_plan() + return LogicalPlan(self.df.logical_plan()) def optimized_logical_plan(self) -> LogicalPlan: """Return the optimized ``LogicalPlan``. @@ -324,7 +321,7 @@ def optimized_logical_plan(self) -> LogicalPlan: Returns: Optimized logical plan. """ - return self.df.optimized_logical_plan() + return LogicalPlan(self.df.optimized_logical_plan()) def execution_plan(self) -> ExecutionPlan: """Return the execution/physical plan. @@ -332,7 +329,7 @@ def execution_plan(self) -> ExecutionPlan: Returns: Execution plan. """ - return self.df.execution_plan() + return ExecutionPlan(self.df.execution_plan()) def repartition(self, num: int) -> DataFrame: """Repartition a DataFrame into ``num`` partitions. diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 152aa38d3..8600627ae 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -22,16 +22,18 @@ from __future__ import annotations -from typing import Any, Optional, Type +from typing import Any, Optional, Type, TYPE_CHECKING import pyarrow as pa from datafusion.common import DataTypeMap, NullTreatment, RexType from typing_extensions import deprecated -from ._internal import LogicalPlan from ._internal import expr as expr_internal from ._internal import functions as functions_internal +if TYPE_CHECKING: + from datafusion.plan import LogicalPlan + # The following are imported from the internal representation. We may choose to # give these all proper wrappers, or to simply leave as is. These were added # in order to support passing the `test_imports` unit test. @@ -485,7 +487,7 @@ def rex_call_operator(self) -> str: def column_name(self, plan: LogicalPlan) -> str: """Compute the output column name based on the provided logical plan.""" - return self.expr.column_name(plan) + return self.expr.column_name(plan._raw_plan) def order_by(self, *exprs: Expr | SortExpr) -> ExprFuncBuilder: """Set the ordering for a window or aggregate function. diff --git a/python/datafusion/plan.py b/python/datafusion/plan.py new file mode 100644 index 000000000..3836edec6 --- /dev/null +++ b/python/datafusion/plan.py @@ -0,0 +1,147 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module supports physical and logical plans in DataFusion.""" + +from __future__ import annotations + +import datafusion._internal as df_internal + +from typing import List, Any, TYPE_CHECKING + +if TYPE_CHECKING: + from datafusion.context import SessionContext + +__all__ = [ + "LogicalPlan", + "ExecutionPlan", +] + + +class LogicalPlan: + """Logical Plan. + + A `LogicalPlan` is a node in a tree of relational operators (such as + Projection or Filter). + + Represents transforming an input relation (table) to an output relation + (table) with a potentially different schema. Plans form a dataflow tree + where data flows from leaves up to the root to produce the query result. + + `LogicalPlan`s can be created by the SQL query planner, the DataFrame API, + or programmatically (for example custom query languages). + """ + + def __init__(self, plan: df_internal.LogicalPlan) -> None: + """This constructor should not be called by the end user.""" + self._raw_plan = plan + + def to_variant(self) -> Any: + """Convert the logical plan into its specific variant.""" + return self._raw_plan.to_variant() + + def inputs(self) -> List[LogicalPlan]: + """Returns the list of inputs to the logical plan.""" + return [LogicalPlan(p) for p in self._raw_plan.inputs()] + + def __repr__(self) -> str: + """Generate a printable representation of the plan.""" + return self._raw_plan.__repr__() + + def display(self) -> str: + """Print the logical plan.""" + return self._raw_plan.display() + + def display_indent(self) -> str: + """Print an indented form of the logical plan.""" + return self._raw_plan.display_indent() + + def display_indent_schema(self) -> str: + """Print an indented form of the schema for the logical plan.""" + return self._raw_plan.display_indent_schema() + + def display_graphviz(self) -> str: + """Print the graph visualization of the logical plan. + + Returns a `format`able structure that produces lines meant for graphical display + using the `DOT` language. This format can be visualized using software from + [`graphviz`](https://graphviz.org/) + """ + return self._raw_plan.display_graphviz() + + @staticmethod + def from_proto(ctx: SessionContext, data: bytes) -> LogicalPlan: + """Create a LogicalPlan from protobuf bytes. + + Tables created in memory from record batches are currently not supported. + """ + return LogicalPlan(df_internal.LogicalPlan.from_proto(ctx.ctx, data)) + + def to_proto(self) -> bytes: + """Convert a LogicalPlan to protobuf bytes. + + Tables created in memory from record batches are currently not supported. + """ + return self._raw_plan.to_proto() + + +class ExecutionPlan: + """Represent nodes in the DataFusion Physical Plan.""" + + def __init__(self, plan: df_internal.ExecutionPlan) -> None: + """This constructor should not be called by the end user.""" + self._raw_plan = plan + + def children(self) -> List[ExecutionPlan]: + """Get a list of children `ExecutionPlan`s that act as inputs to this plan. + + The returned list will be empty for leaf nodes such as scans, will contain a + single value for unary nodes, or two values for binary nodes (such as joins). + """ + return [ExecutionPlan(e) for e in self._raw_plan.children()] + + def display(self) -> str: + """Print the physical plan.""" + return self._raw_plan.display() + + def display_indent(self) -> str: + """Print an indented form of the physical plan.""" + return self._raw_plan.display_indent() + + def __repr__(self) -> str: + """Print a string representation of the physical plan.""" + return self._raw_plan.__repr__() + + @property + def partition_count(self) -> int: + """Returns the number of partitions in the physical plan.""" + return self._raw_plan.partition_count + + @staticmethod + def from_proto(ctx: SessionContext, data: bytes) -> ExecutionPlan: + """Create an ExecutionPlan from protobuf bytes. + + Tables created in memory from record batches are currently not supported. + """ + return ExecutionPlan(df_internal.ExecutionPlan.from_proto(ctx.ctx, data)) + + def to_proto(self) -> bytes: + """Convert an ExecutionPlan into protobuf bytes. + + Tables created in memory from record batches are currently not supported. + """ + return self._raw_plan.to_proto() diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index 0cdd19a51..dea47acca 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -28,10 +28,10 @@ from typing import TYPE_CHECKING from typing_extensions import deprecated import pathlib +from datafusion.plan import LogicalPlan if TYPE_CHECKING: from datafusion.context import SessionContext - from datafusion._internal import LogicalPlan __all__ = [ "Plan", @@ -156,7 +156,9 @@ def to_substrait_plan(logical_plan: LogicalPlan, ctx: SessionContext) -> Plan: Substrait plan. """ return Plan( - substrait_internal.Producer.to_substrait_plan(logical_plan, ctx.ctx) + substrait_internal.Producer.to_substrait_plan( + logical_plan._raw_plan, ctx.ctx + ) ) @@ -181,8 +183,8 @@ def from_substrait_plan(ctx: SessionContext, plan: Plan) -> LogicalPlan: Returns: LogicalPlan. """ - return substrait_internal.Consumer.from_substrait_plan( - ctx.ctx, plan.plan_internal + return LogicalPlan( + substrait_internal.Consumer.from_substrait_plan(ctx.ctx, plan.plan_internal) ) diff --git a/python/datafusion/tests/test_plans.py b/python/datafusion/tests/test_plans.py new file mode 100644 index 000000000..0283a4e6a --- /dev/null +++ b/python/datafusion/tests/test_plans.py @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datafusion import SessionContext, LogicalPlan, ExecutionPlan +import pytest + + +# Note: We must use CSV because memory tables are currently not supported for +# conversion to/from protobuf. +@pytest.fixture +def df(): + ctx = SessionContext() + return ctx.read_csv(path="testing/data/csv/aggregate_test_100.csv").select("c1") + + +def test_logical_plan_to_proto(ctx, df) -> None: + logical_plan_bytes = df.logical_plan().to_proto() + logical_plan = LogicalPlan.from_proto(ctx, logical_plan_bytes) + + df_round_trip = ctx.create_dataframe_from_logical_plan(logical_plan) + + assert df.collect() == df_round_trip.collect() + + original_execution_plan = df.execution_plan() + execution_plan_bytes = original_execution_plan.to_proto() + execution_plan = ExecutionPlan.from_proto(ctx, execution_plan_bytes) + + assert str(original_execution_plan) == str(execution_plan) diff --git a/src/physical_plan.rs b/src/physical_plan.rs index c97c1a96e..9ef2f0ebb 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -16,9 +16,13 @@ // under the License. use datafusion::physical_plan::{displayable, ExecutionPlan, ExecutionPlanProperties}; +use datafusion_proto::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec}; +use prost::Message; use std::sync::Arc; -use pyo3::prelude::*; +use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyBytes}; + +use crate::{context::PySessionContext, errors::DataFusionError}; #[pyclass(name = "ExecutionPlan", module = "datafusion", subclass)] #[derive(Debug, Clone)] @@ -54,6 +58,35 @@ impl PyExecutionPlan { format!("{}", d.indent(false)) } + pub fn to_proto<'py>(&'py self, py: Python<'py>) -> PyResult> { + let codec = DefaultPhysicalExtensionCodec {}; + let proto = datafusion_proto::protobuf::PhysicalPlanNode::try_from_physical_plan( + self.plan.clone(), + &codec, + )?; + + let bytes = proto.encode_to_vec(); + Ok(PyBytes::new_bound(py, &bytes)) + } + + #[staticmethod] + pub fn from_proto(ctx: PySessionContext, proto_msg: Bound<'_, PyBytes>) -> PyResult { + let bytes: &[u8] = proto_msg.extract()?; + let proto_plan = + datafusion_proto::protobuf::PhysicalPlanNode::decode(bytes).map_err(|e| { + PyRuntimeError::new_err(format!( + "Unable to decode logical node from serialized bytes: {}", + e + )) + })?; + + let codec = DefaultPhysicalExtensionCodec {}; + let plan = proto_plan + .try_into_physical_plan(&ctx.ctx, &ctx.ctx.runtime_env(), &codec) + .map_err(DataFusionError::from)?; + Ok(Self::new(plan)) + } + fn __repr__(&self) -> String { self.display_indent() } diff --git a/src/sql/logical.rs b/src/sql/logical.rs index d00f0af3f..fc398ff89 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -17,7 +17,6 @@ use std::sync::Arc; -use crate::errors::py_unsupported_variant_err; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; use crate::expr::cross_join::PyCrossJoin; @@ -35,8 +34,11 @@ use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; use crate::expr::unnest::PyUnnest; use crate::expr::window::PyWindowExpr; -use datafusion::logical_expr::LogicalPlan; -use pyo3::prelude::*; +use crate::{context::PySessionContext, errors::py_unsupported_variant_err}; +use datafusion::{error::DataFusionError, logical_expr::LogicalPlan}; +use datafusion_proto::logical_plan::{AsLogicalPlan, DefaultLogicalExtensionCodec}; +use prost::Message; +use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyBytes}; use crate::expr::logical_node::LogicalNode; @@ -125,6 +127,33 @@ impl PyLogicalPlan { fn display_graphviz(&self) -> String { format!("{}", self.plan.display_graphviz()) } + + pub fn to_proto<'py>(&'py self, py: Python<'py>) -> PyResult> { + let codec = DefaultLogicalExtensionCodec {}; + let proto = + datafusion_proto::protobuf::LogicalPlanNode::try_from_logical_plan(&self.plan, &codec)?; + + let bytes = proto.encode_to_vec(); + Ok(PyBytes::new_bound(py, &bytes)) + } + + #[staticmethod] + pub fn from_proto(ctx: PySessionContext, proto_msg: Bound<'_, PyBytes>) -> PyResult { + let bytes: &[u8] = proto_msg.extract()?; + let proto_plan = + datafusion_proto::protobuf::LogicalPlanNode::decode(bytes).map_err(|e| { + PyRuntimeError::new_err(format!( + "Unable to decode logical node from serialized bytes: {}", + e + )) + })?; + + let codec = DefaultLogicalExtensionCodec {}; + let plan = proto_plan + .try_into_logical_plan(&ctx.ctx, &codec) + .map_err(DataFusionError::from)?; + Ok(Self::new(plan)) + } } impl From for LogicalPlan { From 1fd3762ce471fe375598740af447e6747f307bdc Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 4 Oct 2024 12:26:30 -0400 Subject: [PATCH 214/413] Feature/instance udfs (#890) * Add option for passing in constructor arguments to the udaf * Fix small warnings in pylance * Improve type hinting for udaf and fix one pylance warning * Set up UDWF to take arguments as constructor just like UDAF to ensure we get a clean state when functions are reused * Improve handling of udf when user provides a class instead of bare function * Add unit tests for UDF showing callable class * Add license text * Switching to use factory methods for udaf and udwf * Move new tests to the new testing directory --- python/datafusion/udf.py | 136 +++++++++++++++----- python/tests/test_dataframe.py | 16 --- python/{datafusion => }/tests/test_plans.py | 0 python/tests/test_udaf.py | 79 +++++++----- python/tests/test_udf.py | 96 ++++++++++++++ python/tests/test_udwf.py | 36 ++++-- src/udwf.rs | 6 +- 7 files changed, 272 insertions(+), 97 deletions(-) rename python/{datafusion => }/tests/test_plans.py (100%) create mode 100644 python/tests/test_udf.py diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index bb7a90866..291ef2bae 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -23,7 +23,7 @@ from datafusion.expr import Expr from typing import Callable, TYPE_CHECKING, TypeVar from abc import ABCMeta, abstractmethod -from typing import List +from typing import List, Optional from enum import Enum import pyarrow @@ -84,9 +84,9 @@ class ScalarUDF: def __init__( self, - name: str | None, + name: Optional[str], func: Callable[..., _R], - input_types: list[pyarrow.DataType], + input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, ) -> None: @@ -94,6 +94,8 @@ def __init__( See helper method :py:func:`udf` for argument details. """ + if isinstance(input_types, pyarrow.DataType): + input_types = [input_types] self._udf = df_internal.ScalarUDF( name, func, input_types, return_type, str(volatility) ) @@ -104,8 +106,8 @@ def __call__(self, *args: Expr) -> Expr: This function is not typically called by an end user. These calls will occur during the evaluation of the dataframe. """ - args = [arg.expr for arg in args] - return Expr(self._udf.__call__(*args)) + args_raw = [arg.expr for arg in args] + return Expr(self._udf.__call__(*args_raw)) @staticmethod def udf( @@ -113,7 +115,7 @@ def udf( input_types: list[pyarrow.DataType], return_type: _R, volatility: Volatility | str, - name: str | None = None, + name: Optional[str] = None, ) -> ScalarUDF: """Create a new User-Defined Function. @@ -133,7 +135,10 @@ def udf( if not callable(func): raise TypeError("`func` argument must be callable") if name is None: - name = func.__qualname__.lower() + if hasattr(func, "__qualname__"): + name = func.__qualname__.lower() + else: + name = func.__class__.__name__.lower() return ScalarUDF( name=name, func=func, @@ -167,10 +172,6 @@ def evaluate(self) -> pyarrow.Scalar: pass -if TYPE_CHECKING: - _A = TypeVar("_A", bound=(Callable[..., _R], Accumulator)) - - class AggregateUDF: """Class for performing scalar user-defined functions (UDF). @@ -180,10 +181,10 @@ class AggregateUDF: def __init__( self, - name: str | None, - accumulator: _A, + name: Optional[str], + accumulator: Callable[[], Accumulator], input_types: list[pyarrow.DataType], - return_type: _R, + return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, ) -> None: @@ -193,7 +194,12 @@ def __init__( descriptions. """ self._udaf = df_internal.AggregateUDF( - name, accumulator, input_types, return_type, state_type, str(volatility) + name, + accumulator, + input_types, + return_type, + state_type, + str(volatility), ) def __call__(self, *args: Expr) -> Expr: @@ -202,21 +208,52 @@ def __call__(self, *args: Expr) -> Expr: This function is not typically called by an end user. These calls will occur during the evaluation of the dataframe. """ - args = [arg.expr for arg in args] - return Expr(self._udaf.__call__(*args)) + args_raw = [arg.expr for arg in args] + return Expr(self._udaf.__call__(*args_raw)) @staticmethod def udaf( - accum: _A, - input_types: list[pyarrow.DataType], - return_type: _R, + accum: Callable[[], Accumulator], + input_types: pyarrow.DataType | list[pyarrow.DataType], + return_type: pyarrow.DataType, state_type: list[pyarrow.DataType], volatility: Volatility | str, - name: str | None = None, + name: Optional[str] = None, ) -> AggregateUDF: """Create a new User-Defined Aggregate Function. - The accumulator function must be callable and implement :py:class:`Accumulator`. + If your :py:class:`Accumulator` can be instantiated with no arguments, you + can simply pass it's type as ``accum``. If you need to pass additional arguments + to it's constructor, you can define a lambda or a factory method. During runtime + the :py:class:`Accumulator` will be constructed for every instance in + which this UDAF is used. The following examples are all valid. + + .. code-block:: python + import pyarrow as pa + import pyarrow.compute as pc + + class Summarize(Accumulator): + def __init__(self, bias: float = 0.0): + self._sum = pa.scalar(bias) + + def state(self) -> List[pa.Scalar]: + return [self._sum] + + def update(self, values: pa.Array) -> None: + self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py()) + + def merge(self, states: List[pa.Array]) -> None: + self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py()) + + def evaluate(self) -> pa.Scalar: + return self._sum + + def sum_bias_10() -> Summarize: + return Summarize(10.0) + + udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], "immutable") + udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], "immutable") + udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), [pa.float64()], "immutable") Args: accum: The accumulator python function. @@ -229,14 +266,16 @@ def udaf( Returns: A user-defined aggregate function, which can be used in either data aggregation or window function calls. - """ - if not issubclass(accum, Accumulator): + """ # noqa W505 + if not callable(accum): + raise TypeError("`func` must be callable.") + if not isinstance(accum.__call__(), Accumulator): raise TypeError( - "`accum` must implement the abstract base class Accumulator" + "Accumulator must implement the abstract base class Accumulator" ) if name is None: - name = accum.__qualname__.lower() - if isinstance(input_types, pyarrow.lib.DataType): + name = accum.__call__().__class__.__qualname__.lower() + if isinstance(input_types, pyarrow.DataType): input_types = [input_types] return AggregateUDF( name=name, @@ -421,8 +460,8 @@ class WindowUDF: def __init__( self, - name: str | None, - func: WindowEvaluator, + name: Optional[str], + func: Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, @@ -447,30 +486,57 @@ def __call__(self, *args: Expr) -> Expr: @staticmethod def udwf( - func: WindowEvaluator, + func: Callable[[], WindowEvaluator], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: pyarrow.DataType, volatility: Volatility | str, - name: str | None = None, + name: Optional[str] = None, ) -> WindowUDF: """Create a new User-Defined Window Function. + If your :py:class:`WindowEvaluator` can be instantiated with no arguments, you + can simply pass it's type as ``func``. If you need to pass additional arguments + to it's constructor, you can define a lambda or a factory method. During runtime + the :py:class:`WindowEvaluator` will be constructed for every instance in + which this UDWF is used. The following examples are all valid. + + .. code-block:: python + + import pyarrow as pa + + class BiasedNumbers(WindowEvaluator): + def __init__(self, start: int = 0) -> None: + self.start = start + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + return pa.array([self.start + i for i in range(num_rows)]) + + def bias_10() -> BiasedNumbers: + return BiasedNumbers(10) + + udwf1 = udwf(BiasedNumbers, pa.int64(), pa.int64(), "immutable") + udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable") + udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable") + Args: - func: The python function. + func: A callable to create the window function. input_types: The data types of the arguments to ``func``. return_type: The data type of the return value. volatility: See :py:class:`Volatility` for allowed values. + arguments: A list of arguments to pass in to the __init__ method for accum. name: A descriptive name for the function. Returns: A user-defined window function. - """ - if not isinstance(func, WindowEvaluator): + """ # noqa W505 + if not callable(func): + raise TypeError("`func` must be callable.") + if not isinstance(func.__call__(), WindowEvaluator): raise TypeError( "`func` must implement the abstract base class WindowEvaluator" ) if name is None: - name = func.__class__.__qualname__.lower() + name = func.__call__().__class__.__qualname__.lower() if isinstance(input_types, pyarrow.DataType): input_types = [input_types] return WindowUDF( diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index ad7f728b4..e89c57159 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -29,7 +29,6 @@ WindowFrame, column, literal, - udf, ) from datafusion.expr import Window @@ -236,21 +235,6 @@ def test_unnest_without_nulls(nested_df): assert result.column(1) == pa.array([7, 8, 8, 9, 9, 9]) -def test_udf(df): - # is_null is a pa function over arrays - is_null = udf( - lambda x: x.is_null(), - [pa.int64()], - pa.bool_(), - volatility="immutable", - ) - - df = df.select(is_null(column("a"))) - result = df.collect()[0].column(0) - - assert result == pa.array([False, False, False]) - - def test_join(): ctx = SessionContext() diff --git a/python/datafusion/tests/test_plans.py b/python/tests/test_plans.py similarity index 100% rename from python/datafusion/tests/test_plans.py rename to python/tests/test_plans.py diff --git a/python/tests/test_udaf.py b/python/tests/test_udaf.py index 6f2525b0f..8f31748e0 100644 --- a/python/tests/test_udaf.py +++ b/python/tests/test_udaf.py @@ -21,14 +21,14 @@ import pyarrow.compute as pc import pytest -from datafusion import Accumulator, column, udaf, udf +from datafusion import Accumulator, column, udaf class Summarize(Accumulator): """Interface of a user-defined accumulation.""" - def __init__(self): - self._sum = pa.scalar(0.0) + def __init__(self, initial_value: float = 0.0): + self._sum = pa.scalar(initial_value) def state(self) -> List[pa.Scalar]: return [self._sum] @@ -79,25 +79,22 @@ def test_errors(df): volatility="immutable", ) - accum = udaf( - MissingMethods, - pa.int64(), - pa.int64(), - [pa.int64()], - volatility="immutable", - ) - df = df.aggregate([], [accum(column("a"))]) - msg = ( "Can't instantiate abstract class MissingMethods (without an implementation " "for abstract methods 'evaluate', 'merge', 'update'|with abstract methods " "evaluate, merge, update)" ) with pytest.raises(Exception, match=msg): - df.collect() + accum = udaf( # noqa F841 + MissingMethods, + pa.int64(), + pa.int64(), + [pa.int64()], + volatility="immutable", + ) -def test_aggregate(df): +def test_udaf_aggregate(df): summarize = udaf( Summarize, pa.float64(), @@ -106,13 +103,46 @@ def test_aggregate(df): volatility="immutable", ) - df = df.aggregate([], [summarize(column("a"))]) + df1 = df.aggregate([], [summarize(column("a"))]) # execute and collect the first (and only) batch - result = df.collect()[0] + result = df1.collect()[0] assert result.column(0) == pa.array([1.0 + 2.0 + 3.0]) + df2 = df.aggregate([], [summarize(column("a"))]) + + # Run a second time to ensure the state is properly reset + result = df2.collect()[0] + + assert result.column(0) == pa.array([1.0 + 2.0 + 3.0]) + + +def test_udaf_aggregate_with_arguments(df): + bias = 10.0 + + summarize = udaf( + lambda: Summarize(bias), + pa.float64(), + pa.float64(), + [pa.float64()], + volatility="immutable", + ) + + df1 = df.aggregate([], [summarize(column("a"))]) + + # execute and collect the first (and only) batch + result = df1.collect()[0] + + assert result.column(0) == pa.array([bias + 1.0 + 2.0 + 3.0]) + + df2 = df.aggregate([], [summarize(column("a"))]) + + # Run a second time to ensure the state is properly reset + result = df2.collect()[0] + + assert result.column(0) == pa.array([bias + 1.0 + 2.0 + 3.0]) + def test_group_by(df): summarize = udaf( @@ -146,20 +176,3 @@ def test_register_udaf(ctx, df) -> None: df_result = ctx.sql("select summarize(b) from test_table") assert df_result.collect()[0][0][0].as_py() == 14.0 - - -def test_register_udf(ctx, df) -> None: - is_null = udf( - lambda x: x.is_null(), - [pa.float64()], - pa.bool_(), - volatility="immutable", - name="is_null", - ) - - ctx.register_udf(is_null) - - df_result = ctx.sql("select is_null(a) from test_table") - result = df_result.collect()[0].column(0) - - assert result == pa.array([False, False, False]) diff --git a/python/tests/test_udf.py b/python/tests/test_udf.py new file mode 100644 index 000000000..568a66dbb --- /dev/null +++ b/python/tests/test_udf.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datafusion import udf, column +import pyarrow as pa +import pytest + + +@pytest.fixture +def df(ctx): + # create a RecordBatch and a new DataFrame from it + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 4, 6])], + names=["a", "b"], + ) + return ctx.create_dataframe([[batch]], name="test_table") + + +def test_udf(df): + # is_null is a pa function over arrays + is_null = udf( + lambda x: x.is_null(), + [pa.int64()], + pa.bool_(), + volatility="immutable", + ) + + df = df.select(is_null(column("a"))) + result = df.collect()[0].column(0) + + assert result == pa.array([False, False, False]) + + +def test_register_udf(ctx, df) -> None: + is_null = udf( + lambda x: x.is_null(), + [pa.float64()], + pa.bool_(), + volatility="immutable", + name="is_null", + ) + + ctx.register_udf(is_null) + + df_result = ctx.sql("select is_null(a) from test_table") + result = df_result.collect()[0].column(0) + + assert result == pa.array([False, False, False]) + + +class OverThresholdUDF: + def __init__(self, threshold: int = 0) -> None: + self.threshold = threshold + + def __call__(self, values: pa.Array) -> pa.Array: + return pa.array(v.as_py() >= self.threshold for v in values) + + +def test_udf_with_parameters(df) -> None: + udf_no_param = udf( + OverThresholdUDF(), + pa.int64(), + pa.bool_(), + volatility="immutable", + ) + + df1 = df.select(udf_no_param(column("a"))) + result = df1.collect()[0].column(0) + + assert result == pa.array([True, True, True]) + + udf_with_param = udf( + OverThresholdUDF(2), + pa.int64(), + pa.bool_(), + volatility="immutable", + ) + + df2 = df.select(udf_with_param(column("a"))) + result = df2.collect()[0].column(0) + + assert result == pa.array([False, True, True]) diff --git a/python/tests/test_udwf.py b/python/tests/test_udwf.py index 67c0979fe..2099ac9bc 100644 --- a/python/tests/test_udwf.py +++ b/python/tests/test_udwf.py @@ -24,7 +24,7 @@ class ExponentialSmoothDefault(WindowEvaluator): - def __init__(self, alpha: float) -> None: + def __init__(self, alpha: float = 0.9) -> None: self.alpha = alpha def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: @@ -44,7 +44,7 @@ def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: class ExponentialSmoothBounded(WindowEvaluator): - def __init__(self, alpha: float) -> None: + def __init__(self, alpha: float = 0.9) -> None: self.alpha = alpha def supports_bounded_execution(self) -> bool: @@ -75,7 +75,7 @@ def evaluate( class ExponentialSmoothRank(WindowEvaluator): - def __init__(self, alpha: float) -> None: + def __init__(self, alpha: float = 0.9) -> None: self.alpha = alpha def include_rank(self) -> bool: @@ -101,7 +101,7 @@ def evaluate_all_with_rank( class ExponentialSmoothFrame(WindowEvaluator): - def __init__(self, alpha: float) -> None: + def __init__(self, alpha: float = 0.9) -> None: self.alpha = alpha def uses_window_frame(self) -> bool: @@ -134,7 +134,7 @@ class SmoothTwoColumn(WindowEvaluator): the previous and next rows. """ - def __init__(self, alpha: float) -> None: + def __init__(self, alpha: float = 0.9) -> None: self.alpha = alpha def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: @@ -183,7 +183,7 @@ def df(): def test_udwf_errors(df): with pytest.raises(TypeError): udwf( - NotSubclassOfWindowEvaluator(), + NotSubclassOfWindowEvaluator, pa.float64(), pa.float64(), volatility="immutable", @@ -191,35 +191,42 @@ def test_udwf_errors(df): smooth_default = udwf( - ExponentialSmoothDefault(0.9), + ExponentialSmoothDefault, + pa.float64(), + pa.float64(), + volatility="immutable", +) + +smooth_w_arguments = udwf( + lambda: ExponentialSmoothDefault(0.8), pa.float64(), pa.float64(), volatility="immutable", ) smooth_bounded = udwf( - ExponentialSmoothBounded(0.9), + ExponentialSmoothBounded, pa.float64(), pa.float64(), volatility="immutable", ) smooth_rank = udwf( - ExponentialSmoothRank(0.9), + ExponentialSmoothRank, pa.utf8(), pa.float64(), volatility="immutable", ) smooth_frame = udwf( - ExponentialSmoothFrame(0.9), + ExponentialSmoothFrame, pa.float64(), pa.float64(), volatility="immutable", ) smooth_two_col = udwf( - SmoothTwoColumn(0.9), + SmoothTwoColumn, [pa.int64(), pa.int64()], pa.float64(), volatility="immutable", @@ -227,10 +234,15 @@ def test_udwf_errors(df): data_test_udwf_functions = [ ( - "default_udwf", + "default_udwf_no_arguments", smooth_default(column("a")), [0, 0.9, 1.89, 2.889, 3.889, 4.889, 5.889], ), + ( + "default_udwf_w_arguments", + smooth_w_arguments(column("a")), + [0, 0.8, 1.76, 2.752, 3.75, 4.75, 5.75], + ), ( "default_udwf_partitioned", smooth_default(column("a")).partition_by(column("c")).build(), diff --git a/src/udwf.rs b/src/udwf.rs index 31cc5e60e..43c21ec7b 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -197,7 +197,11 @@ impl PartitionEvaluator for RustPartitionEvaluator { pub fn to_rust_partition_evaluator(evaluator: PyObject) -> PartitionEvaluatorFactory { Arc::new(move || -> Result> { - let evaluator = Python::with_gil(|py| evaluator.clone_ref(py)); + let evaluator = Python::with_gil(|py| { + evaluator + .call0(py) + .map_err(|e| DataFusionError::Execution(e.to_string())) + })?; Ok(Box::new(RustPartitionEvaluator::new(evaluator))) }) } From 108221413cc15caeddb0a266028cbc824b5b46fd Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Fri, 4 Oct 2024 12:26:41 -0500 Subject: [PATCH 215/413] chore(ci): remove Mambaforge variant from CI (#894) * migrate away from mambaforge in CI It is being sunsetted and will stop working entirely in Jan 2025. Ref: https://conda-forge.org/news/2024/07/29/sunsetting-mambaforge/ * add conda-build to conda workflow --- .github/workflows/conda.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml index 52888cb14..c2b8fab02 100644 --- a/.github/workflows/conda.yml +++ b/.github/workflows/conda.yml @@ -70,17 +70,16 @@ jobs: - name: Set up Python uses: conda-incubator/setup-miniconda@v3.0.4 with: - miniforge-variant: Mambaforge - use-mamba: true + miniforge-variant: Miniforge3 python-version: "3.8" channel-priority: strict - name: Install dependencies run: | - mamba install -c conda-forge boa conda-verify + conda install -c conda-forge conda-build conda-verify which python pip list - mamba list + conda list # Clean the conda cache - name: Clean Conda Cache run: conda clean --all --yes @@ -89,7 +88,7 @@ jobs: # suffix for nightly package versions export VERSION_SUFFIX=a`date +%y%m%d` - conda mambabuild conda/recipes \ + conda build conda/recipes \ --python ${{ matrix.python }} \ --variants "{target_platform: [${{ matrix.arch }}]}" \ --error-overlinking \ @@ -99,7 +98,7 @@ jobs: - name: Test conda packages if: matrix.arch == 'linux-64' # can only test native platform packages run: | - conda mambabuild --test packages/${{ matrix.arch }}/*.tar.bz2 + conda build --test packages/${{ matrix.arch }}/*.tar.bz2 - name: Upload conda packages as artifacts uses: actions/upload-artifact@v4 with: From ec8246da3b45e766fe6fb515ade01e0bae73af98 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sat, 5 Oct 2024 09:57:22 -0500 Subject: [PATCH 216/413] Use OnceLock to store TokioRuntime (#895) * bump rust-version to match upstream datafusion * use std::sync::OnceLock to store tokio runtime instead of round-tripping to python * stop exporting TokioRuntime to python * remove unused argument from get_tokio_runtime * remove superflous Arc from get_tokio_runtime * add #[inline] annotation to get_tokio_runtime I also included a reference comment in case future users experience problems with using datafusion-python behind a forking app server l ike `gunicorn`. * fix clippy lint * cargo fmt --- Cargo.toml | 2 +- python/datafusion/__init__.py | 3 +-- src/context.rs | 2 +- src/dataframe.rs | 4 ++-- src/lib.rs | 6 ------ src/utils.rs | 23 +++++++++++------------ 6 files changed, 16 insertions(+), 24 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a0723984f..4f2602316 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ description = "Apache DataFusion DataFrame and SQL Query Engine" readme = "README.md" license = "Apache-2.0" edition = "2021" -rust-version = "1.64" +rust-version = "1.78" include = ["/src", "/datafusion", "/LICENSE.txt", "pyproject.toml", "Cargo.toml", "Cargo.lock"] [features] diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 63c19b3e1..e0bc57f44 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -36,7 +36,7 @@ from .catalog import Catalog, Database, Table # The following imports are okay to remain as opaque to the user. -from ._internal import Config, runtime +from ._internal import Config from .record_batch import RecordBatchStream, RecordBatch @@ -75,7 +75,6 @@ "literal", "lit", "DFSchema", - "runtime", "Catalog", "Database", "Table", diff --git a/src/context.rs b/src/context.rs index fde442ce4..5317a3eda 100644 --- a/src/context.rs +++ b/src/context.rs @@ -982,7 +982,7 @@ impl PySessionContext { ) -> PyResult { let ctx: TaskContext = TaskContext::from(&self.ctx.state()); // create a Tokio runtime to run the async code - let rt = &get_tokio_runtime(py).0; + let rt = &get_tokio_runtime().0; let plan = plan.plan.clone(); let fut: JoinHandle> = rt.spawn(async move { plan.execute(part, Arc::new(ctx)) }); diff --git a/src/dataframe.rs b/src/dataframe.rs index 1f7f2e643..e77ca8425 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -543,7 +543,7 @@ impl PyDataFrame { fn execute_stream(&self, py: Python) -> PyResult { // create a Tokio runtime to run the async code - let rt = &get_tokio_runtime(py).0; + let rt = &get_tokio_runtime().0; let df = self.df.as_ref().clone(); let fut: JoinHandle> = rt.spawn(async move { df.execute_stream().await }); @@ -553,7 +553,7 @@ impl PyDataFrame { fn execute_stream_partitioned(&self, py: Python) -> PyResult> { // create a Tokio runtime to run the async code - let rt = &get_tokio_runtime(py).0; + let rt = &get_tokio_runtime().0; let df = self.df.as_ref().clone(); let fut: JoinHandle>> = rt.spawn(async move { df.execute_stream_partitioned().await }); diff --git a/src/lib.rs b/src/lib.rs index 98821833d..0b57e0999 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -66,7 +66,6 @@ pub mod utils; static GLOBAL: MiMalloc = MiMalloc; // Used to define Tokio Runtime as a Python module attribute -#[pyclass] pub(crate) struct TokioRuntime(tokio::runtime::Runtime); /// Low-level DataFusion internal package. @@ -75,11 +74,6 @@ pub(crate) struct TokioRuntime(tokio::runtime::Runtime); /// datafusion directory. #[pymodule] fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { - // Register the Tokio Runtime as a module attribute so we can reuse it - m.add( - "runtime", - TokioRuntime(tokio::runtime::Runtime::new().unwrap()), - )?; // Register the python classes m.add_class::()?; m.add_class::()?; diff --git a/src/utils.rs b/src/utils.rs index 0d72eaf75..7fb23cafe 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -20,20 +20,19 @@ use crate::TokioRuntime; use datafusion::logical_expr::Volatility; use pyo3::prelude::*; use std::future::Future; +use std::sync::OnceLock; use tokio::runtime::Runtime; /// Utility to get the Tokio Runtime from Python -pub(crate) fn get_tokio_runtime(py: Python) -> PyRef { - let datafusion = py.import_bound("datafusion._internal").unwrap(); - let tmp = datafusion.getattr("runtime").unwrap(); - match tmp.extract::>() { - Ok(runtime) => runtime, - Err(_e) => { - let rt = TokioRuntime(tokio::runtime::Runtime::new().unwrap()); - let obj: Bound<'_, TokioRuntime> = Py::new(py, rt).unwrap().into_bound(py); - obj.extract().unwrap() - } - } +#[inline] +pub(crate) fn get_tokio_runtime() -> &'static TokioRuntime { + // NOTE: Other pyo3 python libraries have had issues with using tokio + // behind a forking app-server like `gunicorn` + // If we run into that problem, in the future we can look to `delta-rs` + // which adds a check in that disallows calls from a forked process + // https://github.com/delta-io/delta-rs/blob/87010461cfe01563d91a4b9cd6fa468e2ad5f283/python/src/utils.rs#L10-L31 + static RUNTIME: OnceLock = OnceLock::new(); + RUNTIME.get_or_init(|| TokioRuntime(tokio::runtime::Runtime::new().unwrap())) } /// Utility to collect rust futures with GIL released @@ -42,7 +41,7 @@ where F: Future + Send, F::Output: Send, { - let runtime: &Runtime = &get_tokio_runtime(py).0; + let runtime: &Runtime = &get_tokio_runtime().0; py.allow_threads(|| runtime.block_on(f)) } From 6c8292f8c6c67d79868a87bc16e0b06ac14d7f0b Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Sun, 6 Oct 2024 23:41:45 +0200 Subject: [PATCH 217/413] feat: expose http object store (#885) * feat: expose HTTP ObjectStore The objective is to allow the user to register CSV directly from an HTTP URL, delaying downloading the file until required * chore: return PyResult --- Cargo.toml | 2 +- python/datafusion/context.py | 4 +++- python/datafusion/object_store.py | 12 ++--------- python/tests/test_sql.py | 12 +++++++++-- python/tests/test_store.py | 10 ++------- python/tests/test_wrapper_coverage.py | 2 +- src/context.rs | 1 + src/store.rs | 31 +++++++++++++++++++++++++++ 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4f2602316..d2d3e79ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,7 +47,7 @@ uuid = { version = "1.9", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" -object_store = { version = "0.11.0", features = ["aws", "gcp", "azure"] } +object_store = { version = "0.11.0", features = ["aws", "gcp", "azure", "http"] } parking_lot = "0.12" regex-syntax = "0.8" syn = "2.0.79" diff --git a/python/datafusion/context.py b/python/datafusion/context.py index b08e62d77..957d7e311 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -450,7 +450,9 @@ def __init__( self.ctx = SessionContextInternal(config, runtime) - def register_object_store(self, schema: str, store: Any, host: str | None) -> None: + def register_object_store( + self, schema: str, store: Any, host: str | None = None + ) -> None: """Add a new object store into the session. Args: diff --git a/python/datafusion/object_store.py b/python/datafusion/object_store.py index c927e7614..7cc17506f 100644 --- a/python/datafusion/object_store.py +++ b/python/datafusion/object_store.py @@ -22,14 +22,6 @@ GoogleCloud = object_store.GoogleCloud LocalFileSystem = object_store.LocalFileSystem MicrosoftAzure = object_store.MicrosoftAzure +Http = object_store.Http -__all__ = [ - "AmazonS3", - "GoogleCloud", - "LocalFileSystem", - "MicrosoftAzure", -] - - -def __getattr__(name): - return getattr(object_store, name) +__all__ = ["AmazonS3", "GoogleCloud", "LocalFileSystem", "MicrosoftAzure", "Http"] diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py index e39a9f5c7..39e5ffe6d 100644 --- a/python/tests/test_sql.py +++ b/python/tests/test_sql.py @@ -22,7 +22,7 @@ from pyarrow.csv import write_csv import pyarrow.dataset as ds import pytest -from datafusion.object_store import LocalFileSystem +from datafusion.object_store import Http from datafusion import udf, col @@ -139,6 +139,15 @@ def test_register_csv_list(ctx, tmp_path): assert int_sum == 2 * sum(int_values) +def test_register_http_csv(ctx): + url = "https://raw.githubusercontent.com/ibis-project/testing-data/refs/heads/master/csv/diamonds.csv" + ctx.register_object_store("", Http(url)) + ctx.register_csv("remote", url) + assert ctx.table_exist("remote") + res, *_ = ctx.sql("SELECT COUNT(*) AS total FROM remote").to_pylist() + assert res["total"] > 0 + + def test_register_parquet(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) ctx.register_parquet("t", path) @@ -494,7 +503,6 @@ def test_register_listing_table( dir_root = f"file://{dir_root}/" if path_to_str else dir_root - ctx.register_object_store("file://local", LocalFileSystem(), None) ctx.register_listing_table( "my_table", dir_root, diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 3ffd9ee49..f85b28311 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -16,21 +16,15 @@ # under the License. import os + import pytest from datafusion import SessionContext -from datafusion.object_store import LocalFileSystem - - -@pytest.fixture -def local(): - return LocalFileSystem() @pytest.fixture -def ctx(local): +def ctx(): ctx = SessionContext() - ctx.register_object_store("file://local", local, None) return ctx diff --git a/python/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py index c53a89c59..86f2d57f2 100644 --- a/python/tests/test_wrapper_coverage.py +++ b/python/tests/test_wrapper_coverage.py @@ -55,7 +55,7 @@ def missing_exports(internal_obj, wrapped_obj) -> None: def test_datafusion_missing_exports() -> None: - """Check for any missing pythone exports. + """Check for any missing python exports. This test verifies that every exposed class, attribute, and function in the internal (pyo3) module is also exposed in our python wrappers. diff --git a/src/context.rs b/src/context.rs index 5317a3eda..f445874d6 100644 --- a/src/context.rs +++ b/src/context.rs @@ -312,6 +312,7 @@ impl PySessionContext { StorageContexts::GoogleCloudStorage(gcs) => (gcs.inner, gcs.bucket_name), StorageContexts::MicrosoftAzure(azure) => (azure.inner, azure.container_name), StorageContexts::LocalFileSystem(local) => (local.inner, "".to_string()), + StorageContexts::HTTP(http) => (http.store, http.url), }; // let users override the host to match the api signature from upstream diff --git a/src/store.rs b/src/store.rs index 846d96a6d..1e5fab472 100644 --- a/src/store.rs +++ b/src/store.rs @@ -22,7 +22,10 @@ use pyo3::prelude::*; use object_store::aws::{AmazonS3, AmazonS3Builder}; use object_store::azure::{MicrosoftAzure, MicrosoftAzureBuilder}; use object_store::gcp::{GoogleCloudStorage, GoogleCloudStorageBuilder}; +use object_store::http::{HttpBuilder, HttpStore}; use object_store::local::LocalFileSystem; +use pyo3::exceptions::PyValueError; +use url::Url; #[derive(FromPyObject)] pub enum StorageContexts { @@ -30,6 +33,7 @@ pub enum StorageContexts { GoogleCloudStorage(PyGoogleCloudContext), MicrosoftAzure(PyMicrosoftAzureContext), LocalFileSystem(PyLocalFileSystemContext), + HTTP(PyHttpContext), } #[pyclass(name = "LocalFileSystem", module = "datafusion.store", subclass)] @@ -219,10 +223,37 @@ impl PyAmazonS3Context { } } +#[pyclass(name = "Http", module = "datafusion.store", subclass)] +#[derive(Debug, Clone)] +pub struct PyHttpContext { + pub url: String, + pub store: Arc, +} + +#[pymethods] +impl PyHttpContext { + #[new] + fn new(url: String) -> PyResult { + let store = match Url::parse(url.as_str()) { + Ok(url) => HttpBuilder::new() + .with_url(url.origin().ascii_serialization()) + .build(), + Err(_) => HttpBuilder::new().build(), + } + .map_err(|e| PyValueError::new_err(format!("Error: {:?}", e.to_string())))?; + + Ok(Self { + url, + store: Arc::new(store), + }) + } +} + pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; Ok(()) } From af2d66581817c21048ad77729069b7af40b4ba98 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 7 Oct 2024 07:27:34 -0600 Subject: [PATCH 218/413] Update version and generate changelog (#901) --- CHANGELOG.md | 49 ++++++++++++++- Cargo.lock | 170 ++++++++++++++++++++++++--------------------------- Cargo.toml | 8 +-- 3 files changed, 131 insertions(+), 96 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a2bfc79..ae3a2348a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,54 @@ # DataFusion Python Changelog +## [42.0.0](https://github.com/apache/datafusion-python/tree/42.0.0) (2024-10-06) + +This release consists of 20 commits from 6 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: expose between [#868](https://github.com/apache/datafusion-python/pull/868) (mesejo) +- feat: make register_csv accept a list of paths [#883](https://github.com/apache/datafusion-python/pull/883) (mesejo) +- feat: expose http object store [#885](https://github.com/apache/datafusion-python/pull/885) (mesejo) + +**Fixed bugs:** + +- fix: Calling `count` on a pyarrow dataset results in an error [#843](https://github.com/apache/datafusion-python/pull/843) (Michael-J-Ward) + +**Other:** + +- Upgrade datafusion [#867](https://github.com/apache/datafusion-python/pull/867) (emgeee) +- Feature/aggregates as windows [#871](https://github.com/apache/datafusion-python/pull/871) (timsaucer) +- Fix regression on register_udaf [#878](https://github.com/apache/datafusion-python/pull/878) (timsaucer) +- build(deps): upgrade setup-protoc action and protoc version number [#873](https://github.com/apache/datafusion-python/pull/873) (Michael-J-Ward) +- build(deps): bump prost-types from 0.13.2 to 0.13.3 [#881](https://github.com/apache/datafusion-python/pull/881) (dependabot[bot]) +- build(deps): bump prost from 0.13.2 to 0.13.3 [#882](https://github.com/apache/datafusion-python/pull/882) (dependabot[bot]) +- chore: remove XFAIL from passing tests [#884](https://github.com/apache/datafusion-python/pull/884) (Michael-J-Ward) +- Add user defined window function support [#880](https://github.com/apache/datafusion-python/pull/880) (timsaucer) +- build(deps): bump syn from 2.0.77 to 2.0.79 [#886](https://github.com/apache/datafusion-python/pull/886) (dependabot[bot]) +- fix example of reading parquet from s3 [#896](https://github.com/apache/datafusion-python/pull/896) (sir-sigurd) +- release-testing [#889](https://github.com/apache/datafusion-python/pull/889) (Michael-J-Ward) +- chore(bench): fix create_tables.sql for tpch benchmark [#897](https://github.com/apache/datafusion-python/pull/897) (Michael-J-Ward) +- Add physical and logical plan conversion to and from protobuf [#892](https://github.com/apache/datafusion-python/pull/892) (timsaucer) +- Feature/instance udfs [#890](https://github.com/apache/datafusion-python/pull/890) (timsaucer) +- chore(ci): remove Mambaforge variant from CI [#894](https://github.com/apache/datafusion-python/pull/894) (Michael-J-Ward) +- Use OnceLock to store TokioRuntime [#895](https://github.com/apache/datafusion-python/pull/895) (Michael-J-Ward) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 7 Michael J Ward + 5 Tim Saucer + 3 Daniel Mesejo + 3 dependabot[bot] + 1 Matt Green + 1 Sergey Fedoseev +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + ## [41.0.0](https://github.com/apache/datafusion-python/tree/41.0.0) (2024-09-09) This release consists of 19 commits from 6 contributors. See credits at the end of this changelog for more information. @@ -63,7 +111,6 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. - ## [40.0.0](https://github.com/apache/datafusion-python/tree/40.0.0) (2024-08-09) This release consists of 18 commits from 4 contributors. See credits at the end of this changelog for more information. diff --git a/Cargo.lock b/Cargo.lock index 0935ea9e0..815323bf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5fb1d8e4442bd405fdfd1dacb42792696b0cf9cb15882e5d097b742a676d375" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] @@ -130,9 +130,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45aef0d9cf9a039bf6cd1acc451b137aca819977b0928dece52bd92811b640ba" +checksum = "a9ba0d7248932f4e2a12fb37f0a2e3ec82b3bdedbac2a1dce186e036843b8f8c" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a" +checksum = "d60afcdc004841a5c8d8da4f4fa22d64eb19c0c01ef4bcedd77f175a7cf6e38f" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f" +checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d" +checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80" dependencies = [ "bytes", "half", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785" +checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd178575f45624d045e4ebee714e246a05d9652e41363ee3f57ec18cca97f740" +checksum = "d3a4e4d63830a341713e35d9a42452fbc6241d5f42fa5cf6a4681b8ad91370c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65" +checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634" dependencies = [ "arrow-buffer", "arrow-schema", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a" +checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d24805ba326758effdd6f2cbdd482fcfab749544f21b134701add25b33f474e6" +checksum = "b198b9c6fcf086501730efbbcb483317b39330a116125af7bb06467d04b352a3" dependencies = [ "arrow-array", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b" +checksum = "2427f37b4459a4b9e533045abe87a5183a5e0995a3fc2c2fd45027ae2cc4ef3f" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb" +checksum = "15959657d92e2261a7a323517640af87f5afd9fd8a6492e424ebee2203c567f6" dependencies = [ "ahash", "arrow-array", @@ -311,18 +311,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858" +checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba" +checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a" dependencies = [ "ahash", "arrow-array", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230" +checksum = "7ab3db7c09dd826e74079661d84ed01ed06547cf75d52c2818ef776d0d852305" dependencies = [ "arrow-array", "arrow-buffer", @@ -542,9 +542,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.24" +version = "1.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" +checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" dependencies = [ "jobserver", "libc", @@ -572,9 +572,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" dependencies = [ "chrono", "chrono-tz-build", @@ -583,12 +583,11 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" dependencies = [ "parse-zoneinfo", - "phf", "phf_codegen", ] @@ -1151,7 +1150,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "41.0.0" +version = "42.0.0" dependencies = [ "arrow", "async-trait", @@ -1161,14 +1160,10 @@ dependencies = [ "futures", "mimalloc", "object_store", - "parking_lot", "prost", "prost-types", "pyo3", "pyo3-build-config", - "rand", - "regex-syntax", - "syn", "tokio", "url", "uuid", @@ -1297,9 +1292,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -1312,9 +1307,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1322,15 +1317,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1339,15 +1334,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -1356,21 +1351,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -1407,9 +1402,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32085ea23f3234fc7846555e85283ba4de91e21016dc0455a16286d87a292d64" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -1659,9 +1654,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "itertools" @@ -1704,9 +1699,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -1717,9 +1712,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -1728,9 +1723,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "0.8.6" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" dependencies = [ "lexical-util", "static_assertions", @@ -1738,18 +1733,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "0.8.5" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" dependencies = [ "lexical-util", "lexical-write-integer", @@ -1758,9 +1753,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "0.8.5" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" dependencies = [ "lexical-util", "static_assertions", @@ -2007,9 +2002,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] @@ -2047,12 +2042,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.1" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" -dependencies = [ - "portable-atomic", -] +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -2094,9 +2086,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.0.0" +version = "53.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0fbf928021131daaa57d334ca8e3904fe9ae22f73c56244fc7db9b04eedc3d8" +checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e" dependencies = [ "ahash", "arrow-array", @@ -2670,9 +2662,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.13" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dabaac7466917e566adb06783a81ca48944c6898a1b08b9374106dd671f4c8" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "once_cell", "ring", @@ -2744,9 +2736,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] diff --git a/Cargo.toml b/Cargo.toml index d2d3e79ed..df72cd40a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "41.0.0" +version = "42.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] @@ -35,25 +35,21 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync"] } -rand = "0.8" pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "53", features = ["pyarrow"] } datafusion = { version = "42.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } datafusion-substrait = { version = "42.0.0", optional = true } datafusion-proto = { version = "42.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` -prost-types = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.9", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" object_store = { version = "0.11.0", features = ["aws", "gcp", "azure", "http"] } -parking_lot = "0.12" -regex-syntax = "0.8" -syn = "2.0.79" url = "2" [build-dependencies] +prost-types = "0.13" # keep in line with `datafusion-substrait` pyo3-build-config = "0.22" [lib] From cdec2025383ed2c6549ffaa37f1dee199e7d1f9b Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Tue, 8 Oct 2024 17:30:57 -0400 Subject: [PATCH 219/413] Ts/minor updates release process (#903) * Add instructions for updating submodule to test a release * Apply formatting to changelog script --- dev/release/README.md | 1 + dev/release/generate-changelog.py | 76 +++++++++++++++++++------------ 2 files changed, 48 insertions(+), 29 deletions(-) diff --git a/dev/release/README.md b/dev/release/README.md index 49fd9de2d..b2c015e1d 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -169,6 +169,7 @@ cd datafusion-python # checkout the release commit git fetch --tags git checkout 40.0.0-rc1 +git submodule update --init --recursive # create the env python3 -m venv venv diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 5645d2f74..0f07457d0 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -22,18 +22,22 @@ import re import subprocess + def print_pulls(repo_name, title, pulls): - if len(pulls) > 0: + if len(pulls) > 0: print("**{}:**".format(title)) print() - for (pull, commit) in pulls: + for pull, commit in pulls: url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) - print("- {} [#{}]({}) ({})".format(pull.title, pull.number, url, commit.author.login)) + print( + "- {} [#{}]({}) ({})".format( + pull.title, pull.number, url, commit.author.login + ) + ) print() def generate_changelog(repo, repo_name, tag1, tag2, version): - # get a list of commits between two tags print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) comparison = repo.compare(tag1, tag2) @@ -61,29 +65,27 @@ def generate_changelog(repo, repo_name, tag1, tag2, version): # categorize the pull requests based on GitHub labels print("Categorizing pull requests", file=sys.stderr) - for (pull, commit) in all_pulls: - + for pull, commit in all_pulls: # see if PR title uses Conventional Commits - cc_type = '' - cc_scope = '' - cc_breaking = '' - parts = re.findall(r'^([a-z]+)(\([a-z]+\))?(!)?:', pull.title) + cc_type = "" + cc_breaking = "" + parts = re.findall(r"^([a-z]+)(\([a-z]+\))?(!)?:", pull.title) if len(parts) == 1: parts_tuple = parts[0] - cc_type = parts_tuple[0] # fix, feat, docs, chore - cc_scope = parts_tuple[1] # component within project - cc_breaking = parts_tuple[2] == '!' + cc_type = parts_tuple[0] # fix, feat, docs, chore + # cc_scope = parts_tuple[1] # component within project + cc_breaking = parts_tuple[2] == "!" labels = [label.name for label in pull.labels] - if 'api change' in labels or cc_breaking: + if "api change" in labels or cc_breaking: breaking.append((pull, commit)) - elif 'bug' in labels or cc_type == 'fix': + elif "bug" in labels or cc_type == "fix": bugs.append((pull, commit)) - elif 'performance' in labels or cc_type == 'perf': + elif "performance" in labels or cc_type == "perf": performance.append((pull, commit)) - elif 'enhancement' in labels or cc_type == 'feat': + elif "enhancement" in labels or cc_type == "feat": enhancements.append((pull, commit)) - elif 'documentation' in labels or cc_type == 'docs' or cc_type == 'doc': + elif "documentation" in labels or cc_type == "docs" or cc_type == "doc": docs.append((pull, commit)) else: other.append((pull, commit)) @@ -114,13 +116,19 @@ def generate_changelog(repo, repo_name, tag1, tag2, version): print(f"# Apache DataFusion Python {version} Changelog\n") # get the number of commits - commit_count = subprocess.check_output(f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + commit_count = subprocess.check_output( + f"git log --pretty=oneline {tag1}..{tag2} | wc -l", shell=True, text=True + ).strip() # get number of contributors - contributor_count = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True).strip() + contributor_count = subprocess.check_output( + f"git shortlog -sn {tag1}..{tag2} | wc -l", shell=True, text=True + ).strip() - print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " - f"See credits at the end of this changelog for more information.\n") + print( + f"This release consists of {commit_count} commits from {contributor_count} contributors. " + f"See credits at the end of this changelog for more information.\n" + ) print_pulls(repo_name, "Breaking changes", breaking) print_pulls(repo_name, "Performance related", performance) @@ -130,17 +138,24 @@ def generate_changelog(repo, repo_name, tag1, tag2, version): print_pulls(repo_name, "Other", other) # show code contributions - credits = subprocess.check_output(f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True).rstrip() + credits = subprocess.check_output( + f"git shortlog -sn {tag1}..{tag2}", shell=True, text=True + ).rstrip() print("## Credits\n") - print("Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " - "per contributor.\n") + print( + "Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) " + "per contributor.\n" + ) print("```") print(credits) print("```\n") - print("Thank you also to everyone who contributed in other ways such as filing issues, reviewing " - "PRs, and providing feedback on this release.\n") + print( + "Thank you also to everyone who contributed in other ways such as filing issues, reviewing " + "PRs, and providing feedback on this release.\n" + ) + def cli(args=None): """Process command line arguments.""" @@ -150,7 +165,9 @@ def cli(args=None): parser = argparse.ArgumentParser() parser.add_argument("tag1", help="The previous commit or tag (e.g. 0.1.0)") parser.add_argument("tag2", help="The current commit or tag (e.g. HEAD)") - parser.add_argument("version", help="The version number to include in the changelog") + parser.add_argument( + "version", help="The version number to include in the changelog" + ) args = parser.parse_args() token = os.getenv("GITHUB_TOKEN") @@ -160,5 +177,6 @@ def cli(args=None): repo = g.get_repo(project) generate_changelog(repo, project, args.tag1, args.tag2, args.version) + if __name__ == "__main__": - cli() \ No newline at end of file + cli() From 840b5de08dad7dd9cd0d8db19d9245dcace721ce Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Mon, 14 Oct 2024 09:55:46 -0500 Subject: [PATCH 220/413] fix: remove use of deprecated `make_scalar_function` (#906) * fix: remove use of deprecated `make_scalar_function` `make_scalar_function` has been deprecated since v36 [0]. It is being removed from the public api in v43 [1]. [0]: https://github.com/apache/datafusion/pull/8878 [1]: https://github.com/apache/datafusion/pull/12505 * remove use of `.unwrap()` from pyarrow_function_to_rust --- src/udf.rs | 65 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/src/udf.rs b/src/udf.rs index 7d5db2f96..ec8efb169 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -24,39 +24,56 @@ use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::FromPyArrow; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::error::DataFusionError; -use datafusion::logical_expr::create_udf; use datafusion::logical_expr::function::ScalarFunctionImplementation; use datafusion::logical_expr::ScalarUDF; +use datafusion::logical_expr::{create_udf, ColumnarValue}; use crate::expr::PyExpr; use crate::utils::parse_volatility; +/// Create a Rust callable function fr a python function that expects pyarrow arrays +fn pyarrow_function_to_rust( + func: PyObject, +) -> impl Fn(&[ArrayRef]) -> Result { + move |args: &[ArrayRef]| -> Result { + Python::with_gil(|py| { + // 1. cast args to Pyarrow arrays + let py_args = args + .iter() + .map(|arg| { + arg.into_data() + .to_pyarrow(py) + .map_err(|e| DataFusionError::Execution(format!("{e:?}"))) + }) + .collect::, _>>()?; + let py_args = PyTuple::new_bound(py, py_args); + + // 2. call function + let value = func + .call_bound(py, py_args, None) + .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; + + // 3. cast to arrow::array::Array + let array_data = ArrayData::from_pyarrow_bound(value.bind(py)) + .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; + Ok(make_array(array_data)) + }) + } +} + /// Create a DataFusion's UDF implementation from a python function /// that expects pyarrow arrays. This is more efficient as it performs /// a zero-copy of the contents. -fn to_rust_function(func: PyObject) -> ScalarFunctionImplementation { - #[allow(deprecated)] - datafusion::physical_plan::functions::make_scalar_function( - move |args: &[ArrayRef]| -> Result { - Python::with_gil(|py| { - // 1. cast args to Pyarrow arrays - let py_args = args - .iter() - .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) - .collect::>(); - let py_args = PyTuple::new_bound(py, py_args); - - // 2. call function - let value = func - .call_bound(py, py_args, None) - .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; +fn to_scalar_function_impl(func: PyObject) -> ScalarFunctionImplementation { + // Make the python function callable from rust + let pyarrow_func = pyarrow_function_to_rust(func); - // 3. cast to arrow::array::Array - let array_data = ArrayData::from_pyarrow_bound(value.bind(py)).unwrap(); - Ok(make_array(array_data)) - }) - }, - ) + // Convert input/output from datafusion ColumnarValue to arrow arrays + Arc::new(move |args: &[ColumnarValue]| { + let array_refs = ColumnarValue::values_to_arrays(args)?; + let array_result = pyarrow_func(&array_refs)?; + Ok(array_result.into()) + }) } /// Represents a PyScalarUDF @@ -82,7 +99,7 @@ impl PyScalarUDF { input_types.0, Arc::new(return_type.0), parse_volatility(volatility)?, - to_rust_function(func), + to_scalar_function_impl(func), ); Ok(Self { function }) } From 3d751728467a4210007bd9d266f22ae0e291d63f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Oct 2024 12:05:45 -0500 Subject: [PATCH 221/413] build(deps): bump pyo3 from 0.22.3 to 0.22.4 (#910) Bumps [pyo3](https://github.com/pyo3/pyo3) from 0.22.3 to 0.22.4. - [Release notes](https://github.com/pyo3/pyo3/releases) - [Changelog](https://github.com/PyO3/pyo3/blob/main/CHANGELOG.md) - [Commits](https://github.com/pyo3/pyo3/commits) --- updated-dependencies: - dependency-name: pyo3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 815323bf4..56c368f46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2342,9 +2342,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225" +checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" dependencies = [ "cfg-if", "indoc", @@ -2360,9 +2360,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3" +checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" dependencies = [ "once_cell", "target-lexicon", @@ -2370,9 +2370,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c" +checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" dependencies = [ "libc", "pyo3-build-config", @@ -2380,9 +2380,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28" +checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2392,9 +2392,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.3" +version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1" +checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" dependencies = [ "heck 0.5.0", "proc-macro2", From 72f274385792d6eee3d9053ed786966b4899b24d Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:20:22 +0200 Subject: [PATCH 222/413] feat: expose drop method (#913) --- python/datafusion/dataframe.py | 11 +++++++++++ python/tests/test_dataframe.py | 11 +++++++++++ src/dataframe.rs | 7 +++++++ 3 files changed, 29 insertions(+) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index c5ac0bb89..a9e4d4d10 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -129,6 +129,17 @@ def select(self, *exprs: Expr | str) -> DataFrame: ] return DataFrame(self.df.select(*exprs_internal)) + def drop(self, *columns: str) -> DataFrame: + """Drop arbitrary amount of columns. + + Args: + columns: Column names to drop from the dataframe. + + Returns: + DataFrame with those columns removed in the projection. + """ + return DataFrame(self.df.drop(*columns)) + def filter(self, *predicates: Expr) -> DataFrame: """Return a DataFrame for which ``predicate`` evaluates to ``True``. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index e89c57159..88c642a7d 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -169,6 +169,17 @@ def test_sort(df): assert table.to_pydict() == expected +def test_drop(df): + df = df.drop("c") + + # execute and collect the first (and only) batch + result = df.collect()[0] + + assert df.schema().names == ["a", "b"] + assert result.column(0) == pa.array([1, 2, 3]) + assert result.column(1) == pa.array([4, 5, 6]) + + def test_limit(df): df = df.limit(1) diff --git a/src/dataframe.rs b/src/dataframe.rs index e77ca8425..db243704a 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -170,6 +170,13 @@ impl PyDataFrame { Ok(Self::new(df)) } + #[pyo3(signature = (*args))] + fn drop(&self, args: Vec) -> PyResult { + let cols = args.iter().map(|s| s.as_ref()).collect::>(); + let df = self.df.as_ref().clone().drop_columns(&cols)?; + Ok(Self::new(df)) + } + fn filter(&self, predicate: PyExpr) -> PyResult { let df = self.df.as_ref().clone().filter(predicate.into())?; Ok(Self::new(df)) From b4b03fe10fab72cc5606a193b58e1c9ae5031318 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:20:39 +0200 Subject: [PATCH 223/413] feat: expose `join_on` (#914) * feat: expose join_on method * test: improve join_on case --- python/datafusion/dataframe.py | 25 ++++++++++++++++++++++- python/tests/test_dataframe.py | 36 ++++++++++++++++++++++++++++++++++ src/dataframe.rs | 25 +++++++++++++++++++++++ 3 files changed, 85 insertions(+), 1 deletion(-) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index a9e4d4d10..60203ffb4 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -21,7 +21,7 @@ from __future__ import annotations -from typing import Any, List, TYPE_CHECKING +from typing import Any, List, TYPE_CHECKING, Literal from datafusion.record_batch import RecordBatchStream from typing_extensions import deprecated from datafusion.plan import LogicalPlan, ExecutionPlan @@ -304,6 +304,29 @@ def join( """ return DataFrame(self.df.join(right.df, join_keys, how)) + def join_on( + self, + right: DataFrame, + *on_exprs: Expr, + how: Literal["inner", "left", "right", "full", "semi", "anti"] = "inner", + ) -> DataFrame: + """Join two :py:class:`DataFrame`using the specified expressions. + + On expressions are used to support in-equality predicates. Equality + predicates are correctly optimized + + Args: + right: Other DataFrame to join with. + on_exprs: single or multiple (in)-equality predicates. + how: Type of join to perform. Supported types are "inner", "left", + "right", "full", "semi", "anti". + + Returns: + DataFrame after join. + """ + exprs = [expr.expr for expr in on_exprs] + return DataFrame(self.df.join_on(right.df, exprs, how)) + def explain(self, verbose: bool = False, analyze: bool = False) -> DataFrame: """Return a DataFrame with the explanation of its plan so far. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 88c642a7d..6330ede04 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -270,6 +270,42 @@ def test_join(): assert table.to_pydict() == expected +def test_join_on(): + ctx = SessionContext() + + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]], "l") + + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2]), pa.array([-8, 10])], + names=["a", "c"], + ) + df1 = ctx.create_dataframe([[batch]], "r") + + df2 = df.join_on(df1, column("l.a").__eq__(column("r.a")), how="inner") + df2.show() + df2 = df2.sort(column("l.a")) + table = pa.Table.from_batches(df2.collect()) + + expected = {"a": [1, 2], "c": [-8, 10], "b": [4, 5]} + assert table.to_pydict() == expected + + df3 = df.join_on( + df1, + column("l.a").__eq__(column("r.a")), + column("l.a").__lt__(column("r.c")), + how="inner", + ) + df3.show() + df3 = df3.sort(column("l.a")) + table = pa.Table.from_batches(df3.collect()) + expected = {"a": [2], "c": [10], "b": [5]} + assert table.to_pydict() == expected + + def test_distinct(): ctx = SessionContext() diff --git a/src/dataframe.rs b/src/dataframe.rs index db243704a..fa6c1d44f 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -300,6 +300,31 @@ impl PyDataFrame { Ok(Self::new(df)) } + fn join_on(&self, right: PyDataFrame, on_exprs: Vec, how: &str) -> PyResult { + let join_type = match how { + "inner" => JoinType::Inner, + "left" => JoinType::Left, + "right" => JoinType::Right, + "full" => JoinType::Full, + "semi" => JoinType::LeftSemi, + "anti" => JoinType::LeftAnti, + how => { + return Err(DataFusionError::Common(format!( + "The join type {how} does not exist or is not implemented" + )) + .into()); + } + }; + let exprs: Vec = on_exprs.into_iter().map(|e| e.into()).collect(); + + let df = self + .df + .as_ref() + .clone() + .join_on(right.df.as_ref().clone(), join_type, exprs)?; + Ok(Self::new(df)) + } + /// Print the query plan #[pyo3(signature = (verbose=false, analyze=false))] fn explain(&self, py: Python, verbose: bool, analyze: bool) -> PyResult<()> { From 494b89a522541bbaf9c3cd5d7b6bd7ab7218a399 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Tue, 15 Oct 2024 13:21:30 +0200 Subject: [PATCH 224/413] refactor: from_arrow (#917) --- python/datafusion/context.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 957d7e311..5221c866c 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -30,7 +30,7 @@ from datafusion.record_batch import RecordBatchStream from datafusion.udf import ScalarUDF, AggregateUDF, WindowUDF -from typing import Any, TYPE_CHECKING +from typing import Any, TYPE_CHECKING, Protocol from typing_extensions import deprecated if TYPE_CHECKING: @@ -41,6 +41,28 @@ from datafusion.plan import LogicalPlan, ExecutionPlan +class ArrowStreamExportable(Protocol): + """Type hint for object exporting Arrow C Stream via Arrow PyCapsule Interface. + + https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html + """ + + def __arrow_c_stream__( # noqa: D105 + self, requested_schema: object | None = None + ) -> object: ... + + +class ArrowArrayExportable(Protocol): + """Type hint for object exporting Arrow C Array via Arrow PyCapsule Interface. + + https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html + """ + + def __arrow_c_array__( # noqa: D105 + self, requested_schema: object | None = None + ) -> tuple[object, object]: ... + + class SessionConfig: """Session configuration options.""" @@ -592,12 +614,18 @@ def from_pydict( """ return DataFrame(self.ctx.from_pydict(data, name)) - def from_arrow(self, data: Any, name: str | None = None) -> DataFrame: + def from_arrow( + self, + data: ArrowStreamExportable | ArrowArrayExportable, + name: str | None = None, + ) -> DataFrame: """Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow source. The Arrow data source can be any object that implements either ``__arrow_c_stream__`` or ``__arrow_c_array__``. For the latter, it must return - a struct array. Common examples of sources from pyarrow include + a struct array. + + Arrow data can be Polars, Pandas, Pyarrow etc. Args: data: Arrow data source. From 0905f5fca4b763fc61e5e2093a85ad05e203d7fb Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Wed, 16 Oct 2024 14:36:56 +0200 Subject: [PATCH 225/413] feat: add fill_null/nan (#919) --- python/datafusion/expr.py | 12 ++++++++++++ python/datafusion/functions.py | 6 ++++++ python/tests/test_expr.py | 33 ++++++++++++++++++++++++++++++--- src/functions.rs | 6 ++++++ 4 files changed, 54 insertions(+), 3 deletions(-) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 8600627ae..c4e7713f3 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -406,6 +406,18 @@ def is_not_null(self) -> Expr: """Returns ``True`` if this expression is not null.""" return Expr(self.expr.is_not_null()) + def fill_nan(self, value: Any | Expr | None = None) -> Expr: + """Fill NaN values with a provided value.""" + if not isinstance(value, Expr): + value = Expr.literal(value) + return Expr(functions_internal.nanvl(self.expr, value.expr)) + + def fill_null(self, value: Any | Expr | None = None) -> Expr: + """Fill NULL values with a provided value.""" + if not isinstance(value, Expr): + value = Expr.literal(value) + return Expr(functions_internal.nvl(self.expr, value.expr)) + _to_pyarrow_types = { float: pa.float64(), int: pa.int64(), diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 0401afbc4..727321979 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -186,6 +186,7 @@ "min", "named_struct", "nanvl", + "nvl", "now", "nth_value", "nullif", @@ -673,6 +674,11 @@ def nanvl(x: Expr, y: Expr) -> Expr: return Expr(f.nanvl(x.expr, y.expr)) +def nvl(x: Expr, y: Expr) -> Expr: + """Returns ``x`` if ``x`` is not ``NULL``. Otherwise returns ``y``.""" + return Expr(f.nvl(x.expr, y.expr)) + + def octet_length(arg: Expr) -> Expr: """Returns the number of bytes of a string.""" return Expr(f.octet_length(arg.expr)) diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index b58177f16..1847edef2 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import pyarrow +import pyarrow as pa import pytest from datafusion import SessionContext, col from datafusion.expr import ( @@ -125,8 +125,8 @@ def test_sort(test_ctx): def test_relational_expr(test_ctx): ctx = SessionContext() - batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array(["alpha", "beta", "gamma"])], + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array(["alpha", "beta", "gamma"])], names=["a", "b"], ) df = ctx.create_dataframe([[batch]], name="batch_array") @@ -216,3 +216,30 @@ def test_display_name_deprecation(): # returns appropriate result assert name == expr.schema_name() assert name == "foo" + + +@pytest.fixture +def df(): + ctx = SessionContext() + + # create a RecordBatch and a new DataFrame from it + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, None]), pa.array([4, None, 6]), pa.array([None, None, 8])], + names=["a", "b", "c"], + ) + + return ctx.from_arrow(batch) + + +def test_fill_null(df): + df = df.select( + col("a").fill_null(100).alias("a"), + col("b").fill_null(25).alias("b"), + col("c").fill_null(1234).alias("c"), + ) + df.show() + result = df.collect()[0] + + assert result.column(0) == pa.array([1, 2, 100]) + assert result.column(1) == pa.array([4, 25, 6]) + assert result.column(2) == pa.array([1234, 1234, 8]) diff --git a/src/functions.rs b/src/functions.rs index 6f8dd7ada..24d33af39 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -490,6 +490,11 @@ expr_fn!( x y, "Returns x if x is not NaN otherwise returns y." ); +expr_fn!( + nvl, + x y, + "Returns x if x is not NULL otherwise returns y." +); expr_fn!(nullif, arg_1 arg_2); expr_fn!(octet_length, args, "Returns number of bytes in the string. Since this version of the function accepts type character directly, it will not strip trailing spaces."); expr_fn_vec!(overlay); @@ -913,6 +918,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(min))?; m.add_wrapped(wrap_pyfunction!(named_struct))?; m.add_wrapped(wrap_pyfunction!(nanvl))?; + m.add_wrapped(wrap_pyfunction!(nvl))?; m.add_wrapped(wrap_pyfunction!(now))?; m.add_wrapped(wrap_pyfunction!(nullif))?; m.add_wrapped(wrap_pyfunction!(octet_length))?; From fc7e3e546dbb03783e66d44837d75a1e6ad62827 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 19 Oct 2024 06:15:28 +0800 Subject: [PATCH 226/413] Change requires-python version (#924) --- pyproject.toml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e10333a0..d327c0ec1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,8 +23,8 @@ build-backend = "maturin" name = "datafusion" description = "Build and run queries against data" readme = "README.md" -license = {file = "LICENSE.txt"} -requires-python = ">=3.6" +license = { file = "LICENSE.txt" } +requires-python = ">=3.7" keywords = ["datafusion", "dataframe", "rust", "query-engine"] classifier = [ "Development Status :: 2 - Pre-Alpha", @@ -42,10 +42,7 @@ classifier = [ "Programming Language :: Python", "Programming Language :: Rust", ] -dependencies = [ - "pyarrow>=11.0.0", - "typing-extensions;python_version<'3.13'", -] +dependencies = ["pyarrow>=11.0.0", "typing-extensions;python_version<'3.13'"] [project.urls] homepage = "https://datafusion.apache.org/python" @@ -58,9 +55,7 @@ profile = "black" [tool.maturin] python-source = "python" module-name = "datafusion._internal" -include = [ - { path = "Cargo.lock", format = "sdist" } -] +include = [{ path = "Cargo.lock", format = "sdist" }] exclude = [".github/**", "ci/**", ".asf.yaml"] # Require Cargo.lock is up to date locked = true From 7cca0283e7837426e6ccdf9d5c8cbfc0c8b239c9 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sat, 19 Oct 2024 00:57:57 +0200 Subject: [PATCH 227/413] feat: add `with_columns` (#909) * feat: add with_columns * chore: add doc * Format docstring to render in online documentation --------- Co-authored-by: Tim Saucer --- python/datafusion/dataframe.py | 47 +++++++++++++++++++++++++++++++++- python/tests/test_dataframe.py | 31 ++++++++++++++++++++++ src/dataframe.rs | 10 ++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 60203ffb4..9c0953c35 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -21,7 +21,7 @@ from __future__ import annotations -from typing import Any, List, TYPE_CHECKING, Literal +from typing import Any, Iterable, List, Literal, TYPE_CHECKING from datafusion.record_batch import RecordBatchStream from typing_extensions import deprecated from datafusion.plan import LogicalPlan, ExecutionPlan @@ -171,6 +171,51 @@ def with_column(self, name: str, expr: Expr) -> DataFrame: """ return DataFrame(self.df.with_column(name, expr.expr)) + def with_columns( + self, *exprs: Expr | Iterable[Expr], **named_exprs: Expr + ) -> DataFrame: + """Add columns to the DataFrame. + + By passing expressions, iteratables of expressions, or named expressions. To + pass named expressions use the form name=Expr. + + Example usage: The following will add 4 columns labeled a, b, c, and d:: + + df = df.with_columns( + lit(0).alias('a'), + [lit(1).alias('b'), lit(2).alias('c')], + d=lit(3) + ) + + Args: + exprs: Either a single expression or an iterable of expressions to add. + named_exprs: Named expressions in the form of ``name=expr`` + + Returns: + DataFrame with the new columns added. + """ + + def _simplify_expression( + *exprs: Expr | Iterable[Expr], **named_exprs: Expr + ) -> list[Expr]: + expr_list = [] + for expr in exprs: + if isinstance(expr, Expr): + expr_list.append(expr.expr) + elif isinstance(expr, Iterable): + for inner_expr in expr: + expr_list.append(inner_expr.expr) + else: + raise NotImplementedError + if named_exprs: + for alias, expr in named_exprs.items(): + expr_list.append(expr.alias(alias).expr) + return expr_list + + expressions = _simplify_expression(*exprs, **named_exprs) + + return DataFrame(self.df.with_columns(expressions)) + def with_column_renamed(self, old_name: str, new_name: str) -> DataFrame: r"""Rename one column by applying a new projection. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 6330ede04..0d4a7dcb0 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -216,6 +216,37 @@ def test_with_column(df): assert result.column(2) == pa.array([5, 7, 9]) +def test_with_columns(df): + df = df.with_columns( + (column("a") + column("b")).alias("c"), + (column("a") + column("b")).alias("d"), + [ + (column("a") + column("b")).alias("e"), + (column("a") + column("b")).alias("f"), + ], + g=(column("a") + column("b")), + ) + + # execute and collect the first (and only) batch + result = df.collect()[0] + + assert result.schema.field(0).name == "a" + assert result.schema.field(1).name == "b" + assert result.schema.field(2).name == "c" + assert result.schema.field(3).name == "d" + assert result.schema.field(4).name == "e" + assert result.schema.field(5).name == "f" + assert result.schema.field(6).name == "g" + + assert result.column(0) == pa.array([1, 2, 3]) + assert result.column(1) == pa.array([4, 5, 6]) + assert result.column(2) == pa.array([5, 7, 9]) + assert result.column(3) == pa.array([5, 7, 9]) + assert result.column(4) == pa.array([5, 7, 9]) + assert result.column(5) == pa.array([5, 7, 9]) + assert result.column(6) == pa.array([5, 7, 9]) + + def test_with_column_renamed(df): df = df.with_column("c", column("a") + column("b")).with_column_renamed("c", "sum") diff --git a/src/dataframe.rs b/src/dataframe.rs index fa6c1d44f..dd5d89ce9 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -187,6 +187,16 @@ impl PyDataFrame { Ok(Self::new(df)) } + fn with_columns(&self, exprs: Vec) -> PyResult { + let mut df = self.df.as_ref().clone(); + for expr in exprs { + let expr: Expr = expr.into(); + let name = format!("{}", expr.schema_name()); + df = df.with_column(name.as_str(), expr)? + } + Ok(Self::new(df)) + } + /// Rename one column by applying a new projection. This is a no-op if the column to be /// renamed does not exist. fn with_column_renamed(&self, old_name: &str, new_name: &str) -> PyResult { From 70c099aad8ec337ef88e27c125a8eeba328d62de Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:21:25 +0200 Subject: [PATCH 228/413] feat: add `cast` to DataFrame (#916) * feat: add with_columns * feat: add top level cast * chore: improve docstring --------- Co-authored-by: Tim Saucer --- python/datafusion/dataframe.py | 13 +++++++++++++ python/tests/test_dataframe.py | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 9c0953c35..3ed6d40fe 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -21,6 +21,7 @@ from __future__ import annotations + from typing import Any, Iterable, List, Literal, TYPE_CHECKING from datafusion.record_batch import RecordBatchStream from typing_extensions import deprecated @@ -267,6 +268,18 @@ def sort(self, *exprs: Expr | SortExpr) -> DataFrame: exprs_raw = [sort_or_default(expr) for expr in exprs] return DataFrame(self.df.sort(*exprs_raw)) + def cast(self, mapping: dict[str, pa.DataType[Any]]) -> DataFrame: + """Cast one or more columns to a different data type. + + Args: + mapping: Mapped with column as key and column dtype as value. + + Returns: + DataFrame after casting columns + """ + exprs = [Expr.column(col).cast(dtype) for col, dtype in mapping.items()] + return self.with_columns(exprs) + def limit(self, count: int, offset: int = 0) -> DataFrame: """Return a new :py:class:`DataFrame` with a limited number of rows. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 0d4a7dcb0..bb408c9c9 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -247,6 +247,15 @@ def test_with_columns(df): assert result.column(6) == pa.array([5, 7, 9]) +def test_cast(df): + df = df.cast({"a": pa.float16(), "b": pa.list_(pa.uint32())}) + expected = pa.schema( + [("a", pa.float16()), ("b", pa.list_(pa.uint32())), ("c", pa.int64())] + ) + + assert df.schema() == expected + + def test_with_column_renamed(df): df = df.with_column("c", column("a") + column("b")).with_column_renamed("c", "sum") From f59dd08bfbc0f01cc16b858465d03c3a01ba647c Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Mon, 21 Oct 2024 23:02:49 +0200 Subject: [PATCH 229/413] feat: add `head`, `tail` methods (#915) * feat: add head, tail methods * chore: add default head/tail --- python/datafusion/dataframe.py | 25 +++++++++++++++++++++++++ python/tests/test_dataframe.py | 22 ++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 3ed6d40fe..e4f8073d3 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -292,6 +292,31 @@ def limit(self, count: int, offset: int = 0) -> DataFrame: """ return DataFrame(self.df.limit(count, offset)) + def head(self, n: int = 5) -> DataFrame: + """Return a new :py:class:`DataFrame` with a limited number of rows. + + Args: + n: Number of rows to take from the head of the DataFrame. + + Returns: + DataFrame after limiting. + """ + return DataFrame(self.df.limit(n, 0)) + + def tail(self, n: int = 5) -> DataFrame: + """Return a new :py:class:`DataFrame` with a limited number of rows. + + Be aware this could be potentially expensive since the row size needs to be + determined of the dataframe. This is done by collecting it. + + Args: + n: Number of rows to take from the tail of the DataFrame. + + Returns: + DataFrame after limiting. + """ + return DataFrame(self.df.limit(n, max(0, self.count() - n))) + def collect(self) -> list[pa.RecordBatch]: """Execute this :py:class:`DataFrame` and collect results into memory. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index bb408c9c9..d73f5ebde 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -201,6 +201,28 @@ def test_limit_with_offset(df): assert len(result.column(1)) == 1 +def test_head(df): + df = df.head(1) + + # execute and collect the first (and only) batch + result = df.collect()[0] + + assert result.column(0) == pa.array([1]) + assert result.column(1) == pa.array([4]) + assert result.column(2) == pa.array([8]) + + +def test_tail(df): + df = df.tail(1) + + # execute and collect the first (and only) batch + result = df.collect()[0] + + assert result.column(0) == pa.array([3]) + assert result.column(1) == pa.array([6]) + assert result.column(2) == pa.array([8]) + + def test_with_column(df): df = df.with_column("c", column("a") + column("b")) From 56b72438004965f36dd4ce7e14d62a533ab2026f Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:23:36 +0200 Subject: [PATCH 230/413] chore: deprecate `select_columns` (#911) * chore: deprecate select_columns * chore: lint * Update user document to use select instead of select_columns * Update all tpch examples to use select instead of select_columns --------- Co-authored-by: Tim Saucer --- .../common-operations/select-and-filter.rst | 4 ++-- examples/import.py | 10 +++++----- examples/tpch/convert_data_to_parquet.py | 2 +- examples/tpch/q02_minimum_cost_supplier.py | 12 ++++++------ examples/tpch/q03_shipping_priority.py | 8 ++++---- examples/tpch/q04_order_priority_checking.py | 6 +++--- examples/tpch/q05_local_supplier_volume.py | 12 ++++++------ .../tpch/q06_forecasting_revenue_change.py | 2 +- examples/tpch/q07_volume_shipping.py | 10 +++++----- examples/tpch/q08_market_share.py | 18 ++++++++---------- .../tpch/q09_product_type_profit_measure.py | 14 ++++++-------- examples/tpch/q10_returned_item_reporting.py | 10 +++++----- .../tpch/q11_important_stock_identification.py | 8 ++++---- examples/tpch/q12_ship_mode_order_priority.py | 4 ++-- examples/tpch/q13_customer_distribution.py | 6 ++---- examples/tpch/q14_promotion_effect.py | 6 ++---- examples/tpch/q15_top_supplier.py | 6 +++--- .../tpch/q16_part_supplier_relationship.py | 8 ++++---- examples/tpch/q17_small_quantity_order.py | 4 ++-- examples/tpch/q18_large_volume_customer.py | 8 ++++---- examples/tpch/q19_discounted_revenue.py | 4 ++-- examples/tpch/q20_potential_part_promotion.py | 14 ++++++-------- .../tpch/q21_suppliers_kept_orders_waiting.py | 8 ++++---- examples/tpch/q22_global_sales_opportunity.py | 6 ++---- python/datafusion/dataframe.py | 3 +++ python/tests/test_dataframe.py | 14 ++++++-------- 26 files changed, 98 insertions(+), 109 deletions(-) diff --git a/docs/source/user-guide/common-operations/select-and-filter.rst b/docs/source/user-guide/common-operations/select-and-filter.rst index 92b4841b2..075909129 100644 --- a/docs/source/user-guide/common-operations/select-and-filter.rst +++ b/docs/source/user-guide/common-operations/select-and-filter.rst @@ -33,7 +33,7 @@ DataFusion can work with several file types, to start simple we can use a subset ctx = SessionContext() df = ctx.read_parquet("yellow_trip_data.parquet") - df.select_columns("trip_distance", "passenger_count") + df.select("trip_distance", "passenger_count") For mathematical or logical operations use :py:func:`~datafusion.col` to select columns, and give meaningful names to the resulting operations using :py:func:`~datafusion.expr.Expr.alias` @@ -48,7 +48,7 @@ operations using :py:func:`~datafusion.expr.Expr.alias` Please be aware that all identifiers are effectively made lower-case in SQL, so if your file has capital letters (ex: Name) you must put your column name in double quotes or the selection won’t work. As an alternative for simple - column selection use :py:func:`~datafusion.dataframe.DataFrame.select_columns` without double quotes + column selection use :py:func:`~datafusion.dataframe.DataFrame.select` without double quotes For selecting columns with capital letters use ``'"VendorID"'`` diff --git a/examples/import.py b/examples/import.py index cd965cb46..c9d2e8cb6 100644 --- a/examples/import.py +++ b/examples/import.py @@ -28,7 +28,7 @@ # The dictionary keys represent column names and the dictionary values # represent column values df = ctx.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]}) -assert type(df) == datafusion.DataFrame +assert type(df) is datafusion.DataFrame # Dataframe: # +---+---+ # | a | b | @@ -40,19 +40,19 @@ # Create a datafusion DataFrame from a Python list of rows df = ctx.from_pylist([{"a": 1, "b": 4}, {"a": 2, "b": 5}, {"a": 3, "b": 6}]) -assert type(df) == datafusion.DataFrame +assert type(df) is datafusion.DataFrame # Convert pandas DataFrame to datafusion DataFrame pandas_df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) df = ctx.from_pandas(pandas_df) -assert type(df) == datafusion.DataFrame +assert type(df) is datafusion.DataFrame # Convert polars DataFrame to datafusion DataFrame polars_df = pl.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) df = ctx.from_polars(polars_df) -assert type(df) == datafusion.DataFrame +assert type(df) is datafusion.DataFrame # Convert Arrow Table to datafusion DataFrame arrow_table = pa.Table.from_pydict({"a": [1, 2, 3], "b": [4, 5, 6]}) df = ctx.from_arrow(arrow_table) -assert type(df) == datafusion.DataFrame +assert type(df) is datafusion.DataFrame diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py index a8091a708..cb0b2f0bd 100644 --- a/examples/tpch/convert_data_to_parquet.py +++ b/examples/tpch/convert_data_to_parquet.py @@ -138,6 +138,6 @@ df = ctx.read_csv(source_file, schema=schema, has_header=False, delimiter="|") - df = df.select_columns(*output_cols) + df = df.select(*output_cols) df.write_parquet(dest_file, compression="snappy") diff --git a/examples/tpch/q02_minimum_cost_supplier.py b/examples/tpch/q02_minimum_cost_supplier.py index 2171a2083..2440fdad6 100644 --- a/examples/tpch/q02_minimum_cost_supplier.py +++ b/examples/tpch/q02_minimum_cost_supplier.py @@ -43,10 +43,10 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select( "p_partkey", "p_mfgr", "p_type", "p_size" ) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_acctbal", "s_name", "s_address", @@ -55,13 +55,13 @@ "s_nationkey", "s_suppkey", ) -df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select( "ps_partkey", "ps_suppkey", "ps_supplycost" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_regionkey", "n_name" ) -df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns( +df_region = ctx.read_parquet(get_data_path("region.parquet")).select( "r_regionkey", "r_name" ) @@ -115,7 +115,7 @@ # From the problem statement, these are the values we wish to output -df = df.select_columns( +df = df.select( "s_acctbal", "s_name", "n_name", diff --git a/examples/tpch/q03_shipping_priority.py b/examples/tpch/q03_shipping_priority.py index 6a4886d83..c4e8f461a 100644 --- a/examples/tpch/q03_shipping_priority.py +++ b/examples/tpch/q03_shipping_priority.py @@ -37,13 +37,13 @@ ctx = SessionContext() -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_mktsegment", "c_custkey" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderdate", "o_shippriority", "o_custkey", "o_orderkey" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_orderkey", "l_extendedprice", "l_discount", "l_shipdate" ) @@ -80,7 +80,7 @@ # Change the order that the columns are reported in just to match the spec -df = df.select_columns("l_orderkey", "revenue", "o_orderdate", "o_shippriority") +df = df.select("l_orderkey", "revenue", "o_orderdate", "o_shippriority") # Show result diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py index 77c3bd43e..f10b74d91 100644 --- a/examples/tpch/q04_order_priority_checking.py +++ b/examples/tpch/q04_order_priority_checking.py @@ -39,10 +39,10 @@ ctx = SessionContext() -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderdate", "o_orderpriority", "o_orderkey" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_orderkey", "l_commitdate", "l_receiptdate" ) @@ -54,7 +54,7 @@ # Limit results to cases where commitment date before receipt date # Aggregate the results so we only get one row to join with the order table. # Alternately, and likely more idiomatic is instead of `.aggregate` you could -# do `.select_columns("l_orderkey").distinct()`. The goal here is to show +# do `.select("l_orderkey").distinct()`. The goal here is to show # multiple examples of how to use Data Fusion. df_lineitem = df_lineitem.filter(col("l_commitdate") < col("l_receiptdate")).aggregate( [col("l_orderkey")], [] diff --git a/examples/tpch/q05_local_supplier_volume.py b/examples/tpch/q05_local_supplier_volume.py index f17f600a4..2a83d2d1a 100644 --- a/examples/tpch/q05_local_supplier_volume.py +++ b/examples/tpch/q05_local_supplier_volume.py @@ -47,22 +47,22 @@ ctx = SessionContext() -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_custkey", "c_nationkey" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_custkey", "o_orderkey", "o_orderdate" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_orderkey", "l_suppkey", "l_extendedprice", "l_discount" ) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_nationkey" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_regionkey", "n_name" ) -df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns( +df_region = ctx.read_parquet(get_data_path("region.parquet")).select( "r_regionkey", "r_name" ) diff --git a/examples/tpch/q06_forecasting_revenue_change.py b/examples/tpch/q06_forecasting_revenue_change.py index 3beb9eb1f..eaf9b0c29 100644 --- a/examples/tpch/q06_forecasting_revenue_change.py +++ b/examples/tpch/q06_forecasting_revenue_change.py @@ -51,7 +51,7 @@ ctx = SessionContext() -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_shipdate", "l_quantity", "l_extendedprice", "l_discount" ) diff --git a/examples/tpch/q07_volume_shipping.py b/examples/tpch/q07_volume_shipping.py index 44c605a9b..a1d7d81ad 100644 --- a/examples/tpch/q07_volume_shipping.py +++ b/examples/tpch/q07_volume_shipping.py @@ -49,19 +49,19 @@ ctx = SessionContext() -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_nationkey" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_shipdate", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_custkey" ) -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_custkey", "c_nationkey" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name" ) diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py index cd6bc1fa9..95fc0a871 100644 --- a/examples/tpch/q08_market_share.py +++ b/examples/tpch/q08_market_share.py @@ -47,25 +47,23 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( - "p_partkey", "p_type" -) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select("p_partkey", "p_type") +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_nationkey" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_partkey", "l_extendedprice", "l_discount", "l_suppkey", "l_orderkey" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_custkey", "o_orderdate" ) -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_custkey", "c_nationkey" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name", "n_regionkey" ) -df_region = ctx.read_parquet(get_data_path("region.parquet")).select_columns( +df_region = ctx.read_parquet(get_data_path("region.parquet")).select( "r_regionkey", "r_name" ) @@ -133,7 +131,7 @@ # When we join to the customer dataframe, we don't want to confuse other columns, so only # select the supplier key that we need -df_national_suppliers = df_national_suppliers.select_columns("s_suppkey") +df_national_suppliers = df_national_suppliers.select("s_suppkey") # Part 3: Combine suppliers and customers and compute the market share diff --git a/examples/tpch/q09_product_type_profit_measure.py b/examples/tpch/q09_product_type_profit_measure.py index b4a7369f8..0295d3025 100644 --- a/examples/tpch/q09_product_type_profit_measure.py +++ b/examples/tpch/q09_product_type_profit_measure.py @@ -39,16 +39,14 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( - "p_partkey", "p_name" -) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select("p_partkey", "p_name") +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_nationkey" ) -df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select( "ps_suppkey", "ps_partkey", "ps_supplycost" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_partkey", "l_extendedprice", "l_discount", @@ -56,10 +54,10 @@ "l_orderkey", "l_quantity", ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_custkey", "o_orderdate" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name", "n_regionkey" ) diff --git a/examples/tpch/q10_returned_item_reporting.py b/examples/tpch/q10_returned_item_reporting.py index 78327c3ad..25f81b2ff 100644 --- a/examples/tpch/q10_returned_item_reporting.py +++ b/examples/tpch/q10_returned_item_reporting.py @@ -44,7 +44,7 @@ ctx = SessionContext() -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_custkey", "c_nationkey", "c_name", @@ -53,13 +53,13 @@ "c_phone", "c_comment", ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_extendedprice", "l_discount", "l_orderkey", "l_returnflag" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_custkey", "o_orderdate" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name", "n_regionkey" ) @@ -87,7 +87,7 @@ df = df.join(df_nation, (["c_nationkey"], ["n_nationkey"]), how="inner") # These are the columns the problem statement requires -df = df.select_columns( +df = df.select( "c_custkey", "c_name", "revenue", diff --git a/examples/tpch/q11_important_stock_identification.py b/examples/tpch/q11_important_stock_identification.py index 391eb45b1..86ff2296b 100644 --- a/examples/tpch/q11_important_stock_identification.py +++ b/examples/tpch/q11_important_stock_identification.py @@ -37,13 +37,13 @@ ctx = SessionContext() -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_nationkey" ) -df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select( "ps_supplycost", "ps_availqty", "ps_suppkey", "ps_partkey" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name" ) @@ -75,7 +75,7 @@ df = df.filter(col("value") / col("total_value") >= lit(FRACTION)) # We only need to report on these two columns -df = df.select_columns("ps_partkey", "value") +df = df.select("ps_partkey", "value") # Sort in descending order of value df = df.sort(col("value").sort(ascending=False)) diff --git a/examples/tpch/q12_ship_mode_order_priority.py b/examples/tpch/q12_ship_mode_order_priority.py index 150870c64..c3fc0d2e9 100644 --- a/examples/tpch/q12_ship_mode_order_priority.py +++ b/examples/tpch/q12_ship_mode_order_priority.py @@ -42,10 +42,10 @@ ctx = SessionContext() -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_orderpriority" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_orderkey", "l_shipmode", "l_commitdate", "l_shipdate", "l_receiptdate" ) diff --git a/examples/tpch/q13_customer_distribution.py b/examples/tpch/q13_customer_distribution.py index bc0a5bd1f..f8b6c139d 100644 --- a/examples/tpch/q13_customer_distribution.py +++ b/examples/tpch/q13_customer_distribution.py @@ -38,12 +38,10 @@ ctx = SessionContext() -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_custkey", "o_comment" ) -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( - "c_custkey" -) +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select("c_custkey") # Use a regex to remove special cases df_orders = df_orders.filter( diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py index 8cb1e4c5a..8224136ad 100644 --- a/examples/tpch/q14_promotion_effect.py +++ b/examples/tpch/q14_promotion_effect.py @@ -41,12 +41,10 @@ ctx = SessionContext() -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_partkey", "l_shipdate", "l_extendedprice", "l_discount" ) -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( - "p_partkey", "p_type" -) +df_part = ctx.read_parquet(get_data_path("part.parquet")).select("p_partkey", "p_type") # Check part type begins with PROMO diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py index aa76093ec..44d5dd997 100644 --- a/examples/tpch/q15_top_supplier.py +++ b/examples/tpch/q15_top_supplier.py @@ -41,10 +41,10 @@ ctx = SessionContext() -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_suppkey", "l_shipdate", "l_extendedprice", "l_discount" ) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_name", "s_address", @@ -79,7 +79,7 @@ df = df.join(df_supplier, (["l_suppkey"], ["s_suppkey"]), "inner") # Return only the columns requested -df = df.select_columns("s_suppkey", "s_name", "s_address", "s_phone", "total_revenue") +df = df.select("s_suppkey", "s_name", "s_address", "s_phone", "total_revenue") # If we have more than one, sort by supplier number (suppkey) df = df.sort(col("s_suppkey").sort()) diff --git a/examples/tpch/q16_part_supplier_relationship.py b/examples/tpch/q16_part_supplier_relationship.py index fdcb5b4db..cbdd9989a 100644 --- a/examples/tpch/q16_part_supplier_relationship.py +++ b/examples/tpch/q16_part_supplier_relationship.py @@ -40,13 +40,13 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select( "p_partkey", "p_brand", "p_type", "p_size" ) -df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select( "ps_suppkey", "ps_partkey" ) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_comment" ) @@ -75,7 +75,7 @@ df = df_part.join(df_partsupp, (["p_partkey"], ["ps_partkey"]), "inner") -df = df.select_columns("p_brand", "p_type", "p_size", "ps_suppkey").distinct() +df = df.select("p_brand", "p_type", "p_size", "ps_suppkey").distinct() df = df.aggregate( [col("p_brand"), col("p_type"), col("p_size")], diff --git a/examples/tpch/q17_small_quantity_order.py b/examples/tpch/q17_small_quantity_order.py index e0ee8bb90..ff494279b 100644 --- a/examples/tpch/q17_small_quantity_order.py +++ b/examples/tpch/q17_small_quantity_order.py @@ -38,10 +38,10 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select( "p_partkey", "p_brand", "p_container" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_partkey", "l_quantity", "l_extendedprice" ) diff --git a/examples/tpch/q18_large_volume_customer.py b/examples/tpch/q18_large_volume_customer.py index 10c5f6e6a..497615499 100644 --- a/examples/tpch/q18_large_volume_customer.py +++ b/examples/tpch/q18_large_volume_customer.py @@ -35,13 +35,13 @@ ctx = SessionContext() -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_custkey", "c_name" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_custkey", "o_orderdate", "o_totalprice" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_orderkey", "l_quantity", "l_extendedprice" ) @@ -57,7 +57,7 @@ df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), "inner") df = df.join(df_customer, (["o_custkey"], ["c_custkey"]), "inner") -df = df.select_columns( +df = df.select( "c_name", "c_custkey", "o_orderkey", "o_orderdate", "o_totalprice", "total_quantity" ) diff --git a/examples/tpch/q19_discounted_revenue.py b/examples/tpch/q19_discounted_revenue.py index b15cd98bf..c2fe2570d 100644 --- a/examples/tpch/q19_discounted_revenue.py +++ b/examples/tpch/q19_discounted_revenue.py @@ -52,10 +52,10 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select( "p_partkey", "p_brand", "p_container", "p_size" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_partkey", "l_quantity", "l_shipmode", diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index 4ced7aaa1..3a0edb1ec 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -40,19 +40,17 @@ ctx = SessionContext() -df_part = ctx.read_parquet(get_data_path("part.parquet")).select_columns( - "p_partkey", "p_name" -) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_part = ctx.read_parquet(get_data_path("part.parquet")).select("p_partkey", "p_name") +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_shipdate", "l_partkey", "l_suppkey", "l_quantity" ) -df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select_columns( +df_partsupp = ctx.read_parquet(get_data_path("partsupp.parquet")).select( "ps_partkey", "ps_suppkey", "ps_availqty" ) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_address", "s_name", "s_nationkey" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name" ) @@ -91,7 +89,7 @@ df = df.join(df_nation, (["s_nationkey"], ["n_nationkey"]), "inner") # Restrict to the requested data per the problem statement -df = df.select_columns("s_name", "s_address").distinct() +df = df.select("s_name", "s_address").distinct() df = df.sort(col("s_name").sort()) diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py index 6b1679e7d..d3d57acee 100644 --- a/examples/tpch/q21_suppliers_kept_orders_waiting.py +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -35,16 +35,16 @@ ctx = SessionContext() -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select( "o_orderkey", "o_orderstatus" ) -df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select_columns( +df_lineitem = ctx.read_parquet(get_data_path("lineitem.parquet")).select( "l_orderkey", "l_receiptdate", "l_commitdate", "l_suppkey" ) -df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select_columns( +df_supplier = ctx.read_parquet(get_data_path("supplier.parquet")).select( "s_suppkey", "s_name", "s_nationkey" ) -df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select_columns( +df_nation = ctx.read_parquet(get_data_path("nation.parquet")).select( "n_nationkey", "n_name" ) diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py index 41fd5de9e..e6660e60c 100644 --- a/examples/tpch/q22_global_sales_opportunity.py +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -35,12 +35,10 @@ ctx = SessionContext() -df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select_columns( +df_customer = ctx.read_parquet(get_data_path("customer.parquet")).select( "c_phone", "c_acctbal", "c_custkey" ) -df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select_columns( - "o_custkey" -) +df_orders = ctx.read_parquet(get_data_path("orders.parquet")).select("o_custkey") # The nation code is a two digit number, but we need to convert it to a string literal nation_codes = F.make_array(*[lit(str(n)) for n in NATION_CODES]) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index e4f8073d3..e59f00d9f 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -98,6 +98,9 @@ def schema(self) -> pa.Schema: """ return self.df.schema() + @deprecated( + "select_columns() is deprecated. Use :py:meth:`~DataFrame.select` instead" + ) def select_columns(self, *args: str) -> DataFrame: """Filter the DataFrame by columns. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index d73f5ebde..7b20e9e39 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -103,30 +103,28 @@ def partitioned_df(): def test_select(df): - df = df.select( + df_1 = df.select( column("a") + column("b"), column("a") - column("b"), ) # execute and collect the first (and only) batch - result = df.collect()[0] + result = df_1.collect()[0] assert result.column(0) == pa.array([5, 7, 9]) assert result.column(1) == pa.array([-3, -3, -3]) - -def test_select_mixed_expr_string(df): - df = df.select_columns(column("b"), "a") + df_2 = df.select("b", "a") # execute and collect the first (and only) batch - result = df.collect()[0] + result = df_2.collect()[0] assert result.column(0) == pa.array([4, 5, 6]) assert result.column(1) == pa.array([1, 2, 3]) -def test_select_columns(df): - df = df.select_columns("b", "a") +def test_select_mixed_expr_string(df): + df = df.select(column("b"), "a") # execute and collect the first (and only) batch result = df.collect()[0] From 7007e0239f0a0dbe863936b3937cb8a787971a59 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:47:48 -0500 Subject: [PATCH 231/413] build(deps): bump uuid from 1.10.0 to 1.11.0 (#927) Bumps [uuid](https://github.com/uuid-rs/uuid) from 1.10.0 to 1.11.0. - [Release notes](https://github.com/uuid-rs/uuid/releases) - [Commits](https://github.com/uuid-rs/uuid/compare/1.10.0...1.11.0) --- updated-dependencies: - dependency-name: uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56c368f46..0835f219e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3406,9 +3406,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", diff --git a/Cargo.toml b/Cargo.toml index df72cd40a..073f82cf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ datafusion = { version = "42.0.0", features = ["pyarrow", "avro", "unicode_expre datafusion-substrait = { version = "42.0.0", optional = true } datafusion-proto = { version = "42.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` -uuid = { version = "1.9", features = ["v4"] } +uuid = { version = "1.11", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" From 0bc2f31d8d9ac46e1732d1267cb8d83c38452f45 Mon Sep 17 00:00:00 2001 From: kosiew Date: Tue, 29 Oct 2024 00:03:00 +0800 Subject: [PATCH 232/413] Add array_empty (#931) --- .../user-guide/common-operations/expressions.rst | 15 ++++++++++++++- python/datafusion/functions.py | 6 ++++++ python/tests/test_functions.py | 4 ++++ src/functions.rs | 2 ++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index 6014c9d2e..77f3359f5 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -77,12 +77,25 @@ approaches. df = ctx.from_pydict({"a": [[1, 2, 3], [4, 5, 6]]}) df.select(col("a")[0].alias("a0")) - .. warning:: Indexing an element of an array via ``[]`` starts at index 0 whereas :py:func:`~datafusion.functions.array_element` starts at index 1. +To check if an array is empty, you can use the function :py:func:`datafusion.functions.array_empty`. +This function returns a boolean indicating whether the array is empty. + +.. ipython:: python + + from datafusion import SessionContext, col + from datafusion.functions import array_empty + + ctx = SessionContext() + df = ctx.from_pydict({"a": [[], [1, 2, 3]]}) + df.select(array_empty(col("a")).alias("is_empty")) + +In this example, the `is_empty` column will contain `True` for the first row and `False` for the second row. + Structs ------- diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 727321979..570a6ce5e 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -51,6 +51,7 @@ "array_dims", "array_distinct", "array_element", + "array_empty", "array_except", "array_extract", "array_has", @@ -1160,6 +1161,11 @@ def array_element(array: Expr, n: Expr) -> Expr: return Expr(f.array_element(array.expr, n.expr)) +def array_empty(array: Expr) -> Expr: + """Returns a boolean indicating whether the array is empty.""" + return Expr(f.array_empty(array.expr)) + + def array_extract(array: Expr, n: Expr) -> Expr: """Extracts the element with the index n from the array. diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 9353f872d..e6fd41d8b 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -309,6 +309,10 @@ def py_flatten(arr): lambda col: f.array_element(col, literal(1)), lambda data: [r[0] for r in data], ], + [ + lambda col: f.array_empty(col), + lambda data: [len(r) == 0 for r in data], + ], [ lambda col: f.array_extract(col, literal(1)), lambda data: [r[0] for r in data], diff --git a/src/functions.rs b/src/functions.rs index 24d33af39..4facb6cf7 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -572,6 +572,7 @@ array_fn!(array_to_string, array delimiter); array_fn!(array_dims, array); array_fn!(array_distinct, array); array_fn!(array_element, array element); +array_fn!(array_empty, array); array_fn!(array_length, array); array_fn!(array_has, first_array second_array); array_fn!(array_has_all, first_array second_array); @@ -1003,6 +1004,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(array_dims))?; m.add_wrapped(wrap_pyfunction!(array_distinct))?; m.add_wrapped(wrap_pyfunction!(array_element))?; + m.add_wrapped(wrap_pyfunction!(array_empty))?; m.add_wrapped(wrap_pyfunction!(array_length))?; m.add_wrapped(wrap_pyfunction!(array_has))?; m.add_wrapped(wrap_pyfunction!(array_has_all))?; From e015482750e9e08bd426bfcf649445d53705c51a Mon Sep 17 00:00:00 2001 From: kosiew Date: Tue, 29 Oct 2024 18:16:50 +0800 Subject: [PATCH 233/413] feat: add `cardinality` function to calculate total elements in an array (#937) --- .../common-operations/expressions.rst | 14 ++++++++++++++ python/datafusion/functions.py | 6 ++++++ python/tests/test_functions.py | 18 ++++++++++++++++++ src/functions.rs | 2 ++ 4 files changed, 40 insertions(+) diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index 77f3359f5..23430d359 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -96,6 +96,20 @@ This function returns a boolean indicating whether the array is empty. In this example, the `is_empty` column will contain `True` for the first row and `False` for the second row. +To get the total number of elements in an array, you can use the function :py:func:`datafusion.functions.cardinality`. +This function returns an integer indicating the total number of elements in the array. + +.. ipython:: python + + from datafusion import SessionContext, col + from datafusion.functions import cardinality + + ctx = SessionContext() + df = ctx.from_pydict({"a": [[1, 2, 3], [4, 5, 6]]}) + df.select(cardinality(col("a")).alias("num_elements")) + +In this example, the `num_elements` column will contain `3` for both rows. + Structs ------- diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 570a6ce5e..e67ba4ae4 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -132,6 +132,7 @@ "find_in_set", "first_value", "flatten", + "cardinality", "floor", "from_unixtime", "gcd", @@ -1516,6 +1517,11 @@ def flatten(array: Expr) -> Expr: return Expr(f.flatten(array.expr)) +def cardinality(array: Expr) -> Expr: + """Returns the total number of elements in the array.""" + return Expr(f.cardinality(array.expr)) + + # aggregate functions def approx_distinct( expression: Expr, diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index e6fd41d8b..37943e57c 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -540,6 +540,24 @@ def test_array_function_flatten(): ) +def test_array_function_cardinality(): + data = [[1, 2, 3], [4, 4, 5, 6]] + ctx = SessionContext() + batch = pa.RecordBatch.from_arrays([np.array(data, dtype=object)], names=["arr"]) + df = ctx.create_dataframe([[batch]]) + + stmt = f.cardinality(column("arr")) + py_expr = [len(arr) for arr in data] # Expected lengths: [3, 3] + # assert py_expr lengths + + query_result = df.select(stmt).collect()[0].column(0) + + for a, b in zip(query_result, py_expr): + np.testing.assert_array_equal( + np.array([a.as_py()], dtype=int), np.array([b], dtype=int) + ) + + @pytest.mark.parametrize( ("stmt", "py_expr"), [ diff --git a/src/functions.rs b/src/functions.rs index 4facb6cf7..fe3531ba9 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -594,6 +594,7 @@ array_fn!(array_intersect, first_array second_array); array_fn!(array_union, array1 array2); array_fn!(array_except, first_array second_array); array_fn!(array_resize, array size value); +array_fn!(cardinality, array); array_fn!(flatten, array); array_fn!(range, start stop step); @@ -1030,6 +1031,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(array_sort))?; m.add_wrapped(wrap_pyfunction!(array_slice))?; m.add_wrapped(wrap_pyfunction!(flatten))?; + m.add_wrapped(wrap_pyfunction!(cardinality))?; // Window Functions m.add_wrapped(wrap_pyfunction!(lead))?; From aedffe0d8a0522fa21a7b545aa885750f32fc218 Mon Sep 17 00:00:00 2001 From: kosiew Date: Fri, 1 Nov 2024 19:42:57 +0800 Subject: [PATCH 234/413] Add empty scalar function (alias of array_empty), fix a small typo (#938) * feat: add `empty` function as alias of array_empty * fix: correct typo in null_treatment parameter documentation --- .../user-guide/common-operations/expressions.rst | 2 +- python/datafusion/functions.py | 12 +++++++++--- python/tests/test_functions.py | 4 ++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index 23430d359..b2a83c89f 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -82,7 +82,7 @@ approaches. Indexing an element of an array via ``[]`` starts at index 0 whereas :py:func:`~datafusion.functions.array_element` starts at index 1. -To check if an array is empty, you can use the function :py:func:`datafusion.functions.array_empty`. +To check if an array is empty, you can use the function :py:func:`datafusion.functions.array_empty` or `datafusion.functions.empty`. This function returns a boolean indicating whether the array is empty. .. ipython:: python diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index e67ba4ae4..907f801af 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -125,6 +125,7 @@ "decode", "degrees", "digest", + "empty", "encode", "ends_with", "exp", @@ -1522,6 +1523,11 @@ def cardinality(array: Expr) -> Expr: return Expr(f.cardinality(array.expr)) +def empty(array: Expr) -> Expr: + """This is an alias for :py:func:`array_empty`.""" + return array_empty(array) + + # aggregate functions def approx_distinct( expression: Expr, @@ -2140,7 +2146,7 @@ def first_value( expression: Argument to perform bitwise calculation on filter: If provided, only compute against rows for which the filter is True order_by: Set the ordering of the expression to evaluate - null_treatment: Assign whether to respect or ignull null values. + null_treatment: Assign whether to respect or ignore null values. """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None @@ -2172,7 +2178,7 @@ def last_value( expression: Argument to perform bitwise calculation on filter: If provided, only compute against rows for which the filter is True order_by: Set the ordering of the expression to evaluate - null_treatment: Assign whether to respect or ignull null values. + null_treatment: Assign whether to respect or ignore null values. """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None @@ -2206,7 +2212,7 @@ def nth_value( n: Index of value to return. Starts at 1. filter: If provided, only compute against rows for which the filter is True order_by: Set the ordering of the expression to evaluate - null_treatment: Assign whether to respect or ignull null values. + null_treatment: Assign whether to respect or ignore null values. """ order_by_raw = sort_list_to_raw_sort_list(order_by) filter_raw = filter.expr if filter is not None else None diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 37943e57c..c65c633a4 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -313,6 +313,10 @@ def py_flatten(arr): lambda col: f.array_empty(col), lambda data: [len(r) == 0 for r in data], ], + [ + lambda col: f.empty(col), + lambda data: [len(r) == 0 for r in data], + ], [ lambda col: f.array_extract(col, literal(1)), lambda data: [r[0] for r in data], From cbe28cb4bb53c26940f5c020981592141030a324 Mon Sep 17 00:00:00 2001 From: David Rauschenbach Date: Tue, 5 Nov 2024 08:35:45 -0800 Subject: [PATCH 235/413] README How to develop section now also works on Apple M1 (#940) --- README.md | 2 + conda/environments/datafusion-cuda-dev.yaml | 44 +++++++++++++++++++++ conda/environments/datafusion-dev.yaml | 3 -- 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 conda/environments/datafusion-cuda-dev.yaml diff --git a/README.md b/README.md index b1d5397ef..83b307e7a 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,8 @@ conda env create -f ./conda/environments/datafusion-dev.yaml -n datafusion-dev conda activate datafusion-dev ``` +Or alternatively, if you are on an OS that supports CUDA Toolkit, you can use `-f ./conda/environments/datafusion-cuda-dev.yaml`. + Bootstrap (Pip): ```bash diff --git a/conda/environments/datafusion-cuda-dev.yaml b/conda/environments/datafusion-cuda-dev.yaml new file mode 100644 index 000000000..1f6f23942 --- /dev/null +++ b/conda/environments/datafusion-cuda-dev.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +channels: + - conda-forge +dependencies: + - black + - flake8 + - isort + - maturin>=1.5.1 + - mypy + - numpy + - pyarrow>=11.0.0 + - pytest + - toml + - importlib_metadata + - python>=3.10 + # Packages useful for building distributions and releasing + - mamba + - conda-build + - anaconda-client + # Packages for documentation building + - sphinx + - pydata-sphinx-theme==0.8.0 + - myst-parser + - jinja2 + # GPU packages + - cudf + - cudatoolkit=11.8 +name: datafusion-dev diff --git a/conda/environments/datafusion-dev.yaml b/conda/environments/datafusion-dev.yaml index 1f6f23942..b4b503dc6 100644 --- a/conda/environments/datafusion-dev.yaml +++ b/conda/environments/datafusion-dev.yaml @@ -38,7 +38,4 @@ dependencies: - pydata-sphinx-theme==0.8.0 - myst-parser - jinja2 - # GPU packages - - cudf - - cudatoolkit=11.8 name: datafusion-dev From 4a6c4d129af3e1eb207f64ee84285419afb26876 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Fri, 8 Nov 2024 15:06:27 +0100 Subject: [PATCH 236/413] refactor: dataframe `join` params (#912) * refactor: dataframe join params * chore: add description for on params * fix type * chore: change join param * chore: update join params in tpch * oops * chore: final change * Add support for join_keys as a positional argument --------- Co-authored-by: Tim Saucer --- .../user-guide/common-operations/joins.rst | 10 +- examples/tpch/_tests.py | 6 +- examples/tpch/q02_minimum_cost_supplier.py | 12 ++- examples/tpch/q03_shipping_priority.py | 6 +- examples/tpch/q04_order_priority_checking.py | 4 +- examples/tpch/q05_local_supplier_volume.py | 13 ++- examples/tpch/q07_volume_shipping.py | 12 ++- examples/tpch/q08_market_share.py | 14 +-- .../tpch/q09_product_type_profit_measure.py | 13 ++- examples/tpch/q10_returned_item_reporting.py | 6 +- .../q11_important_stock_identification.py | 6 +- examples/tpch/q12_ship_mode_order_priority.py | 2 +- examples/tpch/q13_customer_distribution.py | 4 +- examples/tpch/q14_promotion_effect.py | 4 +- examples/tpch/q15_top_supplier.py | 2 +- .../tpch/q16_part_supplier_relationship.py | 6 +- examples/tpch/q17_small_quantity_order.py | 2 +- examples/tpch/q18_large_volume_customer.py | 4 +- examples/tpch/q19_discounted_revenue.py | 2 +- examples/tpch/q20_potential_part_promotion.py | 11 ++- .../tpch/q21_suppliers_kept_orders_waiting.py | 8 +- examples/tpch/q22_global_sales_opportunity.py | 2 +- python/datafusion/dataframe.py | 95 +++++++++++++++++-- python/tests/test_dataframe.py | 66 ++++++++++++- src/dataframe.rs | 15 +-- 25 files changed, 240 insertions(+), 85 deletions(-) diff --git a/docs/source/user-guide/common-operations/joins.rst b/docs/source/user-guide/common-operations/joins.rst index 09fa145a7..40d922150 100644 --- a/docs/source/user-guide/common-operations/joins.rst +++ b/docs/source/user-guide/common-operations/joins.rst @@ -56,7 +56,7 @@ will be included in the resulting DataFrame. .. ipython:: python - left.join(right, join_keys=(["customer_id"], ["id"]), how="inner") + left.join(right, left_on="customer_id", right_on="id", how="inner") The parameter ``join_keys`` specifies the columns from the left DataFrame and right DataFrame that contains the values that should match. @@ -70,7 +70,7 @@ values for the corresponding columns. .. ipython:: python - left.join(right, join_keys=(["customer_id"], ["id"]), how="left") + left.join(right, left_on="customer_id", right_on="id", how="left") Full Join --------- @@ -80,7 +80,7 @@ is no match. Unmatched rows will have null values. .. ipython:: python - left.join(right, join_keys=(["customer_id"], ["id"]), how="full") + left.join(right, left_on="customer_id", right_on="id", how="full") Left Semi Join -------------- @@ -90,7 +90,7 @@ omitting duplicates with multiple matches in the right table. .. ipython:: python - left.join(right, join_keys=(["customer_id"], ["id"]), how="semi") + left.join(right, left_on="customer_id", right_on="id", how="semi") Left Anti Join -------------- @@ -101,4 +101,4 @@ the right table. .. ipython:: python - left.join(right, join_keys=(["customer_id"], ["id"]), how="anti") \ No newline at end of file + left.join(right, left_on="customer_id", right_on="id", how="anti") \ No newline at end of file diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 903b53548..13144ae9d 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -18,7 +18,7 @@ import pytest from importlib import import_module import pyarrow as pa -from datafusion import col, lit, functions as F +from datafusion import DataFrame, col, lit, functions as F from util import get_answer_file @@ -94,7 +94,7 @@ def check_q17(df): ) def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): module = import_module(query_code) - df = module.df + df: DataFrame = module.df # Treat q17 as a special case. The answer file does not match the spec. # Running at scale factor 1, we have manually verified this result does @@ -121,5 +121,5 @@ def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): cols = list(read_schema.names) - assert df.join(df_expected, (cols, cols), "anti").count() == 0 + assert df.join(df_expected, on=cols, how="anti").count() == 0 assert df.count() == df_expected.count() diff --git a/examples/tpch/q02_minimum_cost_supplier.py b/examples/tpch/q02_minimum_cost_supplier.py index 2440fdad6..c4ccf8ad3 100644 --- a/examples/tpch/q02_minimum_cost_supplier.py +++ b/examples/tpch/q02_minimum_cost_supplier.py @@ -80,16 +80,20 @@ # Now that we have the region, find suppliers in that region. Suppliers are tied to their nation # and nations are tied to the region. -df_nation = df_nation.join(df_region, (["n_regionkey"], ["r_regionkey"]), how="inner") +df_nation = df_nation.join( + df_region, left_on=["n_regionkey"], right_on=["r_regionkey"], how="inner" +) df_supplier = df_supplier.join( - df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner" + df_nation, left_on=["s_nationkey"], right_on=["n_nationkey"], how="inner" ) # Now that we know who the potential suppliers are for the part, we can limit out part # supplies table down. We can further join down to the specific parts we've identified # as matching the request -df = df_partsupp.join(df_supplier, (["ps_suppkey"], ["s_suppkey"]), how="inner") +df = df_partsupp.join( + df_supplier, left_on=["ps_suppkey"], right_on=["s_suppkey"], how="inner" +) # Locate the minimum cost across all suppliers. There are multiple ways you could do this, # but one way is to create a window function across all suppliers, find the minimum, and @@ -111,7 +115,7 @@ df = df.filter(col("min_cost") == col("ps_supplycost")) -df = df.join(df_part, (["ps_partkey"], ["p_partkey"]), how="inner") +df = df.join(df_part, left_on=["ps_partkey"], right_on=["p_partkey"], how="inner") # From the problem statement, these are the values we wish to output diff --git a/examples/tpch/q03_shipping_priority.py b/examples/tpch/q03_shipping_priority.py index c4e8f461a..5ebab13c0 100644 --- a/examples/tpch/q03_shipping_priority.py +++ b/examples/tpch/q03_shipping_priority.py @@ -55,9 +55,9 @@ # Join all 3 dataframes -df = df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="inner").join( - df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner" -) +df = df_customer.join( + df_orders, left_on=["c_custkey"], right_on=["o_custkey"], how="inner" +).join(df_lineitem, left_on=["o_orderkey"], right_on=["l_orderkey"], how="inner") # Compute the revenue diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py index f10b74d91..8bf02cb83 100644 --- a/examples/tpch/q04_order_priority_checking.py +++ b/examples/tpch/q04_order_priority_checking.py @@ -66,7 +66,9 @@ ) # Perform the join to find only orders for which there are lineitems outside of expected range -df = df_orders.join(df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner") +df = df_orders.join( + df_lineitem, left_on=["o_orderkey"], right_on=["l_orderkey"], how="inner" +) # Based on priority, find the number of entries df = df.aggregate( diff --git a/examples/tpch/q05_local_supplier_volume.py b/examples/tpch/q05_local_supplier_volume.py index 2a83d2d1a..413a4acb9 100644 --- a/examples/tpch/q05_local_supplier_volume.py +++ b/examples/tpch/q05_local_supplier_volume.py @@ -76,15 +76,18 @@ # Join all the dataframes df = ( - df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="inner") - .join(df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner") + df_customer.join( + df_orders, left_on=["c_custkey"], right_on=["o_custkey"], how="inner" + ) + .join(df_lineitem, left_on=["o_orderkey"], right_on=["l_orderkey"], how="inner") .join( df_supplier, - (["l_suppkey", "c_nationkey"], ["s_suppkey", "s_nationkey"]), + left_on=["l_suppkey", "c_nationkey"], + right_on=["s_suppkey", "s_nationkey"], how="inner", ) - .join(df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner") - .join(df_region, (["n_regionkey"], ["r_regionkey"]), how="inner") + .join(df_nation, left_on=["s_nationkey"], right_on=["n_nationkey"], how="inner") + .join(df_region, left_on=["n_regionkey"], right_on=["r_regionkey"], how="inner") ) # Compute the final result diff --git a/examples/tpch/q07_volume_shipping.py b/examples/tpch/q07_volume_shipping.py index a1d7d81ad..18c290d9c 100644 --- a/examples/tpch/q07_volume_shipping.py +++ b/examples/tpch/q07_volume_shipping.py @@ -90,20 +90,22 @@ # Limit suppliers to either nation df_supplier = df_supplier.join( - df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner" + df_nation, left_on=["s_nationkey"], right_on=["n_nationkey"], how="inner" ).select(col("s_suppkey"), col("n_name").alias("supp_nation")) # Limit customers to either nation df_customer = df_customer.join( - df_nation, (["c_nationkey"], ["n_nationkey"]), how="inner" + df_nation, left_on=["c_nationkey"], right_on=["n_nationkey"], how="inner" ).select(col("c_custkey"), col("n_name").alias("cust_nation")) # Join up all the data frames from line items, and make sure the supplier and customer are in # different nations. df = ( - df_lineitem.join(df_orders, (["l_orderkey"], ["o_orderkey"]), how="inner") - .join(df_customer, (["o_custkey"], ["c_custkey"]), how="inner") - .join(df_supplier, (["l_suppkey"], ["s_suppkey"]), how="inner") + df_lineitem.join( + df_orders, left_on=["l_orderkey"], right_on=["o_orderkey"], how="inner" + ) + .join(df_customer, left_on=["o_custkey"], right_on=["c_custkey"], how="inner") + .join(df_supplier, left_on=["l_suppkey"], right_on=["s_suppkey"], how="inner") .filter(col("cust_nation") != col("supp_nation")) ) diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py index 95fc0a871..7138ab65a 100644 --- a/examples/tpch/q08_market_share.py +++ b/examples/tpch/q08_market_share.py @@ -89,27 +89,27 @@ # After this join we have all of the possible sales nations df_regional_customers = df_regional_customers.join( - df_nation, (["r_regionkey"], ["n_regionkey"]), how="inner" + df_nation, left_on=["r_regionkey"], right_on=["n_regionkey"], how="inner" ) # Now find the possible customers df_regional_customers = df_regional_customers.join( - df_customer, (["n_nationkey"], ["c_nationkey"]), how="inner" + df_customer, left_on=["n_nationkey"], right_on=["c_nationkey"], how="inner" ) # Next find orders for these customers df_regional_customers = df_regional_customers.join( - df_orders, (["c_custkey"], ["o_custkey"]), how="inner" + df_orders, left_on=["c_custkey"], right_on=["o_custkey"], how="inner" ) # Find all line items from these orders df_regional_customers = df_regional_customers.join( - df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner" + df_lineitem, left_on=["o_orderkey"], right_on=["l_orderkey"], how="inner" ) # Limit to the part of interest df_regional_customers = df_regional_customers.join( - df_part, (["l_partkey"], ["p_partkey"]), how="inner" + df_part, left_on=["l_partkey"], right_on=["p_partkey"], how="inner" ) # Compute the volume for each line item @@ -126,7 +126,7 @@ # Determine the suppliers by the limited nation key we have in our single row df above df_national_suppliers = df_national_suppliers.join( - df_supplier, (["n_nationkey"], ["s_nationkey"]), how="inner" + df_supplier, left_on=["n_nationkey"], right_on=["s_nationkey"], how="inner" ) # When we join to the customer dataframe, we don't want to confuse other columns, so only @@ -141,7 +141,7 @@ # column only from suppliers in the nation we are evaluating. df = df_regional_customers.join( - df_national_suppliers, (["l_suppkey"], ["s_suppkey"]), how="left" + df_national_suppliers, left_on=["l_suppkey"], right_on=["s_suppkey"], how="left" ) # Use a case statement to compute the volume sold by suppliers in the nation of interest diff --git a/examples/tpch/q09_product_type_profit_measure.py b/examples/tpch/q09_product_type_profit_measure.py index 0295d3025..aa47d76c0 100644 --- a/examples/tpch/q09_product_type_profit_measure.py +++ b/examples/tpch/q09_product_type_profit_measure.py @@ -65,13 +65,16 @@ df = df_part.filter(F.strpos(col("p_name"), part_color) > lit(0)) # We have a series of joins that get us to limit down to the line items we need -df = df.join(df_lineitem, (["p_partkey"], ["l_partkey"]), how="inner") -df = df.join(df_supplier, (["l_suppkey"], ["s_suppkey"]), how="inner") -df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), how="inner") +df = df.join(df_lineitem, left_on=["p_partkey"], right_on=["l_partkey"], how="inner") +df = df.join(df_supplier, left_on=["l_suppkey"], right_on=["s_suppkey"], how="inner") +df = df.join(df_orders, left_on=["l_orderkey"], right_on=["o_orderkey"], how="inner") df = df.join( - df_partsupp, (["l_suppkey", "l_partkey"], ["ps_suppkey", "ps_partkey"]), how="inner" + df_partsupp, + left_on=["l_suppkey", "l_partkey"], + right_on=["ps_suppkey", "ps_partkey"], + how="inner", ) -df = df.join(df_nation, (["s_nationkey"], ["n_nationkey"]), how="inner") +df = df.join(df_nation, left_on=["s_nationkey"], right_on=["n_nationkey"], how="inner") # Compute the intermediate values and limit down to the expressions we need df = df.select( diff --git a/examples/tpch/q10_returned_item_reporting.py b/examples/tpch/q10_returned_item_reporting.py index 25f81b2ff..94b398c1d 100644 --- a/examples/tpch/q10_returned_item_reporting.py +++ b/examples/tpch/q10_returned_item_reporting.py @@ -74,7 +74,7 @@ col("o_orderdate") < date_start_of_quarter + interval_one_quarter ) -df = df.join(df_lineitem, (["o_orderkey"], ["l_orderkey"]), how="inner") +df = df.join(df_lineitem, left_on=["o_orderkey"], right_on=["l_orderkey"], how="inner") # Compute the revenue df = df.aggregate( @@ -83,8 +83,8 @@ ) # Now join in the customer data -df = df.join(df_customer, (["o_custkey"], ["c_custkey"]), how="inner") -df = df.join(df_nation, (["c_nationkey"], ["n_nationkey"]), how="inner") +df = df.join(df_customer, left_on=["o_custkey"], right_on=["c_custkey"], how="inner") +df = df.join(df_nation, left_on=["c_nationkey"], right_on=["n_nationkey"], how="inner") # These are the columns the problem statement requires df = df.select( diff --git a/examples/tpch/q11_important_stock_identification.py b/examples/tpch/q11_important_stock_identification.py index 86ff2296b..707265e16 100644 --- a/examples/tpch/q11_important_stock_identification.py +++ b/examples/tpch/q11_important_stock_identification.py @@ -52,9 +52,11 @@ # Find part supplies of within this target nation -df = df_nation.join(df_supplier, (["n_nationkey"], ["s_nationkey"]), how="inner") +df = df_nation.join( + df_supplier, left_on=["n_nationkey"], right_on=["s_nationkey"], how="inner" +) -df = df.join(df_partsupp, (["s_suppkey"], ["ps_suppkey"]), how="inner") +df = df.join(df_partsupp, left_on=["s_suppkey"], right_on=["ps_suppkey"], how="inner") # Compute the value of individual parts diff --git a/examples/tpch/q12_ship_mode_order_priority.py b/examples/tpch/q12_ship_mode_order_priority.py index c3fc0d2e9..def2a6c30 100644 --- a/examples/tpch/q12_ship_mode_order_priority.py +++ b/examples/tpch/q12_ship_mode_order_priority.py @@ -75,7 +75,7 @@ # We need order priority, so join order df to line item -df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), how="inner") +df = df.join(df_orders, left_on=["l_orderkey"], right_on=["o_orderkey"], how="inner") # Restrict to line items we care about based on the problem statement. df = df.filter(col("l_commitdate") < col("l_receiptdate")) diff --git a/examples/tpch/q13_customer_distribution.py b/examples/tpch/q13_customer_distribution.py index f8b6c139d..67365a96a 100644 --- a/examples/tpch/q13_customer_distribution.py +++ b/examples/tpch/q13_customer_distribution.py @@ -49,7 +49,9 @@ ) # Since we may have customers with no orders we must do a left join -df = df_customer.join(df_orders, (["c_custkey"], ["o_custkey"]), how="left") +df = df_customer.join( + df_orders, left_on=["c_custkey"], right_on=["o_custkey"], how="left" +) # Find the number of orders for each customer df = df.aggregate([col("c_custkey")], [F.count(col("o_custkey")).alias("c_count")]) diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py index 8224136ad..cd26ee2bd 100644 --- a/examples/tpch/q14_promotion_effect.py +++ b/examples/tpch/q14_promotion_effect.py @@ -57,7 +57,9 @@ ) # Left join so we can sum up the promo parts different from other parts -df = df_lineitem.join(df_part, (["l_partkey"], ["p_partkey"]), "left") +df = df_lineitem.join( + df_part, left_on=["l_partkey"], right_on=["p_partkey"], how="left" +) # Make a factor of 1.0 if it is a promotion, 0.0 otherwise df = df.with_column("promo_factor", F.coalesce(col("promo_factor"), lit(0.0))) diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py index 44d5dd997..0bc316f7a 100644 --- a/examples/tpch/q15_top_supplier.py +++ b/examples/tpch/q15_top_supplier.py @@ -76,7 +76,7 @@ # Now that we know the supplier(s) with maximum revenue, get the rest of their information # from the supplier table -df = df.join(df_supplier, (["l_suppkey"], ["s_suppkey"]), "inner") +df = df.join(df_supplier, left_on=["l_suppkey"], right_on=["s_suppkey"], how="inner") # Return only the columns requested df = df.select("s_suppkey", "s_name", "s_address", "s_phone", "total_revenue") diff --git a/examples/tpch/q16_part_supplier_relationship.py b/examples/tpch/q16_part_supplier_relationship.py index cbdd9989a..a6a0c43eb 100644 --- a/examples/tpch/q16_part_supplier_relationship.py +++ b/examples/tpch/q16_part_supplier_relationship.py @@ -56,7 +56,7 @@ # Remove unwanted suppliers df_partsupp = df_partsupp.join( - df_unwanted_suppliers, (["ps_suppkey"], ["s_suppkey"]), "anti" + df_unwanted_suppliers, left_on=["ps_suppkey"], right_on=["s_suppkey"], how="anti" ) # Select the parts we are interested in @@ -73,7 +73,9 @@ p_sizes = F.make_array(*[lit(s).cast(pa.int32()) for s in SIZES_OF_INTEREST]) df_part = df_part.filter(~F.array_position(p_sizes, col("p_size")).is_null()) -df = df_part.join(df_partsupp, (["p_partkey"], ["ps_partkey"]), "inner") +df = df_part.join( + df_partsupp, left_on=["p_partkey"], right_on=["ps_partkey"], how="inner" +) df = df.select("p_brand", "p_type", "p_size", "ps_suppkey").distinct() diff --git a/examples/tpch/q17_small_quantity_order.py b/examples/tpch/q17_small_quantity_order.py index ff494279b..d7b43d498 100644 --- a/examples/tpch/q17_small_quantity_order.py +++ b/examples/tpch/q17_small_quantity_order.py @@ -51,7 +51,7 @@ ) # Combine data -df = df.join(df_lineitem, (["p_partkey"], ["l_partkey"]), "inner") +df = df.join(df_lineitem, left_on=["p_partkey"], right_on=["l_partkey"], how="inner") # Find the average quantity window_frame = WindowFrame("rows", None, None) diff --git a/examples/tpch/q18_large_volume_customer.py b/examples/tpch/q18_large_volume_customer.py index 497615499..165fce033 100644 --- a/examples/tpch/q18_large_volume_customer.py +++ b/examples/tpch/q18_large_volume_customer.py @@ -54,8 +54,8 @@ # We've identified the orders of interest, now join the additional data # we are required to report on -df = df.join(df_orders, (["l_orderkey"], ["o_orderkey"]), "inner") -df = df.join(df_customer, (["o_custkey"], ["c_custkey"]), "inner") +df = df.join(df_orders, left_on=["l_orderkey"], right_on=["o_orderkey"], how="inner") +df = df.join(df_customer, left_on=["o_custkey"], right_on=["c_custkey"], how="inner") df = df.select( "c_name", "c_custkey", "o_orderkey", "o_orderdate", "o_totalprice", "total_quantity" diff --git a/examples/tpch/q19_discounted_revenue.py b/examples/tpch/q19_discounted_revenue.py index c2fe2570d..4aed0cbae 100644 --- a/examples/tpch/q19_discounted_revenue.py +++ b/examples/tpch/q19_discounted_revenue.py @@ -72,7 +72,7 @@ (col("l_shipmode") == lit("AIR")) | (col("l_shipmode") == lit("AIR REG")) ) -df = df.join(df_part, (["l_partkey"], ["p_partkey"]), "inner") +df = df.join(df_part, left_on=["l_partkey"], right_on=["p_partkey"], how="inner") # Create the user defined function (UDF) definition that does the work diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index 3a0edb1ec..d720cdce6 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -70,7 +70,7 @@ ) # This will filter down the line items to the parts of interest -df = df.join(df_part, (["l_partkey"], ["p_partkey"]), "inner") +df = df.join(df_part, left_on="l_partkey", right_on="p_partkey", how="inner") # Compute the total sold and limit ourselves to individual supplier/part combinations df = df.aggregate( @@ -78,15 +78,18 @@ ) df = df.join( - df_partsupp, (["l_partkey", "l_suppkey"], ["ps_partkey", "ps_suppkey"]), "inner" + df_partsupp, + left_on=["l_partkey", "l_suppkey"], + right_on=["ps_partkey", "ps_suppkey"], + how="inner", ) # Find cases of excess quantity df.filter(col("ps_availqty") > lit(0.5) * col("total_sold")) # We could do these joins earlier, but now limit to the nation of interest suppliers -df = df.join(df_supplier, (["ps_suppkey"], ["s_suppkey"]), "inner") -df = df.join(df_nation, (["s_nationkey"], ["n_nationkey"]), "inner") +df = df.join(df_supplier, left_on=["ps_suppkey"], right_on=["s_suppkey"], how="inner") +df = df.join(df_nation, left_on=["s_nationkey"], right_on=["n_nationkey"], how="inner") # Restrict to the requested data per the problem statement df = df.select("s_name", "s_address").distinct() diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py index d3d57acee..27cf816fa 100644 --- a/examples/tpch/q21_suppliers_kept_orders_waiting.py +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -52,13 +52,13 @@ df_suppliers_of_interest = df_nation.filter(col("n_name") == lit(NATION_OF_INTEREST)) df_suppliers_of_interest = df_suppliers_of_interest.join( - df_supplier, (["n_nationkey"], ["s_nationkey"]), "inner" + df_supplier, left_on="n_nationkey", right_on="s_nationkey", how="inner" ) # Find the failed orders and all their line items df = df_orders.filter(col("o_orderstatus") == lit("F")) -df = df_lineitem.join(df, (["l_orderkey"], ["o_orderkey"]), "inner") +df = df_lineitem.join(df, left_on="l_orderkey", right_on="o_orderkey", how="inner") # Identify the line items for which the order is failed due to. df = df.with_column( @@ -102,7 +102,9 @@ ) # Join to the supplier of interest list for the nation of interest -df = df.join(df_suppliers_of_interest, (["suppkey"], ["s_suppkey"]), "inner") +df = df.join( + df_suppliers_of_interest, left_on=["suppkey"], right_on=["s_suppkey"], how="inner" +) # Count how many orders that supplier is the only failed supplier for df = df.aggregate([col("s_name")], [F.count(col("o_orderkey")).alias("numwait")]) diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py index e6660e60c..72dce5289 100644 --- a/examples/tpch/q22_global_sales_opportunity.py +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -62,7 +62,7 @@ df = df.filter(col("c_acctbal") > col("avg_balance")) # Limit results to customers with no orders -df = df.join(df_orders, (["c_custkey"], ["o_custkey"]), "anti") +df = df.join(df_orders, left_on="c_custkey", right_on="o_custkey", how="anti") # Count up the customers and the balances df = df.aggregate( diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index e59f00d9f..efd4038ae 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -20,9 +20,8 @@ """ from __future__ import annotations - - -from typing import Any, Iterable, List, Literal, TYPE_CHECKING +import warnings +from typing import Any, Iterable, List, TYPE_CHECKING, Literal, overload from datafusion.record_batch import RecordBatchStream from typing_extensions import deprecated from datafusion.plan import LogicalPlan, ExecutionPlan @@ -32,7 +31,7 @@ import pandas as pd import polars as pl import pathlib - from typing import Callable + from typing import Callable, Sequence from datafusion._internal import DataFrame as DataFrameInternal from datafusion.expr import Expr, SortExpr, sort_or_default @@ -368,27 +367,105 @@ def distinct(self) -> DataFrame: """ return DataFrame(self.df.distinct()) + @overload + def join( + self, + right: DataFrame, + on: str | Sequence[str], + how: Literal["inner", "left", "right", "full", "semi", "anti"] = "inner", + *, + left_on: None = None, + right_on: None = None, + join_keys: None = None, + ) -> DataFrame: ... + + @overload def join( self, right: DataFrame, + on: None = None, + how: Literal["inner", "left", "right", "full", "semi", "anti"] = "inner", + *, + left_on: str | Sequence[str], + right_on: str | Sequence[str], + join_keys: tuple[list[str], list[str]] | None = None, + ) -> DataFrame: ... + + @overload + def join( + self, + right: DataFrame, + on: None = None, + how: Literal["inner", "left", "right", "full", "semi", "anti"] = "inner", + *, join_keys: tuple[list[str], list[str]], - how: str, + left_on: None = None, + right_on: None = None, + ) -> DataFrame: ... + + def join( + self, + right: DataFrame, + on: str | Sequence[str] | tuple[list[str], list[str]] | None = None, + how: Literal["inner", "left", "right", "full", "semi", "anti"] = "inner", + *, + left_on: str | Sequence[str] | None = None, + right_on: str | Sequence[str] | None = None, + join_keys: tuple[list[str], list[str]] | None = None, ) -> DataFrame: """Join this :py:class:`DataFrame` with another :py:class:`DataFrame`. - Join keys are a pair of lists of column names in the left and right - dataframes, respectively. These lists must have the same length. + `on` has to be provided or both `left_on` and `right_on` in conjunction. Args: right: Other DataFrame to join with. - join_keys: Tuple of two lists of column names to join on. + on: Column names to join on in both dataframes. how: Type of join to perform. Supported types are "inner", "left", "right", "full", "semi", "anti". + left_on: Join column of the left dataframe. + right_on: Join column of the right dataframe. + join_keys: Tuple of two lists of column names to join on. [Deprecated] Returns: DataFrame after join. """ - return DataFrame(self.df.join(right.df, join_keys, how)) + # This check is to prevent breaking API changes where users prior to + # DF 43.0.0 would pass the join_keys as a positional argument instead + # of a keyword argument. + if isinstance(on, tuple) and len(on) == 2: + if isinstance(on[0], list) and isinstance(on[1], list): + join_keys = on # type: ignore + on = None + + if join_keys is not None: + warnings.warn( + "`join_keys` is deprecated, use `on` or `left_on` with `right_on`", + category=DeprecationWarning, + stacklevel=2, + ) + left_on = join_keys[0] + right_on = join_keys[1] + + if on: + if left_on or right_on: + raise ValueError( + "`left_on` or `right_on` should not provided with `on`" + ) + left_on = on + right_on = on + elif left_on or right_on: + if left_on is None or right_on is None: + raise ValueError("`left_on` and `right_on` should both be provided.") + else: + raise ValueError( + "either `on` or `left_on` and `right_on` should be provided." + ) + if isinstance(left_on, str): + left_on = [left_on] + if isinstance(right_on, str): + right_on = [right_on] + + return DataFrame(self.df.join(right.df, how, left_on, right_on)) def join_on( self, diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 7b20e9e39..330475302 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -321,14 +321,72 @@ def test_join(): ) df1 = ctx.create_dataframe([[batch]], "r") - df = df.join(df1, join_keys=(["a"], ["a"]), how="inner") - df.show() - df = df.sort(column("l.a")) - table = pa.Table.from_batches(df.collect()) + df2 = df.join(df1, on="a", how="inner") + df2.show() + df2 = df2.sort(column("l.a")) + table = pa.Table.from_batches(df2.collect()) + + expected = {"a": [1, 2], "c": [8, 10], "b": [4, 5]} + assert table.to_pydict() == expected + + df2 = df.join(df1, left_on="a", right_on="a", how="inner") + df2.show() + df2 = df2.sort(column("l.a")) + table = pa.Table.from_batches(df2.collect()) expected = {"a": [1, 2], "c": [8, 10], "b": [4, 5]} assert table.to_pydict() == expected + # Verify we don't make a breaking change to pre-43.0.0 + # where users would pass join_keys as a positional argument + df2 = df.join(df1, (["a"], ["a"]), how="inner") # type: ignore + df2.show() + df2 = df2.sort(column("l.a")) + table = pa.Table.from_batches(df2.collect()) + + expected = {"a": [1, 2], "c": [8, 10], "b": [4, 5]} + assert table.to_pydict() == expected + + +def test_join_invalid_params(): + ctx = SessionContext() + + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], + names=["a", "b"], + ) + df = ctx.create_dataframe([[batch]], "l") + + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2]), pa.array([8, 10])], + names=["a", "c"], + ) + df1 = ctx.create_dataframe([[batch]], "r") + + with pytest.deprecated_call(): + df2 = df.join(df1, join_keys=(["a"], ["a"]), how="inner") + df2.show() + df2 = df2.sort(column("l.a")) + table = pa.Table.from_batches(df2.collect()) + + expected = {"a": [1, 2], "c": [8, 10], "b": [4, 5]} + assert table.to_pydict() == expected + + with pytest.raises( + ValueError, match=r"`left_on` or `right_on` should not provided with `on`" + ): + df2 = df.join(df1, on="a", how="inner", right_on="test") # type: ignore + + with pytest.raises( + ValueError, match=r"`left_on` and `right_on` should both be provided." + ): + df2 = df.join(df1, left_on="a", how="inner") # type: ignore + + with pytest.raises( + ValueError, match=r"either `on` or `left_on` and `right_on` should be provided." + ): + df2 = df.join(df1, how="inner") # type: ignore + def test_join_on(): ctx = SessionContext() diff --git a/src/dataframe.rs b/src/dataframe.rs index dd5d89ce9..ee8fbbf9d 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -271,8 +271,9 @@ impl PyDataFrame { fn join( &self, right: PyDataFrame, - join_keys: (Vec, Vec), how: &str, + left_on: Vec, + right_on: Vec, ) -> PyResult { let join_type = match how { "inner" => JoinType::Inner, @@ -289,16 +290,8 @@ impl PyDataFrame { } }; - let left_keys = join_keys - .0 - .iter() - .map(|s| s.as_ref()) - .collect::>(); - let right_keys = join_keys - .1 - .iter() - .map(|s| s.as_ref()) - .collect::>(); + let left_keys = left_on.iter().map(|s| s.as_ref()).collect::>(); + let right_keys = right_on.iter().map(|s| s.as_ref()).collect::>(); let df = self.df.as_ref().clone().join( right.df.as_ref().clone(), From 3c662010f9d133cda65749b119a1b1731edbe4e5 Mon Sep 17 00:00:00 2001 From: Michael J Ward Date: Sun, 10 Nov 2024 11:02:01 -0600 Subject: [PATCH 237/413] Upgrade to Datafusion 43 (#905) * patch datafusion deps * migrate from deprecated RuntimeEnv::new to RuntimeEnv::try_new Ref: https://github.com/apache/datafusion/pull/12566 * remove Arc from create_udf call Ref: https://github.com/apache/datafusion/pull/12489 * doc typo * migrage new UnnestOptions API Ref: https://github.com/apache/datafusion/pull/12836/files * update API for logical expr Limit Ref: https://github.com/apache/datafusion/pull/12836 * remove logical expr CrossJoin It was removed upstream. Ref: https://github.com/apache/datafusion/pull/13076 * update PyWindowUDF Ref: https://github.com/apache/datafusion/issues/12803 * migrate window functions lead and lag to udwf Ref: https://github.com/apache/datafusion/issues/12802 * migrate window functions rank, dense_rank, and percent_rank to udwf Ref: https://github.com/apache/datafusion/issues/12648 * convert window function cume_dist to udwf Ref: https://github.com/apache/datafusion/issues/12695 * convert window function ntile to udwf Ref: https://github.com/apache/datafusion/issues/12694 * clean up functions_window invocation * Only one column was being passed to udwf * Update to DF 43.0.0 * Update tests to look for string_view type * String view is now the default type for strings * Making a variety of adjustments in wrappers and unit tests to account for the switch from string to string_view as default * Resolve errors in doc building --------- Co-authored-by: Tim Saucer --- Cargo.lock | 373 ++++++++++++++++++--------------- Cargo.toml | 9 +- examples/tpch/_tests.py | 4 +- python/datafusion/expr.py | 4 +- python/datafusion/functions.py | 11 +- python/datafusion/udf.py | 1 + python/tests/test_expr.py | 16 +- python/tests/test_functions.py | 67 ++++-- python/tests/test_imports.py | 2 - python/tests/test_sql.py | 7 + src/context.rs | 2 +- src/dataframe.rs | 8 +- src/expr.rs | 2 - src/expr/cross_join.rs | 94 --------- src/expr/limit.rs | 22 +- src/functions.rs | 18 +- src/sql/logical.rs | 3 +- src/udf.rs | 4 +- src/udwf.rs | 29 ++- 19 files changed, 338 insertions(+), 338 deletions(-) delete mode 100644 src/expr/cross_join.rs diff --git a/Cargo.lock b/Cargo.lock index 0835f219e..497c5b850 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "apache-avro" @@ -130,9 +130,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9ba0d7248932f4e2a12fb37f0a2e3ec82b3bdedbac2a1dce186e036843b8f8c" +checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" dependencies = [ "arrow-arith", "arrow-array", @@ -152,9 +152,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d60afcdc004841a5c8d8da4f4fa22d64eb19c0c01ef4bcedd77f175a7cf6e38f" +checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" dependencies = [ "arrow-array", "arrow-buffer", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f16835e8599dbbb1659fd869d865254c4cf32c6c2bb60b6942ac9fc36bfa5da" +checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" dependencies = [ "ahash", "arrow-buffer", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a1f34f0faae77da6b142db61deba2cb6d60167592b178be317b341440acba80" +checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" dependencies = [ "bytes", "half", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "450e4abb5775bca0740bec0bcf1b1a5ae07eff43bd625661c4436d8e8e4540c4" +checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3a4e4d63830a341713e35d9a42452fbc6241d5f42fa5cf6a4681b8ad91370c4" +checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" dependencies = [ "arrow-array", "arrow-buffer", @@ -235,9 +235,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b1e618bbf714c7a9e8d97203c806734f012ff71ae3adc8ad1b075689f540634" +checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" dependencies = [ "arrow-buffer", "arrow-schema", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98e983549259a2b97049af7edfb8f28b8911682040e99a94e4ceb1196bd65c2" +checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" dependencies = [ "arrow-array", "arrow-buffer", @@ -262,9 +262,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b198b9c6fcf086501730efbbcb483317b39330a116125af7bb06467d04b352a3" +checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" dependencies = [ "arrow-array", "arrow-buffer", @@ -282,9 +282,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2427f37b4459a4b9e533045abe87a5183a5e0995a3fc2c2fd45027ae2cc4ef3f" +checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" dependencies = [ "arrow-array", "arrow-buffer", @@ -297,9 +297,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15959657d92e2261a7a323517640af87f5afd9fd8a6492e424ebee2203c567f6" +checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" dependencies = [ "ahash", "arrow-array", @@ -311,18 +311,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf0388a18fd7f7f3fe3de01852d30f54ed5182f9004db700fbe3ba843ed2794" +checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b83e5723d307a38bf00ecd2972cd078d1339c7fd3eb044f609958a9a24463f3a" +checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" dependencies = [ "ahash", "arrow-array", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ab3db7c09dd826e74079661d84ed01ed06547cf75d52c2818ef776d0d852305" +checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -351,9 +351,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.13" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" dependencies = [ "bzip2", "flate2", @@ -482,9 +482,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -515,9 +515,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bzip2" @@ -542,9 +542,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.28" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e80e3b6a3ab07840e1cae9b0666a63970dc28e8ed5ffbcdacbfc760c281bfc1" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "jobserver", "libc", @@ -557,6 +557,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -725,9 +731,9 @@ dependencies = [ [[package]] name = "dary_heap" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7762d17f1241643615821a8455a0b2c3e803784b058693d990b11f2dce25a0ca" +checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" [[package]] name = "dashmap" @@ -745,9 +751,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee907b081e45e1d14e1f327e89ef134f91fcebad0bfc2dc229fa9f6044379682" +checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" dependencies = [ "ahash", "apache-avro", @@ -804,9 +810,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c2b914f6e33c429af7d8696c72a47ed9225d7e2b82c747ebdfa2408ed53579f" +checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" dependencies = [ "arrow-schema", "async-trait", @@ -819,9 +825,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a84f8e76330c582a6b8ada0b2c599ca46cfe46b7585e458fc3f4092bc722a18" +checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" dependencies = [ "ahash", "apache-avro", @@ -832,6 +838,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", + "indexmap", "instant", "libc", "num_cpus", @@ -845,9 +852,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf08cc30d92720d557df13bd5a5696213bd5ea0f38a866d8d85055d866fba774" +checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" dependencies = [ "log", "tokio", @@ -855,9 +862,9 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86bc4183d5c45b9f068a6f351678a0d1eb1225181424542bb75db18ec280b822" +checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" dependencies = [ "arrow", "chrono", @@ -876,9 +883,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202119ce58e4d103e37ae64aab40d4e574c97bdd2bea994bf307b175fcbfa74d" +checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" dependencies = [ "ahash", "arrow", @@ -888,7 +895,9 @@ dependencies = [ "datafusion-common", "datafusion-expr-common", "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", "datafusion-physical-expr-common", + "indexmap", "paste", "serde_json", "sqlparser", @@ -898,20 +907,21 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b181ce8569216abb01ef3294aa16c0a40d7d39350c2ff01ede00f167a535f2" +checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" dependencies = [ "arrow", "datafusion-common", + "itertools", "paste", ] [[package]] name = "datafusion-functions" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4124b8066444e05a24472f852e94cf56546c0f4d92d00f018f207216902712" +checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" dependencies = [ "arrow", "arrow-buffer", @@ -936,9 +946,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94acdac235ea21810150a89751617ef2db7e32eba27f54be48a81bde2bfe119" +checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" dependencies = [ "ahash", "arrow", @@ -950,16 +960,16 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "half", + "indexmap", "log", "paste", - "sqlparser", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c9ea085bbf900bf16e2ca0f56fc56236b2e4f2e1a2cccb67bcd83c5ab4ad0ef" +checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" dependencies = [ "ahash", "arrow", @@ -971,9 +981,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c882e61665ed60c5ce9b061c1e587aeb8ae5ae4bcb5e5f2465139ab25328e0f" +checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" dependencies = [ "arrow", "arrow-array", @@ -994,21 +1004,34 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98a354ce96df3ca6d025093adac9fd55ca09931c9b6f2630140721a95873fde4" +checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" dependencies = [ "datafusion-common", "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", "datafusion-physical-expr-common", "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] name = "datafusion-optimizer" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf677c74fb7b5a1899ef52709e4a70fff3ed80bdfb4bbe495909810e83d5f39" +checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" dependencies = [ "arrow", "async-trait", @@ -1026,9 +1049,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b077999f6eb6c43d6b25bc66332a3be2f693c382840f008dd763b8540f9530" +checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" dependencies = [ "ahash", "arrow", @@ -1037,30 +1060,26 @@ dependencies = [ "arrow-ord", "arrow-schema", "arrow-string", - "base64 0.22.1", "chrono", "datafusion-common", - "datafusion-execution", "datafusion-expr", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "hex", "indexmap", "itertools", "log", "paste", "petgraph", - "regex", ] [[package]] name = "datafusion-physical-expr-common" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce847f885c2b13bbe29f5c8b7948797131aa470af6e16d2a94f4428b4f4f1bd" +checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" dependencies = [ "ahash", "arrow", @@ -1072,13 +1091,15 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d13238e3b9fdd62a4c18760bfef714bb990d1e1d3430e9f416aae4b3cfaa71af" +checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" dependencies = [ + "arrow", "arrow-schema", "datafusion-common", "datafusion-execution", + "datafusion-expr-common", "datafusion-physical-expr", "datafusion-physical-plan", "itertools", @@ -1086,9 +1107,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faba6f55a7eaf0241d07d12c2640de52742646b10f754485d5192bdfe2c9ceae" +checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" dependencies = [ "ahash", "arrow", @@ -1102,8 +1123,8 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", "futures", @@ -1121,9 +1142,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585357d621fa03ea85a7fefca79ebc5ef0ee13a7f82be0762a414879a4d190a7" +checksum = "f730f7fc5a20134d4e5ecdf7bbf392002ac58163d58423ea28a702dc077b06e1" dependencies = [ "arrow", "chrono", @@ -1137,9 +1158,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4db6534382f92f528bdb5d925b4214c31ffd84fa7fe1eff3ed0d2f1286851ab8" +checksum = "12c225fe49e4f943e35446b263613ada7a9e9f8d647544e6b07037b9803567df" dependencies = [ "arrow", "chrono", @@ -1155,6 +1176,7 @@ dependencies = [ "arrow", "async-trait", "datafusion", + "datafusion-functions-window-common", "datafusion-proto", "datafusion-substrait", "futures", @@ -1171,15 +1193,16 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad8d96a9b52e1aa24f9373696a815be828193efce7cb0bbd2140b6bb67d1819" +checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" dependencies = [ "arrow", "arrow-array", "arrow-schema", "datafusion-common", "datafusion-expr", + "indexmap", "log", "regex", "sqlparser", @@ -1188,9 +1211,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "42.0.0" +version = "43.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92b1b80e98bf5a9921bf118816e0e766d18527e343153321fcccfe4d68c5c45" +checksum = "8b9c768d2b4c4485c43afbaeeb86dd1f2ac3fb34a9e6e8c8b06180d2a223d5ba" dependencies = [ "arrow-buffer", "async-recursion", @@ -1530,9 +1553,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -1568,9 +1591,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -1684,9 +1707,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -1763,9 +1786,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libflate" @@ -1793,9 +1816,9 @@ dependencies = [ [[package]] name = "libm" -version = "0.2.8" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libmimalloc-sys" @@ -2011,9 +2034,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a0c4b3a0e31f8b66f71ad8064521efa773910196e2cde791436f13409f3b45" +checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" dependencies = [ "async-trait", "base64 0.22.1", @@ -2086,9 +2109,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.1.0" +version = "53.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310c46a70a3ba90d98fec39fa2da6d9d731e544191da6fb56c9d199484d0dd3e" +checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e" dependencies = [ "ahash", "arrow-array", @@ -2228,9 +2251,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -2261,9 +2284,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", "syn", @@ -2271,9 +2294,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -2342,9 +2365,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e89ce2565d6044ca31a3eb79a334c3a79a841120a98f64eea9f579564cb691" +checksum = "3d922163ba1f79c04bc49073ba7b32fd5a8d3b76a87c955921234b8e77333c51" dependencies = [ "cfg-if", "indoc", @@ -2360,9 +2383,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8afbaf3abd7325e08f35ffb8deb5892046fcb2608b703db6a583a5ba4cea01e" +checksum = "bc38c5feeb496c8321091edf3d63e9a6829eab4b863b4a6a65f26f3e9cc6b179" dependencies = [ "once_cell", "target-lexicon", @@ -2370,9 +2393,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec15a5ba277339d04763f4c23d85987a5b08cbb494860be141e6a10a8eb88022" +checksum = "94845622d88ae274d2729fcefc850e63d7a3ddff5e3ce11bd88486db9f1d357d" dependencies = [ "libc", "pyo3-build-config", @@ -2380,9 +2403,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e0f01b5364bcfbb686a52fc4181d412b708a68ed20c330db9fc8d2c2bf5a43" +checksum = "e655aad15e09b94ffdb3ce3d217acf652e26bbc37697ef012f5e5e348c716e5e" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2392,9 +2415,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a09b550200e1e5ed9176976d0060cbc2ea82dc8515da07885e7b8153a85caacb" +checksum = "ae1e3f09eecd94618f60a455a23def79f79eba4dc561a97324bf9ac8c6df30ce" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -2456,10 +2479,11 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -2517,9 +2541,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -2552,9 +2576,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eae2a1ebfecc58aff952ef8ccd364329abe627762f5bf09ff42eb9d98522479" +checksum = "1541daf4e4ed43a0922b7969bdc2170178bcacc5dabf7e39bc508a9fa3953a7a" dependencies = [ "hashbrown 0.14.5", "memchr", @@ -2562,9 +2586,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "base64 0.22.1", "bytes", @@ -2649,9 +2673,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags 2.6.0", "errno", @@ -2662,9 +2686,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "once_cell", "ring", @@ -2698,9 +2722,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" @@ -2715,9 +2739,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "ryu" @@ -2813,18 +2837,18 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", @@ -2844,9 +2868,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -2974,9 +2998,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.50.0" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2e5b515a2bd5168426033e9efbfd05500114833916f1d5c268f938b4ee130ac" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" dependencies = [ "log", "sqlparser_derive", @@ -3042,9 +3066,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.41.9" +version = "0.45.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a3bf05f1d7a3fd7a97790d410f6e859b3a98dcde05e7a3fc00b31b0f60fe7cb" +checksum = "a127ae9d8e443cea5c2122eb2ffe5fe489e802a1e746a09c5a5cb59d074c0aeb" dependencies = [ "heck 0.5.0", "pbjson", @@ -3055,6 +3079,7 @@ dependencies = [ "prost-build", "prost-types", "protobuf-src", + "regress", "schemars", "semver", "serde", @@ -3073,9 +3098,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.79" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -3112,18 +3137,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", @@ -3167,9 +3192,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -3297,9 +3322,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6beec125971dda80a086f90b4a70f60f222990ce4d63ad0fc140492f53444" +checksum = "b4c644dda9862f0fef3a570d8ddb3c2cfb1d5ac824a1f2ddfa7bc8f071a5ad8a" dependencies = [ "typify-impl", "typify-macro", @@ -3307,9 +3332,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93bbb24e990654aff858d80fee8114f4322f7d7a1b1ecb45129e2fcb0d0ad5ae" +checksum = "d59ab345b6c0d8ae9500b9ff334a4c7c0d316c1c628dc55726b95887eb8dbd11" dependencies = [ "heck 0.5.0", "log", @@ -3327,9 +3352,9 @@ dependencies = [ [[package]] name = "typify-macro" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e6491896e955692d68361c68db2b263e3bec317ec0b684e0e2fa882fb6e31e" +checksum = "785e2cdcef0df8160fdd762ed548a637aaec1e83704fdbc14da0df66013ee8d0" dependencies = [ "proc-macro2", "quote", @@ -3447,9 +3472,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -3458,9 +3483,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", @@ -3473,9 +3498,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -3485,9 +3510,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3495,9 +3520,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", @@ -3508,15 +3533,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "wasm-streams" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ "futures-util", "js-sys", @@ -3527,9 +3552,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 073f82cf0..11ce08c75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,9 +37,10 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync"] } pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "53", features = ["pyarrow"] } -datafusion = { version = "42.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-substrait = { version = "42.0.0", optional = true } -datafusion-proto = { version = "42.0.0" } +datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-substrait = { version = "43.0.0", optional = true } +datafusion-proto = { version = "43.0.0" } +datafusion-functions-window-common = { version = "43.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.11", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } @@ -58,4 +59,4 @@ crate-type = ["cdylib", "rlib"] [profile.release] lto = true -codegen-units = 1 +codegen-units = 1 \ No newline at end of file diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 13144ae9d..3ce9cdfe5 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -25,7 +25,7 @@ def df_selection(col_name, col_type): if col_type == pa.float64() or isinstance(col_type, pa.Decimal128Type): return F.round(col(col_name), lit(2)).alias(col_name) - elif col_type == pa.string(): + elif col_type == pa.string() or col_type == pa.string_view(): return F.trim(col(col_name)).alias(col_name) else: return col(col_name) @@ -43,7 +43,7 @@ def load_schema(col_name, col_type): def expected_selection(col_name, col_type): if col_type == pa.int64() or col_type == pa.int32(): return F.trim(col(col_name)).cast(col_type).alias(col_name) - elif col_type == pa.string(): + elif col_type == pa.string() or col_type == pa.string_view(): return F.trim(col(col_name)).alias(col_name) else: return col(col_name) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index c4e7713f3..b10724381 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -51,7 +51,6 @@ Column = expr_internal.Column CreateMemoryTable = expr_internal.CreateMemoryTable CreateView = expr_internal.CreateView -CrossJoin = expr_internal.CrossJoin Distinct = expr_internal.Distinct DropTable = expr_internal.DropTable EmptyRelation = expr_internal.EmptyRelation @@ -140,7 +139,6 @@ "Join", "JoinType", "JoinConstraint", - "CrossJoin", "Union", "Unnest", "UnnestExpr", @@ -376,6 +374,8 @@ def literal(value: Any) -> Expr: ``value`` must be a valid PyArrow scalar value or easily castable to one. """ + if isinstance(value, str): + value = pa.scalar(value, type=pa.string_view()) if not isinstance(value, pa.Scalar): value = pa.scalar(value) return Expr(expr_internal.Expr.literal(value)) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 907f801af..5a2eab56d 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -297,7 +297,7 @@ def decode(input: Expr, encoding: Expr) -> Expr: def array_to_string(expr: Expr, delimiter: Expr) -> Expr: """Converts each element to its text representation.""" - return Expr(f.array_to_string(expr.expr, delimiter.expr)) + return Expr(f.array_to_string(expr.expr, delimiter.expr.cast(pa.string()))) def array_join(expr: Expr, delimiter: Expr) -> Expr: @@ -1067,7 +1067,10 @@ def struct(*args: Expr) -> Expr: def named_struct(name_pairs: list[tuple[str, Expr]]) -> Expr: """Returns a struct with the given names and arguments pairs.""" - name_pair_exprs = [[Expr.literal(pair[0]), pair[1]] for pair in name_pairs] + name_pair_exprs = [ + [Expr.literal(pa.scalar(pair[0], type=pa.string())), pair[1]] + for pair in name_pairs + ] # flatten name_pairs = [x.expr for xs in name_pair_exprs for x in xs] @@ -1424,7 +1427,9 @@ def array_sort(array: Expr, descending: bool = False, null_first: bool = False) nulls_first = "NULLS FIRST" if null_first else "NULLS LAST" return Expr( f.array_sort( - array.expr, Expr.literal(desc).expr, Expr.literal(nulls_first).expr + array.expr, + Expr.literal(pa.scalar(desc, type=pa.string())).expr, + Expr.literal(pa.scalar(nulls_first, type=pa.string())).expr, ) ) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index 291ef2bae..d9d994b22 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -229,6 +229,7 @@ def udaf( which this UDAF is used. The following examples are all valid. .. code-block:: python + import pyarrow as pa import pyarrow.compute as pc diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 1847edef2..77f88aa44 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -85,14 +85,18 @@ def test_limit(test_ctx): plan = plan.to_variant() assert isinstance(plan, Limit) - assert plan.skip() == 0 + # TODO: Upstream now has expressions for skip and fetch + # REF: https://github.com/apache/datafusion/pull/12836 + # assert plan.skip() == 0 df = test_ctx.sql("select c1 from test LIMIT 10 OFFSET 5") plan = df.logical_plan() plan = plan.to_variant() assert isinstance(plan, Limit) - assert plan.skip() == 5 + # TODO: Upstream now has expressions for skip and fetch + # REF: https://github.com/apache/datafusion/pull/12836 + # assert plan.skip() == 5 def test_aggregate_query(test_ctx): @@ -126,7 +130,10 @@ def test_relational_expr(test_ctx): ctx = SessionContext() batch = pa.RecordBatch.from_arrays( - [pa.array([1, 2, 3]), pa.array(["alpha", "beta", "gamma"])], + [ + pa.array([1, 2, 3]), + pa.array(["alpha", "beta", "gamma"], type=pa.string_view()), + ], names=["a", "b"], ) df = ctx.create_dataframe([[batch]], name="batch_array") @@ -141,7 +148,8 @@ def test_relational_expr(test_ctx): assert df.filter(col("b") == "beta").count() == 1 assert df.filter(col("b") != "beta").count() == 2 - assert df.filter(col("a") == "beta").count() == 0 + with pytest.raises(Exception): + df.filter(col("a") == "beta").count() def test_expr_to_variant(): diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index c65c633a4..b3a5a0652 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -34,9 +34,9 @@ def df(): # create a RecordBatch and a new DataFrame from it batch = pa.RecordBatch.from_arrays( [ - pa.array(["Hello", "World", "!"]), + pa.array(["Hello", "World", "!"], type=pa.string_view()), pa.array([4, 5, 6]), - pa.array(["hello ", " world ", " !"]), + pa.array(["hello ", " world ", " !"], type=pa.string_view()), pa.array( [ datetime(2022, 12, 31), @@ -88,8 +88,8 @@ def test_literal(df): assert len(result) == 1 result = result[0] assert result.column(0) == pa.array([1] * 3) - assert result.column(1) == pa.array(["1"] * 3) - assert result.column(2) == pa.array(["OK"] * 3) + assert result.column(1) == pa.array(["1"] * 3, type=pa.string_view()) + assert result.column(2) == pa.array(["OK"] * 3, type=pa.string_view()) assert result.column(3) == pa.array([3.14] * 3) assert result.column(4) == pa.array([True] * 3) assert result.column(5) == pa.array([b"hello world"] * 3) @@ -97,7 +97,9 @@ def test_literal(df): def test_lit_arith(df): """Test literals with arithmetic operations""" - df = df.select(literal(1) + column("b"), f.concat(column("a"), literal("!"))) + df = df.select( + literal(1) + column("b"), f.concat(column("a").cast(pa.string()), literal("!")) + ) result = df.collect() assert len(result) == 1 result = result[0] @@ -600,21 +602,33 @@ def test_array_function_obj_tests(stmt, py_expr): f.ascii(column("a")), pa.array([72, 87, 33], type=pa.int32()), ), # H = 72; W = 87; ! = 33 - (f.bit_length(column("a")), pa.array([40, 40, 8], type=pa.int32())), - (f.btrim(literal(" World ")), pa.array(["World", "World", "World"])), + ( + f.bit_length(column("a").cast(pa.string())), + pa.array([40, 40, 8], type=pa.int32()), + ), + ( + f.btrim(literal(" World ")), + pa.array(["World", "World", "World"], type=pa.string_view()), + ), (f.character_length(column("a")), pa.array([5, 5, 1], type=pa.int32())), (f.chr(literal(68)), pa.array(["D", "D", "D"])), ( f.concat_ws("-", column("a"), literal("test")), pa.array(["Hello-test", "World-test", "!-test"]), ), - (f.concat(column("a"), literal("?")), pa.array(["Hello?", "World?", "!?"])), + ( + f.concat(column("a").cast(pa.string()), literal("?")), + pa.array(["Hello?", "World?", "!?"]), + ), (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), (f.length(column("c")), pa.array([6, 7, 2], type=pa.int32())), (f.lower(column("a")), pa.array(["hello", "world", "!"])), (f.lpad(column("a"), literal(7)), pa.array([" Hello", " World", " !"])), - (f.ltrim(column("c")), pa.array(["hello ", "world ", "!"])), + ( + f.ltrim(column("c")), + pa.array(["hello ", "world ", "!"], type=pa.string_view()), + ), ( f.md5(column("a")), pa.array( @@ -640,19 +654,25 @@ def test_array_function_obj_tests(stmt, py_expr): f.rpad(column("a"), literal(8)), pa.array(["Hello ", "World ", "! "]), ), - (f.rtrim(column("c")), pa.array(["hello", " world", " !"])), + ( + f.rtrim(column("c")), + pa.array(["hello", " world", " !"], type=pa.string_view()), + ), ( f.split_part(column("a"), literal("l"), literal(1)), pa.array(["He", "Wor", "!"]), ), (f.starts_with(column("a"), literal("Wor")), pa.array([False, True, False])), (f.strpos(column("a"), literal("o")), pa.array([5, 2, 0], type=pa.int32())), - (f.substr(column("a"), literal(3)), pa.array(["llo", "rld", ""])), + ( + f.substr(column("a"), literal(3)), + pa.array(["llo", "rld", ""], type=pa.string_view()), + ), ( f.translate(column("a"), literal("or"), literal("ld")), pa.array(["Helll", "Wldld", "!"]), ), - (f.trim(column("c")), pa.array(["hello", "world", "!"])), + (f.trim(column("c")), pa.array(["hello", "world", "!"], type=pa.string_view())), (f.upper(column("c")), pa.array(["HELLO ", " WORLD ", " !"])), (f.ends_with(column("a"), literal("llo")), pa.array([True, False, False])), ( @@ -794,9 +814,9 @@ def test_temporal_functions(df): f.date_trunc(literal("month"), column("d")), f.datetrunc(literal("day"), column("d")), f.date_bin( - literal("15 minutes"), + literal("15 minutes").cast(pa.string()), column("d"), - literal("2001-01-01 00:02:30"), + literal("2001-01-01 00:02:30").cast(pa.string()), ), f.from_unixtime(literal(1673383974)), f.to_timestamp(literal("2023-09-07 05:06:14.523952")), @@ -858,8 +878,8 @@ def test_case(df): result = df.collect() result = result[0] assert result.column(0) == pa.array([10, 8, 8]) - assert result.column(1) == pa.array(["Hola", "Mundo", "!!"]) - assert result.column(2) == pa.array(["Hola", "Mundo", None]) + assert result.column(1) == pa.array(["Hola", "Mundo", "!!"], type=pa.string_view()) + assert result.column(2) == pa.array(["Hola", "Mundo", None], type=pa.string_view()) def test_when_with_no_base(df): @@ -877,8 +897,10 @@ def test_when_with_no_base(df): result = df.collect() result = result[0] assert result.column(0) == pa.array([4, 5, 6]) - assert result.column(1) == pa.array(["too small", "just right", "too big"]) - assert result.column(2) == pa.array(["Hello", None, None]) + assert result.column(1) == pa.array( + ["too small", "just right", "too big"], type=pa.string_view() + ) + assert result.column(2) == pa.array(["Hello", None, None], type=pa.string_view()) def test_regr_funcs_sql(df): @@ -1021,8 +1043,13 @@ def test_regr_funcs_df(func, expected): def test_binary_string_functions(df): df = df.select( - f.encode(column("a"), literal("base64")), - f.decode(f.encode(column("a"), literal("base64")), literal("base64")), + f.encode(column("a").cast(pa.string()), literal("base64").cast(pa.string())), + f.decode( + f.encode( + column("a").cast(pa.string()), literal("base64").cast(pa.string()) + ), + literal("base64").cast(pa.string()), + ), ) result = df.collect() assert len(result) == 1 diff --git a/python/tests/test_imports.py b/python/tests/test_imports.py index 3d324fb62..6ea77b15f 100644 --- a/python/tests/test_imports.py +++ b/python/tests/test_imports.py @@ -46,7 +46,6 @@ Join, JoinType, JoinConstraint, - CrossJoin, Union, Like, ILike, @@ -129,7 +128,6 @@ def test_class_module_is_datafusion(): Join, JoinType, JoinConstraint, - CrossJoin, Union, Like, ILike, diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py index 39e5ffe6d..a2521dd09 100644 --- a/python/tests/test_sql.py +++ b/python/tests/test_sql.py @@ -468,6 +468,13 @@ def test_simple_select(ctx, tmp_path, arr): batches = ctx.sql("SELECT a AS tt FROM t").collect() result = batches[0].column(0) + # In DF 43.0.0 we now default to having BinaryView and StringView + # so the array that is saved to the parquet is slightly different + # than the array read. Convert to values for comparison. + if isinstance(result, pa.BinaryViewArray) or isinstance(result, pa.StringViewArray): + arr = arr.tolist() + result = result.tolist() + np.testing.assert_equal(result, arr) diff --git a/src/context.rs b/src/context.rs index f445874d6..c2a263fa7 100644 --- a/src/context.rs +++ b/src/context.rs @@ -287,7 +287,7 @@ impl PySessionContext { } else { RuntimeConfig::default() }; - let runtime = Arc::new(RuntimeEnv::new(runtime_config)?); + let runtime = Arc::new(RuntimeEnv::try_new(runtime_config)?); let session_state = SessionStateBuilder::new() .with_config(config) .with_runtime_env(runtime) diff --git a/src/dataframe.rs b/src/dataframe.rs index ee8fbbf9d..e7d6ca6d6 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -402,7 +402,9 @@ impl PyDataFrame { #[pyo3(signature = (column, preserve_nulls=true))] fn unnest_column(&self, column: &str, preserve_nulls: bool) -> PyResult { - let unnest_options = UnnestOptions { preserve_nulls }; + // TODO: expose RecursionUnnestOptions + // REF: https://github.com/apache/datafusion/pull/11577 + let unnest_options = UnnestOptions::default().with_preserve_nulls(preserve_nulls); let df = self .df .as_ref() @@ -413,7 +415,9 @@ impl PyDataFrame { #[pyo3(signature = (columns, preserve_nulls=true))] fn unnest_columns(&self, columns: Vec, preserve_nulls: bool) -> PyResult { - let unnest_options = UnnestOptions { preserve_nulls }; + // TODO: expose RecursionUnnestOptions + // REF: https://github.com/apache/datafusion/pull/11577 + let unnest_options = UnnestOptions::default().with_preserve_nulls(preserve_nulls); let cols = columns.iter().map(|s| s.as_ref()).collect::>(); let df = self .df diff --git a/src/expr.rs b/src/expr.rs index 49fa4b845..bca0cd3fa 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -65,7 +65,6 @@ pub mod column; pub mod conditional_expr; pub mod create_memory_table; pub mod create_view; -pub mod cross_join; pub mod distinct; pub mod drop_table; pub mod empty_relation; @@ -775,7 +774,6 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/expr/cross_join.rs b/src/expr/cross_join.rs deleted file mode 100644 index 5bc202aac..000000000 --- a/src/expr/cross_join.rs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion::logical_expr::logical_plan::CrossJoin; -use pyo3::prelude::*; -use std::fmt::{self, Display, Formatter}; - -use super::logical_node::LogicalNode; -use crate::common::df_schema::PyDFSchema; -use crate::sql::logical::PyLogicalPlan; - -#[pyclass(name = "CrossJoin", module = "datafusion.expr", subclass)] -#[derive(Clone)] -pub struct PyCrossJoin { - cross_join: CrossJoin, -} - -impl From for PyCrossJoin { - fn from(cross_join: CrossJoin) -> PyCrossJoin { - PyCrossJoin { cross_join } - } -} - -impl From for CrossJoin { - fn from(cross_join: PyCrossJoin) -> Self { - cross_join.cross_join - } -} - -impl Display for PyCrossJoin { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!( - f, - "CrossJoin - \nLeft: {:?} - \nRight: {:?} - \nSchema: {:?}", - &self.cross_join.left, &self.cross_join.right, &self.cross_join.schema - ) - } -} - -#[pymethods] -impl PyCrossJoin { - /// Retrieves the left input `LogicalPlan` to this `CrossJoin` node - fn left(&self) -> PyResult { - Ok(self.cross_join.left.as_ref().clone().into()) - } - - /// Retrieves the right input `LogicalPlan` to this `CrossJoin` node - fn right(&self) -> PyResult { - Ok(self.cross_join.right.as_ref().clone().into()) - } - - /// Resulting Schema for this `CrossJoin` node instance - fn schema(&self) -> PyResult { - Ok(self.cross_join.schema.as_ref().clone().into()) - } - - fn __repr__(&self) -> PyResult { - Ok(format!("CrossJoin({})", self)) - } - - fn __name__(&self) -> PyResult { - Ok("CrossJoin".to_string()) - } -} - -impl LogicalNode for PyCrossJoin { - fn inputs(&self) -> Vec { - vec![ - PyLogicalPlan::from((*self.cross_join.left).clone()), - PyLogicalPlan::from((*self.cross_join.right).clone()), - ] - } - - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) - } -} diff --git a/src/expr/limit.rs b/src/expr/limit.rs index 876e154c1..84ad7d68b 100644 --- a/src/expr/limit.rs +++ b/src/expr/limit.rs @@ -46,7 +46,7 @@ impl Display for PyLimit { write!( f, "Limit - Skip: {} + Skip: {:?} Fetch: {:?} Input: {:?}", &self.limit.skip, &self.limit.fetch, &self.limit.input @@ -56,15 +56,19 @@ impl Display for PyLimit { #[pymethods] impl PyLimit { - /// Retrieves the skip value for this `Limit` - fn skip(&self) -> usize { - self.limit.skip - } + // NOTE: Upstream now has expressions for skip and fetch + // TODO: Do we still want to expose these? + // REF: https://github.com/apache/datafusion/pull/12836 - /// Retrieves the fetch value for this `Limit` - fn fetch(&self) -> Option { - self.limit.fetch - } + // /// Retrieves the skip value for this `Limit` + // fn skip(&self) -> usize { + // self.limit.skip + // } + + // /// Retrieves the fetch value for this `Limit` + // fn fetch(&self) -> Option { + // self.limit.fetch + // } /// Retrieves the input `LogicalPlan` to this `Limit` node fn input(&self) -> PyResult> { diff --git a/src/functions.rs b/src/functions.rs index fe3531ba9..e29c57f9b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,7 +16,6 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; -use datafusion::logical_expr::window_function; use datafusion::logical_expr::ExprFunctionExt; use datafusion::logical_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; @@ -33,6 +32,7 @@ use datafusion::common::{Column, ScalarValue, TableReference}; use datafusion::execution::FunctionRegistry; use datafusion::functions; use datafusion::functions_aggregate; +use datafusion::functions_window; use datafusion::logical_expr::expr::Alias; use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; use datafusion::logical_expr::{ @@ -758,7 +758,7 @@ pub fn lead( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::lead(arg.expr, Some(shift_offset), default_value); + let window_fn = functions_window::expr_fn::lead(arg.expr, Some(shift_offset), default_value); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -772,7 +772,7 @@ pub fn lag( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::lag(arg.expr, Some(shift_offset), default_value); + let window_fn = functions_window::expr_fn::lag(arg.expr, Some(shift_offset), default_value); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -783,7 +783,7 @@ pub fn row_number( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = datafusion::functions_window::expr_fn::row_number(); + let window_fn = functions_window::expr_fn::row_number(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -794,7 +794,7 @@ pub fn rank( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::rank(); + let window_fn = functions_window::expr_fn::rank(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -805,7 +805,7 @@ pub fn dense_rank( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::dense_rank(); + let window_fn = functions_window::expr_fn::dense_rank(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -816,7 +816,7 @@ pub fn percent_rank( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::percent_rank(); + let window_fn = functions_window::expr_fn::percent_rank(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -827,7 +827,7 @@ pub fn cume_dist( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::cume_dist(); + let window_fn = functions_window::expr_fn::cume_dist(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } @@ -839,7 +839,7 @@ pub fn ntile( partition_by: Option>, order_by: Option>, ) -> PyResult { - let window_fn = window_function::ntile(arg.into()); + let window_fn = functions_window::expr_fn::ntile(arg.into()); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) } diff --git a/src/sql/logical.rs b/src/sql/logical.rs index fc398ff89..40f0a6a65 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -19,7 +19,6 @@ use std::sync::Arc; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; -use crate::expr::cross_join::PyCrossJoin; use crate::expr::distinct::PyDistinct; use crate::expr::empty_relation::PyEmptyRelation; use crate::expr::explain::PyExplain; @@ -68,7 +67,6 @@ impl PyLogicalPlan { match self.plan.as_ref() { LogicalPlan::Aggregate(plan) => PyAggregate::from(plan.clone()).to_variant(py), LogicalPlan::Analyze(plan) => PyAnalyze::from(plan.clone()).to_variant(py), - LogicalPlan::CrossJoin(plan) => PyCrossJoin::from(plan.clone()).to_variant(py), LogicalPlan::Distinct(plan) => PyDistinct::from(plan.clone()).to_variant(py), LogicalPlan::EmptyRelation(plan) => PyEmptyRelation::from(plan.clone()).to_variant(py), LogicalPlan::Explain(plan) => PyExplain::from(plan.clone()).to_variant(py), @@ -92,6 +90,7 @@ impl PyLogicalPlan { | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) | LogicalPlan::DescribeTable(_) + | LogicalPlan::Execute(_) | LogicalPlan::RecursiveQuery(_) => Err(py_unsupported_variant_err(format!( "Conversion of variant not implemented: {:?}", self.plan diff --git a/src/udf.rs b/src/udf.rs index ec8efb169..4570e77a6 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -31,7 +31,7 @@ use datafusion::logical_expr::{create_udf, ColumnarValue}; use crate::expr::PyExpr; use crate::utils::parse_volatility; -/// Create a Rust callable function fr a python function that expects pyarrow arrays +/// Create a Rust callable function from a python function that expects pyarrow arrays fn pyarrow_function_to_rust( func: PyObject, ) -> impl Fn(&[ArrayRef]) -> Result { @@ -97,7 +97,7 @@ impl PyScalarUDF { let function = create_udf( name, input_types.0, - Arc::new(return_type.0), + return_type.0, parse_volatility(volatility)?, to_scalar_function_impl(func), ); diff --git a/src/udwf.rs b/src/udwf.rs index 43c21ec7b..3f5ad0b1d 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -20,11 +20,16 @@ use std::ops::Range; use std::sync::Arc; use arrow::array::{make_array, Array, ArrayData, ArrayRef}; +use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; use datafusion::logical_expr::window_state::WindowAggState; +use datafusion::physical_plan::PhysicalExpr; use datafusion::scalar::ScalarValue; +use datafusion_functions_window_common::expr::ExpressionArgs; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use crate::expr::PyExpr; +use crate::utils::parse_volatility; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::{DataFusionError, Result}; @@ -33,9 +38,6 @@ use datafusion::logical_expr::{ }; use pyo3::types::{PyList, PyTuple}; -use crate::expr::PyExpr; -use crate::utils::parse_volatility; - #[derive(Debug)] struct RustPartitionEvaluator { evaluator: PyObject, @@ -90,6 +92,7 @@ impl PartitionEvaluator for RustPartitionEvaluator { } fn evaluate_all(&mut self, values: &[ArrayRef], num_rows: usize) -> Result { + println!("evaluate all called with number of values {}", values.len()); Python::with_gil(|py| { let py_values = PyList::new_bound( py, @@ -299,11 +302,25 @@ impl WindowUDFImpl for MultiColumnWindowUDF { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(self.return_type.clone()) + fn field(&self, field_args: WindowUDFFieldArgs) -> Result { + // TODO: Should nullable always be `true`? + Ok(arrow::datatypes::Field::new( + field_args.name(), + self.return_type.clone(), + true, + )) } - fn partition_evaluator(&self) -> Result> { + // TODO: Enable passing partition_evaluator_args to python? + fn partition_evaluator( + &self, + _partition_evaluator_args: PartitionEvaluatorArgs, + ) -> Result> { + let _ = _partition_evaluator_args; (self.partition_evaluator_factory)() } + + fn expressions(&self, expr_args: ExpressionArgs) -> Vec> { + expr_args.input_exprs().into() + } } From e3e55b7cb70ac27e209edfcad2e008f685687e90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Nov 2024 10:43:22 -0600 Subject: [PATCH 238/413] build(deps): bump tokio from 1.40.0 to 1.41.1 (#946) Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.40.0 to 1.41.1. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.40.0...tokio-1.41.1) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 497c5b850..f483a6a2a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3192,9 +3192,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 11ce08c75..d86948b3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ protoc = [ "datafusion-substrait/protoc" ] substrait = ["dep:datafusion-substrait"] [dependencies] -tokio = { version = "1.39", features = ["macros", "rt", "rt-multi-thread", "sync"] } +tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync"] } pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "53", features = ["pyarrow"] } datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } From 53cdb11637ac1945bdb8291a72bf4c7fbad95c49 Mon Sep 17 00:00:00 2001 From: kosiew Date: Tue, 12 Nov 2024 19:20:55 +0800 Subject: [PATCH 239/413] Add list_cat, list_concat, list_repeat (#942) * Add list_cat, list_concat * Add list_repeat * docs: add examples for list_cat, list_concat, and list_repeat functions * Amend list_repeat code example - literal * Amend list_ to array_ in documentation --- .../common-operations/expressions.rst | 29 +++++++++++++++++++ python/datafusion/functions.py | 27 +++++++++++++++++ python/tests/test_functions.py | 12 ++++++++ 3 files changed, 68 insertions(+) diff --git a/docs/source/user-guide/common-operations/expressions.rst b/docs/source/user-guide/common-operations/expressions.rst index b2a83c89f..e94e1a6b5 100644 --- a/docs/source/user-guide/common-operations/expressions.rst +++ b/docs/source/user-guide/common-operations/expressions.rst @@ -110,6 +110,35 @@ This function returns an integer indicating the total number of elements in the In this example, the `num_elements` column will contain `3` for both rows. +To concatenate two arrays, you can use the function :py:func:`datafusion.functions.array_cat` or :py:func:`datafusion.functions.array_concat`. +These functions return a new array that is the concatenation of the input arrays. + +.. ipython:: python + + from datafusion import SessionContext, col + from datafusion.functions import array_cat, array_concat + + ctx = SessionContext() + df = ctx.from_pydict({"a": [[1, 2, 3]], "b": [[4, 5, 6]]}) + df.select(array_cat(col("a"), col("b")).alias("concatenated_array")) + +In this example, the `concatenated_array` column will contain `[1, 2, 3, 4, 5, 6]`. + +To repeat the elements of an array a specified number of times, you can use the function :py:func:`datafusion.functions.array_repeat`. +This function returns a new array with the elements repeated. + +.. ipython:: python + + from datafusion import SessionContext, col, literal + from datafusion.functions import array_repeat + + ctx = SessionContext() + df = ctx.from_pydict({"a": [[1, 2, 3]]}) + df.select(array_repeat(col("a"), literal(2)).alias("repeated_array")) + +In this example, the `repeated_array` column will contain `[[1, 2, 3], [1, 2, 3]]`. + + Structs ------- diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 5a2eab56d..88ea7280d 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -147,6 +147,8 @@ "length", "levenshtein", "list_append", + "list_cat", + "list_concat", "list_dims", "list_distinct", "list_element", @@ -162,6 +164,7 @@ "list_prepend", "list_push_back", "list_push_front", + "list_repeat", "list_remove", "list_remove_all", "list_remove_n", @@ -1145,6 +1148,22 @@ def array_distinct(array: Expr) -> Expr: return Expr(f.array_distinct(array.expr)) +def list_cat(*args: Expr) -> Expr: + """Concatenates the input arrays. + + This is an alias for :py:func:`array_concat`, :py:func:`array_cat`. + """ + return array_concat(*args) + + +def list_concat(*args: Expr) -> Expr: + """Concatenates the input arrays. + + This is an alias for :py:func:`array_concat`, :py:func:`array_cat`. + """ + return array_concat(*args) + + def list_distinct(array: Expr) -> Expr: """Returns distinct values from the array after removing duplicates. @@ -1369,6 +1388,14 @@ def array_repeat(element: Expr, count: Expr) -> Expr: return Expr(f.array_repeat(element.expr, count.expr)) +def list_repeat(element: Expr, count: Expr) -> Expr: + """Returns an array containing ``element`` ``count`` times. + + This is an alias for :py:func:`array_repeat`. + """ + return array_repeat(element, count) + + def array_replace(array: Expr, from_val: Expr, to_val: Expr) -> Expr: """Replaces the first occurrence of ``from_val`` with ``to_val``.""" return Expr(f.array_replace(array.expr, from_val.expr, to_val.expr)) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index b3a5a0652..c14cfc2dc 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -291,6 +291,14 @@ def py_flatten(arr): lambda col: f.array_cat(col, col), lambda data: [np.concatenate([arr, arr]) for arr in data], ], + [ + lambda col: f.list_cat(col, col), + lambda data: [np.concatenate([arr, arr]) for arr in data], + ], + [ + lambda col: f.list_concat(col, col), + lambda data: [np.concatenate([arr, arr]) for arr in data], + ], [ lambda col: f.array_dims(col), lambda data: [[len(r)] for r in data], @@ -439,6 +447,10 @@ def py_flatten(arr): lambda col: f.array_repeat(col, literal(2)), lambda data: [[arr] * 2 for arr in data], ], + [ + lambda col: f.list_repeat(col, literal(2)), + lambda data: [[arr] * 2 for arr in data], + ], [ lambda col: f.array_replace(col, literal(3.0), literal(4.0)), lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], From 5e32ada2565cebec3df54a1bbf9725f3a434b24d Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 15 Nov 2024 11:23:27 -0500 Subject: [PATCH 240/413] Add foreign table providers (#921) * testing FFI for table provider * Was able to get round trip schema from datafusion -> delta table -> datafusion * Expand file structure * WIP on execution plan * Working through execution plan FFI * Using datafusion-proto for execution plan properties * Adding plan properties parsing from ffi * Standardize naming for FFI structs * Intermediate testing and troubleshooting * Adding record batch stream ffi representation * Mimimum viable product demonstrating foreign table provider * Move ffi module to datafusion core * Modifications need to compile against latest DF * Set DF to 42.0.0 * Rebasing and pulling in a few changes for DF43.0 * Add wrapper for register table provider * Suppress deprecation warning * Add example for FFI table provider * Add pytest for FFI module to CI * Add license text * Change the name of the FFI table provider test so it doesn't try to run during the first pass of pytest when the module hasn't been built * Build example in build stage to be used during test stage * Combine pytests into one stage * Fix path for unit test * Installing maturin for ffi test in test script * Need to install the wheel for unit test * Add online documentation about using custom table providers * Raise an error if method is not implemented when it is expected --- .github/workflows/build.yml | 4 +- .github/workflows/test.yaml | 9 + Cargo.lock | 627 +++- Cargo.toml | 1 + docs/source/user-guide/io/index.rst | 1 + docs/source/user-guide/io/table_provider.rst | 56 + .../ffi-table-provider/.cargo/config.toml | 12 + examples/ffi-table-provider/Cargo.lock | 3175 +++++++++++++++++ examples/ffi-table-provider/Cargo.toml | 36 + examples/ffi-table-provider/build.rs | 20 + examples/ffi-table-provider/pyproject.toml | 33 + .../python/tests/_test_table_provider.py | 40 + examples/ffi-table-provider/src/lib.rs | 115 + python/datafusion/context.py | 8 + python/tests/test_dataframe.py | 1 + src/context.rs | 29 +- 16 files changed, 4078 insertions(+), 89 deletions(-) create mode 100644 docs/source/user-guide/io/table_provider.rst create mode 100644 examples/ffi-table-provider/.cargo/config.toml create mode 100644 examples/ffi-table-provider/Cargo.lock create mode 100644 examples/ffi-table-provider/Cargo.toml create mode 100644 examples/ffi-table-provider/build.rs create mode 100644 examples/ffi-table-provider/pyproject.toml create mode 100644 examples/ffi-table-provider/python/tests/_test_table_provider.py create mode 100644 examples/ffi-table-provider/src/lib.rs diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f52913ce8..084a96192 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -156,7 +156,9 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Python package - run: maturin build --release --strip --features substrait + run: | + maturin build --release --strip --features substrait + - name: List Mac wheels run: find target/wheels/ diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f9383db5f..21faedecd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -110,6 +110,15 @@ jobs: pip install -e . -vv pytest -v . + - name: FFI unit tests + run: | + source venv/bin/activate + pip install -e . -vv + pip install maturin==1.5.1 + cd examples/ffi-table-provider + maturin develop --release --strip + pytest python/tests/_test_table_provider.py + - name: Cache the generated dataset id: cache-tpch-dataset uses: actions/cache@v4 diff --git a/Cargo.lock b/Cargo.lock index f483a6a2a..7b57b330a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,54 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "abi_stable" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" +dependencies = [ + "abi_stable_derive", + "abi_stable_shared", + "const_panic", + "core_extensions", + "crossbeam-channel", + "generational-arena", + "libloading", + "lock_api", + "parking_lot", + "paste", + "repr_offset", + "rustc_version", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "abi_stable_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" +dependencies = [ + "abi_stable_shared", + "as_derive_utils", + "core_extensions", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "typed-arena", +] + +[[package]] +name = "abi_stable_shared" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" +dependencies = [ + "core_extensions", +] + [[package]] name = "addr2line" version = "0.24.2" @@ -63,9 +111,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +checksum = "611cc2ae7d2e242c457e4be7f97036b8ad9ca152b499f53faf99b1ed8fc2553f" [[package]] name = "android-tzdata" @@ -84,9 +132,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "apache-avro" @@ -349,6 +397,18 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "as_derive_utils" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" +dependencies = [ + "core_extensions", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "async-compression" version = "0.4.17" @@ -367,6 +427,15 @@ dependencies = [ "zstd-safe 7.2.1", ] +[[package]] +name = "async-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" +dependencies = [ + "abi_stable", +] + [[package]] name = "async-recursion" version = "1.1.1" @@ -375,7 +444,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -386,7 +455,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -542,9 +611,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" dependencies = [ "jobserver", "libc", @@ -637,6 +706,12 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "const_panic" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -668,6 +743,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "core_extensions" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c71dc07c9721607e7a16108336048ee978c3a8b129294534272e8bac96c0ee" +dependencies = [ + "core_extensions_proc_macros", +] + +[[package]] +name = "core_extensions_proc_macros" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f3b219d28b6e3b4ac87bc1fc522e0803ab22e055da177bff0068c4150c61a6" + [[package]] name = "cpufeatures" version = "0.2.14" @@ -686,6 +776,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" @@ -917,6 +1016,24 @@ dependencies = [ "paste", ] +[[package]] +name = "datafusion-ffi" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e923c459b53a26d92a8806d1f6a37fdf48bde51507a39eaed6f42a60f2bfd160" +dependencies = [ + "abi_stable", + "arrow", + "async-ffi", + "async-trait", + "datafusion", + "datafusion-proto", + "doc-comment", + "futures", + "log", + "prost", +] + [[package]] name = "datafusion-functions" version = "43.0.0" @@ -1176,6 +1293,7 @@ dependencies = [ "arrow", "async-trait", "datafusion", + "datafusion-ffi", "datafusion-functions-window-common", "datafusion-proto", "datafusion-substrait", @@ -1238,6 +1356,23 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + [[package]] name = "dyn-clone" version = "1.0.17" @@ -1268,9 +1403,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" [[package]] name = "fixedbitset" @@ -1369,7 +1504,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -1402,6 +1537,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generational-arena" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" +dependencies = [ + "cfg-if", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1477,9 +1621,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -1631,14 +1775,143 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", ] [[package]] @@ -1648,7 +1921,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -1786,9 +2059,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.161" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "libflate" @@ -1814,6 +2087,16 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + [[package]] name = "libm" version = "0.2.11" @@ -1836,6 +2119,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -2289,7 +2578,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.87", ] [[package]] @@ -2328,7 +2617,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn", + "syn 2.0.87", "tempfile", ] @@ -2342,7 +2631,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -2365,9 +2654,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d922163ba1f79c04bc49073ba7b32fd5a8d3b76a87c955921234b8e77333c51" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" dependencies = [ "cfg-if", "indoc", @@ -2383,9 +2672,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc38c5feeb496c8321091edf3d63e9a6829eab4b863b4a6a65f26f3e9cc6b179" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" dependencies = [ "once_cell", "target-lexicon", @@ -2393,9 +2682,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94845622d88ae274d2729fcefc850e63d7a3ddff5e3ce11bd88486db9f1d357d" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" dependencies = [ "libc", "pyo3-build-config", @@ -2403,34 +2692,34 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e655aad15e09b94ffdb3ce3d217acf652e26bbc37697ef012f5e5e348c716e5e" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn", + "syn 2.0.87", ] [[package]] name = "pyo3-macros-backend" -version = "0.22.5" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1e3f09eecd94618f60a455a23def79f79eba4dc561a97324bf9ac8c6df30ce" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", - "syn", + "syn 2.0.87", ] [[package]] name = "quad-rand" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b76f1009795ca44bb5aaae8fd3f18953e209259c33d9b059b1f53d58ab7511db" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" @@ -2479,9 +2768,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e346e016eacfff12233c243718197ca12f148c84e1e84268a896699b41c71780" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ "cfg_aliases", "libc", @@ -2584,6 +2873,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "repr_offset" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" +dependencies = [ + "tstr", +] + [[package]] name = "reqwest" version = "0.12.9" @@ -2673,9 +2971,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -2788,7 +3086,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 2.0.87", ] [[package]] @@ -2812,9 +3110,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -2852,7 +3150,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -2863,7 +3161,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -2887,7 +3185,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.87", ] [[package]] @@ -2971,7 +3269,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -3014,9 +3312,15 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3048,7 +3352,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.87", ] [[package]] @@ -3061,7 +3365,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn", + "syn 2.0.87", ] [[package]] @@ -3085,7 +3389,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn", + "syn 2.0.87", "typify", "walkdir", ] @@ -3098,9 +3402,20 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.85" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3116,6 +3431,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -3124,9 +3450,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", "fastrand", @@ -3137,22 +3463,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -3175,6 +3501,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -3214,7 +3550,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -3266,7 +3602,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -3284,6 +3620,21 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tstr" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" +dependencies = [ + "tstr_proc_macros", +] + +[[package]] +name = "tstr_proc_macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" + [[package]] name = "twox-hash" version = "1.6.3" @@ -3294,6 +3645,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typed-builder" version = "0.16.2" @@ -3311,7 +3668,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", ] [[package]] @@ -3345,7 +3702,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn", + "syn 2.0.87", "thiserror", "unicode-ident", ] @@ -3363,31 +3720,16 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn", + "syn 2.0.87", "typify-impl", ] -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -3420,15 +3762,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", "percent-encoding", ] +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.11.0" @@ -3492,7 +3846,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -3526,7 +3880,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3560,6 +3914,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.9" @@ -3569,6 +3939,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.52.0" @@ -3690,6 +4066,18 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xz2" version = "0.1.7" @@ -3699,6 +4087,30 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -3717,7 +4129,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -3726,6 +4159,28 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zstd" version = "0.12.4" diff --git a/Cargo.toml b/Cargo.toml index d86948b3a..02707b957 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ arrow = { version = "53", features = ["pyarrow"] } datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } datafusion-substrait = { version = "43.0.0", optional = true } datafusion-proto = { version = "43.0.0" } +datafusion-ffi = { version = "43.0.0" } datafusion-functions-window-common = { version = "43.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.11", features = ["v4"] } diff --git a/docs/source/user-guide/io/index.rst b/docs/source/user-guide/io/index.rst index 05411327e..b885cfeda 100644 --- a/docs/source/user-guide/io/index.rst +++ b/docs/source/user-guide/io/index.rst @@ -26,3 +26,4 @@ IO csv json parquet + table_provider diff --git a/docs/source/user-guide/io/table_provider.rst b/docs/source/user-guide/io/table_provider.rst new file mode 100644 index 000000000..2ff9ae46f --- /dev/null +++ b/docs/source/user-guide/io/table_provider.rst @@ -0,0 +1,56 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Custom Table Provider +===================== + +If you have a custom data source that you want to integrate with DataFusion, you can do so by +implementing the `TableProvider `_ +interface in Rust and then exposing it in Python. To do so, +you must use DataFusion 43.0.0 or later and expose a `FFI_TableProvider `_ +via `PyCapsule `_. + +A complete example can be found in the `examples folder `_. + +.. code-block:: rust + + #[pymethods] + impl MyTableProvider { + + fn __datafusion_table_provider__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let name = CString::new("datafusion_table_provider").unwrap(); + + let provider = Arc::new(self.clone()) + .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + let provider = FFI_TableProvider::new(Arc::new(provider), false); + + PyCapsule::new_bound(py, provider, Some(name.clone())) + } + } + +Once you have this library available, in python you can register your table provider +to the ``SessionContext``. + +.. code-block:: python + + provider = MyTableProvider() + ctx.register_table_provider("my_table", provider) + + ctx.table("my_table").show() diff --git a/examples/ffi-table-provider/.cargo/config.toml b/examples/ffi-table-provider/.cargo/config.toml new file mode 100644 index 000000000..91a099a61 --- /dev/null +++ b/examples/ffi-table-provider/.cargo/config.toml @@ -0,0 +1,12 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + diff --git a/examples/ffi-table-provider/Cargo.lock b/examples/ffi-table-provider/Cargo.lock new file mode 100644 index 000000000..3b57cac75 --- /dev/null +++ b/examples/ffi-table-provider/Cargo.lock @@ -0,0 +1,3175 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "abi_stable" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69d6512d3eb05ffe5004c59c206de7f99c34951504056ce23fc953842f12c445" +dependencies = [ + "abi_stable_derive", + "abi_stable_shared", + "const_panic", + "core_extensions", + "crossbeam-channel", + "generational-arena", + "libloading", + "lock_api", + "parking_lot", + "paste", + "repr_offset", + "rustc_version", + "serde", + "serde_derive", + "serde_json", +] + +[[package]] +name = "abi_stable_derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7178468b407a4ee10e881bc7a328a65e739f0863615cca4429d43916b05e898" +dependencies = [ + "abi_stable_shared", + "as_derive_utils", + "core_extensions", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", + "typed-arena", +] + +[[package]] +name = "abi_stable_shared" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b5df7688c123e63f4d4d649cba63f2967ba7f7861b1664fca3f77d3dad2b63" +dependencies = [ + "core_extensions", +] + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "const-random", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "arrow" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "num", +] + +[[package]] +name = "arrow-array" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.14.5", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "lazy_static", + "lexical-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "flatbuffers", + "lz4_flex", +] + +[[package]] +name = "arrow-json" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap", + "lexical-core", + "num", + "serde", + "serde_json", +] + +[[package]] +name = "arrow-ord" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "half", + "num", +] + +[[package]] +name = "arrow-row" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "arrow-select" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + +[[package]] +name = "as_derive_utils" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff3c96645900a44cf11941c111bd08a6573b0e2f9f69bc9264b179d8fae753c4" +dependencies = [ + "core_extensions", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-compression" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +dependencies = [ + "bzip2", + "flate2", + "futures-core", + "futures-io", + "memchr", + "pin-project-lite", + "tokio", + "xz2", + "zstd", + "zstd-safe", +] + +[[package]] +name = "async-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4de21c0feef7e5a556e51af767c953f0501f7f300ba785cc99c47bdc8081a50" +dependencies = [ + "abi_stable", +] + +[[package]] +name = "async-trait" +version = "0.1.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" + +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "cc" +version = "1.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" +dependencies = [ + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-targets", +] + +[[package]] +name = "chrono-tz" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" +dependencies = [ + "parse-zoneinfo", + "phf_codegen", +] + +[[package]] +name = "comfy-table" +version = "7.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" +dependencies = [ + "strum", + "strum_macros", + "unicode-width", +] + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "const_panic" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "core_extensions" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92c71dc07c9721607e7a16108336048ee978c3a8b129294534272e8bac96c0ee" +dependencies = [ + "core_extensions_proc_macros", +] + +[[package]] +name = "core_extensions_proc_macros" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f3b219d28b6e3b4ac87bc1fc522e0803ab22e055da177bff0068c4150c61a6" + +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-ipc", + "arrow-schema", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "dashmap", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-sql", + "flate2", + "futures", + "glob", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "num_cpus", + "object_store", + "parking_lot", + "parquet", + "paste", + "pin-project-lite", + "rand", + "sqlparser", + "tempfile", + "tokio", + "tokio-util", + "url", + "uuid", + "xz2", + "zstd", +] + +[[package]] +name = "datafusion-catalog" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.14.5", + "indexmap", + "instant", + "libc", + "num_cpus", + "object_store", + "parquet", + "paste", + "sqlparser", + "tokio", +] + +[[package]] +name = "datafusion-common-runtime" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" +dependencies = [ + "log", + "tokio", +] + +[[package]] +name = "datafusion-execution" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" +dependencies = [ + "arrow", + "chrono", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "hashbrown 0.14.5", + "log", + "object_store", + "parking_lot", + "rand", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap", + "paste", + "serde_json", + "sqlparser", + "strum", + "strum_macros", +] + +[[package]] +name = "datafusion-expr-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" +dependencies = [ + "arrow", + "datafusion-common", + "itertools", + "paste", +] + +[[package]] +name = "datafusion-ffi" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e923c459b53a26d92a8806d1f6a37fdf48bde51507a39eaed6f42a60f2bfd160" +dependencies = [ + "abi_stable", + "arrow", + "async-ffi", + "async-trait", + "datafusion", + "datafusion-proto", + "doc-comment", + "futures", + "log", + "prost", +] + +[[package]] +name = "datafusion-functions" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "hashbrown 0.14.5", + "hex", + "itertools", + "log", + "md-5", + "rand", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" +dependencies = [ + "ahash", + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "indexmap", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", + "rand", +] + +[[package]] +name = "datafusion-functions-nested" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-physical-expr-common", + "itertools", + "log", + "paste", + "rand", +] + +[[package]] +name = "datafusion-functions-window" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" +dependencies = [ + "datafusion-common", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-optimizer" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "paste", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-string", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "paste", + "petgraph", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" +dependencies = [ + "ahash", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "rand", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-execution", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-plan", + "itertools", +] + +[[package]] +name = "datafusion-physical-plan" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" +dependencies = [ + "ahash", + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap", + "itertools", + "log", + "once_cell", + "parking_lot", + "pin-project-lite", + "rand", + "tokio", +] + +[[package]] +name = "datafusion-proto" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f730f7fc5a20134d4e5ecdf7bbf392002ac58163d58423ea28a702dc077b06e1" +dependencies = [ + "arrow", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-proto-common", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12c225fe49e4f943e35446b263613ada7a9e9f8d647544e6b07037b9803567df" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-sql" +version = "43.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "indexmap", + "log", + "regex", + "sqlparser", + "strum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "doc-comment" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" + +[[package]] +name = "ffi-table-provider" +version = "0.1.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "datafusion", + "datafusion-ffi", + "pyo3", + "pyo3-build-config", +] + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flatbuffers" +version = "24.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.0.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generational-arena" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877e94aff08e743b651baaea359664321055749b398adff8740a7399af7796e7" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", +] + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashbrown" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +dependencies = [ + "equivalent", + "hashbrown 0.15.1", +] + +[[package]] +name = "indoc" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "lexical-core" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" + +[[package]] +name = "libloading" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +dependencies = [ + "cfg-if", + "winapi", +] + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "humantime", + "itertools", + "parking_lot", + "percent-encoding", + "snafu", + "tokio", + "tracing", + "url", + "walkdir", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "parquet" +version = "53.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.14.5", + "lz4_flex", + "num", + "num-bigint", + "object_store", + "paste", + "seq-macro", + "snap", + "thrift", + "tokio", + "twox-hash", + "zstd", + "zstd-sys", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + +[[package]] +name = "phf" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prost" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "pyo3" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "repr_offset" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb1070755bd29dffc19d0971cab794e607839ba2ef4b69a9e6fbc8733c1b72ea" +dependencies = [ + "tstr", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "0.38.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustversion" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + +[[package]] +name = "serde" +version = "1.0.214" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.214" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "serde_json" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "siphasher" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "snafu" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + +[[package]] +name = "sqlparser" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.87", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "ordered-float", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +dependencies = [ + "backtrace", + "bytes", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tokio-util" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "tstr" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8e0294f14baae476d0dd0a2d780b2e24d66e349a9de876f5126777a37bdba7" +dependencies = [ + "tstr_proc_macros", +] + +[[package]] +name = "tstr_proc_macros" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" + +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + +[[package]] +name = "url" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +dependencies = [ + "getrandom", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.87", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" + +[[package]] +name = "web-sys" +version = "0.3.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "zstd" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.13+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/examples/ffi-table-provider/Cargo.toml b/examples/ffi-table-provider/Cargo.toml new file mode 100644 index 000000000..4e54eaf03 --- /dev/null +++ b/examples/ffi-table-provider/Cargo.toml @@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "ffi-table-provider" +version = "0.1.0" +edition = "2021" + +[dependencies] +datafusion = { version = "43.0.0" } +datafusion-ffi = { version = "43.0.0" } +pyo3 = { version = "0.22.6", features = ["extension-module", "abi3", "abi3-py38"] } +arrow = { version = "53.2.0" } +arrow-array = { version = "53.2.0" } +arrow-schema = { version = "53.2.0" } + +[build-dependencies] +pyo3-build-config = "0.22.6" + +[lib] +name = "ffi_table_provider" +crate-type = ["cdylib", "rlib"] diff --git a/examples/ffi-table-provider/build.rs b/examples/ffi-table-provider/build.rs new file mode 100644 index 000000000..4878d8b0e --- /dev/null +++ b/examples/ffi-table-provider/build.rs @@ -0,0 +1,20 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +fn main() { + pyo3_build_config::add_extension_module_link_args(); +} diff --git a/examples/ffi-table-provider/pyproject.toml b/examples/ffi-table-provider/pyproject.toml new file mode 100644 index 000000000..116efae9c --- /dev/null +++ b/examples/ffi-table-provider/pyproject.toml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["maturin>=1.6,<2.0"] +build-backend = "maturin" + +[project] +name = "ffi_table_provider" +requires-python = ">=3.8" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +dynamic = ["version"] + +[tool.maturin] +features = ["pyo3/extension-module"] diff --git a/examples/ffi-table-provider/python/tests/_test_table_provider.py b/examples/ffi-table-provider/python/tests/_test_table_provider.py new file mode 100644 index 000000000..56c05e4fa --- /dev/null +++ b/examples/ffi-table-provider/python/tests/_test_table_provider.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datafusion import SessionContext +from ffi_table_provider import MyTableProvider +import pyarrow as pa + + +def test_table_loading(): + ctx = SessionContext() + table = MyTableProvider(3, 2, 4) + ctx.register_table_provider("t", table) + result = ctx.table("t").collect() + + assert len(result) == 4 + assert result[0].num_columns == 3 + + result = [r.column(0) for r in result] + expected = [ + pa.array([0, 1], type=pa.int32()), + pa.array([2, 3, 4], type=pa.int32()), + pa.array([4, 5, 6, 7], type=pa.int32()), + pa.array([6, 7, 8, 9, 10], type=pa.int32()), + ] + + assert result == expected diff --git a/examples/ffi-table-provider/src/lib.rs b/examples/ffi-table-provider/src/lib.rs new file mode 100644 index 000000000..473244d88 --- /dev/null +++ b/examples/ffi-table-provider/src/lib.rs @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ffi::CString, sync::Arc}; + +use arrow_array::ArrayRef; +use datafusion::{ + arrow::{ + array::RecordBatch, + datatypes::{DataType, Field, Schema}, + }, + datasource::MemTable, + error::{DataFusionError, Result}, +}; +use datafusion_ffi::table_provider::FFI_TableProvider; +use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyCapsule}; + +/// In order to provide a test that demonstrates different sized record batches, +/// the first batch will have num_rows, the second batch num_rows+1, and so on. +#[pyclass(name = "MyTableProvider", module = "ffi_table_provider", subclass)] +#[derive(Clone)] +struct MyTableProvider { + num_cols: usize, + num_rows: usize, + num_batches: usize, +} + +fn create_record_batch( + schema: &Arc, + num_cols: usize, + start_value: i32, + num_values: usize, +) -> Result { + let end_value = start_value + num_values as i32; + let row_values: Vec = (start_value..end_value).collect(); + + let columns: Vec<_> = (0..num_cols) + .map(|_| { + std::sync::Arc::new(arrow::array::Int32Array::from(row_values.clone())) as ArrayRef + }) + .collect(); + + RecordBatch::try_new(Arc::clone(schema), columns).map_err(DataFusionError::from) +} + +impl MyTableProvider { + fn create_table(&self) -> Result { + let fields: Vec<_> = (0..self.num_cols) + .map(|idx| (b'A' + idx as u8) as char) + .map(|col_name| Field::new(col_name, DataType::Int32, true)) + .collect(); + + let schema = Arc::new(Schema::new(fields)); + + let batches: Result> = (0..self.num_batches) + .map(|batch_idx| { + let start_value = batch_idx * self.num_rows; + create_record_batch( + &schema, + self.num_cols, + start_value as i32, + self.num_rows + batch_idx, + ) + }) + .collect(); + + MemTable::try_new(schema, vec![batches?]) + } +} + +#[pymethods] +impl MyTableProvider { + #[new] + fn new(num_cols: usize, num_rows: usize, num_batches: usize) -> Self { + Self { + num_cols, + num_rows, + num_batches, + } + } + + fn __datafusion_table_provider__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let name = CString::new("datafusion_table_provider").unwrap(); + + let provider = self + .create_table() + .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; + let provider = FFI_TableProvider::new(Arc::new(provider), false); + + PyCapsule::new_bound(py, provider, Some(name.clone())) + } +} + +#[pymodule] +fn ffi_table_provider(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 5221c866c..a07b5d175 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -685,6 +685,14 @@ def deregister_table(self, name: str) -> None: """Remove a table from the session.""" self.ctx.deregister_table(name) + def register_table_provider(self, name: str, provider: Any) -> None: + """Register a table provider. + + This table provider must have a method called ``__datafusion_table_provider__`` + which returns a PyCapsule that exposes a ``FFI_TableProvider``. + """ + self.ctx.register_table_provider(name, provider) + def register_record_batches( self, name: str, partitions: list[list[pyarrow.RecordBatch]] ) -> None: diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 330475302..b82f95e35 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -306,6 +306,7 @@ def test_unnest_without_nulls(nested_df): assert result.column(1) == pa.array([7, 8, 8, 9, 9, 9]) +@pytest.mark.filterwarnings("ignore:`join_keys`:DeprecationWarning") def test_join(): ctx = SessionContext() diff --git a/src/context.rs b/src/context.rs index c2a263fa7..8675e97df 100644 --- a/src/context.rs +++ b/src/context.rs @@ -28,7 +28,7 @@ use object_store::ObjectStore; use url::Url; use uuid::Uuid; -use pyo3::exceptions::{PyKeyError, PyTypeError, PyValueError}; +use pyo3::exceptions::{PyKeyError, PyNotImplementedError, PyTypeError, PyValueError}; use pyo3::prelude::*; use crate::catalog::{PyCatalog, PyTable}; @@ -67,7 +67,8 @@ use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; -use pyo3::types::{PyDict, PyList, PyTuple}; +use datafusion_ffi::table_provider::{FFI_TableProvider, ForeignTableProvider}; +use pyo3::types::{PyCapsule, PyDict, PyList, PyTuple}; use tokio::task::JoinHandle; /// Configuration options for a SessionContext @@ -566,6 +567,30 @@ impl PySessionContext { Ok(()) } + /// Construct datafusion dataframe from Arrow Table + pub fn register_table_provider( + &mut self, + name: &str, + provider: Bound<'_, PyAny>, + ) -> PyResult<()> { + if provider.hasattr("__datafusion_table_provider__")? { + let capsule = provider.getattr("__datafusion_table_provider__")?.call0()?; + let capsule = capsule.downcast::()?; + // validate_pycapsule(capsule, "arrow_array_stream")?; + + let provider = unsafe { capsule.reference::() }; + let provider: ForeignTableProvider = provider.into(); + + let _ = self.ctx.register_table(name, Arc::new(provider))?; + + Ok(()) + } else { + Err(PyNotImplementedError::new_err( + "__datafusion_table_provider__ does not exist on Table Provider object.", + )) + } + } + pub fn register_record_batches( &mut self, name: &str, From 92b093c767a8ec76918db6abeae35f85ee19fa60 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 23 Nov 2024 23:21:24 +0800 Subject: [PATCH 241/413] Add make_list and tests for make_list, make_array (#949) --- python/datafusion/functions.py | 9 +++++++++ python/tests/test_functions.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 88ea7280d..6ad4c50c2 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -184,6 +184,7 @@ "lpad", "ltrim", "make_array", + "make_list", "make_date", "max", "md5", @@ -1044,6 +1045,14 @@ def make_array(*args: Expr) -> Expr: return Expr(f.make_array(args)) +def make_list(*args: Expr) -> Expr: + """Returns an array using the specified input expressions. + + This is an alias for :py:func:`make_array`. + """ + return make_array(*args) + + def array(*args: Expr) -> Expr: """Returns an array using the specified input expressions. diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index c14cfc2dc..0d40032bb 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -576,6 +576,37 @@ def test_array_function_cardinality(): ) +@pytest.mark.parametrize("make_func", [f.make_array, f.make_list]) +def test_make_array_functions(make_func): + ctx = SessionContext() + batch = pa.RecordBatch.from_arrays( + [ + pa.array(["Hello", "World", "!"], type=pa.string()), + pa.array([4, 5, 6]), + pa.array(["hello ", " world ", " !"], type=pa.string()), + ], + names=["a", "b", "c"], + ) + df = ctx.create_dataframe([[batch]]) + + stmt = make_func( + column("a").cast(pa.string()), + column("b").cast(pa.string()), + column("c").cast(pa.string()), + ) + py_expr = [ + ["Hello", "4", "hello "], + ["World", "5", " world "], + ["!", "6", " !"], + ] + + query_result = df.select(stmt).collect()[0].column(0) + for a, b in zip(query_result, py_expr): + np.testing.assert_array_equal( + np.array(a.as_py(), dtype=str), np.array(b, dtype=str) + ) + + @pytest.mark.parametrize( ("stmt", "py_expr"), [ From 54e5e0d9cc876ca31eea6f79a623e5163eae75f9 Mon Sep 17 00:00:00 2001 From: Daniel Mesejo Date: Sat, 23 Nov 2024 16:22:03 +0100 Subject: [PATCH 242/413] fix: udwf example (#948) --- examples/python-udwf.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/python-udwf.py b/examples/python-udwf.py index 05b3021d8..55de2bdc7 100644 --- a/examples/python-udwf.py +++ b/examples/python-udwf.py @@ -185,35 +185,36 @@ def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: df = ctx.create_dataframe([[batch]]) exp_smooth = udwf( - ExponentialSmoothDefault(0.9), + lambda: ExponentialSmoothDefault(0.9), pa.float64(), pa.float64(), volatility="immutable", ) smooth_two_row = udwf( - SmoothBoundedFromPreviousRow(0.9), + lambda: SmoothBoundedFromPreviousRow(0.9), pa.float64(), pa.float64(), volatility="immutable", ) smooth_rank = udwf( - SmoothAcrossRank(0.9), + lambda: SmoothAcrossRank(0.9), pa.float64(), pa.float64(), volatility="immutable", ) smooth_frame = udwf( - ExponentialSmoothFrame(0.9), + lambda: ExponentialSmoothFrame(0.9), pa.float64(), pa.float64(), volatility="immutable", + name="smooth_frame", ) smooth_two_col = udwf( - SmoothTwoColumn(0.9), + lambda: SmoothTwoColumn(0.9), [pa.float64(), pa.int64()], pa.float64(), volatility="immutable", From deb1f255ac314d569ee9299dfd421028763bbbb9 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 29 Nov 2024 11:57:42 -0500 Subject: [PATCH 243/413] Documentation updates: simplify examples and add section on data sources (#955) * Add a simple example to the introduction page to demonstrate loading a dataframe from a csv file and displaying the contents * Update basics doc to be a little more straight forward * Move downloading of data files for examples into the build scripts and just point the users to where these files are located instead of adding url lib requests to the python examples so we can focus on what is most important to the user * Handle a few errors generated by doc site builder * Switch example so that there is not confusion about the single and double quotes due to capitalization * Add section on data sources * Build pipeline doesn't have polars and it isn't really necessary for the example, so swith to a code block instead of ipython directive --- .github/workflows/docs.yaml | 2 + docs/.gitignore | 2 + docs/build.sh | 11 +- docs/source/images/jupyter_lab_df_view.png | Bin 0 -> 150303 bytes docs/source/index.rst | 25 +-- docs/source/user-guide/basics.rst | 74 +++---- .../common-operations/aggregations.rst | 10 +- .../common-operations/functions.rst | 6 - .../user-guide/common-operations/index.rst | 2 + .../common-operations/select-and-filter.rst | 11 +- .../user-guide/common-operations/windows.rst | 6 - docs/source/user-guide/data-sources.rst | 187 ++++++++++++++++++ docs/source/user-guide/introduction.rst | 34 ++++ docs/source/user-guide/io/avro.rst | 2 + docs/source/user-guide/io/csv.rst | 2 + docs/source/user-guide/io/json.rst | 2 + docs/source/user-guide/io/parquet.rst | 3 +- docs/source/user-guide/io/table_provider.rst | 2 + python/datafusion/dataframe.py | 2 +- python/datafusion/plan.py | 4 +- 20 files changed, 300 insertions(+), 87 deletions(-) create mode 100644 docs/source/images/jupyter_lab_df_view.png create mode 100644 docs/source/user-guide/data-sources.rst diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e47497b2a..86288e2d8 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -75,6 +75,8 @@ jobs: set -x source venv/bin/activate cd docs + curl -O https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv + curl -O https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet make html - name: Copy & push the generated HTML diff --git a/docs/.gitignore b/docs/.gitignore index 41e135341..6e8a53b6f 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,2 +1,4 @@ pokemon.csv yellow_trip_data.parquet +yellow_tripdata_2021-01.parquet + diff --git a/docs/build.sh b/docs/build.sh index 5afe85812..31398d195 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -19,8 +19,17 @@ # set -e + +if [ ! -f pokemon.csv ]; then + curl -O https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv +fi + +if [ ! -f yellow_tripdata_2021-01.parquet ]; then + curl -O https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet +fi + rm -rf build 2> /dev/null rm -rf temp 2> /dev/null mkdir temp cp -rf source/* temp/ -make SOURCEDIR=`pwd`/temp html \ No newline at end of file +make SOURCEDIR=`pwd`/temp html diff --git a/docs/source/images/jupyter_lab_df_view.png b/docs/source/images/jupyter_lab_df_view.png new file mode 100644 index 0000000000000000000000000000000000000000..9dafb4f61d3a82b6f51e03fa43f3c97a4f8bde0f GIT binary patch literal 150303 zcma&M1yozj)&NQkDGtTGP~6=q1b5fs1P{SoT8b5?c(I}_UR;W_xCgfYDeeTfATRyy z{bjwk*1P{%=j6kBs( z6cj8e2LM1*2>_tgbO+lyI0I2o6hEaJpc(2666cy|)77Iq)s_K=zE45Xma%_QMDkl+ z@o5-=HEq=VgxrfcPU0W=<*%cu@Wia_@DGBtWTlNrFuga`UhcG32ZB4mM|W3we2*d? zMi+BE^z3nFk zo>9Q4bjbze;nA4?CRCgnMUlV9;ZEF)vnLYe7n`4zk|c^O8CX)loW1+UQxFCg-dZ?s zcK)wa-t3ZJhz-G$8t162C$#16EuXParWa`;%zuDyu^S|#a%2PZ(NVlmdpcTLl5g<7 z$v8g)_23PkXF5pj*xae{ffSR6M@hS)TLv&iG&7jSR@D(wQzxA&F7ck@kwg_`nV-iE z7qTr^T>HQHR&sy=sPFg;!;!(&@ zvJZWV)jW0B?Xtk>aSi_(-W8}W-s6wQ^=jiqFAGReg|VMCDBo+BKu~4S{EzZRFtCGK zxsK3K`F<|*X}mU#4%<+yMrg*%8o3YLJOow(QB-HO>s_gWg)zJfNgo9X94gp@F{7+s z6eOaAOX??bV#Mn$rf+RwkC8yLSvF)@Pd?m@Tx-e&|FEStpq9$UoMptTAwczd?q^Mp zy4^+c9F=7&-V0L}!%aB{ckS-IVXt8}>)0p##dX`zY&wC^#H_UR94S_>4G3I_|`fu~)wE4OC8-*zxi-_brHdy7JuGV&*i z!7l;bZ2c|%HR0{EV^MFEG)d&Js`@AJ0X z?!8bxH^^_lz$PZ~H}!j>gL{L57-dg}UCxb^`xV#WpK1NwWD+2)E zkqDvSG_##|cin%r6B?*ZbsMhjx>iDf`L0(Tm=BrNU#0NIu4&ZV_!WK0`2tOhW}lPV zQowgw)a+OQNJt^<9utEkox4LCdx)*jYR={z4qkg>m7`%DFwDMhu;>T++IvYPRaZ%a zYfgP@^W%?c9#Hct*pvO#1=Jdfg7U}1-NNF}jt}Pn0}4Xo&!yC9PMu|!M&wHrlKA7I z&re@O4s$-gPScDOQ==h$=3(({d;RIs2ccs$X<97V559?5j9oi?tD@=eOD#|{FqPku z8KUZcys#jgf9V}6@dxEQTK_KxSDbgKpg*Y1Xa*W2zE5s{i5-(k;}OUJcA_M$LJePE zMqzfz1}4!mJpCm1IZ2n{89tr14Jt1#3qYTCKZ!$8pq54?G+UnY_2B^32L9y7HhIZ> z(z0mBLiz7Rq6EMKc7Dn@1*WfLJFn`|a}qeE=L+<8T_g1U#WI5>uH{4yex^y zw89}Jw6@kVe3=%n^ux%IML%B8vcZr(aBS`eP6Y`fiqK7T>=)IampHwIuGEq*^1`|L z=GMhdMBB(ZBc!?z$KdDWzY{Gqj$gZ?TVs%gm4*?>h**lPa<8&4shEcO$iCJ}w^uuV zAHb8DA(yTGPIc++l8U&TGm|#~LW$wCWr-jP`=|D2RxN_=q%8Ns01@C{r6)5xSh;#kFs~mlZpH8`7t$X4Nw-Xa!uP z>`~1K@sp26Z)vFHZ%(C+mpRzk<2ukx?>NRgQVMh|Iyby*5N#-TK#o8mzlQ*c)6rzx_k~Ki$=b=6lWR_+ zbBkqO^5x0#6QjG%q6v}-9dW^FE5&Nljk8J{vEja#gH+I#2w z&D)DxTrAgTVjoFAmZDN%kCC6_%uydnWDB)V+H8#e!##J@q;G319Mb<&b z4|?r-O%1sM{jCG7eYYLhqkRzDXtmT)FOvd}DM6_BM^F}(Y`8`6b`rV=Fiz*6J@Xv|mzGe%S;J4LATFL9Sp&kFmhvA*(~%b}ZMIiO(r8oRi2rAtPdp!okf$t62u%sufjOObi>S~c4tk&Bd2Z^KPT>w>R#9Nxa|_} zvW*akERX09uf`3bv?QVXWKa1q9)qIJedJzoSW(L+=vCC3Y-OxJ9*9cf&8n(>0kAv9xGq>|*CAXhzE4^qvTydT07Rj_~_R6{%lzE0Z*ODyU4szQ4^GVir@$K2p ziR`K_igd*cCC(bVG$i~k7tK{CF6%0Nuxty7eAlktPpE_*jMRBWlApwOEm-LUSiB7O zNL*euMMLxYl*(22*K7Qoe7UY4?)2`cKO|U$|1oOSxQ#wP!pe(HC6mYu&fLswmuYlu zKi1Q?pDn4lTF_b$t3`AF?0&iR*+a)8A*x<>pE&00K*nU223ZgQAP z<2Ici;VJoP!)iic%(VP%%(>PFu(aIo$kIcCB)Iq?~>`R&ZjTtg=gyOQ~2 z>vYg=AFu~l02E3`ASH;5hulQ;SgN)DY^7BNtLcUJMbQk&u<5W+Wqwss)vh5$O`~N_ zZ!2)WqXP0Sr114uD{!&pX2H-E#Wh8h%Q?*O=b*jLLWxFJcorCpv7l`L)YI!oZEmpG zp8|b?jy?9>Rr>$@eOY^G?i1zhc@Vq2@a|B{w0+ohZj!uS-gv4m;O?kV;zIzrBb8&* z!OXHrBkn=+Q9y^3*W>83RkSZSytuQ}wc@&d8NtooBa^8*Ctd=H%&j`8%us(O@EHI13ZohkJ2T_eKjA(k1c0t z8bb~aUE#tH%*R^8>&^XFqJBACxf>EUlLC({hYmM<$#yf#oXh?Gp#h7z2>~k?GIwhp z;K95vA^IKlettV#kDbQ1eV1PCpWDcN+ppsuvZ%nOFu@CjE2+SA#L*dI68DzIJJ=>z z{E=mMst%&lnb=Wr?}%tPF}ZF+JZpLO%&IDk?UCr0h1}HC6W{h{D8{@fp1zWW@jBU$ z8!xo7jW*Hmm@0(nhhoG~8jc?&N=P1-m!4X{1by#9Z}--pM3p?jgJX} zei0xa->*;q3mUcd>$CqVKeb1;p-Afhl$4Nf9cyBFlfPxn9%$1LEl<@!C*LlNJDW2hs|1a&vOOmV8M|ODpbfV=JmHEC28A$S;Z4 z_MV=uqFh`)K0cg2e4Jo+J1!m(5fLtKUM^l<4kQGJho6h5r7wqz2mQYW`LA(gfgaZG z4z8XKU>DlI<62sQy*wpezy3SXe_#LlpFm%S|C!0f$vS|LG(*lIDaa!z{tx{Br{#ZU{9io{|EDL95D)+V?)tx){@-2oJb>;1 zFbFxPr{w>v*S|ae@5X<36zBSj`u`%uzv%o=Et1fZFU7h3d(k9cPI6u|A=i=8K~_T- z`9|)tztfT|r4ZxbN56!z!=s5(Ab0597X#U~hl+n)c~`yseeqMpV=UvJa)8N4@`#Hj;!Re2j`M$*96k08614BMhFRx7qJ;$Dwb`Zm&t!dqPpX0h zjoY-=D*6A%tnbq=sfUwt6>ZBo2*14S6>-+Cv z`r>|dKHkF*GVOA;?Chd&x6({p202@I3TBDs%tP)EQmX@j`%U_?7P`+n9xkfwq#kb| zCh0Ix({e45!MmCZ6SmFg!58J`K%r$%oxW3_PK2r8enp*ab;$jG9Y@nr?H$7*8S|69 zrf+${q5(%9d1B-Hs0c@VLkkGx3U~?azzKJEvlyr|V|66M{LC zLH>e>gSsnkYn}6*7_JUs#I8ojV(U*+?}axez*Ifs&VI|y#|QXhv>oHKA^Y4uZzw2z zIT&%++iQM1nQlvB=DTXoTTrfU^fWo2lKjP8AGQ&4UPJFmB_^UPnK!;W`H{r?AveS3 zwd)B|9fti@_p7)e!U0HL;)@QL6_9)T+}8DJg4BcWC350cIm^O0!689d;Mpv^`E=j> zQ512S-nq1RSDW6>9x&=;Hg#K>>i@jsG_~5s2NvXG{;BQAt9n@9Jm{ihEXSECpH}p0 z$%Ai2A-R{>gdGa*yg%3;PnWu#NT?m=>TIjtA5(-)#0jlrn1telZS~k;$ zcg`z*9jhv-$cx97jqygH}AJ)c77$xyL7d|5m)Z>U?aUZ$ljYgdRD^i);<( zn}8ae7wa7sV$9A9W-r~0{Qi7EtK}OLE*uyW+0IVpsOcdfi^vYTTK0KYD|{T-cGV6C z@zv%A-|kXD>7PL~2VZFZlDgSQ9&G1>h=O+p_X1 z)>qy2YUU@=1~y{wHVI52(w2~x5{A?alV=^iDIOfxljKmux2Mzb%@`D7J z^#vU}3qhUh3WCXg#%_~oXU^Q0j#q!Z|J5{{$Fkv8@|ArX+)w4hzDTgFljW}v_~#kT zfPGVDw*Ou!y-%psb37Qsp)ppWey(9&s345k#5sxM$FHYpAPzLy_opX%)&6=WdvotR z)EcL?RT@B9dr}2vaUBX}L4q?PSbGnBHI(Gj?U$cj)*~v)t2PA{i zX8TpX+rHRVulzkB@_hL~hkrc^4%Xp6z1!?SEW>Sw+3R9q)5e{bi%xv6@)&*Zg4YQ1 ze&iX?GN71{UePn<`mSMF>fYVML-O*E`QIefoL*FbD7H>9&97y=r?nI=yasc;7g@x% zGOTY!eRy-itXiQ#9k(;pobhDg3Y*{aR*~EXjTO&myYA&0yPUViMJx=szr!+OUZ^x3 ziKBAs!C#<&UH44}+|Bn(rUjkEm^&YaY>UA%PwdFk4+9=dmgj)Hp#|bs&(VMFi1BhR$@uhixj378)h6i7TdFwZw zl^xmXDrP>5!7|cgnU%tMb3LIzv#c*)tlfJoz3_3WuNkxTr6;jjS$$nu8ej5YVt8^- zz{(kQ!t0iJ5BDiA?^78G7v0PmvUx%c@gyqWKOYEJu6&8%k9Jas1+4xSGPORn53zhp zCM&0DX(wd|et!&Q5M{}Qx(h9f59}XUfue;8$mJW=hoT?tG?P(D(km%fd&AqW#uU*} z{j1mYaj?&A0~Fib0K|ngk{6{KL`GkEt|tkS|8wx#ld?$&9zW}6mlvko?rHIC@A zM0Nkly)X2MWv&k2*S!RgTTiVy5zB}$k-t?sqGC+A^CS*z0}p^9KBMVr{b0~=wqO4q zM~l7O_v`b`fP&=bpj{R7vJ^?Pl&~VwOZy8gscxFb>;8qRpGS*UVYSIgk!sOcMvzI_ z{1N77D-CXVvC2qxgoRc7VGiw9%sf9G8;4zQFi9;azwAADf<`%hE3ubaYCphR!n{O^ zp~Mi+7_LhDqHMxxJBVHz{?M;~3fan3BjOD_~bne9HrvM_4`J`{?Z zxh+hP{L+56$)&MXnp{W0q|0H)7(<82#FJ~Fx8MPhsO!d+YTH`{h*XgP(t!5zEOqF; z${B46)P6lK?dzX;FSB5Gwx!*6{mD!xV{Y2CeA@MM=#;V13wC)yj3w7>W&&>B?K0EV764w~oq!mE$OC+hhp1I3m{ReG`NCO<=6EQs?=&L@UpG zasSObgo=92HXm@XHg#T+)xJ)$4dG*@i=XUW7xn&uo`n~_y$JyI-Y4G(t&B0h!=Te* zEEKd^t%><&`-{FS>v1;2c`=DqDPcsY2s*bhPxSII^Q)l3;D~n8 zH{#`Hvvv#sVmcsR*RPu&e7Pj_IK;;Gbe#@6`<+Pg@X5a}c@E@7ZQ|YC%qcC4W&Sz) zwMj!CKumv^!M1Jwcxy08>aY|qemXD``Lq7eD=p05_Epqi*2;Ma*0(2v$LN8FPUczT zW@grL0@1gJ;aq{1-VlkaS9C4+%pF=y2CWmncZ&`r*2IDkP?*^l*%^0RUea3aAchW~ z*#5BZK}ca1eW~Ipvm^Mq@zu(EZ&^UvP{reizWk4kc_cDhg<(v(?1u)6YGy09vrV?l zx^@g?Brka7y-VI6Zv%Cv#KPyzoF zK1y#c?*7|CVlLV$x$RiqG$$|e(GQl-KjQl#x@jIX`2$JJ15aW_y81v?E>;cjAKf?+ z#Tg;=5i@l;O)LIReKt*uDx57gXsuScn+UZ+0f;)J&VVn)!y{YFth7qlO?dP`eln*5BbJN4><<1GS|05A^ z1up=7Q+sPz#_XF=AkMSe^1FW5Zf-+JWt67X z+fey*6x=1E;#;nN8OTiFv|C69Jks9S87z+iC~)rSy?I{{cY@A)ReMy%=JvW^YcKsJ-NZWOy&%U%|{{f2>0-B?qN3qXVTP z^E;_Q9wD8L)pPD8S#+4TOHCwLVatm-57({xVRgloRx(g*I~%My#(E7hs;(wb*LOwv z8(JU?55dfKO>EyeQB8NZK)Z#JH3B5gMtxI1%KWnpPkOKfrRH#iWE&u_y{&nCH0E=pN$HU0BLSE?hD%M#zgmRcC56uFXMiH04E{VpliYF zLOIG{O{j&fx~Dhx;&{7qHM~$G&nMXj#q^0N$4bBu!=fSS*2FIDUCFZTW`;)L+`iC_ z1iu!}1o1c6(}2Ys6Irv(yRd{ONa6#-<(-?@D60Tcl5TJY647@}SFr)pG(mXH|<=3fnbs5IDfZL?JfG z(Kki#=HSbNd_h41zTwG&LOA}=TTF$g#%sVJN)+U%B$Km zvNq1=O$SBW&#*3jz!xeA!LwKRN^2tCSRlbgud^|+YLFv`n|_z3rmbOqd!Kz%3U5j3 z0E&T?;rR5=(M^oFgYEC#JN6#H;>Awpr#gow9TI!PugV!+2seMyG`tnV>&B+*SL5E2 zedbiXsmcT(Mr}iyhmCri?xv@*na|QKG2$8O6V$lX7tZk-YpW24X%q~urf}@41kBW+h>hk(dm!M9lL zDZBL(J1}1*GS_Q$(g_L}pIE|eM29Q?T`xFR#GA##ZR?YaRA#W-9z{&Q;JsL}fr{IX z=W_6hf(qB=iL;pcAK`h~Nb+1Xd1|ivcb%p1d>4^0>1h1Mu&axBz{3OcM+ZLtZlAJT z`E4b!o^KzYq;)7#nPx#$boLvtjeLy)ijBF*_KPwO-cV7o{q>Tx<018&hrkrfk$pSt z9+wFASSqZ;tMCpDNc9xTi5XjsR^JY@{8&r0;pD>1)$a$I3tN>y1EYuP}Nb)&X`}rht6X|}Z`kBko!g-?>Ct7Rm z?0}6Wiosvu5ZEK?Q%g~6zw>2s_DF0@?gS~q>W-Of@7dMzr>E;EdW2-Q^X#I=2YTpP zy}6MVWk%O89m#ZtVpMRW0?_m#)HY!&nFFF;6+-^++oe_Vdm)gbby{0&2s9`w}_=teP1hvPGCb&d!i@(P`-osR;FhU1J45yV|J_oWYN0Jr@m3;W{Wb+4jzsDN&LQE1fRws-sekfmk zF5%ETRvd_1EO zz?1iw$0uBC%WNwY9+wNR+{T;Jf#LNzgoK<>9(mng%q3(Y9wf+yD>}f9k%T%!J{@+H zIQ}<{1)tlYR}}k_wM$@AizXYX=t4@EdKm@enjDNMuYtUv!72n|@u2fIrPF8`Z)f(4((9FE(3Qg^0bD@;J(FnE zY^3d7_Z>3q`SMbATtW-i4*?6Thhy zG<)!e$Q&k>8@!U30_e@|(8D`(x$s$;VaB&dWpc=US}oY z;R}DF14`itU#4RO`%#RRqHk5!IPIB7+4y^zQahP`bR0!Oh>I>v@f>A4Sd%!jNwiae zMGwn}Kh#gZCiyKHfNUd=?W(J;8$eU-|Ekm1i6B=rKCp-`?#~58QM*W8M63$Q+S4n) z%$B|*y<_6sY6Y*E$-Vz>lR1-S`N;z3(82Ev-mRMN*OQb7q+oVmAI?z=%~r^|GCU+N z+N)LkxG(}f{+g1vi1%WjyI7q;jyQ2W;5SWi z2|5nQ(AHy7&9%qv#itZU<7@5#XL`6EOYB7IU)`rxp zx8&|EB`btE-CXT1R2pe17=>hjgc&R-1@hn7sSC^NojJGLHPY1dHIRMU>3mi-*fiQ< z|6ZNU;NZ*0ck%KDnR0Z0b6GEKZLl@eQDq%oS!Zbf1$Yq-{0Pn}V+P z?yBz|PZ^TE)NG5CE<=fOiom161oa}EO$inq{ke9S`nH0?GqE(@&!S|EDjW^E3WW;a zzyp&pDl0THUo-1emSFqvL4Df`{}8Wu7*eTW5>`7M(~{TH=ZP-K(h_A|DBoZjo0|rk z{%Y7uIXSOpw$r)r_=X@2r(;V?7XPN3F{OTUiP$?g5W8p@+Uwh4Fxk#jAb#j@MsB?m z(T(b8my&eecBGmqfmwKLo%KOb_>dHaL;E;=R*$HRXF>E08R{q$7knYjlIGCyZzw~a zb#Qu|!fBNxa1-Jj`88LK_^WsE5PkA_y0KDP=q8@uPKlT!Jw&gJ(yjb9(LEl#-F`6{ zQ#G15H}CJ#09HK>we~k_CJyoI9ak3O!ZU`RC6ZMh8sqH86)Id zJ|}#293}}A8~p2RX6TyGi*-F4&`vAtB9_tQJ ziIaH$LBL_d;Wsr)?W={PG3Jh>C11WAyKRvZoP^}-wp593@f204ge&D{v64$-?hh%0 z=krHaWWzrKJq@722(53~qZCIVq`yVfs1{&aGL$sXG6n8 zZZ_lnh;&r_WTF(Rz@C;$seX_=8yD%_+@-%hZ27>JSy=|hV%lDPSezW8+F>kGk^p%L zs|?ZZnI~UAY(#h=ZLg*!_u)pTV~%dBL9ys553at(`wpxaFThGE=N=Q}!$n|`chbuD zhhDohF{2dQ@K(MpDIZ~$cnA?L;pJw|p}z5rebH{<{zIU7i7V~JPZ-?KSjKC}YtddZ zXx3wW#Pc*>^QvDkygFw6=EvD#mG~>|6SdQoRZ zI2^`~Z7c^F6=$zL&$xCvIXF$60zrlQuILvz`NL>Zf?7@+I5Z?76=w%$6An`z-wFzH zDC>wipNQp&#+>G!zW1&LHN83fUBb|jhji-;c)|202jdP1=b?N*-T;k{Qn)4Z5|}b= zj>7op;>6GYoG%$lN#v;%+hZCVH=Z9ZwR%bC@Jh}IeOs{;0}6O)-|4qRBHiWUgI0+Y z)(AS>6MtXAS^t zlL9krV!fgx1nC9ZyL8bAZy^Zdp6UVFNiMBdwfR+meF5cA4gh+~RU<)(APo*Nl2FTH z$ir>?qj&=bRUxeY!{|Cm(-7+r z`zw-`^!~6uUw#{}pXu{>Cpo3_%rVi5qNn=7i@e^hXM?`b_(6%o7BHK86`Q%PbOB#COg7llMJ#J#Jd3 z7_!jQ_wHphpW=9{#rK)orE}K(>Q)95<-Q3~YcS0-kO$ho%bX7KF1(iCIqn6=!(ZX7 zGn`}=<>rH7Zh?}Ji7=C2q^nZ}Q@h0^g<(=$RSG2?*>Mq3rH%t#zhs!wq*j;~mq<}1 zLT0tB;*{5fVhZNNz|d_0y)<`cF1RWYFwC^mnM^Dou$r+RKQPxwNG_TxS>wR8o{9PD zrf^K~Q;up!x*^z_o4TP#{07dp#i3VXef_pzjgE$lN`oubJD2693K#MS_8PvpZW~mQ z{L~ETI=6Q{g+Et*l@~JWkY3ZCB`YT_DFf_cxyi-_?Ix#dp~K$>a;*hH!9)3hR_Be* zeV1Ggo78)t{-ReBrLEr&i>O03%pO4E?NcC0N%F?x zk@~w{K@wOq+{Epjr}vsEPpJ;cG{<@&U4p+zn#RpQPa~L+vyj`S*W26EK9p%_ciN*4 zvB)-vWB1-jzs#f;rTp`#mi2PG_5bW zNVN-a`mZ#ss0-F?bbFJBYb%B(7OfjX)YsS!S+J=1oAwpxq*s*Lx(6& z{ra0hxvZFVDMAso?*6dy@p|vt?Th(yj{ziFZb6HEZfZP zG?#=`CE?RVgn`f#KcGpNP%R9L3BqQEFk)Zd*b{`KC5LavwKZCE#D0clP$=9%V(+$c z;u=BS4-+c((x+(#eIUB)d3i&=En?bO!c}`!_JjBL@qlifSP?<+w5=anm-_JOm|R)_ zfG9sWG%)cEkA;(ukNu z>Xy)PnngE>2;x{YoubX)F zP#zN5Yn(@of=dNCVum}+#e*72_a@^h`gnyVi3uS(eYfM;rA||3medTGM#Rw0mMLZY zGOk<-qg=N>GP$UqzVDMcgjgzGyw zGy>7WJcd2ZF+GsI0u1DBEX7Y=KtlO~G9Xv#~k6hg2aKBar+1)%aMh5H z^DUvw@(MEQTe;z2!s``5vgK-Gm*aI;GD~UnkAR4Rf%c$`>nCLCserKZqQG8Gz~dXC zF_McW-1Z&7__sX=8J)E7@Tx*5rvMb}`|rSkA&H3Sr=Qx!rBAG)1tk$EZ0@oF4OF=ONe6hPAAC@3iOXrQ^Ws1T8w>txP*&4}tObSIXS4I3>rx&QY?( znad&0$6phz*4!R^UD~5rF8xKoW#qMMf!u=dex&n$vE-{wf!F0Aw2?+ zf926>_@(<@!E^!5A2B`8an0E*&0K%W_j!poz>20d4AKuCTpP`PJDmYqFZ)lkN4~6mIE9MQ>$W8A{2Cuk)N6XD z;dONAXA*p-slM=%t-?YordSH~1rIzlWDbfm7Gg2~H!A<~8!41zSQonDU{rFq<2goq z>GoK^d|g<%mP2{q-!{B8j|c4x#B6E$LA7WszsYo_Hm9%zM49x0}p`b9wx0WP6?%HFpxh-X?N39!ALcvB|@}xo7Ox4=f{3^3oq|MQ!XOgfo{D z7GvG4w^I=vD+zRL=oI~}Gjar)9%hAbvqpE^jDm>^kAqfpqFJR{^P{y2T>dfPbvS*^f+a** z&YcDp?F-|g@oAac?6N?DA)oTA)|c}l)%SP5q!b5k3x*~nm$&hGC$SEp4{dQY&RfZt zf*gwW-1ZN8Phma;hj;43^DL@VS&`U7q?NqNpJ1)v{+_^Itu=8#)_cls2CuyI+S|#- z!iwX*Fk}>`b<fg?swg2uLE7cxz zbru2;_iL*6-KHO|Gx+)T%M8L5R&S!jkq+I71*BK-4C?aHvjDAwre@jOH9=~viGK{J zpU@&WjU+~>VDBKo`ag%U8mc`qlZmQW7n*dbKlnz6GNeWk$w;bsD z9@YI3<1i`9rIjTM>3;y}^qmcLPd5Y?(q5@+UOHO^gcDX!JLF_nr(76V=MCTfepoU* z3IOQm*o_HTQYu6||Kn$G5$#wJ}4TzVi&X z9c(G15YxY~U(4R-3Co=L^f%#4GwE&Yrjv)w!fua%t%*@fnt`~tJr7HR6sHjTD-SyE z1fpG9w|D}YPT{@)$-kjdhdLQbSVl1P$qla6Q-rm9F~|-5%=7wi15~S6?P23)-g2b} z4a8PG|KbycRO?r&2=RKZe{Y6K8Jny z;LehZg(4`pylOjVtg%Ah&+Z^>iKqDRON)D#cSD+6E_uhCIK$z&xU4G=89!lZf7{|9 zPoKnvy7KA7Os~y`s$wpMfN^AxNNqC?gvX3aD7t=(tXFv?8Qgz;w?T!#@Pp)7Bintx zh(!6PE*+11V@vr7ILvU&>71eVh)=FWL71tnJN-i36Bc%toFqTnclrHj5L95-xryjk z`#$9!ZoHr1->m9;nVSp-_g_}$aY4TIw?#|FEiv!;T(#qP!1|8ocIwB@hKSyR<|>if zTPAgXVs`bOJXN#gX4?sFJEC!9IDPrf!uBkyq$A%jKH03w8LZOXNps@g{*a42v8%82 zw|#Hr9&XZ7H{X3k2F)-#unD%|CmzE_58vT;Hx*|tv|f9mt-!zvKt?&on-ePXUP`0s zCrOsmMAvQG`R9z3U=Q=T!@o!Kg;9$5JA3Gb2U^)IomZ~NrykM5SirL6v$-u1PU`JdX zo06wPvmZ-)8kKU>{}K!qxR zQuMV0Cg;!Xo`>T8RIGTMF)(jG6f97{7F*9Ioc8&fK?6m+jpE$WwWi7EQ$7CLan!mW zebeGhp(1YJM}sH6%id;FlfTxvgOTi0Thc>mv=n@<90Q5}VLqa=ReV%OaW3>(C@n@a zKlUx}>b(UHbJoZ1#sny@f7sbkZz6VzSFAVhZ^HN=D#1P0lx_q7pS}r!H4*HvDe#14DWn=U>7pGG40L9pT)I819sxl?RI{+4*FB0YM47J1Z%78p!@v7 zV?4IZrHZ%pgMXH5X3cUBC3)XR(To5=B|p#g&5<4KJd`datRwRd^bFQ#ebN9hFF_(p0Iy&?(z8l=N;ib$7!c#OnH1|vI z(nrhRtG~y0V1$C2Or5GG<{vgmS56w{%%C`2U7Ebw!L%qO?}qVA(na7vQZGS}AIc(tTQ|>_g(GCCJyFQ?@4~4&o{z=}e(V(C zG9gcw<3Hi%q~1o5lBq5@0!cmQG^=jGIX~j-*W2+PlI-4+twCQrWK7>8RekLnn(4-Q z>H>~06!wdYpm!8o)1Y%V)u?$(FH1@e8EBm6-&_WxM9XFJ(=Ud1xF)sav=JQ!OwBe%b@0QOWGGgn;u5?zeOS0 zh$t^qpd8kVVYiyN@p6|4{<4(a9C@4K6|F9tsXo5^%0!A7=$=P)NEFOaIg&3^!Dc3i z1~SrJeby0RIn8_sy3JQ5vD!?P>XWyb`O8prxf;{~ERovqyydKAuGTU_b>ZK^#I0)k z*2R0f^Ru9w9sS#hhL$M`lS<$~?YpnX0?&w`NX<3=^+WK{;JhHNl4Gct3P9zb#2Mlb ztP$*|1VsC3Qs4n)FLX`qsjuI#ilo=JG3q@>qb4e~^!pQg8!{69r0A&ERFL324~zNH z+~v8k+Z@nKNc4#G%0fl$Ua{Si*J^uY8$Dp;y)C5k4%5NL7_-DdWki3|^HdqLTG&d; zlERtba(-SGV@RCEwJ2XYa=>Q{A@EiW2q4DxZ%)DJ``t@o-UjkPMl&>F=rE}qjg)ya z&AYFzDj1e9?)dnRx;!?2lT``VrgU=g>T(0}17qbsE?ILHVhPYS`ba+Q3no_S;;=}e zLlu3w*t+Q~E=4ZC$SL`YhI#2s%c+u$pUT)Z8{0^0Hd{L}tPZ8+2-C~ESenp330iIv(ed2$opXNDjtQ_^qt^~WZ zC4}6qldb+q^abf8DxgwkeJT8K*q!+ORbT1*7H#*0jWz1hEO*=u45I~PM#7@#M2%WE zMj#0(;p*tSP-ezJW|2Lsq36lhs)ckh>w78tdZJf3SVr|X%+{**0mlz#9dta|MnVmnr7?RwgSr=2lSg-C5Zp`9tq8joRW9(q_e&Kb9_NG^4cur5p*@TBJniZAB+P!lr$`Q@Y6-}L2G12@?S5xB6tGFQsSS(suF zfO43J*~Nk6EKbM8_auU#u@LXufdaF}-^4>z?&eowgpZ6SnW@d;ADpeIiLmg@d?7p4 zR?}UHDpHw^CX#EvX!OrxDj&%a5|oM*xvqir%jkoVHN7!HHglDj$oNRebvF%Tpvg3J z8o-{(G!g>y#x3NvSw?0B4|Y7c$G9z>=BSs>pkdjgy!Lj(;{ODxOn-1Z@VIasC_{`` z=SX*o0tcc&!7KY_o_s%$BJCR%MhD3E?xf$&mY@57A7J!T?Gj9>}URzBLBd*5*xJ1Crna4EL3oe2tXfXKTQ*1!XU*U#~ve&%o zr-5CTghgQpjrHln37Kjss^GKf@z$CZ=gG z9bu#T%Yr+RZ}kAg4*eqj^IOR6N%uQAj2E)SA_<-vqOKi|1{OfsjN765Eo$uE} z35WdN4(IX4w*of#7WrB<@E8OgszCHF<{CPJt8c*wKk9jiSnu<#nhDuRX4bbNxr1)j z3ZD;`!_+(BiPxe{(3b8c^Ir7NL%3(wf7)qYXe>=%GUhIS?Z6lPT^jG&)^z<-nVvV7 zFAAoYOBZMV%2mrqD~*PH+*jm_bm)49)Ze7-K^@f^!=~)G9AmQD&&_D9Jn{ZA3trc& zkxbQ}cP8|%WRjMK97Q9W~-qFa~!kt3)1JZFa& zK)hYFNE7U-qw15`S*YDS)NtIPa6+konJR+e7U)U3D#|R&^CeUEh0ai--nVzvlPz$Y z0hyL?E8z|?=Wat1qByr#%0u5jSVdg%G%{(j7A~Cv%iPE}J|{JiR{7(f!WGIDifvNs zzw7XXWms$g+}2+J(phTnygPFv`&qI<_d?K8(=wR4oyXN3bx)>i9q`U$JJg$bv zzX_IlkVLIf_uC>fXh(&$&TN)~znh4*4X1zu=@k#qepIIW$uXH^_TFQwxW4*rR6t@W z`Q>jCyO**qN|U#z;fb<8PB~J1u&lRuQ#00ZGyE{qlLrpG4iT2r135wOE$%k6PA>pC z5030;0RlEj)uIsbSu!BfN(e$dH7Ztx+YS^2=LaWb1lJyFu<}q^H^Q)y%D0n2|ORqZ#l51Fn4({lXVY^ zq8ny&YI9E*!hw*7C8{!WG5j?oxE-JB8vAx<;MJnZ*(uvI_glhN5_?5>+R`*4<9Cob z%QLegR3&sVZGWDmLd2%Fc#h8O_rvbLF6Da78iA$;(rTVzDs+kI;ft?oDatAdpolOE z${zPgYwbzw1R3*3pNjZI<1+;Z=mBk&cekc?>m_HfjhI`kqNsdRy!5~Ldw;x^w~8#E z^0+{>g)V_{kL zF<-6jS9b*QutVtSBb}Gz!=GCnUj3AS@YHn3Y%>NxgoJ91P zxgv5gMe64A;7M{Gb!oOtggtzy+6uSAwiBkJ99mht>77Sy_%bhkDs~V0g?$N^C6S`? zXZNaFvnzMbNXAf2b}3~<{Q#Z42dv@fD=9=Lt|Rff)6c`>a(k`0eZ| z@(_rFMhed(^PTR1(&ZX&mtgh7lNn{O@Vr;eQ3*bhWfsFMAxHPWN>clqe|lrzQ{;HX z+#LLMNmJP6P&lO7awLE*yxL>@%j8jtv5?zqLXanqgtD8*3WQcOXZpHM=uvOUxvNuZI6Sq+pLaW zKXchvaMzPe)9LUW;N5QFU6JS~0YrX;tftuW2Wc9F&skU4G8$L1g-yrUtQSI3Al$Pn zu;`6p8yPIY?F!XT)|!MLJ0jocVw?`Cy9|hMu6bS5>-LZYDi-IwC+mI!iAc$9uB-5KG$P}jm7t(YbJGKXVTmfryFu znsD+sC~nECjUne7-x}$=ii~$@uC`0@`ZbGjgDGE;Qw}a>0xrW3I)?oHp(Uf5vBcW( zl1`~{p@*zGS|8t)74qG`%Ndl$*B-v~a_GsFQRBq$!sO>Z&sOh|<7DsSNjt|1RW>GY zE9kGCIGw4}Z*K1iJ8N^B-E25M2(LEbybN)4C`?jaAU_}1FrYfeSdnO_1v81-X-LKnR$|h(kyIqX zARYy?+&Za{{0!6UUwP;|>ift2w7IzD!4j!+X2wC!aRFjeLt;-}OXwoz%a86wajrO@ z4YZMs0(h;LndQSVDu(7JgXd9vVJ`mIK|cYTHsTX$8LIAgUe-SuyF>i@&iPMu?{lB? zVh7V!-4y--wy;O&3|0uAboWhtg|yHXXqu#d-e{?6nRyRA7gBpB4}l+K4?pJ`C;#R! zC1t&XW#_&qF}uv@dIpzv|a|BR+*R9%)1m|taWb53S zm>1t089Yikf3Th)C$hTm%ld^|J@>lsxk+(JrHT=4Tdh%1wew?!mYHH8{g@u}UxDNgmE{rN4oQArUn>jf!H49i;gqjJ~X)&{C|IA2A*EvpyYrvx)Q&+U#GZargRHj;Jg&agGypY=A+2PxH zqnp~jpwmL};dnJCip=gdX`>IR%K9mVH&w~0zo;F{HM052$1vin@yfN$6SIDqR_!lb zbA7r=e+d2j%zp*guY>aB;CB5=diHOBv`}|tn;V2OG8%S}w%ceU*BWmBY4qZoaNp&x zM&W}&Dc{MZps!6vqWok1V~voU9XD641g(=1vaXOmR#`{Rki;dI5@pVT)oL29Fk8S zaLo4ZJ(0PH6AM2nO`$^91 zmUAB+5`CEkNOv=g5O)eC)mz-dSB{Eo!&ij&jDkv-S=sSWNh#rn7-F+U$_P117B_Ma z0xL13ssVM=2d?%sIGqbT!Jpy1wRjq@*svRe=MMkpbw!>o}c*mma(QmUi`h$floGu+1a*t6HMzP3vl= zm~h4>l219d8&|J>4@T1!i!;#B{q;q8meJ=^DI^;LR_LiMlRJVm*_T}bT4kCc{mSiY zo{C$qI`7HRCLED~%4kqYjXPFPw4i(X(=yR`x?+9vxE(j}?PvLQA{EToiOdkxS^L^^ zg*qpp7W^-|u>HB#(xhojjCS-ld;oy*n_>;?AM3!&qN!I;ER}LPoEEgz2jQCvHnQrO z0@BRzdSS6Zq2Vg3t*X&+fN!GSluTNH&K}9xy-Q_~`?2Vz?)(mT2hn(l>4Ohpqp2!? zyVTTw?(N*YnB;rVL83vOvfEC>FP<;Hu{wp}@PyA2&#!2N6h#0NLD+9IFFXdf_R9L@ zj~om&;=W~oeC1FR(NGd+w5`4W z!_BYsBe(x!DO}JsB}^Q@3zm8fTN%0jb~bAH!sm`n$afyX5ZROT{9Y^F8o1)5H45jZ zYCL>>wKr+b=u728cp?*6v19Y1wzq}5%V!h!215z zBB28HzyI?|4UpVZSn>a!;>HU3_7D{;SVby2e}g5@6av`wD+b+v#`-@O`M>$e{Q=Pa zgd4=;fK<(K;ZY458Agn$4l!t+DfAOHIwnD7G>MEE!s zl!^Xtgzb<3i13fhMTh@~pYVVCnJMWW%IPsc;zRzeTLMtp4x5=A#vsY%-&qP?!D=kIOyl^ zf4l+BV0t_2eF6iz9n+fmFb*s6&2s;8RB(RFrIVh9fzn}`z}ZtM=9Bj|xbxF>{!Q81 zJAvQ81v&%XVrZ3|mpJOQ$w9kQ$4mHv&S{HFih&V(zXqH`Hya|f86oj@tg!K>Cq+- z=Y%vmxoI4gG6>(xq(B(CV-*8+vFZ_ zvSt9Ei&2%Mw#O&G;?jJt&jPcfU9A9$$@%^6L=JA<)$w#y#?_{E=aE~to{97L>vlZh ze&DA-9-l5ADLw#NzMCkEj2J=OecJP(V3qhuFI(2>uf69RWf^uglUBd+N3SW-fv3ea z_r+m>)NhXmPdF>&Yq}qJBJ!1fA8pzDieGdIH)}bW-&~)xSl{ES*3^=m&#mfZPC=v> z)5Q6I18GpAH$YkJVJ<+dF_>WLu?GAq+FWnO%X|bcfJ4IQD?pE&uPYdNf*|_7{L%Ka zTbK_)%(vO=cgAO8Gu zzRs{ESo#}svCMnRw(}rQ65Q?5>N53T7is*AoIDbIHM+#dBL!zHU|s`rr`*5t--KU{6vqP!=_UTV8$5hQc+ zkp*#bzNbC?hKUpC_d%{VgijW1m^}5^#weo7#LniW(at$BXhMlQyF^p%+y9}q|Y4XD+8 z6exG-_znF{!&c4fo2!#SV6zaq7@vO9xB>%k?5i5ET+AmL8Iq6X=$sZ~6l;zxLL}F6 zSWtTUx@N>{>nLnf=6e+Z%vZN6ht8kxdKNWLx=$! z)Cp!kfN&=Fx@j{?0ru%r^&?pRR{Q;+p2`(DAxU5+VugAhEQ_o8DR0=*w*qKF@iM@v zi-QU8cw|TU@HQL)R0kS!ggP!1Q%O6%Hy3d^)c`>khK-H?&c$Ye)!O!lTzI0M)JLM( z=w9E!rsF8)8ybx(@o%knF-79^fxijG0B`l(jSyTz?~(TKO#ok~3=IKhUn5gY4ZRC4 zty^se-oaApk!bmf)w?iUoRoZda3(6LZ#!jc-@bdcVS4hljYqbQ9Bu?GT3!@Cq70a` zd`Cj9y4}R*3OrU`1tPO=U-vLXkunf$B3YN45A(PMt@`P!A3_PC{>{sRWfgrkqKs&W zE7o&W@Xn-|lajUYzBTbVTdFCB*eO{2?nCV!o4f4<_6MpKzR7eP?3lGb&BM(F?#)fs zJ;<_BQB)|MR2 zZtXYx>`_|!Fj)aK_={L8>_hFDqOCDb&paVFH1G&9G8jym2B_>S!tHyVS=h*6luOAexCv_wYojv@-?s0eN7p_qx#@d;9eahcka3=>o90t6 zmSVK*Sr48X6AP}FcbR%wpl3r)D(S|qsPO1$1e+yQ(@B<2GoLWynG@7gcqaM?>=x#- zO4Z;6VsBqa8aBX6yAFT!)uuZ6$2b+8G?oNh(JQzh^_@x$FgvNu+lY5}lWYMVC~vm9 zSry#w>X7ZJ>DXtAa9+I8Vm9&G>r|i&Xy}Yy>ZJM#xvK@s&6M)}xdE4DNTv7F#`VY0 z*7F5UGijyK;bVOedB%1YY3~4%yDt(QlB7dJWvlz61L@{7{qfoPRDQBgY2D*2m*!Y# zJP3%fc%&KNe#7%l834?Ej~YtyO1j#QfsYa^Uk0CLmVCixWqA+nr{y>4Rn=%vNm;5ya9 zHr?m-Ynt>eOI-FZMa@mXOHGpBS3Y#?HbD}k$tQuR?HOa8PAQQOFDvfwD%)|QHr9)w+Ea2Z*y20V!l89HWdMaO&tlU`8UKs50 zkGt0{c$ffzrw6z^-s2XPU_luoSt^C-K1iBaLUkr16?vD$^9mR2s8q0_cX(aN9Rr>| zCb%m#=xt|f3_sM+(S=_-3$3|@G1iOmLts=3B%}?>17@PWb`w` zdp9x(x%5DN9!e6Z7w=arIf{c!-L%by3|Ie4smtAVik-draBkDHX9*K8xEm-cIi&(< zYph3)vzzHjxefGO?_mV>J*pIw1vo5g+CdYjuRAiym zoHB>Q#5rrM(r+uue%Po#)rv`*I(MYnp3cV$>&f;q1w}+0IdYoY@Eo=m0g0jpb7s82 zbEyEI>Z~TkzPVm7gLqDxZ7>p;PtiXIuQc>krm>thbiq9M8ZY(M&P*>$hOu!l4;^bM zMW~Kdk1Y0eT*hUtNP4{su>PsQcF!%TAasJQsfs95b@|;=u}6v`<3bHaV!#qfuT=IJ~QKxNp;H+KNq&iso@3g+tbE48bNJn?uLt3esGj zO@CsSU8szx$0;6&hpoL5H9)Ig;gv|HJ8PJ|dh_mq#-VY58q{D)`1=m%ZqsrVs0V%t z>eA7h8M3|kbbUX*tsHB}KEL#IS-7SC?eeCJ3EUL*f9`n?0Shog z54WdsdM)RK7+jjxv6$4SRw;HhBaMfZIV!A2$?rOUIuR=vP9T!#!qo-aJ+;K#39!M1 zgTB;B-mDrO=zbf8y$%z)t4IlS%&$i2G(rQhLsXDOK%2=>|Cz0CixM1}M!W?ole;FQ znr>s0@&l>C`(CJIjFi~PQpXQd2*c~1+KX;ND0=5>6~jRD3y^}!Zr`1$q_pO?^Llw{mtQ; z<_oM(@eEPMTVZEdb(n~6Wc1&+Lp4=cz1)OMS@~_o3btoSy}im1Qd&H2=IBeJA9qWM z2DVIgJoJYjR&DuAYM5SnD){iO>(sctkSE6JFACE1-L5OmQRUHa#MMBsu+3d8cq&m; z@;uOiPI?tt9jZ`N+IW>!bWs8?SJ(BjjR;(XJ?w0M1=lmpa}pSq-OHh2aDrs@u%?2( z4YY+9C(<>^!;sGl4U&(2FJx%hERx!`0YQ;jvM^;QP?RS(1w%ePR_+RaPut|bfa`4s zjb>nzBRh%>oZS}4prza7Ov{&Zi6U>^L*nIFey{5R5ZOdin5eD+#qh?Nn9o~upR0}M zwX#~H_?7~%%Vcl9Pu(m`4XhreGxzN^CDi^(oKlZlT4T>3%p{n6a-%fXmGTYbyE=zw z$afTSVHN_}_WeCRWdzwRN;eiQSD@2oJo-I)yi77c$x}qvordj1CIxX;v>`Sh1rEYr zA}6!HzI_adDY10-aEOVLo}r=%E72mX&h~SCK2JI%^vzw)?z#AZFWD8vpq)2BuVQge z2R8AANyQOvJX{>BB`Nu%iN^j3=^`oz>pK{zvG{a@I*z1RI7JEURcCKQ-IY~=KU#}- z+g{CW*baqt>M%qc&eN|f@1u!kWNi9b*bvunhIRZ&2o0_r1iz3GmHJW6o6kUSsT!@F zY_7x??k_s2b5(0>ESCFy2(@0%1AAArwo|^he`i6jSeh`j6rQ3hXxaK}dGp0KmQ=;d z`#;zMi6c1f)d!3WqDL9QYi2K(h3!TInDAsr;1T6M2W?FbQIPrM=j~L3m!0D7e=ZjC z+s=W}oN50>Ta&y$Uzi8{WTw3&jxJmP&O3-}!H{Ft#PjN2*-Lm9!X2Vvk-QT{J1*vo zgY&v$#`d{yLPU=kFej)4OwA)B@1yZ>Eop>3f#j*#Txl(kU8}$KgN$FPjc;gwHVauk z+x6MQ=bInI)dX=dh*bay5M3;ZK$r)7pjo<0oc2Mx+je>t`uufO$L{33-<(4!yB(S1Lbv1mRllEdB)?H8o0xTX0$2DYQpJGOfOOXc3WTYIam{3K9Fvicc z#QWtn888~Tggk%!J4kx5E0j4<5K%v_q7x^9$e&x{0tXL>sWn!dJMZiF^Nh^CZO03w z3-hP&wm3-IO^LSW_MGf9;DXCQ+BJ2vs+Stu@x@DilCRjDO-(c>pe)ilC2?&bOiAcT zy)(&Ax}dS0^_J`9{Bn8@!Wc}AP(2RkJda6b4R-75Z|{~6C>IK(JsJFTzwD5M-;&7X zd)qg&%K*AT@)auF2#9OUYJ?FB_g<51VXtTB#;6QYr56`v^Dtg)HjZk1Fl`DjZ8fNYMw5Q zux1{63ddsBZWE#0<;;k4?(Gr9UF->EN&mUu12iJ@22}AqJvOOc$c~BbLiaf2!07QX z;VuJ*gDib68(k^L3@%qKaYrFMI`8_pYAx-(7O0FG8mpkTIeL^VW*D&4mbZx3wfFmz z_i{LrS2&j_zB`a9yeFzgl{Sp%u(@7QKk zq=9g(c5OaLBS}sBX@5mLB_uy-CMl4z5PcG#VDI2++oGH zse){y_m7JS_O=YpAFXOQ{Af?c^d#{1K+mDv`U#9SCo4xrE5p=nA79wr392xiJw!W9-zs2tIWJUIlV=A*I7c#h6e?C$vgj`F zGdxtCR9tCamTn^%yV6rBXNtH^6lb4$>vJ(T8qNQwq@VUP5mT z%g}Fz*sX}b$#U-u+K3dl4^UO|><>?Ve2t?n)mDhj?QH+{kEoejM4SAKm9)&{>zziL z6}ds4{Wt!2LaZ)F2X8Jx4Z@bf8S=Z6m76;N#<2K+$j_a;v2v7FxabH77bzX!-(+9D z=TQt1G`&2?j3L`0=;S_$=!kX%372f(R3oWFIwO*+`5rFklcgX_QZ;7h{rN#APNJnq za`+WwfDkcfWpmK2y6lY9p`4YlrTk{K+UeFQuHE@oKQE`N7P|%FH}X@fc^)y;d+|Mz z<~Y_q$v$=Gz1N795Q7^KsqC<)t{obrrF;$UefFbZ8)jUyCpp^n-jkZ**Z1h9J~sF& z8U5&N3V~sd%SiQ^;ibbfT3R$8OQsx&*NbWTw{oGDd-#)=eh>kkgbo)9*xj6KyJo3v z%Ju~4KJN9I5vwMy+894?Zsg!?I&*Ap$_!c@W+eO#OmOgLo8r0C7fBJc>Wa0CVWHM~ z0?b0bRvCN%4%xf+EseRXlYCXDqC$Imu4oZw z2+Kvh|H7^9XgT+-4~uom7ZUL{V0XTKp18By35QxIpJP*>)_g!@%zm{I)OKEG#LGs# z+|mi>9C4BEq|Lv#?o@|sn6_#Ij54GZ(T1a+y6`-(aeKh8{O(YigTu`GnKPcDsp$6T z(V1d>#C&GhDG_6H^zlsfuS9oUUFE7#-euk{-hfh`oglHzM!I6;FYUB_;Ue#VXC3bY z3E3kFpJ{2L^HNx;@ED(4TVi0cTjL_SM4jJE)__V}nF40g6`6X-k!p^P zvj*9kWlx(;2P(g*u5uxD+ z)iLcZ!82LNnwOKDx2HIZKYx8?l#TczpZbva&Xxl58Psx0qIU1>;%j<EK9ih#j{FePk%Mn;aqDKJc2Yi!Wj_!G4sbX|m}R={_p*dbLG6?0 zk)B!?L;|_(SbU^JiyyR1eB=!pBd?gzX)4B)!=>2K8^!EM@6fOv?3T0-;2J{gG~es> zrYt!aC-}%GoOm3cRbiyhT5Y(Q@a?yD;NqwHqh*B=zKI?7mf)BnNDzv}+@U!5_@W`O zpW%1WPEcU_JoSEG4|ITOEUyW3lS&a_Va@LL0sIP4K(Oau?gx6Q`W4uLgeQ?5FZMgT zZmtg;DUHGXdeZOSBZEygeoJ?Mddz)qVhpT2DY-)(ZDHaSAxha4-MFCrIu#HV*Q=xel=vz->H<~$@^Nk{`11lYrS zJ>49=!A+TV)!^5a=?QhUJ` z+Bq)|+VR%ZU2Xix0P^lrn*$uhl{-=`?~twd7R)%(%f<>tV!DRu)vsC121{o5_3R2< z($59Yu$(q%)tbdW@;d$kUY}`P{a5uGO zm%Q8WPNcZ~JzAl;uDh4jXNF!@k#C+GXU|`YGN^E5po|I&;ZifFxwwX2 zRGW%C{Ra$CTP{Y8d$^*(BeQU=-teyzBNAo|G;lzEMIB7k((mrKAEHi|^_nPv35mExF0}^)w-wzzl??QIL^*Za0 z1;lZe9&rF0K==8~Y>&mEKGyPqX!pg!OC3afuMerF6X!QW%oQQTTiBK!^^^WRk{k#_ zE0BKJWTQ46*KIM2Ml7dw+6aC1_!M>Y67Lust{(Sl+9AcE?e^hPRgja4wCHF8ch@$d5E8h{cEZIpufw!Xr3no7o{x+R$dL3f%QNZ#AdMm^)7Z=}2DI`!G z!U-i$a^L`8g|_So<~ko@Q>6VK}<8#&i~MKt@JQ zXPkC@IK(I>gS`LveMmP;eI;9u@TbG8pK(5j=>xB0{0`#+)F!%lbWf2 zp>OaB`Dj}bjG>~7NrH+Y!Qrj6+c88k`&;M$hAhSV{Wy!dduXxjU8j&(X1p zsUK!6~o4-ArgEx3{{C51z zfFbjq6+HkwFS6h&$bm1iM&?Y~=UW>~vw+MgMCIdfa+@jtTOTf=mr ze7Q3ix^evI$kRvF9(wWH$$dIBV@e$)8?&FVQE?$=|JBDC4ImT!F@d826SwEa0=9dd z;>ap%$mPiDHxy{+k3O#|Bxf@)!8w5|`gj!(=jLJsr-|iH3sF>lB^vPohto534_?a` z9hL5)UUeA!mYp%rBrTj+d9XU$Cdoey>->gVKE>k-A?HG>p=%h7$Z2-vTpp;n9Zxor z0bP&7yzkBMu#Bj}N)J^oPeK-ach8C-sJ!&L1EGaD zL~9dXpJ^dtcUX7D=JA_T3%fL}TdXZw#=O1x zp|lZ})6k~FAkLc^z1~EEveC7f#@1pLo7S-}dzmm(bhti5obX0%{QHym6QmfuPHS|a zY`Rq2`EUpqiIk|FFf3zFnWD0h+*x?XS@tB$#_A0*X+REHvxRxc3H$Zqtp19aEQCJ{ z_VC0m%PHA$fONgd)TmqP$+zW|jd;wQmyoo{@KcX&akQh@phz0d+o?vH>L={g7pk|l zj*O@9SsxQlW$W0Pb{0_wd8ghiJFtJ77GZ*qZokCCS0ua)Ktgz7wi&nX9W-_{b!BI`)wj^o1xd9Y^kC zXk_Z#*zD?0-2aFYlP#sW2m{7Bd$ta=S-zN1rjbRy)zW|E@q^u2P1rkR!WUnxG8os; zE=pet>;)g0hr;Rls4P&$V?DRWRE2?Pk%!qsL%Af1-H*uzz?PRAtmsrEeRw9k@G+yW zCu(H>Y@gv1NW?DPO(|eVchXfdJwLZIw@GLPCv-FTnW?xY&

x6C^NjF2J3m(Je<2+;640kKUYK6*|wB&(X9( zc+|DKqnGYGn-Y=k43a*nxCCS)M}E6Kf&?_e75hTGK?9r&iu~$cxNmBx*g?gr?VgG| z?R6I7N2Xw>;&tY^+lpx&@PUhDA&$?Imy&45EjKQ)k$bvWV6|_5WYMKeWP{1%CSCW4 ztSfF1>V+yHl*X7f8S8gnN4<}Kd^4?W_kBVfj>A+vyjeA9+CRKTW56kR<0W|B^$9*( ziQUPL@pAcEUg)Qpe_M{Ab3uTFMF)8{kq4Ws|0MKObDcf6q-j7>c0#+09=`t~9V>TT ziK9)M_!afuRi3&3H6HvHtRj$=d*jBzp-t`$DBUSY8axZ}6f!Vkf_kyyU+=V_<~wvM zJ@V-)KfV?zO+@YeN6xk^*6d%>kQx5R@bvfbqJ`JPtBwWKx=i572gWrhEN9M|5m|6< zfe{a1?>~}USMlVvH1vR-6V68E6Lffe_K!u(2$|aEOXQ4Bo{s5gbb3czH-nnhv$*JC zrRP9fPpS4iG!*#yWxUsyH-BgVGP$_q%+}Bk7=dqfMLaD?8B`(IbAkJQqIA&)A{UN* zP)i5Xq8w>Ozca4>xU?X4IZJQ4h>1HG7~H674O(v>`~?OCl-Be9+{QD!zV?I(gK(4JrnGM~!oqn#>C9HY+jj3SPxGIhRdepA z!@5s#@vVBym4xTgQG+yo_Mu^*;z`+U2XD7%^;P3u8r-JJ7phXh;^T_<|@zai#{t7P!GBPrsG4j3o zkujKVu`{}jS6rFay&ZZwZE@BWmSa>RugwHWE_XdoyKx|ug113(7r=YRI6%hl`+c_2c#nI@UJ!7Wl0+%w~$f1+I z8dOSEm$i`zSN#@30CnVMHQPw@xXwZ0`*GytvsrcxqruN{b@SI2wxiHT+p zbjV9NZ}p1|Ax4I_1qmfo!E)HDHb&nkx6U;zYb%IBucR1K2-gX2x+7wpi~H+nTi!a6 zDAPoVChYX>xQoOrtDR>mVGM+QMm%rNr=(`;mCLJ&T&}juS3=xYuO2nkkQ_27XNK-4 z1dO|=%#U{|%cC|X;@@4k8RP0foMjryYZqS6>DN_2R(Q28$I1D~gJ<7-OW*T-|LxFG zMxuhZSaV`mpqG86IVuXYD)HEE47hV@sGHf3dI3I)Y~Z;=H@!eW|F~^cXGE16PhS;{m`g=(WIDwhppCA3GZ5GVQ6qY$d_5iNO z-${h6oR@xhZP#(#?Jg&cIu!FED=W{Vr2VwHX4Xfmzea9%y@g_uv7jw0OV?q9l>r@x zuqW!OMc3w_tpEs(_BTb4%0Ezru-k3u79@kX!Pt$uJQ&2t;g(<_w84QFhMj+;yU7S_ z0$g8#wi5eHVZ%;ygXX6LwInxGoCamwfx9mJw20!8%_gc0Aar9D+}PV-v2!0?n*rm%1-)iTh>$wOEgI^p_s)%K0v-^3|eLr3i9n@0hAS-&T(lEmk6oz zxfUpw$Z({a?<28r+nFILz|q~|LiEQI`%mFVsjy$c+Y0q+lhR)ANHe`W$8z-ab_Y^q zW(OwT@u1rnX|f<9Zsq25H~JiJ&)-*XYLj^87Ve80Q4#npcgv5(O4o~kc}2yU5dDnc_k89x{bIGKX@m8>EUf(TPTq z&U-F-rE1t*43Sl6CBn-)f8LK6_pd9X;Qh|NR63>3KcB+;WxE~(=d0gqW6dcbIGsYO zSqRr@;Bq3$61(5-C@>OJwSK?;#KFaHc~nf$`eHWUJPS|W^lf$igLXsW;CHq;A4et6 z)}MmrdwfzPIp7pS-wRtl6?Fg`IY-H%DUsX?^#Wc*FG5p^PdluQ_phUrHE1tHV5i3| z;xB9M{WFQP$X62Sv#cLETYwd|A5EF7&QSgU%;@mX0nW{ssDSqFSA|BIO#FsI%O1ab z2lr73Mv2RPs2Y*%BNbG3t+t!M<(YBWFo$R8?;%`=Nl6MZ?pC|>+?t&>-Cx6d4_nEj zQ~|X<1TI8Wbr_*S;xC5LCvv0E@{dE)5=8N-q#KJ?qYE+a0EB=S>Ozb9V1)Vi;{RevZi5ed`s!s@LX^s&fg<`lzt)N6*Ni&DW*;(F$ZNud2cS8VSK z{j3`dYAs~yll^_B`OoE_%Tz}qJ_AX?Q$<=o9h2Hm%!RBvO}i8vplm*tOk&2zM&$9~ zvxR~gl>X0(tIBU-Wbez-{QEPxa6a~w@}##k5WU^l6AZM*pFHZ+K)|L@AlXIT9fxNx zya=W+u`MT;d;JXIl#L>4PaUQr+y5T^%FPyU(@cxtCSLuzmquA;R*=(i1xnQ4J8h^V!!bViOwO%^Vq)U( zNri>js_ND6G+%_pV8NSJLYp*DaTQQeNA|3_p&z{%riEk;(Z>+POtZ z6;vZr<;Lc)UKjr>k#wRzes#YnMX-%i9wf%A;da`6ZzCwd!mazxd7E;sG3&u&!@-$y zt;q~QW6NUlLs^3+ielqs3EM-mQu7Ug?S^s=(dr>VEuk9NdcA_$sO7V!|y6#Wdq- z8|`49$V+&}-tHy{G#|nrpO7BmZa{hA4proL9YC5hv0$?Jc>TyDV&*i2nBf=eEQTZQ z?&Elh6p!8qvs8Pj%4LZ=c7*P0OtW!Y21rvC%Eor$5kd{Lo*m?hsv^*arEhkmZWB5DLa#kQM|L02lTXp$IdLS*= zVT=c5zJDv>m(D@y_el$g|CW9~4CwdOVsyp+XOYqWzp}`yKp(goAD{F$p7ndIF-D%z zL*rNZS)+f4F_O=8K*fHfE_T?zjidKi0x}@>|C5CK*C@z`E^H9{BHB!>e<$|QQCLOh zCxL$(_^54y1;qY>Qhes$SPy9uRP6g@kNjJ31$=@{DvDiUcuOC~#-C*H@;!`a03{=mJd3 z)Z4r~pJ*{PX7SFg!H@k0EUi-ZtSeX2J^z}dc$HedNYxBZ^Ge47(E=N(ci)JgaK_j* zpDj-U=s4p$Xn>LoRN#S6f8#P?;yO7vvSc34l>mnD_~czE)-TkyT-l+7&M+s5(>H`u z1Aj3rz#bh21W_8@n+^kP1*x&WfqY2e44?v!xs9C1pH`tJWg?x;#QY$`11JweRBf^n z)D;9hf!$5`^&lWG3?<}{-_*l+XRhqMH;gD={ycd8rhX~R5v5jt4W@n24x6l1UJ-vf zkbL_cSZYY3CZIlR)IiOf`v)MD{v0*ww9KuOgR)HcqG^y~s}VJ5wE>_M;bLL-G)zQW zedBsYTscj>2W!mi#SKK1TRv07V>zfgUlmwbF3-7p&kYV4O!s3GEx`oPr+;Z~Eig8! zVPecBY07BM)9MF>sVJNBe}Hht#`@ClwWNQ*!_xjKAyC; zR-|^2UyFEy;-G-ItQaQ?5$yufi$j$^U{s>%F-p%Wg8FWKKTJKhOyh?3;!P!mw&iSaREIPR2tC@dmN;#JxoJs#u`mSTN0F)6;(S+9n6!&HsnFt5#% z@+?|Zjp7)|)u*m%Y z3`J`~EhH?vwrR^QaDqR52R-}%~879i}JMua)^^~&l+#=wFoLv&M?5AtdVc6u ze+60!17rqfAO7Ol-hQmGcSdnrJW!~XAx$WJ!k)Ah;AO@fI`FO+x$QSquFP*|ffZr~ zvEehu5!qPntWCxkR`I6EeXLLf8L^6jF1tLD%2BjzF+Kg1lB`m=QVfp!5Iv%DcE@YQ zyKL?2p!Y!krS)>l@4=mOuWip|slH_XS;uA03FkCNz@W5{^5gNU+{;R}M(5Iz3LN>u zMG{mn`x4D$S{-$g@gK(K=j4|8T@za{I+uOlEZ^&*vKHCkH))bb?^6DQ!sL#@-O`Z3 zH6#kxa6vVGd+_+`7Bvir7?QU?EYtN{%aN%xn=;bMK91AK;EfZmv)(UEv3djMz!=WW zfD#2i3Z4o-Fg^fhmh%tyE7a-zp$2_F=(S-Dpf99ssNmZvvS{ws7Jm)+O}?&2b7jt; zHy=p(<fgx&>VlJm`eR1NV#-^mGm{t4qP$97`XhhB&K4&cSLmj*40Z(6F!|PJakzMxM`;D^4UpZSrb((BAH>)! zo&8GUUAY_(-!V=f#A~Y>7=1AKIIQpBtZ}1iU_((D?aTul>*q4{m7aR<&uog5jVlaI z-qo;W^rAajXx`On?<}4sr=NwK8V+VXI)-t5#R$cM$4N_WIT=E=1~U)JkDCrD z7gUqC2Zh9Oyq6_m7hAR5ND*Y^8FLdwrB`Q~XSjmsS>!3$U8$O&n16W=Xrjfv>d7X8 z$A%4X(d6d)`DO44o}9n-c)~qFH0$Lws=A&0V`5UW2z)7FPFCb!1qh%nDi^bXV{# z!3#Zz3shd=(k0wa`?$|zh4qnCsKU(){7=q>bdB#c4Y?xFm7qxVW~(2ZcV9LCP;r-v zw5+h_qWz*}>WK19%g7?}CjsU1$0{{!4C?^V)e||0@jLZz)^hf{U1yD*Ola@@pZuZ%8INVfVCHAvjvgqw~=~(HB8j{;a&tlWhp1C@vFVQFi zf<*tjXxSZl9u+jm=MuOW$mPPXb41&EqXm(O)f>Yf-JQz$p_$w=zA`tpDkZWyQA$-_tL+P4u;HaSUjwa4`cfxsVriqLe^IBs z|LYGQ!YD75a0=V#pN~5NPeLec`m3sn9KQ2o+k(>E7sdP21Ypc|rvv3)fZCm&bY<#% zg-aVw+&j+iK)#c>L%|-ghG=^qY13SHP$JO3(wA{xU~mdlfRN?z-+;p5J@``QZTI8L zwPH%+hYE%aE4u{xj1{6;0F?D$N@-oF;Q{dv{JF@p59G31)-Nq4Dwt*qW~4IIOR?gD zUa0QB#I$x-CwVYS>7HGr^^^G#nd|=jr6BLPYm?JsOx}R5nU~s;Tg?~SFb{LiW4jgv zT_2=Ejp7Q-lO9Z%Gk<$Be?LGVzd-qQ2c^59-oQlDJ*0NuldSo%?Y!$Mj|dOB5rW&) z6eR~Qw}NHlBtcO>{95Pf<8*G>=lQ%yyGRyX4OL~+F+}ee==7NJC^&&HW3Q;qK4P%-39f@x~*hl}` zlf+<=Mps-axmZ#)@KKN ze)Ihj(mLKVaEcn6I)eO_Jdwva?sqT|9kW~8Pr+sxhxt&Jgzl}zS?x2k{@vIze?I@) z9nZ2$6?1ahjeF1_fJnx{!kT7R~Hh| zoPZCy5?RQ|f*RIr*_Uy`yBN4soOApK?$hmxY8^ZIL7cPE;^(V5)}2k*w#{>zk^rJb zyj&d2vN_D5l1;eGmw`}G{1AGSPzG8*Qpd)W1*4Bli2@w(3;8W$n7bGR?HPd+${J1s z7$i{`G!u5y-h#GinT6)&Irb zTSis+w(H)4h;+xKg(*mPHzE^|4(V?fP3EebzgCwzvDO%UAQSNrGsDlKaIxQjxnpN{*79TX?G+- zlr)@MH*K@$C!kf6P{$m0IE0VO9^$`CQWq~xVQS=V@_vJOjZEaXxp4f= z8<<$-Unoe_iq39wm8W9T<*)FaFVNbdb8mK#b$JV910$>Xo?SsUD&>16>H^W$hNjX; zrwSCJ2kVi2L5G{IXBAXme^JBuItWuG#|G_(pgqw-$6bGn_ix=?TG!xgy%0kRaf(9% z@1SDI9=_VeX8qQPn66EuWk8*#u3p3wmV-~`aip%^ zFmLPe$7~VqZ*!x#1?^5JQqv`wlBf!$7t{lzX2!RFtt^~rS;e~F8tq4SlwcdQt3YdE zYHe31+tGYFq;F;#EqtF#*!e2w*g!QM|K^)_&L>=AFJ&K`6p;Q&oFs4j7I7{u^@s6N z&CWcZFm!o%p}2?AdY;$T26V?gjGA7q@~$XbMh9VW+F<~0jn`Y_=eR@=`V1c7s)q^! z9yP+YC&2;XQ3ZAmZ)@-LW0B*)2=Pd-FPeB6xceb&Z~5nNr(z6g-7pUm%d)P8?68z$ z1`4|7!Gejbc(r*$=dbb~V22TqKXqM` z@3xx!@QyJWMyun!;O85Xr4g6rmg0!D-}Vs=2ylIv@5v)uHR*X99!SaqHE+|oAL>;4 zcfe{Tj@Gs{%aI%*^bwgC?7T+P$_F$+jLp@99I1yD`D4g1vzEdjQ)|mhF2_{~Oli2= zue#;T-m&T!>;5h-TQkU_#orH_ z@(pHH`w97|Y=L;X0i`zl+46(|EbPI%vdOXxE{|4zmSxP$$7RSmauNqCgR1(>!y?#K zlR-AAF%-97bLz~0jWjpyglLBFY@hIr+*AEIJq9ZkUn29tRn0WhzBPX5?!QWShJoVr zX3H%e-ITKc?FN-Z`MkmSO(}w|KVH{NRry+797Ts+$gTk*jt@5(L9CAZ=fGdX?Tt!t zz=85XD~)za(i;O(d5S_Ugcd}|x<%)p!$=u*6h`u>LIp%M#r#G?J}Sphqh?}c!x@yW zxbk-ezfI?RE#1@zAB`DWfujR;djaM~0{T8P9ya-z{0Mp@ep}=d(jDseU;M*r7be97 z^Mu~5JTwUrNo~Cg&25>cs@24sYYoiMV+h};85-~Gh|_++Ie4&F)m)Xvlz7>>C6KI{ z-yIYtv&$9Z@8`9+<{_KKrn3F;yrHuxT)2VK-Qx#cc@ItC%|L&dpGQ?jkk@1#vapTO zlug9J`aB_LnrmYSu}0@M_g+;vxa}mQ+K=;1jcq;QC|}|Psl{`#mA1p3E@}Ef(PGW& zh=N-2hx|V6j+xqjGite2%>#81{RqzAg{P&a? zG2|XN>B8iE&c{kX1>5IK<=D0-+jsk$m5(dmVGM8UYEr^VgO%HsGBUoEeZ3O=Y_}cG zDXI4>%<;QVk!upxE;Z^wl|W^Sy9d)X#YU2egug^3D4HGm{*Y#m8&~>Ke1eT@Wqc@B z!Fi2E-KD^&+>d#1HfORY&D|JxpL&ByILqHTGHQ6r@O@ts649wtR|rAT$N$VKb=en%}gR zBDvX(G()IcMlvv&Rk+bGn#<*@<-+GZAl{KE^d)k)&GLxD2_g=WO^h&tSR7MxR>tA*;klFoo)DyG2?4`$VZuI+oO?0 zJ=&2xNq!l~d{O*Px`*(N=tjJ}9XAL_*v?IPzxs=9)z9NbN*ByXQYvJPHLLcf=CWsC zvW=8YVZ;>&Wl)=WtcD%w?Q7!*K_MLUUV*ThGpoZXoW2=OYz|D0VIK;*hcND*P{(am zD(ADF#^s=dvMm2|&v4E)p~|)5BT-OOEU50JcJSu8k|}==CUEu>_Rsr`i#Tzr0@6K` z@2pX9v&%7N6sgmP@G(nzTWaDY_T_l;zNNo*3lCj;)&g3BqdY^j*G}$i1_8Up-iw#W z*xq~0q}kKPFFyCaWotBDw+gQ)^S5=Bj0xwI*_GB~0b`9<(vaH1Ss6PDvpKdjv*ibK zoOUUD^|=uFz}=(=f#O)pmCx%5F8Y@lBJPt83dhc-Yz9)(N&bAegneFWqa4Csa|&~a zgfFd2U1{Z9vgupT?#o|>jXH%3s%MR`8qAl z$&YzQKNJH(!#S;Z9hQI4!yLwa`hHlCA&udl`CC9dXWYUV-LRp#JI4^iDEUs?EdEx) zdSWWsTRzp?yc9&@8bFedSZWD^wv>JO_w$u!n*_h=Q^Yw6*l1y9yv^Z{c1^6E{OiMG zONRVV%pj$BD}4KGrRFS(IZOC4tq;XupE8l?TytF|i41xQNEV_g=}PE*jc+Wu?v1&_ zyb}?m|9qw>S43L3C@B{`@p>>rGI8etFZ`Ow&z-im;(Kos6O}X?tj5TYPF(xEcnT(R zbASIa4|};(%-2OlVrNHMas7+AOW}O4JZf9xzJ7bR?+1nZ8&`zS)Pn;Bn`bCZY26oX z+Ux!>*bZd;@+gbWZlJLW=e)$rCy3me;Nhi-5Np(VgAYts)kC)Bc|RD^DYwaMP+hH4 zd*q>Tv?V7maTzjsij-(J9G`?QrB>pt!7nJ#hz)*WgQez%vrVoaTW^cj z-)dUuR0%y3UB6JjejJkQnRqS0D#8PPk(f^Jx#K(VmnHA(SoPI^w5ls@LKv1}kq`!dF3tf~1qzmLpuY8l|! zjD1pD^Ev9Je1D+-=Y_Wd&3#u|a0s~&<;}R?9eLJxx}y8fgPI|BG^#p3k8{r@e$w|Q zp363y((jGrjLu`}kM`d31hYw}kn~CuwJ=LZIWcKAD;&q=%E1Twtr1!sY2FdhbM;ZL z>3lNmLoxj&s`r0j_}>;M88Qn~hX@`*cdww6mAo9iRygD<6yH`|5>hlpC=S5xoP(wW>k-`-0h{<9+HqQ+yo$S=*g zua7!;9;;W?S8Goe@g(8s^eg1?pzHkv%~r~ZqfqNV8pH9MSD4$R)&yX(re*3cZw3(2+CSMX`}feo+KwGY)mx(17RV@4Vti zIysRcxHEYn|IVJ)GKoY7l0a{U$!9Yk03kkfuHE7fBb^lDRy$7d1djjmsb_*JYVEsI zEVM;d*JW_Qz|I%2e$^-gG4-}`{wF@Y4QDu~IwFXl&@)?w=j=<$t9)uldBeQ@Qo9S& zz6!LTzKC)HFL5fW7q*0!R%t1!l)+UOopIY1YB}sRjaWX2!8>N;;iE#2e$Zj>!Z0%L zQaM^rrJp=Z`8MJcxBk=|rXQfww$gX$VD_ca*56>Enq|CU|4FN}C%4PMoRZ^0Oe&AV zkjfK_Ni2iEis2Q?D(RhT)|;x}iOjx03iQp9zKpG_<(xhP?@{ex$zT86MI@v1z8L5+ zz)htaU8(`E*WGs1*L6mM+MGaygIU02fBY3@tp&N#qOI|^g48|E91IIcR^b#O8vr?Y(*6=l06EGaaok@8$PB% z_BZFe4H{3?L6S})^Wc;6wEasK%=DL6eblTITNgW(?B&czINjg4GIKS zvK?NuI?&nkGeer4Bjas2By#yCdg85nNHqGY4{V3J~X=ai--C zSWys3Xnreeqg6EfLcn(wvFC0Dj3-M6fnJyge82xp5x-IcFDVcK(LPpZtciy{2yLw;vFAACkl3}mlL4QMPx|$h0r?oblyIGK$U}bON3kgWR?TKOAIppTg z0nRjw6%Mc8sU-OW=^j`XN8To%uaXC$+{cjYlO2$FOitP82r5h(gzm@MBzHdfVb4yo z2*}v^EjA^zL|>0L%qGcEsgRg%Qm&dapG!>&)||oLq2btjwJ07Z&2Huc3;d_!ItAH7 zg{AYG_rrsvY4-`$9|)3Ze=_>qbsBV+fb~L0FRTw6A#!|?NnBFeM%rHv+sCK(Tbd9- zO$D<;{s;B0O706$8;|UWe>B|`4NkYMsH`ptDTw{rq160cFpz&E+q%(;_#FP)oRz&? zo+;^MQGz=$u%39wexk4Vd!K{tzSwBtIPPL2n6Q)Nt;MY-!5r}=G4)zYW?fc^nt#0y zcrl3XnQUK+(uI-@U8r|ls)Le}*HfKaO@dxLLm)8CV?^d`hwwhX;E%tO4=^8$|E9?>0ITmr|jy@dX$mGCCZ!e{v4<>+L{ z`9`5i)KvE%4MvWLSsfrLx)rUNP6OIbFJ8wF1pFyCA31KYWq17=LovV3@b}zs_OSJM zi3xxLE zb55>=tj&S%Y;)AB`rk#*^k#`OhEG1Pt7=i|H^@g`OaT)Lu_O)K^p!a+ zzWa0udW>0{qfRM221UjtEn1HhECw(t{wkT1-Xz+HV^_4_UMl>3-}c~(6whaC>r>#g zB0R77#N4ggZ;7Mqpl~cpLYBgp1j)=|ieyAG?_Dx}zUz~!(clzEGuS91KLcJrQ`V`B zK9ZuOh^Yqd+$wl9@dI;-AL5X zF8omP?p*M$c$sPvoGd_qJz(vCHaiFSC2V``|A)2k4rBindQMNAKkoUa&>GdNtk`d=b zB0L+t038~kikVYR27P})`Sg%bF!IiSntcq@V)y_1Nc!lV!j?Pm9L*xl5|Q?kr=a|4 zmSQMv#5q*_KGRlEZsKo+y;M0b4yqdsj?c20sFQ-E2*a~vm)1WshSCMl%U_OCX>pN* z$Bvj$+dAAEA5y$G{CNZ1%om-uzzr6{(r8f7fRb*>G)fUdpt&*B0whxzeL8k`djblI&57x zIvMvx6x}>Wb+bcb<)#04r7a`3wiR`LK7C?A0*;3XZHPagAMxcx=?I~DP>U)TXC=8s zBHDurlUzyHWN%`&yY zA)#$M7>=4&#j^jjP0_J7ruxbI@C)9`If9*aj8%#TTiOJpkzKkiKUo5by5A4bwYug@ z+y%amuKRy(-};YyD0pRToAQjG>{E34cxfkVyvC<~`~h}GxO>I4m{q)nZ1 z+fb2#Ghuj>M_{@S*&Tdqy!Z~%@tVdhy}(rkmnBiwr&E)us7ljMRLS8|5`U)lh6zA?8jZXkl9X3Z-jR>>XYisfq3#09Rovx1Zq5an&I1?{5AY5a@>vF)4 z@!Db-#BBn=hRb?H{K>FX#e@Mf$^L(ZPArC{$fUE9{?b%71g&;H>4 zxnr8>9K_Tman@{t8{8w=$ zT9<)uP%@WMJ*%->FL@nG6sluT*y6((UV*ZTofq56N&>dVGDF<$*T=sA!zzDTq9|2% zzKBY&-)CjqHUxj8@KJ|X`^(j?wN4GNkFK?sT&HSC$Xf-lT_CS@GdM!n;!<#yv?@|Y z20fBp76ajWbvhbSLd_@!9OXXUyMx-P&STOzYhHs&-55LE_*QzKFrovsyIsp{1GVwm zN+9*7@FGLstNdhkn$|DLVr(5FWS6}M6|VOERvsmcWf=fyxiR{yx&Bk)ims9e9cQnR z6N>sKZnJEYTb6`=bEfydFB1kHhWzTnS4S4E|MPFNN_# z{3(?E^wCmtUO_X;&FeH=q=nJ>snOtBm+bi*RWFtMFiPDa`^Au|Vnr!(VIJ$+ueR~;ukpB88gdUkUUAcL55O~(y z+nRj;A7mR?Ar*rSE5&$KpO)ezLokXPS6Gsd90T!D0t>NhC_*QT4%9;CQ4;JMVAjX6 z%U1PfDswqm3i!%vEa}nf8=4kC=0`zkC@I_j_-mNfyVI>9v(ws*I#;kM|K|(|!2W7m zl%repo|67=AF%B=+k5!fC$Nvo-UgtnR$7=ucUB~AFKAZI`4(x@lnPn{jFG@q1 z-HZt1jrs(tJ5@oC_-CoT{I6SLLjbo2Cg|OrUmO##ODKw z(uo1PI=N-|`@w|%7N-EI7x&wiXE5?}nZj*h+zTaDcd)^5pl+;+Cha&3?fmeL7StTV z_(fVy&)4I|2*kJZ4qske-Q)3=d)1*&5oPl?-61$D&hBLUkrHgg%fuB9f^!L%GSEA+>sp(YQgg(mK|MXAw8!4)Hpd==- z=(|^$wB>2_3d^F&cgT1-5sce6Phq=&mevsWNnnr$yLqz0C>^wxZ3Uh|6nXRuQ0ex& zBy_GyKfqY@JIyByZ~#t-0lmbEv%9s6mP?kzQl=LrywG&W{^+}mqZ<%(QGCp`Uo zk^8r_*<{P_K@iC;-GR3vvGx+{_iV4_&k(i^hO6{3f0iq7-ZY`@a!>b=pO81w7D-FU z=(}Nzpj~t2I?X-F+9E47Xo}F%XS2lk0Sd!8;!?-!^_g0}Mpx+*6!5twFgNFFH3ooJ zihHQ)pELzeTUJ!yMlEIgdGDqr+_%1)iaBEPTh}rW#d5@$+TK}kRLzS+E&{%}4s`(V z{)wtB!CP7;pK{oO`A#9<m4H6Dvw8&Fa$aDk0uT06$}$3hQz)+gyNpv-~~ zwU!@v5M;pDl){SEL5E-R{ZSThB^%#)LKXYA6W0fCQ>4&6UiZ<=O5x254A6Srk?#6) z50!}nPY{FmJ?51ipGa1Hgx1CdlPKVoofnC;cxm_h1swf*+$Vu>TGCkS3Snyso$9dV z@i_ktX!OvLgZz;M*IRzv`#cy$cmg!%u3?`?N@dg-yG!a`lJzYvpEf@3RD*bDTMvj> zHin|28H)JuLa^P7R4y6Eg>I@^PZ!`5LRe2l5eBC_ack3dwH*VFo(3{C`m6z@9d}*< zgMZ4-Kmg>9@Y?tsg;yE)QFgahzLO2Kz$p8>>_%%_f4{Je$WCA!%Uz@CB2W}P?O+@a zA8=CpN|RHxcT2WPs#UL*I6UYorgl6TO^Rz59Pr-^ed@zwNHs$emdH#f6kXndl%M}3 zI=%~qs-qU+#CdA(}cHIMFCbiyaK&^7jl!acly|AGl{ac*jxh) z2hIk)u0ai^u!%)@#CsOWjXv|wqZGDXh%7+#dpr!f3-x?9XjtcR2ymJQ_Bvm5*?6~2 zGSO7%b}%(P-(jhWp(We1(jO^V??)tO`ThP(m9vI5<@eHD^&i0k{m(N{!o(RXm!qsL z)*2%7@OI$>o0l3zq%T~Z@Id2{9Y_#LpUPL0f=A(PLg|}Y_DVv}3phnB=}=9_!7 zxyG`RXA`9H~XBuNhY)0dbt;&M|12 zOrj*pH4CPobg5Nr4c1Y$Mcu@iuMdvjo4iuix~1S*yUjOQ0^v2$Q257V+|$|_9H=+Z z!VkV`YrGKQD7C^`9NpHNy^KJ!gg(|R1F4XF(lFMpOSj?IjOmi-6(PO8PPxH28 zv7t{O%(4Clek+;Z=xue!PaE3Fh zspUuUqgh7Dll6XgrLUmvPy6%RtG)SG{VP1n3}QrzthP?b$JnoW#xBoHNKX#e#8~lf zPMXqo8n;zZo7UdeTL!nE^pgD*A>BUkAn&bO;i$98pIL6vo(R#nIWnPq?J)ZZw2-?v zRt6nZoez4DLcpL|+Ncey)!O=~KmOf&fi5h<8D>648NMaDLMkQm4v&384zQt`-`;Mlo+w^41BSLbu3u0Yp;X(1LUzL} zdo&aV#Ezw6F6XW?`H8G4>7Da6>GN%BwYam2X*(lFBh!-6KC7Adw=uPOUoF&YQScyy zeV6vkM2`BDqg|byaA*g|&RP!wYE%;Y(^AI~0|noUh6%z-Ga&)*G-EISylr_KUmMNu zY!vz`@>PD>_tx{ZXk3k#%h-f&G@acCs3uD<7XGO`l(L^loK-k6sP^mb=3j~Ozou9o zYBFTos==uAr!lQ4qZZIhYWnpfgpfP1mSO;GgzSs!vmc=Q+Y2|#&ebWvH?cd=v)s@leq!~ruPtl)$lLWuUz~q`*kW%o9iRu8GYMswH zcB1S}F;yQ&QOz^{k{7Qv5&7Jrum6&KjyS`6bSTb%_z|FLO`xE=v<+t#@@~PEJlycQ|TD=+a(<2ToGnwfiD<+ zw2+%%K-`2u6}La=Qd;NcY8N){c8yUkIW)|N2UU31-X&RXev&bp40T+s zto5z`983HRPIR_u3*zo2hCblMw4mw0GU{Tzh7kduZ!h;KdD3@JUQN z%X~}DzrK`NlUfgS`6_L8szA%?%M;bMo>&AD&w*W8RIGTmucTTxL9@B&g+T6wD(wDv zj|e}NSUm_XBE!rB2d~n!vT^Z5GNiW<4=B_3RsvsLwB&n&uNY3!`pLG^Q+r!1MCH}xzk=KMWn6XDVLZsyo%NC!MOwSu>IV{_CF?YplLyorf zH5I)V0@FJth2k_N&$6yG4 zG%v{{O@1Zk10Kz|k9zM@GEFpmRDA_%g?=iG39XYP7ke*RRT1Y&V+%Bvk5rJgxYHS(2h;b~`G9 zP^9&aOrc-TxWWp9T!ieUZ~t3RBJzfNMqnW|^2${A{yO5+XrL!sPSaooI9vvc{xPOW z49x!Hc{PJl4Ag%cAsDQDr|!fGtqlZamiH%qQx5h9B9E=c9b5~0AjnP6R&WxxAv$y) zoI)ckre8v(*c90ekK0CR4HC@ETHpHe2&a(u``a@K+VmpUEImli9<+5&g4`}jUsyuh z4kD9cbx~$5SOmYTpkA;1Xm$Xjo zje8lzRVMB5y|pvArI)orGtGArj}^{uXGG1jp_BRLZGso7@9OqNZTYZ}e5YbU`J~Ik zrb6M#G9!;_#!<~b@&a)7*nSZBTq~Cz;?aF}vYeK$&15_97|mf%B+Mzv0^VbwlxZ6y zG?TWeyu{3XYj%tYt!_C_fU#9;zq$GFTVQP%ulyMli>l_i4u)_5C0@_zQjIVk)S%Vk zj-VpiZuImCRm>A9K}RRNPW=A+#vG&Vnfp4oQ=gBbAfxlt4W5M<*B&s{%}7hLNy1?6 z9ewh&2Q^Z&d&_3H4Q#zh3l6kw+S-~@%0$8H!N436%-%X1PC4+ z>IxM^AG%D_pOJ)rphLSHA0F(Yu4H2zVN=uxIhBU*J=6{igLfjZ;O}Igr3)iicWSt> z46=!w%<<$=%7aNg5ee4B8t~>tit6@}_>j4=zSMlE1WM5qgy^hfF3IFkZ*+u@PvI5( z^0J}@KY>7&GscXsm-+1KV=MX`P3_}*B4>;cG=oL-qrGDfKAVXnzAr4}VfA5> z#vKKbzO!V@j4!diBZD{7G#6j`&ASIwld(ggJ8>vt{63|75B{tJp7oF7!1~e`pJBG1 zO3#t)<;iU`IS=*fX|&&J&1X#;oDAEAcd$*u?)1{K26)eKf2g!kPq}S5%_O$i-9=SO zdX-)e4y7HozNqP?=pClM^`AwaS~{9z^eKS3fkrhJTO~+{@&#HW%;b|5(c;4 zdQDa4Fbl}pUcNW~&-8=Vg%(B*A(y+G$!yv;;)bR`zk-49W>d)wlc>S`76JnHUeO0K z{C&wFkCnZHa_c>Zs^a-5HH|)F0CfdnRd;JAjL@U>jg`PBdV@Mk)ZR5`ShG&`vCLov%Cx;gmrtT(gJ8+Dtz`b#W3R;ieVN8#G$O4e~ zi$R**t}TnuAI-Tlp0)nzLlV4~(t;R%v8vf#ziRr!AwRF%fLjLt43*q~yegxk<1Zcr zUcYEq4}1Z#z^cd>#i$mO*4rz8+m8%}Ylbkst)qm#Pe(rj4S06=E+S8kD&(w@r+P#p z*0&GQ2Unotw!$A{?@N$OiDHw&A#SEe-_S(7!vBXOmWB8sFxXCAaZ|YV`RR7iM3DO&QcTi2epBRv+Wbh++%jpJjNtZPcf(U_&Z!e z6&=y6+DwiCDe4%|wRIiICqFYUW&rC z2QH+Hm5kv}i6Z&(c220d>MVoC{*;of4e?q^8JhwcCRXNUH4g~8BH+M`Wfc6#_c(9* z6jNf9PyAk(WMA8~uAqqsp1huDV%U~y^tq#?=J{$7Esj%he%4`^pc?C2a0GzUY`jah z_$Uqn3vr9QA&u?*G_|73EUtMEdkvhj`52^6Sd(&jaa0R{pX#$~NXl2;T)Un%BiFpH z^A2i7z9XLwnilYDy08=v44UAlorT!INP3f zVfHl^DO^{Dr?uWPr_R=w8*boVUbBWN_&3(-)^&~9vuqbPwuflGKP`rhCxnHYi0+K{oM%BCA5C z_oKe9$(jk`yc`6l;(1$15_Y3L(FMfLxUCOLk}I%)d!+B>p>9GmETC6$inr4)`;<$ z&ZCDc6%9j5oDZ}3rC+zk_F|u(KD=SvRb4s#5NJ0j2kU3D%@UHOC#ek`xCL0%3 zWIZg2)IROAIvo(jH$3%EK2Y@u&*vs<&mpC^ zt?=_i$xYwq-!TNvOVRh}QRLFAC*^pC%mr5Wp<_U2uJ>!!XW&x81*y_R#V&ywU8taq8<$U=zYAhR@Aztv)7$$+pRZcqRZ<*i>*`R3JeT5T`9ESdz&nU;b(8CdRra4=0P1ZT zh(P|Z0;wQm4~G{B%;qHUjQJ08Rmg^_`3sj%?jL!g_9{#76;#KhtHT;fs4lu1T98Fp zO@kECkw4nki;1NX9Oj;_LvV(3et4R%K=;(=UffGAPS?DD7OVh!=@+a%yz$diE!R>g zlw9#LOyf{TcFnod>v4uMOYWV`_vyJ6-U>1OSbUP-DT)^)E$lSGq)OvjSo;OxbxwOj5$r~Unk9%Cr0VFv4rhkf zGYqIczHHmwFX9IYy|{U6_|bW)1X~AZ#GjsIkTJiF%RB{uRk&iuSnr_V9glM@52$yB z^_|16$YG$u2aml3-bE_eC45d3x2dp?;^BgEoOD9*l2Ggh3w?-#c-zE||Fvy+2K zQ?f9$R-Jh4&@Bq2n$kP(9f!BioTfj=+GLC3n3?8N-h3t@6AOU&8_+;3b*2eDI++@O z)|x58FO+V?>rc?`*nX&H-BO#tcbmCO{hs9lqyFb^ z0zu3AZYv;h%uqc1S*2GK7*R@ zNJnbvi3KA*`56X4+Ey^z_$R=wgsCxu}c$u0tG>I{Ej7pJ??jXp(2{V zT3BC+IHBJ`4aKqq1D+wTmB(UN7%r}f8e&e)aJXeVGFLls1+men6+(;Ko7~CD&zlwM zlzM&%%0HXkv77jBq#F-tgn{>kY51f~;)k+2b4m@kzu;~>`tSG^v2kcX zI{|4s^F9`+i}+vhJu?&F&nhh@UN{qWy3>9mQdLXMR7mN>(UaV!y42(Q8Y>o~Uh7&F zowk@RI4Ambe(q$B`vA%^XEck*Y5iqP%r9$(LW+I;uND~Bg0ZEpUmx4}%E1PhDteni z>!0E(;DZ-&{O!9A2)1sDqcuF0Ug`mu3-l;W8N&nZd=FhlzLXRp8Qb>7OI|eoPG7wM z>x5=1e=i!oEaJRtk@56pY~kEvOs&H(9>fa36{eIs#FssYhWC;Q7^VpNdJw@+E91H{ z9(hpz2A*PF=k)-0qUZ!?l8bOIGeQWYCC}l3nXpj@O|525;WgsATZ!bgdW#6cXVu46 z2I>sKUh5?UKHFNiCcl8S0jKDDCJnK@=RPdpGK*)Q3P@?u@Kh ziDt4p4)NUTmRuIAAxaF8! zXU25m2Zc3reqjuMGloBp-vr< zd*@#2>D$t+I@h=^tt^4wrlufnp^98nM4n`a*WEd9@N$F}NLnR-d^h+U%GXxe(9k7Z z*c<7Z+D1y^9(yt>x*93*yVLv6s8?IkQb{2T^kVxH)3aES-g}qtAO6UeyqMwTuLfD4 z?p6I_DX)u1Aa~xh>&nd{b53OGh=PepOE{)rv~*Hc%&0T;Y;g}Vz>ERXh^-S|(Rdl( z_+?(MrpH;?64F|kuqhRf7LwxFlm0JpjSMVtRV&0Qlh90m0OKFs0skCngkD%vNn`Dob9T#Bp#yV)t(7t0{yl7Xij!2jj~>9FV0w zd>u&T%O^j?+tdcJbkr?3I_GSm8E^9xZP^whN~ zpP*K&`x@4A48O)R-$jQvLddzsRNPMAv|L`V2;HPDM+!Ywo9OfU2#_S^LM1UpY895a zJGOMtmlaus@C6VAQU`kMpLiE^Ty-c`KUk4am-}bNJH2emllq!GRgcTY;jAmRb)#!| zom&40NPn4Zs?QP22(!Wp7MHo=DtRjyL2P+Nc-MK)f)N`KVPj9c=wz{m~=)-;$6BNOT8--24e&?nsc-^dXlROb()i)UR3 z6BQ6JH(-}=YVPV`M@lA&(1n)IQ7mXf;h3-aHNo zE0&6RC9cmW#}>6^_-pwBWMeHE(%4nEo<>=wx#KC~BI0qu_J|Sr8@Ico$#w?>1%Q@; zrJuFtIpSm&<<8;;L^)KQX=_82V1Wv?axj5J>#egrENDua^m;6H*u{Ns?k_E-JU#yut zTb58youbpv+IrXRnUhg%AZg*na1aS@h;Xym^S;qN11YpN=9vU!_|3E_a%3l|x9w}y zPPQ)srA5aQWSfE!6G|1*{Lhum5-6<{7$#xhb~=04>9-0yOjTF_s{0Wb&wGa99qd`2mz?{ zO3pxHEY%TP4UH(4;Kwxp&`i92FKY1!CCSKo<`E${Dcj!b6CV+@^%GX1{jEKe&_Q{I zOHG25w+VN2sJL}yA`63< zmLBsMH@B;ywnsE&V_1Bdx05+yYKw1x;bgDF1Ni40u7e^6j8(*N1`d3~O-pn*41YP71TS zXxO#Nf30?4kS5N8Uie_VUX@0{1PMRI7~dRkR1$mi=^J^*GI=)kr7{nYNrmj%-MDA| zitCNHB`!<|FCvs>b+K*X=U)J#hI;0XNNyaTnP+m0pOBL(pYY3$0=(sT|J=b}OVKl8 z+Z{>4feV?ZEeH8&CGr5UwNxc~-TOVDR@;X(F{^6_*;D9N4P;fddIWqZ#!O6lcNqly zt58YztUuaLaA8l5G3f(UuSidjDn*@nJ|Qo?=`TY31-DPMkJ1ppdC&*sJd@FnY#b%m zIL3PwNOqQ}c@Ocl`>z{E5Wnvu%zcYX#r2dXe#by9eO&Q>u=kfyQMPaVF0Oz`N=i3U z0+NE#jnaseNVkB1N(loZCEeYf(j^St1JaFvfOHHD-T!Nz@7})`|2O(-zu0T9#ahpL zT(f3wt~u|UE6(F{e0~aXnoSES$sKGxlNPQny|SY?DKE>uGWo;ehLbB8R3_;g>Bu^w z25V1qQ6-1p$JCw@U2t2$bioXsyQSm>|e-?kW^4drm86aQu#7;NN|em zA@#qgE69XF9d)On{fo)qn?wk>3ZBQ{!TtANklPv5QK%>;@jp~PzBd`T3X|8L{zW`u zv;*ph!kzG66bW=FJm4z23O4>l1JOE-M7T~Ksr^IcOQ4y6s|Zpb`6r`E1rey`otGK^ zQ2AdlI>9~lFgwfi??X(7VhXGdlhe@>e}A}u`{Ek`cr44w_Wb?6mO$%P0VPzy;{7id z@&8xIN}}FK8NduOrXC4eYTQ2g?A?ZOwWZwxLnd_Z5QMwRfD^4I+wx>5CwsI8PoG=R z|6bX+c)i!uW$B2uk$!t+=^XC<%hd7nYM|Rx>CXB4|KJg#LzSQxf+&ys8xE3>8XI%q z0|QFkf-U2(6qPS*p2Qt&Cy9X;WiP&)nC7ua$m?4bc_Fqq2JS_SE3mQ|^aRl?7TN}G z=FVfvg699l-~)i-)90T^WON2eD(u{xws_To^L=M5o6+}s^AX3OrI_)uTd+A&9}p)q zAiF>58oXwR2KWUmL8!fFEiyxz(zd)Uhlfnp1G$yGgFFR`DMGU46l|9NoyA?zgRM=- z2|!(~%nl}emeJJIpx`sviT|*Yzv!2w$d{*ud~ybTQ)c`jAVz-97VMmdh+74PBj&vj z(diruM&Kl+FYUO*a-ta*w+HpXvy6=aBoQ%WaJ{8HwUCW1(@I2#2ih#%X8KstQ?8MX zB1p8DI3EzT9XmYXZUAf-1x=Yxn)SWXPXeTW0~=)D1HcyS0BS|K7IZ)lEvw9&1H6FO z{nrXjyy#ITUaC`R_%!q=my2_j&ev(HFrQ>-lku&ey6SIaytOk(UNh%Ud+Gj%P1Q}| z1SIn{7aRh7-w!6y488-)MSw_DfnZzKOF*hbeFstJGf1eOAtWC9hv{WUbxp-%)FIVU z&~rG0T++DA^bU=Aih&b`U>BrbGbrr-uI22{7~^orj;YNG8eA153eF&N@MFA?bbHXr zDU#oCoANI78|k=*lnP+`JX3fPvRvk|${wBRZ0-f}pe?~C=2s#AWd^L3wM8tE)&}sA z#kK%r=2c+s3qR?kbIY>RwXi=*g?q4EA^{Y2nkytjgtRCwyXSFa=!a&9c?r);Q6Fw=^ zUN9nC!Ycu0bts!U$fpG~!hqZt=tuI{JD~aucPfM?qwbimGR41*()T=H^?N`l`t|tI z$Y6cA;G&MnbzVRV#b;dGG_-)mK=IBsB6;y@4QD>EFPrALAWPM#WzKdJPRI<|N5WEe zUlXR?KR-1yexCCoZzMGe2pt67cWn%lg+$B3X+I9|_+_QVo86KRxytQcCh=M6AJ#PTWCS3-5k4i%_OKeSLZR`Pz?Q({ej1$ZPu z_n$Tf8r;*MjTgIP7kW2~lw9a;%z7DE)+pXH+VVy=^E)HmvdFnHz&zlAcT&QE`)hVs z%fM|lt7FjU>ux!U%>De-*DT80ISmdMBDWwLvVyzRxL&bOZ}DQ%7HAw^{ztBWAqM1%Hgx&n z40sB7vb99}?g@Ek6ZnrnPG<5&UI-6SeS-Piwv5G!+DSv7*n>_o?u}(Zx44t#5B;2^ zqGdiEEZkOU8?WMc8Z43ENk~DVV8^s~_O}Z^;I5^nHNSWTZb3;*a-xf?4a~H&PL^(! zZ5?bBa64K$4@=(vd6D-@sIYWXn${JdRqRv>ChANOb=Ktqx=pz|_vvOskboU8-30_B zcf6erNASw2KcM#C5Hz8cD^UwWY+H4`UGcwY5?ytD6JSWpo(0`VvA#`P+f z7aLTmgvkGtm8RY`Qk@YBXpu=@i}MD5Kj8{1a$5K*ADsA9i}3Gqpy;B_$?T!qhh z>E|FTUEdid2yT<300$B2cElAxoX4>;WY$bk=_)R?1b_tFgCf_~3ARGePVN$yhT@9I z{uTF4f(Rw#6OsWjAXE8~roy0LhoA}-h0c-ZnmY}10QoC?S6ilezdMnOheuR%K`5wVQzTnt4v3v62W7#=_;d1rZJG;a$Hk^>~8(( zf_uJ7_|+Nmu5Tyro*g1PC+oWct?wCD;-=qfDuW|oYvTStym%omzVyr%Ng9DJz;bd;{MnD({R{A4LS*^&gA8*PT#ca9*1&!r1 zv+0n#8>qwa+*S7(&!9a=jAyp!KL0ScTG4j)2XBgu`K%Kj&M zA@vq9BvQ%o0t9+5-BC7rfXd~<8F`}zkAyrEeC3~1;fiF}V1o_~1`{p2Guwj)_+-`1 z6sjD@K#4!WF24PADx};tpoF(7lgX~0!!$;Fer9E|> zl%b{vmZ^Fvo_}C|Q5vNQ1s=!04y&L7w5wWf-&MlGT9J+KdBLma9-?ux7=IK36In!- zFst@{#=6yY*n7HgS1K>YaEDX2H%Tam7eC>JsKVM=Qu>NV1NAcVP3z1AJi+jPigum*imn!!G_ z9`?LyaD8k)=Gav1dR=MnjBnec^lg&c8eh8Z!tMX;3!XZV6!w6%l;l6Cz(o+^E@twM z`AQ2>_|4MD1TL>j0{cH21eK1_*+t*Tp)~GosOK+PI+rZAL;8{e)e!xn!3~(0pjNDP ze7FUao%fu^jb5-be*I8oujKH8#{*O+A#M@E&_R%X??cc_{aA6qLmNB_^v;D{a1w}grjGhTEzHlV1KZ)J z$So|lw3D(NQSVp$F5VtWJ`u8$sj*B3b zdw0hOG%SwU&PQcrLDRiS^GkdG5)W7u3SsJQ3=~kKPuQcoY8bCU{`_~dR?pak<*)u~ zR%F$eYG`W<%zOae;>v>QH9o1yjUH(F;o^ZHFLa-yS{GG=y#-yXeCOljp+3Lg?DaJf z$P=p4^2A7-_j*U!4@CGO3RRi`2N!Q`pE@oit+nUGBb}Yl{YK2Ad5-|-COWLvF1E|K zM&qs}bGa7sT+Q^DSZP9EHjBw1Mr%blBq(b3AEl^cERrs=xQQL(_AIx#D2sf=yPn?6 zq-JbB-^fe+W#ecyv?2W4<06(9uBPQrO@1pfY6Lt-oMrYVUzj=1ZVn6OJk(aj6xc`O zKhfkU*+dJ@#-*E^$wOB{UqkH`Jd$3ox{Z?LlWl!XsNj9*IpN4oRFp>9*aBP(Jg^4A zlLru*%Vp9Q9Z&6Zey+X@>x?m-A<Re0b0uaI=bY@`zx3Y&R%&H9PPEb3|PwkE{w z#y4&J;YFU2Y}J2$X{38PX-~VLXk7f-5Z=)lh?sZ^L5%2_pH~MdN$mK-P({{hl5=m!qvJ*jVNag9Hb>x^#b30vp5q^#9Z@bSu zZdRl6HAg|*ie2WX)Q{4hzh${Tt2>H@atQTTT$0}-(dd4YQY<&ALxp&%z9y39Zz@*Z z?BUWF_iPorVd%%ggNzbHHfMc!J@>o&e(tQu>3QFyhwq#5t=vLsH@gF@4nVy4a&Jf= zl_09(K!j|{7DYBI9f!vpI6VM=5GGSj*U~(MauSQ|S<fzWqb(nX$%lgIh>eOr0V;wRviZFj!8A{=&|=U;-&1ug$W zcFC+ht~*O+S^MYm_49Pqd@bLXGgQ$Y*U7r8!b=C=BUwz$Bm72y>ETqssZ7*mv~KMf z3$CN9NqXynm21-Z5GJ}+l<-G`Gv*xao6u^A%^ur8OC#ga`X9uGbVsXz8gGEz}{m4Sok9*XXTCE;CLlXj~`rX z54*;tNi1v?6Ec^GSF7}sc?sR%lOD!r2q&vQvrV49U#NpYivT0minaooq|wr_Tg9~w z6&u-(hpi%`um!(qXIyAV$b3MX3G&FR-Cv*8&NaZ;w}mI+&0CW+wYTYIW!eT+VA|v> zIZ(Z{G)B0R-bG)F8+%V5fggD~_Np`O{-wib9}nC_avZvf?On1(+iA73;*@WAi^9gf z-A@#eN%FmE9kmLX3JK8+3#?tj{12b;20iBR`u^58mJJdb4ALnhYn;V?El(3DBtu7P zuW-^u{{vW>>0Lm8z);XTqR9pNw)Zz+CHHF0TDUuo;M2K~q8t{9q|g<8i7 zkBPj!tlQ}%JijDCR&(A3h&arw}BHN9Y z(5zo8m>%_26~QW*PcIR4Pu~>-H;09V1>sn{0EW5HwdhMhl%oTvWT3P6L_E*Dbaa01 zXuwIXU+UH&p_o&>K8y4xqU8e{2M@lmXX8Dc+xP|YO$9f{{MF*PX#VPk${p@QQ_n8E z*|p-%zr!`M*)g3oZr+^y5KC!e&;7rc6terC`JW~r`70oCUI(wK?snP0I$Nhg#?r1r ztyc%&G9A98I&V;exXIeQm#N|E;un5lf>WP)%#%cu-G7A3QOi+i$T+5{vl;B93HWnp zn6J}mO^Pynw1W`+e6gf z=1m_lR;tR#D({b>yUU_^&&zaBkr90Y!0cRefGh*_wc;aSFS;O zFzZ_nJ2BU>{i@C5guDG|iXzbJ)!mL4yOgw8iwXH((xZY-k4EF<6XKv;Lv8zQw2auH z_YT@4%to2;+=jTCC5d;Se2h3(LSL7JjCyFjJjiJuQz58eQyvxPx*AJ<)Y-3B&EV~m z%}7zqGkli3z2)okWOQS+%r4pvS#8Yz7tPOMG_DCB@8F^>Mmz%&xFELWjf6_E9P#y7 znBAd&E-ief`f3N3S$2uUUJB3+nP1W9T(m#TMNMHNEoj-HS`^d}{```>@=4RtC_*eG zJ%@df)jUJ{0gYoPfmnSOPVG7_MLf45?yJ+mpibW#qYHgg>C~4^L=-*GTa4PielR=C zD3hXaGxf6awID_vqT`AXx02CInSMo6C1+57GMRBe)_q*v_N}ff{lm|^!5o_M6V!EG zJ{IDSVGJRUE7a?KiHd`Z_w-YSB3+C~C?h9go6;Gc@T+}ZYYqt^-@C4vr4{oo5>Y=r zYN#yP@|-1jh8;zFo;?_Kq-ykbvp-o!U+zLHbaw0ZJ83w6BSmG=sO3nO>}nl3g|djg zIN`UG41goRUQvz=o=VXU;9PFBUVR$hcsE-NAiH>75q^%o#=oxIvavDE zvfo^M_O{09p)69{@s(nk;C-AphJ9!;`yA(C=5IH%^^WdH?L5C=Z=wfSdX0idwiPQr z;0v*$jN!@LFJ*rq+ts;+=C#S~u_DLQ;(27wLS|dOe2MKb0AZq@gI;%RyMRHO@t-l+Irar~JA~lD zmEi=hA1m)Wy#8j|(i{v)2IdoIwFCctKWx&$AM?UkIuJ^+_{AJQIN1QEmC}7Ob0SL2 zy~QPfaHeV6>%88*%-ky@Y$^CbzE)g1_2ZUNE%(s|Y{835Q{%(uvY+LK8J{RbkrhOm z+)z2N7`_zbOml5rR*>;FLuCJf9SDYTVdO44!Zi&fT$sJIh1?22wOZx~HcWbp6w{I0 zj^^n(?|eCV>YTB{@}f@AZa|REU^!ZVJ-gp!h@2%!nA+=Gf0-k93d~Q1WqAjXb(@s$ zD&M|G%09QeKukwTdq$wyNprF38|y({=*^deYZzWi^O$8qvDpbcO?xGlo+Kt#91H)J z%7z7Nmj{IE9E#BM<#$5kq`T7C#?I7CQfY~U;@)Qx4u#>p>uoQd-jjXa8!#P^1J!6G zqlX?at{vMRJiyg;s0s^6_Z%oarI7?PZ~f5-Ka?q*E@CI&bvNFgT1>{LI|~Wz@!LlB zevjrPA9XdpBoiB?Pe-QZm_s`Vp6a`vRRi)QxQdp1b82-nD&XpkxaM}BE>+U|CYGjK zXN}lsyt+Von>YRDazAyNRhv-FcfnR5(KYs1L21a*Ta8OL)qVlD4liLyTzX;sDFg~~ z8)sV#XAX`Od1z{$zM*1v+fKFL zaqn~p(F&Y)IgXF&2KO-kla7Y4 zhT(PH6)QbxprZgonXkz%-a>@}bV=}2R$+s^qqz2e^#J1<{h`3!{KlZ*U4XV<=ErM- z0vM5|s`ab5^GnFQ>E=+J7#1*iEa8)BUR+GC{Ia-q89f`|8#>CbStRnV1$OAomCH+> z`wF*MlTT#I7_C*md@`PO9@cCCgd+C5hy%w82yt2ntiAP19T8&&nX~oKCD@s=iYJ+xK8VT6>!Z+J`c{p&d#+;D zyPR^0j@9(8<15oshW?)5O(Ik4ejaymc-LxvXAs*t#|@dpj}}jPHb~XlM z^%aK9e=d-{v2B*qwO~yLglK@H~UrWsoL-(2@U-tNSGH`je4d(`!EdU zGAgE;j76T=*nt%!hha^jJLiLF=&^^9f+!ImvD?YgQBmQksgQ9$`ycxB?i(7z``sb4 zG0C-Z#7j?~np{}o*racwLdaD;+ctepNW#b_nIA2L%;NUj9EM@%XFXVP2m7G09NRIr zOHILt_Arj=o)nXY%F{79|HE1LwQBEOVXMZ{U_8~!d5qVqfv@uf?`iHS$Ne zaQacO{C8yWdY&tkrij~@-dE5PCYvU6^xkqWGS;Hu2$oHDvB(h4d zcZC&vBPc{DuKey9YIF^!lzt{WT0>6Yzxwls&-K7F$tq0KGdglA!aq7s@+ZI znsIeIZQgXP7f-(~hE4(Lsg5hxN}H{hR-W&oVl;0QHyf&2O7fC;oC*~vlD+?JGKkp! zTE^97lcTak+DI4{79?a76A9DrL)+NRdl^!YG7{WeegzXl`wx)+bO~DW{!Uy&>7NcX zC|aW$9x*MTl4L_-jp%CaW9@7k^c&L&l$BbvVzsZ{7-V*Y`tfaXuKpmGHd>#3im8K3i*ap}MK zFqefRINC!{7H+I?mR`nC`27x*=lGmNXy1#lAw*J1!9wwoWid5koSdSGC&M}a^~wkL z@KaPmHSSy$b%tj>y4GlC)iNAfxi=XwU8g8)LfBciT}1pQ=CZp(Sp49@g2bx4NC13& zE*`{Ss}Pmj^S%D+D6f6ee5LT*>3~W+No6}((hT}4mZd!bH=Dy$h|@;H6eIpf5o|y! z-cHTZRntN!TNdkHS>cn7M?8ITF|1WrU9uH@c4H5lc~z*X+*%iCGFX1I2Yl-lN~9J& z-0IghkE&BGkVj8rS80c$dy_q2oxCyUC-kXQzNOWcDi1H`hP*T4-k9Xk|GxGX0~}q~ ziC1ig+EaI`kfcjHcZLA&nCeqmpMbEyr(e|W_@h_~FLo!_2lM_+qYNfJIQnQY*%eyI ziwt)**`mUZ|9+ZRcdz>S;P|`D59lkg^T|M;w=be7z7<+1rj$e`UN0cP5WKQr8fP6$ zW^CkmG&YxuqkEI;0mW!kfvOYxI4Ye9d|&Bz*~sOzrU>$|!_4D=}L69Ad`zAMNkzB}t^-$dWPKF553u$j(n zmTosj3tqk3Y#l8m9jD(DHzE>~>1h^no7d2t`?9$*tQS;(k(Cp+wQ(2~70}GJX32cR z+_dVpXfICA3gE84&sY^(T;cHC!dl(@N_Ad1WCgjROZ3{#6F$nGDG*e;iXp79ez+@y z;1`ibZBCsMP-$)c3E(~UOW9zO*iG7x8x>?uY>J#Xc9)E5u`E{_voYqA>7|rUtp`7P zjQLEKiG|IkD;gziZl|wZO0pKVH zy&wd6K{~&~Jz52%3bEyyy_&(%pS#4=nD|zY@QI~DM!Cb3B8iAJWU7}eu~Gyyu#=)5 zJC&V2E(HU8idf*bcw{<@vl8(RNM>GjO|8!Rg4j3hr*uwoyiqlL<~Ll zYO0$%-)_L^vZVqeQQ3=A4eX24lkL12h^Ag1XN5~U>%(oF%<4KDD1E=Ui`s&wBi0a4 zds|vKqDLne05=B|G4j#XGHpf&2mU2b+*m!<_v?_9%>t#CMi7ViYKFGXxZCu{@0uaT zT>^lu+3!j)+w2w8W(mQm+=euKG>qmQg&DzQLKHIp$J4+=|Kn+B*~Lw#Lbr8DSJ@Rr z6j`QoI%6(#0^%icbe(C<>jk$o(CPJ8Mj!oO%%v7dq=jZTahXSGD2FkH=!&e)5^;Z= z*3#seCb_jkLb-@6wJ7E?uDes(%vwyV@b5FT<$P{tYLn%DESTrZk0^0zVWB?7=S=(DD&0hpz+F zM)onCd%4k8Zn5r=6!0K%^#tOu=i)-GAzqp79Ok{to_u?)&ueKg(0PQ)a&psazkFJU zaN0*0{NW7gVV(SGWxS5Ln1gMPoxM3Jv=BG+ZOCmsH6Gf;8P05-2zJ5xq*;d!Tr)H% zwo_c;c5-fX3w-wA6!W7Fw^aXVD5tgEa=@7ZINW!x|8-#vZ4dg1kg>YMYM&NGjVRN+ z**C@-i%9{=mQ0EZb2ty3A)ty7+(S6qLhN3gO&+3C%=Xpb>7vhUJl7Q=d&MdGn_N$O zjH`tDaUuX4U@x21^B~8IQ3sz-8yEH02n?s30)yx0^7d+6l`kfR@2E|>I+M5n>6k4Z zh{aP~(9zHo)Jdq#6|08tfmrry!;?YOt+1@C4piSx2`LO;e8QwQ34F;Qe4MYH=-ue( z_g)9x8~KujPS7SEK}IPjW9w@gg8L8+7Yj>A?Ji}(3-w_P6beO|FkAxcTgckJ!J_a? zlig`8{PO4XMHl$q;NYNGqYG@V+j&atN1Rt}b(c>>LGeU#g!hrYTi+=WR8{PN=2GnG zQ{ODrL!JrqyL+2YZ{ORF3Hb9B8`;wrxZxotUmRwBPmpIjPM64b`8peG-49j2v!LtN zM>&vV+6cXxoucu6FJ}1e>quG6rR=r2`%|qwElk%R(hUnavFdB*C2wkqPHYcxp6YVV zr9l`wD2s!|4l!NZI1hPniVg`)HKKc3sl!pz$8SGVT!@9BB^_&bxTH zfBnvg?|U@uQqIjmq2tJf)9T?FBe!oZPErSjo=}Zq+LIe6krGy{J@`skf=m9)m5F5P zB#5KMA9pDAF4el@VoHBFY-^h{{_5s%NL;)dE_2`j6d_`^xV5S44|TbP5WS2x=7hf^7zp~Sj6STk;{zse`Ow=hUwp&i{UXc!Z>Jf)@RhgGPa zB|jBeggow;iyzx@Z+Jm>+*>74`$1?`T`sb#Mi>&H_Jq=2Vd^g#TjbE-%G=5D@{ewUzx|E{*0W?POg-|b=U%9rwb8H2c=-%ds_;eRY7c`)K3Dd8*6V|-YD-I-b6x%Z@!Z%`*zC#px6V>= zoOFZVVZ7;TZYAE`?mN;OZDdS2s()6fR#Xi`<3EoWHlz*fyUc3}$Mf#LUp^hGlEvKe zdw3N6-IxCOczewS?x@%jPKP5+!Y6YO=lV-8jUbvd?Vx!nS6NCIJ~bTkyX2b1y5*;c zEJ<#x`4HK}g8MA*nkux;_3Crkov_?)(%r?-!Gf=#G_!2=)-EX)u)771+(RC2mT8sf zsJk0J+mklebhLlG;w6{r)^c7sB&Z*q&|FlIZt4~3=#!>liI&7x2@|88eQ!g124NtQ z-2BReyOe25(_8V0W^sl(bfJp-i+}dDF%HZXPZ!6IX+`BiS$Hr(TzSh&gJG6N0A+IF z52#pYuIwMplUl26l~?unKB^&!NO4X+opb9%R4(~K6{)=~~Ri(=tE%qdxU zpu15Sf%%mpF79raTP--u^FNmxYg3vg<;{G%8^_nBl)!7QU?+SXrEJyy^&YmAK;cWg ziCQG35z&S7z_dH0a4sJ6vK39XFYFMTT$_F60?}hq|VMnc69_;zg<2P=c4x5 zZ==`X?UH6m_xPX#v|F5dBp2*vu!33Vh4_*KTx62*u9`(4H8MwS*7<@-1L~{zKj8kVJz@s->f#-NWb zIDl8uDMKWkH&eYAUDa`_M7K(cIc7gVIxhxGH=pR#oM@ceW$#zRN}}2$v$#vA%h6CJ zR@%z_dz*U^xWgS$Op22~*6!Y?!Y|Oa@S|A_4i#K@X6Dm0dv5MJg=M{@IJw~udRdC& z_^Qw`bfw?S4PK67hHIsn=W$>q4s#pu>b0;_edwjH>36w!V6N~Z0uxI*4_Xo%d&;E2 zd*3rVF6mnCO1R(N0pwSG%eNrPb^twglb+aGX%LlU0Zyex|E5^pH1c>FiA(pR%-2&3sfy4$xmIj z=QH-JuR}l_rvEXckVpC|58Ts^=QM@(RN}5n0kL=OlI{R&|y_d6L~H=Ubm}MNTMQseI#Kx z#j#qkLgkOJOmUR6+lkbEaD8_wLsjiv>X&PQbLugd{6ee~(8i_P8?$ab;xv~S;52QS z4%s@f&K5_bXd=?AHvb-G$(yT{AZTykMHlkB*!l2$)-^-Z<47CnvMHX4EyKLq?I{2k z1M`CR5qVmD7rBQ+0SE7yZEace`$FQ{4|-?~UF__t<-at$tioa)#pZUD)?N%u__xaZcMaq|LV(j|YTA-JT&{Lq!$N}SvWVfF0TFbt(z6b`+_2H%DbcSR9V#Iz;5Qf4S~oY2(Tc>^PNpF<`5smuz5P!FprHHUcebYEm$cJI7x#L|CG3DYEgJ3(6#Lb6V!x}ez;*#ESAwdaJYWV;5ny zs!NzRGQH-Pxb=rRYfooI-=Xc^L7mm=A?C+lu1)KiE!qpp3-(?Q;J~Qapb`ToYB}bs zipXAoh4a>v&_6sg6Rzp3bMZcuQ~3J*^!+}EXmr+P%}Jg*W#12-Pir+T9M3{(O7#Wp zu!SLlLe}EJdlm7Qkhd2{O?LS-)m109A)M*e0kU=lh95baNG6k9H*z85E#eo=at3&* z<6T<|8%^)VPeHWiAGkKY}jCgzL z)30{XVL4N4w;pPjKl2~#Xm>QQ%}pg*f?bMUU83#OC6Y436pFu!65JoL1;B@i2E*8 zv&;!|Sa4Vs|0gpSy&*%p-0UE|9$l11By{Lc>5;tkV@5n{ZGmKi#-Lr8Nad^DHzd*G zW|;<3dA~9EEC+F-nUxi)tQt{JT+NsKq-R0jVn=5n5j$&*BZ0`A5*^_gva79Qy26w) z{dbr}^tJ0A_XZ9C{*_pv5@wwD-q%X8vi{ge7mZbgj~)zKr>eM{?kwlwmFW;>Jt-p335$gaIKsO9$BwNGu0 z`m1?fJq{7G5{6Dk9b#{Jvx9cX0ZhW}wzaIVt%#6KxMzcPP6!N`?KNIYU`cie1mB=l zwk2J~KJ_Hz#LwBUwmkJDH`*mT`lM>iowD2XA%;DR@&acMJm9h>JBj46XlY~MZwZ<6v~fV(?4rT=_nD_wE#)yC z9ih(HWi!r%>*fk&U#55u&v(Js$-%)W*B}kWJejUUQpzLS;>ZRIgEN zCQAYhIzx8PG=MjT>;a0aCBZ{FJ|Ud3d{w+E%b^F9cGV*euRSqr0(+=;nIc1LXDpIy zyCGX%0sG#?Fa~ZI2Hg}x11T?kw~wodW+V(IosiETrKzpnQyo&uRym4o4Un4*cQg!_ z1YLw26S@?a6o(xMxpE>MfOfnyzCyIKjhj#U|AisIxpSAT68$%Q0TzSWJ#X{=I6saO zH4ZF1ev>am_7AbMIrwFJr5<;ivNn9XA8MWu^|!aP< z!pzjl<7AVB#w?LpuD{SAK2j1vC6aE0MFAMEW$5(igv_whyN_S$^dsA!vB8dy;xt%8 zb)-E1_&7NCjk=R&$X_msm@tB&7(evEzd%Pv(O-yluXD<5{sry&|K9lj4C(*dheAR* z@)%4~kE&=IYX4Gh$T4YUWhe(|^3x_gw#p=iSLpu$+;|sZzKGO(svi3oumgyTv?-Y;pHBCP2_Q)>JL{bnUh*yS!sq$o*KBF3UOm6B zzcGff?i^A~6vp?I0)Oc~@N+yv#zlIIdE}=$V;#*Qtr{KRM08q>Er9zth`kg8;CbsS zG8;%)%QTp3ntI*-S{D;cTkLlySDspiG;S3YziC+2PHgM4~iAOujENrhhvno4P|fBl_GS@x;ir7spZ={$oAu*^|1($1ZL5u{Uc4 z*iz!R|TaYNkhZT**HJXolG&hcmH<)z<)Wa+(K)dtB~dR43F#9`(hDZ+%v(y-IqH9o!s4SkjZ=<9zb+$@ShJ`&_=s0C%}>gdc8mS*w0CNS z4jKTx1qk6@!rgwGz)SMvEg9~I$>#~Xt*ivzCXA~fnSM52^-kA>q5&f6%Ylme=WV@3 ziZoDHWgqK<)|*r^XX#P&nfHM-C?J@t`5EkT=aniDA9a9vqGj?06)y_1waj(#azVTJ zX?Hs73eO*oXt1vpu>dHr1Ba6t%c)dOSuN@ZcHbm&(GoZgDNUoyFf)Y=bbQI$^^XBK zXHorn{n_Mfe>@jnmBr7>odb=a5+N$cBKVW$72e8TdDirseuS0&Tir`;l4%_N=j##D z53D~Wg$1^%5$&&t`tq~tZ)oqe+}^MboP*nq432kqQ5&dmoML#8Fe0jfbX=I&#YQ<` zlH|)u@E{w~HDcR2(JC-aXX6Tgh91JHMR2y>0kAyM(jRL{8>^i>o*k^%2oaN=U!ERm z4(*r~_G`1gQLUz*fv*G;yp8?QQQPNqU&{wDiS&(ARj`9%b^X(mj3~$iJQPBg*!GDl z50}2SD;2&$!RS+yYNx^L%#zvQ5%Itz+93jaLLddz5rjzWK15oexSh7f9YB6X=Lcm8 z7huNn?Z6ZNm0kpwmT0XD*P53*T;kpJjx8J7!#|s>Ykn9@tR)09_dLWCJ2N_x%k!!_ z0>UQ3aQce(Q{#5u*2^!1F~EZi zq)dH${TWVYFMB!Gv?t{eJ8M6aNzG||SCNu3UG{S%v*y~(m!>|qrucBEu~Y(lfAN^y zWhNXOqAvA0#Jc+Ei>u4mBEOL&-wt09@}H>HI#+Li+{i*CnwBJ0uyCYR)B^Z+@V&z@ z@!O3yM{{cq3HFTd!HK?}^9_mcy#?W!j)G~rv;g6S#F3!~QC~Do8gI_|6YD~xX`u|k zZLP%y$3-`qa@r)`cOQLW@GA$f10ylT0vn&jK*2=J3ajeA!XsqouQ8wj%c$3}wSJN$ z;f9+)%Q&t9mKI`znv{X!End;gnVBP*VaA5L61ci7yl@cau;fSD&Ed*K4e-mjCU`n4 zloMTO#p1%DY~u`Moe|#4_oaG7;Oq#N;)N5J!G>2RM{CCSrTBc;d_PsA38Gn(P~Y{# zR(O)Tg#iVUv(Q7TEH!?_cyHq}K!3>}wot8{6Rmx#nddn8wZF%-v}D}EC|o+!#yq^| zQE%(cJqjARNKZSD8Az9c~eLq(d!%kxbKq4qdPTyQ?M zvboyLWa(}IIDScfbK*9ouSCf(Y>whv!F)w^$VsSpV#M0}%rL0rNeBGH!9cKEsjCSA zPjS&BS*_$p1Qf&1vz0re{1V``^$BPzcI~g739KUTv_EO_M}gsS{Zb6B@DIJ#`SvsA zc~u=_f&Sd8A>>UI6G)7 z?|ND1PWDF?n=v>d>0DK~kCJYQ2Ob-t^}zTZ!M7+KpVFBNVdpflOGV@#*O`-y78jAq zvb|7r40E|>!%*LhNYBU@WUf}aURt$f{f%)anu@If5VP+J?#|y0?x_ZXbYFgpp~E{S zoGfUA770H1v+NCTn!*r%&w2#1-C|xNB|dwH?cXp_9}}uuLA~u~*1xaQp)@`E-Q-M0 z%6lSnAi2mOtHqTrb38Q*Hf~K?THnU<$W)CNc})Ci`nl%GTqooF4X0w_2w0<@f39U% zkzKQ{?AIFNf=wxqT^(0So^%lJgEbm)?AVDX2V;2zo$+xmdFPyj9du+zrJbjM3~*(&hy#hM6IV zHIP^R`jB0#^E3Bz6(TIwl6~#cx4KoM^mxZ|1t&q;&@9u1%^JBmXL8v%%pBJTH$M)- z2!y`LB=Cu_xUK(ehXrJB{+S7@wamHP<+^)8=-y(jcZgnr6M>m1r|^9oQo!7fk+c-u z!akVKwJiA68yyBwM4qkK+0pwF%3>8EsrivPzr&5OwV`x-*`wp=6QtNJj3_{ai7M7q zpGhMkCx|2YYFIE6jYC1~?aUOgCzd0lR?R1F1Tcr|_!i2as|x2IJ|Vn`s#$pjcV+Jh z$dlIT$y!kGQT<<88SNM1*y9pB;VCm#lBe7fHP!o3TnW)8erJQ0n|2mDbhS8AIdtj8w3&?YDg90m z#~OBTtu0k=%8Q_qy()P3L*z6KXx7#rlBD|kz-5|el_LFcg78b8J7=5sArwqN>OB^~xY}^>E4GhMur*oyg}6b1FtWf)Ll8g86}sC#|ZmXm|s? z6{MXgO-;BH@+N_U!d}QXD`uVZOBTvq(-B`xCbF*z=yQcqou#Hqw!+s+4`Zq7J|-Qg z^i$Z)nG@@N+GnkL(fO`UW?hv<)g+?5>bQ@+a_9N^%8mO%)o~B_MbN$PbfZ`Q$N?*+6mv49Bg#YgcT??# zxUDkt#r7xtH(p*!3JG_jM=gIJZo0dpmESj$v>25aLqu+n51EaLG$2(hXp`NKqmd@e zDJjzh*OgE|qJG|~;U5!(8*3i*1)>Ef4_+s+Na!h!WB;KPJHn$8@V(EJg<|K(2u}jD zhO+Qxmw;9ze{Hp4V*J)^57<-fO{%eH5ppm^krqx{E`OE~T|^X~cmhSS{Db&^Dq|tIGFDxBkiPht%`hu zN>F;j$0tN`&^oZ9?6UZd`|^-SmyY%;L(B{Qda=y63=7Y-E_{WVc6^v<1={swhaLDN z(nQ^zf!p;4U-}$B?2B(J7CX11L1dR0`uL~+){_HYr^uFSUwlIIB|pGz^uzu03|E{6 zH>JM{3QqBC($J|+eVl4vx;oAr8X23QRgWhR66B`%rDUnZ*s-nZbj~5dCx_m%SN>KI z4~!4;YcygIu^31QOW(DEXf?pw7+WP0#X3arG3~yTFV=aN1j;5SO*Nkj^V+{XIj@~# z5219Upy#fvVbdAg26lrCE>F4^Mk?8}m-8PP${ zOHD5ZlEa@ZfBQv=fS{`jCa|;DsJ4cDn8j$xPpy48BIf0ZdoB(dpN`krA8G#ct}tB;a#bRZ80&GeXBWQrr-6Et&}D>m z^w}Fk(8FZ9yl6DUg@>D?8RYB{tnIKrz**v(!mkg6C@dU1FIayQh((erLMD$8eJ*qh zHguY1tA5v+5Auu-jUP>x=xQ7)NhC_Hz`tMPqjX^O-+%RZqv`hMvLia}D906t31g^B zZ*!QQ;SYb^B9JVxb?>aB4OI}pb8>L%`hkwo@pl(F;adu6^ zS=@bEHpx|dEOPWk<6?J*A{@d5rhc*qo>R)D7rxCLI2>#YI|-C11G~2qHp}xr(rPT- zDu41KA0}~vdskqYO~kC6^tzE}En{FRaJd3RY~G)JsmS`v>i@^yTSis6_TR#Sh@^BW z(%mK9NOyNhOGzv`rKBW8y1S&MOS%MUK|;EuOVV@Qd%Vv(&WH1C$N6x+{l7W3%f0S( z-BLrdRD{;bqeYkR?Eh&m%Lj18n$P;z`unUQ4h&jc?AcBc9 zR~f7L{e!M2I1VPBFTZDV4Ej|{+B)2VMAJ{q+{**IOqEaYMTj?&b^AYzF=`<_3keWm zz48ghQ@~R)`aamKb_4RE4B3$szG7#DZ?}4$?!G6m|NZ@<-Si@&JU4{9I#D|3WtYJ4 z%+QG~^~Uta+hFEuZyBHq3W&7Q%KY45JTMcMen$N4xbR6HpQof2=abeS<#U#UsL7GI zy6ao5TpvB_ZFJ}H1C6kGL8wF?{ID?&HQrJ#lE0c5e2%V-v<<;m@98YtWnT~RWux5u zE|ewgrJ|`XxWd7n5>9Kg%yxxbwKZOM;n=~03aL8Z(%Ic!YlsUk7gg`y(AKBC`PqbK zCLi5JqD$giFZC1h(>bfJS?~y`eZXTIo5Q=&_W5&V1dX4Y!}X2?$`gIwkuxKyVYiQ` zD@yZ0<6TX&mplE(Zu@*Ln=gAGWyXVsTw{abH=x*aoaJve~j%^f)mq9{3%UU^PXEA$T&DTZ~q!8eO?Z=C|dU?_!im z5Go47I<1KR9Jjvn>be%oKl7}v968ge(c`rluBQnbHXF5I5+F?AU-x)h>qsiHu}CIjjTa$l$angg zX`)4ZVhRov#?6}k&Nv#u^m9=DH$vBHHrh{&0yK2eaO7-c_wU@jC~rw2LArpONvWJ6%*@YDgaI{ z&S3$bWa8zw)2LxVedG7-kzi3|6Vg&K;+xlr)Y;R>hz7pYmu%FPw_FUN#j>|m1*-m; z&vGFzOE?1Soevh8s!2AdUmAD9su!pe7?zrX5_M{^F%%~NT1nV69+Q%Uq1B5drzlIn zY5C`Np7B?AHvu`uI`k*iwj!LQE?x8{!z$rbNQzqRz=MgC z9nLJ#(t%e#VOV2lJCl1#CTJ1x!!yH{<-86Hje40JV=Y?mpuG{=E~J0s88ttb)Drk- zvmh;d^NcWc6)kWjG%N2$-&|PHTaMxy*(|-)6~yfHEm{b^_bwbb5ucx;SacXgSW`16 z!t#v}{ftmfnKz~H&}8vkOj|(PcH{E3rwkbG;d)+ar@5NpX zXml&0lNy_Ig%*IQzLtH^{TKlalyz?9QVfk>-X^L9ex0YN{N4X9@ZuXS=}iCOmH-M@ z0$T9hm*1VN*=d{4>+I1)k;2XPSiouHx#8}GvA~Lhs$Ux<-1{oM$n*0e30xlvVS6m5 zA3nAI80GVu^$n|A^ThjLikEcrL-{O~O#6G!1C2()^zAXc+-LLFDnd*=}>A8@{n8+9goaUWr_}w2`&wziy zbQg1h>BZ@Zt+6m1&aHEu^*!iLRS74NkQ1627Ch9+z7H48LQcMA&Cow@(rflfLns#- zMrD>!%^1=Kn`FWcdgb@}9s|UTCoa;Pn~Cr4dwlAqU%E0!=?lny8u6SNo~iLn?n3@L zVZ&j{w1uO_rW|RNQws_CJ&L3c)G=AZ!rrN_YFKu2#D6e;h+akY7 z^71yB&&6^92CzBAC@z@+8Qo{XtmN)g>m1D_GhYv=&K2S;4#xMjPJpjp1m?Vu@)fY) zoT2{tsajR-cgW)G*XQv3c28N#H;(mAY#o#1OARFx8{X0WLQGtP@WPhX-aHa6$wcNZ z1q)B1wHSxxGe{<8O}Sy~y}3Hs7!Qh)o3Qb4%+~8gZ3m?B1Dd&gp&G9e z=nZ|)|5k|P`muBVv+Y9^D#VOvgaam&7W2V)9Rk*c?wZ8}x#*d_TFf$=#zHN}(Mx4X zQwx3OvLPRCr({9T6MiGhd;Um#W|VSN*=E3V&Fb2YY<;~sxOvw4{+Uo&>*;6KtOvGL zrt9K=BJuoHyPiGU^Nq)l-Z;H+PmrZBi{=un8LK=DlFW{{dQ&+c^o4?l`1A1?K~Q)u z<~CvGoL7l(k3r>i^GY_K$7ge4JM!&q(#fhgEi+Oz;SAa|l@u6r60^Q#(UkARA3YCU z5&Y8O{OCf2)-AKV>b%Q@Q{)=9+RHnMIR7a7GQ{Lw>7+*GFq#-c>sC3UJR-1^7mLsw-Y+|J>q+9CO}VwPKj~9A zc~5-*hN%*YNZ836b?O`{PW1MVkyVr8Y-sTmkt)oVCB|G=CPH|)2KlA9x*io75Zk^2sB{WL%SK zA@8mZR&x7PTc=(hh+#bDJqS~H{-{bn*VMm^5fBZJ2_rzZ(9Yz53w-3%h9w*itn`}$ zR*RPh_1|nUzdYEpiV(c;BxmqtUUAa7gp=}&d(tZDL>a^4FXZbO{lJtPsX^>d)a;Mn z$7?Rib>~hKehVBo%VOhRq>0*DL9(5U{PQI_L~dH+By}!^8}=~)o{wC#D8DrE@E>JT z-198S8qXTpe{3;?Z_lz@Z(7@Oh$LWr6879pXJ2h(hgYc?)RB4G9t&^&w65ew$q^@W zcU;5NRf%y>7+DQ{j(Y>QUY?M=(G)yv-LuI;I*SMDfVTX_Hrp3!<{#rK>+=R`TDs>} zKtpkO1mi_Ybz73`E16E8R{at@$(cV7A3XT;=ORBI@jvlUzcP>)gw(G(F(RnU3H-%% zDQct=3dD=4n9Xg;<>R>JRpT`B6(i&0DL#Iy_(nl-AJ>!hPAySfvOxUq{xrj@!euL? z>9YMW!@cS7&|5g`GNTD--_ox#Q)&grTrxSw(kb#O7X1I*+0o!ad9Y`oi~pk44QyNt zV^CqK;Qgywy6I0atsptbV)PNJJk3X-V&e^jSgIWmnXcsdnH=Vm!+3o@~&8=#1EUH$ZVMsMT({^l^t=w^P_?fsSj)>d}V6FKNg?z#Jr z^tUK*R1?-b@IQ_;R#+#;D9ZBl;=lNAHY@>Azt@25TjaZ2x@_mk0GfE4<6Z(~5l}Pm z_m>IWt@;A4QW<(do^zLgy{!Yj=I+$TfUu&m2VCbyHHQE1?j7ARFRg397{qhWnSVI> zzM1BjDGZwJ-3`IXcbRaGKZ`xEh4{=)+z|eA z(7abs2WAo}b0PUu5+IYZv57sbqBH6t%Z?u@6C@JuPy)0LGmJ3_K58 ze2mXJWrq6>xV!8dNPC_vVnPkJLU6l0fKTq=~fj9MsrLqk?-;fzZQ%?e& z<1F9UF#)hR+c<<;8YlZ)DR2d#dkfSz;oUd5ro18i%S49vu=R8Tg|VD%^8$p5YR&^@ zZg_3Fp3_l#`MkotfS~o~BeRMk#e91zu3jRSxW|YToj^;S)X^GsQ^?as-?oZZNeCR4 z1ov43-OryNdPp~s{@MO;?0dWK=+Af~3Td48y)lm$M-|}hqr$>IVOY|n%*B-fm(dr% zPS0(lTLr@(L<`BscunOslN_u@vK~Z*Z~yQ;O-Oc;rm^_4de&MN}O{zhQy z`xOu$#2P`t(EXVgbw#kg5yQA!cFaKVK{_GeajG2kHSRgE7BrTLDmd-r^@gl8qo$EH zn>wpDGM0yZkrf`p?Gp7V57frm5=njNR0j@%&S*#AX*eJ%=$F?!)Ah1}3ZSky{J3V> zH-;5^Jdv3bP;Duy(g`M6T;-9flGLxB4J+QzDUyevj>NWuI4DKCm#` zD=D-*S+eU<#kCdFy`b@Y4*sS>`-tih5KNc|85Jk_x}hzxbqVYi=ka4mEDJ_`U;SBS zZ1k>^F zJ@%0gOF5Ni{;c8GxFCJ3+&u0{g0s@`SQu={iE?3ur<1T-TjLu&C>r!;DU;{{-KWwV z;~PmIro@aLb?Vts$7{n@fA--v7R`A_OGW8r`c*bj^NZW;K%yh|Joj(5Q9Le4FRys$ zYX)|z&cBR!&d()%04H7LnMPo2tF~xzP(bz5Q2V{oohxw7h#F+e925$jT7<)ph-xcc zvsztpom4DPrGWoZ81Tg$>0#QchYuUPhnK0PS7p^!rgx#l6Fzf=<=W_z1DNrctVs=W z@j$9wm-+`kR>>K*lW7`B{MynZvT@smw_ZoOG3}2fuAMP80<QtKY8GkOvVC?s+cy!Y{Z1|gv(q@o4DXRRVKGQ-LFh5SX(ahNdDI2;d>R57;q8<@x zNabtQT4d|^h>#*<9{jf9HP=EXdrfqOTJ#mlD~#Dtte8Odhef5B?%))s8%B0Vh6uS& z6p>up@BVS~4oT1y<4D9sYA;py2d8A~*M0qk1BbeCDeBPOmZca);XDR+=48~2o3Bq& zWaci0leVbYh!BdslTC-va$oloc0aO=gkq7R8#WE4Jzg5>pc{l?z5@51vdrCEOH0Bb zHp?2Vhxm4No-5e(UVb#Uh3|O*(S@_Gvg9^F!=aya!Ry|n^ObW*&;TiPIPX@_9(gFH z^hbu^`#ytbd$>o(h?}-nYT$?ne8XTK1yWuRKh1!;e(vf6-LC8H^DDzjm zfX*_(mq|K;R)vamT9_@G_4nI?FCBtR;?p#Dxri;|np{ zdvh8O%@tYi(Q+GH)6fefwk0cWEf`u99B7Y|LJLT!^5dfog;i;W@++_0PTRjOADLUE8ExU3|dWA2Z7?2_Z&AO*4w zh$01I`vH2Hm^WxDPv_-ilacGVVZI3!3=7@u(1+;aMHC=m}2Q zjy*vUKlrE*$sd((&j!{8Ze^rTKvfx+n*CqKpD2>^Pn zJ$D|lzs)u{WHd38Og>4%RWH_$|80{0=SgEb^TGUhL9e5z@wX{wq64N}DE0RFU#Hv$ zaZ$;ce3KWde|ywGQ?M|eCT7pd|7~F?pg@i7*kY3{zyUw!f{6{~65xH-pKU1};g0v71@`_kkSHF;kdz_shyA({oc+ zV82d16&QaYHaj@HcRgs@Dtt`p!}bZtS8V_!lA5MEGCg(=PFscyzw4b1@1tCTHtPz@ zm2P>HR)l*XV5Yq%IesS{b*Z){Z=#1BRTT5$6Vdr5c5G)IPE~vI1;ut6+;A~#t}f0B zs7ti3n1%!_72~eSunCd>=@7UE!4Gi|e&z}UuLV%T+dlA@m)N)i;DJ-~e*KS$czKo- zsIxF%+MEsAsEeAamRk_ZY*uRCnNxrnwx0O$U5_{(K~fAJ>gQNFEws0!N2 z^gfW4XR3ZV52P#*YLHaF;5>W=z@nTBOK1cpAxtl`+PckD5cI zkDt4$@PivBsqo6u0G7#-7a(^M;m^f5{!Apb*X6Y?0MYU!NGjGea+}e$Dge*pA4?TQ z8K^7qkBoJdwF^KwC%ETk#KR3pnASzYpad_1z>AF%JZ%GZD7QrkGK|#&fZXY};Z5Gc zV*o|TPc{G%dBv8^^)Y}+UNBd6!hbIE2Tt7ujFqh%h9CxF2b}w|0Orxt1D(s0>Y(zC zUjbbj`ra9j!)V`JZ6pqHhfDtg_o}fvyQ@dESWj*_uk7J!@2i;K5 z;$nQLAQ*86u#*a?%ylYy-us^?fwd#&Ua;Z^BeD-~9$r%z0}M^=j1kn~9R*OJojvIO zumPH9`X7^pKl)xMZml(*RFSdYx_xgSgWi)m48U(oVA5xfg45q1MyFw(VuY~`4CSS1mhF{(9F{; z7wo1%`G6pDk+1utP2L3kwElKf9plR+w8b|@pSKlJ4k`6fG7&?pTXn^L0zA)` z{@37-ObE5G0Z?0+uQ2~HfNk9Azn4V@WXCA+c7-&v4f%fi2jqjw{6({9zR>vBfSWkr ze{X^p^Z_WCYxg|v9|m3_)27Elch2>&Mbh{O@WIe*C}7K>FNx-!r&H~N@FDetiZ$>i zn^`XU>#(y)Y)kV>i9l29-2{-Gs0pRh`3Gaolbsy_K=*|%85E<>*V0J9gd=uqtY<^O#q%>rI=Ia0; zO6Qq}h;%TxCh9)TNTwQ@MVlB|lOg1h6wV@z8dA^oxD0PcATZehZ6+RD4FY+R53a^< z02ZoL=ni^x)$W6xhS|gngx!)Y;mf8CradSWz=Fu+v#btm zT63}iV4O~4PZ(G)jF|%g48xM)Iw8Bkm+%Sg>CQv8;kPMAjmYGknY!*lbR>N@`w2Wp z%TZ*aO^(ewg%(E1e7yqWxb;A`IAP)LcMq^6GjTFJam`j5JJ1|#Q&Ny92U4CSy69-4 z9&Rs_`kswnZ5$|51ac3BHYcA%L$F-U+50cvK{WzHcZ!Z)>U^ktt4$eC;YWI2X>~&# zCnup(Kk6zQExHVU+Z1KY&{HPp9(wc(^qJcw+)$X2ScZat>BCb9#3E;VudCCz)P;l< zgm7K9qPe!C?C&K`(#3&_h^#;xC~~)>Tu{!E)g<8uj(e*TH2*{Gwq4EV7&L_JpDkQ@ z)9`dlzHQgzqkQQmzDFUKu(e5z1+@e_e8>YorIne&Tbe5tRPvSdE@vBsS0Oj4d?BAz zX^q&+Y8XcWL^sc;-AMO~e=EidUX{QPSnwCe{2Y%XKLUwN1*i8}z_E(*gCi$H;0~2a z54yGvJN=#7lfWgUY2 zC#(J1bABDVGJZRN=p}u<0$T9}yyuCYM7M{3?j2`6{T0$h$-n=qD(cWgYHq=1oEc)e zF*n~H2`a|?FOH1$UA zy`GYpkA`t~;|$;}6|%YQCM-e#b;a!+XeHTx8iQwR?wWDgzm7v^>FYie=GJt5r`k`j zB+u`J@@PF-LDE+w!Z(f`gEI)DZv2uyK{KPqr~BE`xAVFLU(?|4d@-#(WY8!O7P}8# zph&z^N)z+7AT)$0ueE5Mv81D+Pk<4^=Il^E(bu5tRVhgtH*%!er~8l~M-c~hDXMOz z4G&FR#Uf0y}G}cz@x2Iwrnc zyWZ{0^@7Xz@ycyGy{o3etF$%oG~`@!|B9X9_EOXoPJJoOb`SUM6^#AePSL66)AyZM7XaBh z!v5^FRwK2S0ifK1F{0D&^7vKdsh&R2 zpsv`xJSqP(H;27#v@dC+2%{NsR-HlZ9df!Z1_LxaWcpLnnU1w(A>rt801f_sloTZ$mMQ zp#eTgJ2O6@|bmU$%rOKC?s$fX}|N67m>tY+YKME z?x!T(|2P(&ZmSsg(R~!N{3$rDcBnDYpGE&5TQ=7T#_2eMTORaTQq7NNocX@ZmD*FV z+S^q%=o}lUw^G}%_q4+7LBP5Cv{(8wf`ejy^cP)sKwT|lh))-y-^1bWDVP~Y#OYa- z&}wQZO?DU}U#Qd^=YFDs7m*0w>9RWjT=@X6U^oji&~KE^m^8N^_PkHU)#bwN)nhZ8 z@abhVQe|K`x!Jn;!coq>Ds1QZWOt<*b0!EWk|=o^csh1@EqqfnA(Yfwp($iCTj~~E zKeVsBOtYnPL3lRJLHqSp&XJy&j2|ligQgLdz@F;9%WYYX%Yx0{k%p_QDqU@_R!+d; z%fDmbe|K&e^J3Tcn{~q|xL{OlJzbY}sCygV!a{}_bMe-#lT8pVICHA|%_3;23{|d@ zF{M~mV&vhDn?VYW59ju#Ah&$*EzA)Gq6zqyIWIXC&;w0}5ev+wx)o7m`Y6Yj&Fn0a z`QbzMMwPB`_dY6Y_vI}Sqy7C8e3--g*iC_#bLKu-z8>HY7XA%jw!5{3dUYEi=43cK~dUH3N zzVBlqu*o1F@TsE(zWLLoaM>(oG09AOvd>kpalAwM=46mI>G08=1pMIxU;YI*?VABu z)5s5>rRwI`{t|}sy;bk>=kR=&;R#>qi_#}bI1hkmq)#E_M(lbow1hIYB}Jk{^5;#j zN_Rdk z((_f{og#T*x#bA<6lf4{d4{JwE`PWtI2MS<62zEUq+{3U@}$@F^9OZJBPZhn z19-NLj9h{WCN+4v2{w~A85o&vz#hr@PIf5$GIRM@m`%MGY(+EyEAWRLhtx2oY8|(f zf>suENySWUG@Mn#Q9O80;tQ9(IhAmq&uKGMk;ZQ-Y56EpQMB=N>NsWYif&bz_rjcv3qT;D>Es&!eSley22C5eveH7*0$HEa5xb#>54|kma*iu2 z#zQz5m{1A7Z;@-Os9JI9if$ZZu4~TX416NRV$+<>$(4ATuHv&nzJPpU$Fu0F?pL~C zb)jaUzyH1OxRinsfjAx1TOGnQU*E`fUm2(jpGv2?>g<+#xvcbAiQLRp)VWA%!JKR; zUajEl@aUeoQ}Yts4dCgv_S4Cw-Iva!28|57_Uf>q%~N>OprxsF!qP7w*Q)Y#L{f4s zZJSn(UF>iOXd!cOy;Oz zIlfiKPRpBU1nS{ES0A{P76I#k3dd&5kDo( zdu=>j(-+ESQ@CAtW)8X^SV`V*xEqSH6DD?OW;N#wuFvq)BPf-&p)K^vbQ&=F0mTdEE$Q`vW9|V6y7qOp73Ta8=&++K{NJ ze@$%3|J@~HeSRbywl7cj6gFX@U6%(&icDo}i$eECvFU3hlPIEP9Xs}SFJ;~_%3LoX zU91wHl$Sbffj3(i>fGW^RCtIZ^*F(*GWHq%%pkUx;T|8d0yhMJDtf%0u!<5j)F>Kw zO7*x-+CW(#G3?tHjsCCy$SCZINT?B!yN9G7H}!k0))fv6J+@VlEkor0v2sc@5CmUV zxaJ6eCXqmcE3RK;bjpk~g{7Nh4G4};bvX4=z6vE?qai}l7nr5FD((9021=<>M5CZl zhW>-X&3JBq7-zP}dqSN}6k-Gwr$tcgvu%MG$g;ag-sw3wCqZrV9P5ZR1vm#e&IHwU zMsXIlJ?eK+k8F0P*2#lAru&#^D|YDdpDW*j3QS&=x4Y4{2HJEkoiE=*JP zk--k-g@cqCzob{)s4JSp2wl+a@$7!uohQA+)g_eaX_(G0j@b&o=GQ)RaY(qyzKYVB zaVbZ=`n3wh8Zx3JdNq})5}NA1|IA)1V-|~QHvbl4h4;Mt6{19!Si14uonYjLq?(4V znEuI*MF)N)X-4ql%U!QOS9F9%-WjB5zWDWW1@GLzr}m5)Gh6L2q$()E+B;071xhKx660uvR zy);+Tl-V4Mxv8K#Ebadwn0bSp`~-fyzw&)#AtmdQ%PK;yBhNbuR0(M`CN{U1QVA`x zy~zw1(B^BBdX3&Id{P3yMaGU+TYsG^YwcIex7LbrAtBiCO+cjcnO)y?DUZDKRWAU= z6`lmPic4U6OSt+*ZGwtU{=>M!jm+ly`rKH=teLr}AS%8SBH1r9`{ZL1TUYbbnIgHbmQmDQ1 zoi;2307)vAhu{1wIQ!mNSv~7I+$Ezno_)BytNDf3p7DMvC?jYweYfYb>@L`dUQ#5e zU4MoT6N|!lTh8Id~V0z*p#5b-;Wj0mDG& zzM$*Hcra;u@qpS2?dR5epdnF21)Q5Heh@U?KX^8e9V#O|QNt)J@1S!d5qcOb>4_Fad@>i>ozm19GOv zSXnQ5=B@KIG^R?bs^mkaZ9SR%HzDwr`SX;TS;d;DDRLYDPuuDMrd-e^*N$Ta@@?ra zBgJeX;CR^gJ6^Oh?I@7!eVZF|m*G6P0-~B+bIp-DxsVAK2*E1RQKpGm`fRKdT-5ct zK~SqPkCEuq4AOKREdtC&jlH{kFZFjdkmMD1VUnZYw7>MKwa`bxn&96nR~J^wK~m@y zb^j@y$3%zI1;mdFO%==D@;y)}G#;nskZtM`p3+JCoAzy{d?*w->QDq(0WYwtksL3m zxaLWL11mzM&64NVezUKsYG%ZHSI?*mD3e?k67x-R3^S>dFk0>vzL+(rU0|JwmVHd( zVKRGu9CRJ-6(PzWk{0ajFZc%_XL?BXYAb#9J_}vgmF7BVb`wBGn1{k1(F7j3w1Ka*Jhi=JXOE{*cOK))EuFkcFnqSGbS`{Pf({>2C4d(y zYhj7H;ZoF*c8RldiC24POG@|W0Ch!rYq2!?*B33MI#q5@QlZ_z=8XoJFwDQ_dcaA~? zs&6q59B#!0-sAB&>|Z}B|Kgbs+sQ*1f%kmDhPoB@m}*A|$I4F86!XPtO&Ys=n*>Ql~>e%J5!MBF6xgo<*pnsVn>A z%13SD2Q_r_t3KTN9oOOY;C`Nj9lCZcuxwxWgj z3J|uz)$UX~gDAxcfpL~@uQf{tiIl(prs-#q2!YMFWo`u)fS}P_CaN}|NeGGo-A=|A zqOCjA<{veD7>bmFb9E4?t=i7V_P1h365babwnF7I=A`gMy=e z@a%rhQZv>`usV8J#iLW*oW}azHH{ZN*X3&|ahi%jM7?!EO;!)CDX~I7#z^to+v~e9 z6OyV{DrvXtP~6|3Lq#0cz$02F{AI>(?5Gxx@oa?YGK(otmsjuj-!%>I z-{FA+V;*=)5qAk8uTCw24Nmqf**{ka(>cDn3zA#S)rF!B$srglw3Jpw)O;ctJH>=G zRy=ZHSOgL!1Spda4ww$?hr6v_4D*SeXY0xSg-9KDAQVC+*w;23CHWvj9#>N$g37@ zpD}dl?C6@o#b4ahvmvt~N=>^QUX2{j4yPL*MLU)zsVXiV)F>H}fTa~L(yy>QHk`eq zA7N#^d-IK3+Ed-3yh#&%fFgJW(PX@p5UC$U;d{;iX@gcdo!)69X&`Ysspg5sZV)qn-my8eS$;t z22`f7cAoZ@t_)p!J-S$(m;CKtOHV9vBgzdhjT=dbT^uc~dxhpksU-1JqR(ZlbsPZl z&VJSJMbeA=Tz|Jiu*XGHw;fnE9SWX;bm{ZkcxO_?-`2E>Oe_>tN3Zf_!>EB8d38N< zW_6YugQ3Rx>DrI13LXpzn~>6y|oCr>tyi>I4s z*w==udsY6etg_u}Bxti{UJOrnfao|@`nk+<{!`~RMF*xAQ9gBEF{b_vg#)E}_!Nzz zzflOw2Tw!c%O;Fi2ts}7Dw6^b?j^`7e{UyKT8jk?T=xG=QR4ro<2pa5^#~aGBx{@V znFTw>Noj;O*@`h(AqeMxz1zb_joOoOk0oLZ>Pg#!ksj5l|ny@V-}j+ecgR$u0Tx5fMEnLRtvxkCWzw4cwrgr4tg8 z_|*K>Y=+=wl7{iFI!f%y6P&&uBQz7bk1LLRo5+6m49%9&WQnpouyeMNzRQ{|nQ}*B zV0-%t|8+%h$xMUb7w~F5FK(}r#iCckt*qcqfI(Wpo_?j?+D3&{2~P2|ZFjSyfNb0( z9d^-G!5|z70ff@GR`ait)eG%h9^wpT&%Ql@@jYGtQPMJMHYrV}y)fyXKbog3^(V#8 zXlbkB2H!>bkwHv6sTqU8rTXPYVw2ApOiFd~@~rk|)ILGcp-ya>I8Oo|Wi&ZMzG;TY zZp5LPb8Y8D_hZ7tHO=eA#@~IP)G7ccWDdh01LcvW6f#<7B=@y&-CzQzf8F>1sddM( zvsyX&B#gz98ZA@8OwQy4x#zpANK?;meJ|s=} zt_Y-j1}T}`N*ptOmrw3_L8D#~!MhQ_*Kk?w8PoBml?`E_2^d*u-Q@``Rq^nBIwMlnujT?iwDgJ~ZS zf*Wt&Umg2oJrY3WaofR)@p$RiXaJH9Czba26aVNq_2Lab$^{-dgST&*9?K*mVu*@t%h<&`iws-W_pMCXfdVvXSX&bA1z){HcJfWi~pcAr6 z(?B}(TCxG|_jErN`vo`I%~qoUmpsxv^7Ennxi2EUW1lJH`DVq2Ud!EpjdX=xlh=jh z<kMeI7$P;1D;4e-No|`e1sx!(_zg z{%VVMzP#%j1B-0P7juRLBcc9`0lc_lEsJiPk1=izOB0}V;bkKO76sU4O*t0Rk`Zk1=D(h{f2$Pna8Hk}7uCu*jeBw<_E_NJ1HSi}T=-zPy&lL#A5f(~6RQWHhlvUl2Y0LRv% zWS}my3+{)IXZ5&;`hFRptfogOK19*hvv7-;6SJUgp?928uS1_6<=@A($L(4+0>?yl zNXxwQk(xb$#}e?G+$q3z1QMawKDS+xrhGr?%KKUp0?#^TK}l5OV3loaoh8bC!~7gJ zrCM;*l*<20)lMNllVl7`y2_5Cqz7-6uODb`aQSlWN?eDUo@pjccA<(g2omML!q=L$ z+3k9TXhm;r3>tu%N3YCYBcbVYk0%uw6(jc16S6UP+@4IU-)D-$_c9Gkczk*@E3jDs z8Z1ubsa+S7`6yzbo1v)&bTf=d(Xn`D^r;e(!McJjl_c0f@ex~=3qfG)GTGjj#$L8m z1aXq^FXU^BujwrRBqBzhp}GVqn|t@_MVB1;B}Js@ZFsQrc!_{A2HV+WqW_2wqnBw? z!{bmcB&n!^i@k8Aje-;;zrXR5weqJp86WLs-mj||i15KXNXCx#XWwTYANK?HK(U)C zR++r}TJYG(i159**F8)Hz+O~Z>JvLI3jb z;Rt`olE4CH5NtTEEKxHzEI1>ZinHdZew+hm7Ew8y1gdQlo|X=I&Oc&fUEiL} z4y-=NQ8j$7Dc|%@3vI=W5(l063~VKW!L(#d;WC`AN^dw8Sy3RctH6zq#PZ?+e#6;$ zACC1+393pW)T^dUSA)hG)LCUAk)k$x`d-CYMO&pWG({qy=1XU2I}dQhqX0b(Q=LS_ z8|?#~7<@q#PbM#O4kdLdX>-Qv)K<_Di3-g=5ovT1TyFM5&i$a67;`GMGGNaunj9b=iiieH z#U9{>@bBYpG7q(qTu`t*{L2SL;bGv*r&W)W|MI&bp&}6Q?n{~={`&xd2irv|9w^OQ zW&hiJ3$y_ec-#5NPPBg&%0u5bg|$sr16htXpZ@(F#ft7BkN}IzOGR}_E#sF`|2|e= zZdj**w_hKgn)$au5M>0T_4)1i%fF7nJPnxk_PK?$e|xFlAjR<~6E3X3j#fRS?oTV; z`RlZ=1G;D}Mnc7xf14)R_Fz8E)4cZj>n!;Msryl+Z~*uA??3&2hVpN-?f+sZJEK(0 zg@AU`2HfEsZ{`$~LF*?wJj6>55s!g3ZeJI_W#7vJ0PJQ*T}D|6QbLfcZ)Hj;()#|{ zq_XnqsDPu_+^|t1zLCdDN%MLBVpz@9{A_z7FrohN4<>su#Hc*qwpkeL2%4x|!O>gz zVn&B@A6(>JCi9dBYoYcXWytx~3p7M&+@pJ|5C3nXchC}uM!A>04D`)^1kgFSQ(KC! z>#&mv15Mkd4|qvDdwZO>HKDoHhwZVQ6E*>xHB5rK3;z)@Lv7DHpx^x9`@1z>XV9}+ z0Q8wc%+)>EHj&Krg#8~UpTYESbv5oGJ>eu|wtfb*!W;mJ@MlDYTiLD!%H%p=*Qm&f zcgO;G31qn4x4{lcIrya@7(+-QSTwnacL;jMQvzNz>JsAKaLx##i~PQcfCdkxAPO<0~# zX5c@}g^(FsF0>P})&@E)Bs%^|(4guJS*-e$P%-)cu9W!cy?kTTydE#l-{HNVy*MT9 z@*gjNZ%aUk%C9=3XZy(+wCrChSNiVT3iQ)9n|UfG7{~#+w&yiuXsrSd-LRo6_3O0> z$NWW5boj*q`}VAlQfMI6Nh!A+>eI`IoaR5zzIgtUZ?J*D6a-~u#XY~m0C($fFOP3W zBUZwk}+`_~a zO#GHeFf387M+lOT!?-H|*$_}YL64+9=P;mxjN*goe)`@VD($`e)nGJP|C{e8_Uj zNvKRx&OSN%mf^rS@N^TkAD@Z}AYlUk$`~|Ug1xHdl=A-=06d$_mQcO}<>@eY-=H#C zGUT|=KV&}ox!`Q%aI#PMEjz-`pwLpRN8*RzZ@4o6g%B;9eGBDlWX=fH#js6Ac~KZUv>;~-l2$&lpZ*5$^^yaFYew3DRicNe zxcFGb^4)aw?|U!INN;qMEvR9~P+ZY?OFfvSS&&kG(#6NM2TM91TKf-Y2@H9T`oAG~ zU_c(;nf4q*yWp~3b72Hom5GnfvDT62|8kE}75oE;i99Z{o;=B5NPAr1WO6&&KzP)8 zJzY790ow;X)}uSXR$mS>7MMN$4RoFE35e;6zHCD~AS}VvzT?Hj8^*vAs!jzX+)Zgx zU)QbrF|V0g?7o@MajGLAy5InY=A^skBpX7APIPNw$P6`z*fVOnmz z{hCrffHM0QB9?}?Qqjc$YM3$WG@?{7pl6w4F#$aMg=v)h0>0pCu_7SoxpFmaf}&T9 zHu(7$2{v7(Pk;ZF>baky_Wha7}*BfK>RmK9;;%R$8XW-V09s zACpXFZ-CmdGF=f^a@+w+G4veFok>59%E}hwOw07Fk_e{j9wc7x5Kw%z1c!!8ilCfC=ZjIuSbd_Dui;t**id-+S+$+~Y;u5)vc=Tt{~em5O-Bu3Kog9Q zI9bGrBt_^dJDSyZ@6S_bTMN_CV@WmRK29b#-4Hsrtvx^7P7`A)Ns$yCK&PeI%h*lJ zY2pG*=@rYG06H4rL>Jybd0Q4-^9=I|p#7yJ9WGK#I#^v<2sVO-J%qo;V@ZNTNbrw| z$1jq3UNjg5DH0u4=d;4Z2erd4U&upkfaH6`9dHIz271ySz$+u`e-ZZ9QB{5I+CL)F z-JMc`q;yJ4cPO1AUD6<_!lqm4l#uR{+;o?8w}5oRJNNTD&-p#$ea|@G{}`}*T;w{;MO@z>;*|g|7kEJZkGGd2Vf~J`NdLxc3!6Xg6o-W`G zLtFjMzZyY<6!wL*HnHt0&L?N|9cWM28)`Sg!N6gv2eu`3IjL)s68s5hxmP>be^)PK z+FVx+H7=0q^VEQ*l|&AXg;)O%_D+{scQr`8aNEQW;0&xM9@G*_=GKduI;XDM_#9TU z6*x$s=Zfe&^KnSW(^3__ffg-E-}6_N5vd(Z<;~*A=@3Hpk`>~l$!Cgb@`^3=P*q)C zI>$C-RFjpH0rusueb+GcYhHO$(BKzoy-^leYFZxGXa2J{IZEFO!%7TCaJON$50xUD zmoVOTL!v`7uvXqz%=vH?5aO$|mWNkiL_ zSLWgzxZn|$viNP)&1n>n>!!6f*`Z>v95h)0NOZEB>B*d4kjO8SD^9`p+<|gP& zWc_U&ZZbOjq*#4MI4^QH%f0*@I*`x2_SqXpA0a=xB{`}g&@?}%>7&hiHCcf;4BEEQ zuw8mY1F8byLIlolE}KrUyBVZ}*TtqD{2Glmzou(@1hzTO)Usxf1d*q;#G^dSm8$hH zjw&zDvn%54TkRE(>?j|=zmpQ3t}`olzuoaun6TR7kSce7ioJ=qGH)9cLGm5$<<_q- z@1#xT2`PHAB#gD4BUwgUEWf+DtWnr2ai!cqM_l;A)q~xG2$nc<;+=I@xr&O2*C|%L5`G1##O5x9je0OtZJz6yZW(`Vf?V|!172oJW0J97Q)pcQ`g zy>bLYR^JE@7fQSgLcB6^Bu739y>l8Z%VXmA=7G%mhp{L>&HP)KkI_$1X>LBi>{O~( zL&*|v>)*(j^s#1q=$)4|rym@CrHJta8V;6qYrRKWO?1AKsU~hRTKr5{Q8NK#?{i;c zlUy{s7ry>cCJIo6qs24cdgx~Rhu;WpSG)-F(2 zv8%dI#H9z}LO&Tl2_dpa=!pHM%#&zvO&zD~k{kCCcS)gY=G~32o86oFa6I@V`2#AGLWC>e@-!rjFXT#|21`@3je9(<3p>f- z1-#l2`2l7?bt0aJ=`9FvCRK;t4R8N*Z19dY(&kF%Ak$!H=?ar!5406#)7}& z37*Vw!hC_u#v49=>QzCOlhKL^N#KxBlb8|@t`Ss?#>S4;DARdr^FBm8;ez(E*eGKE zs2$}GsCzBG50rc>M=dkF{{^_3ynVMwjYyw$C7KM3zn~1HGfD0kqzDWvVfc{aMxKM; z`g#swiR>oQlWgfii07?qK3iQ@Z^t|Idrk=S#1AOV!$n<@D|a>M{uUzkyTQD=u(+SH z#9N%TNSgHBv}JJ=8>(cmFPy3;ZKM6tS-$}uBc5V8f}#CK!GUNwWbo&r%TB@i4eBC9 zP7Bj`nHkVOKIm@Pxm$>eZegsg?Hth#T)0_^W`y+jemAQ)x-Y1qtu4Vly}VA=1{7pc zg1~D^3xT-PxJ2v7o+&0Xz?Ey?q7ll4-+1Jm=73uDR=k3)xEbATMZI?p^O;J8N{Sx4ro^U zywUnd+%);%yRl^1RQ(+yAcdG2+r`+I8_s%;?5+i5C6c>)^uHMJ^^y#|tP6!#^hHO9 z?zgsQrVet-`-z@fl~0UFP;Qd#V{Zk#=DaJ%?aZXsaoq?i2M$`aoR0RW#8jEpbU%9c z@?Sw{!x2TN73;yViV(`g)kH7ePHylTJ?%yq>r#Shb=S4eyu(L%N_U3Cx}UwKGW^;} z!Ga;2J*PL%!g=?IJE%89aPxFLH51Tt2W*KE-gy?pt>X~;aM;8T8_%;nTceC?ut3B| zb(cV%-?i;g{0)O-B-!6)j2Q1&Jbz7rEU8n*er~HRe8RCXw+o3N#<%7Ct9zdO`P0Cmv6@w{*T21z%lA6oA17?3! z>EoF#*u6!4G(hcSiB-|(iEG*L0D1<>y02w-3VOq8_NVn-hQ{HTON;G0D+P}-Ow2=L zV;v{eWb=qxcBH2U1b^kUtck~Ul;F*#X`^)2qtfUvdZhxJo)tto31}a^qlMJh zBS+}{CC^ZUM5Mr6uI9@Kyl-%^K0%IvjZ7fd=}}F;vG%(3fv+M3zvPw z7sjRUU{Gz)G^~$+Z|h7EIFl>_JraJrW8iofDDAgPzK#fKEPfLs-Rj7#P2|YaUV?Vm zUS)ibZIhfiC@DvRf%jDZ@XgcC7HssMh7YnLi26<{CDc1B<*n%9Ql{dn1leXolH|j2 zqNpWu8Lpb<&xRgt8Q;Zp8+b6Pf^#dH5^%Ve=jJHtVTwqLO0;AlD5^V>nfd$`O4&dh z_npyv7^`J^W_12z00D!1)Q7fMUVSKs5F7qZ5L2aGI3}s~Li`L2o~~5>>SkMX*=5JL zNr>v*segsJ5fTFYb{_xGJ_}e}~S$ zScu#VtX0LGF*vFiqi(C~zYUb;Iq-AiB~%%k7UCh5veag~Wv-Mk)=isL^9}o(7+sbn zMS&yFJ^Tyo8GaaQKjgE0ne8FZjKV(9Ie12Cn*bLUri+#&O1W8mvsawdPcZE+v(;_u z55LVcAROHDYz|JCxfzHyURce`lDB$K*R(sg+RGSqx!h(na_Gy?4E3V2bCgHw-McNE zq{LspZN9!-GXEmsbv?*FL;REOPsQ$Y?1NbD%KR#PhK~+6!Ne@M9V-Nv$76_Ubs`rP zQ7=cMG!Fq|%=QVYF~|VCmb65R2!#8lF9G)pcFfs7%JBDT^e2HPiGG zNHV#Y$`D$sM@^v^0ESF|0^iyF_0;eQ2;Ygu-okTy9oWW)#Wdf6fnSB@_>`DN$9#1f zqNh9c$utb{t12IMFB#X`kKVwJPVf_&_vSKE8sapuBcha2oPnb!AQ6qiY$ZTwh{*3YHe zB*=@(B2C#NcQ*b!ws5bio9A}T8dIwnxjmgg zB+_@?f>%0om({T$p$$@l+m{V^5sTxrp^kO*M!y3En=`{gR z^8mZM7~QCY3?bqt;!$oH)K%+`o>qL9FJs7OMk?XV%vT-Qv{=_%DiA(ZQR(;U<`s9S(1PH-w^6-3J2|GV zH845v^ff)(0i_%50hyWCG}oYHYv=|?*dw!sxk-w?!OXKw!u=tR2-4Tg)9Yg?IqfEh zI-&vsW_^nc+++ zIL2&qE{|9V1vh2!Q-!gT;>A0=X=9D8Xj+v@liHs0Xal*ox%v#L-l7%sq3G=_W;StQ z)z0(HI}sc8qVdD=2V7!dGakVYVAP!t2(b}3hkT54rqvnNUPRk+Lu)-Dfs-Q6ZQhG- z*B=>#Cf|1tj$kVX)22AYpA>2EZh4YQOKTfmcEX3CpF-;R#?_D_?AmIBk zm7j?J!g-2f$-QCkLyIXPvnMFHNUTURNi4NXGBYx1CraIGm8^9N41&}6f`utVDVWw% zB5dAA!D)Y~K5bAJR-p)&W~Hb|FY$JFk>*&&a8pDe(0grqR;RmExdqp6jV{L7SmaIR zF>bP4I99+Tq7og&QWVBIN7x?~-SoV4fZ6xH+!FKBxvP`-B?Lp&JTui0G$#k1W2@4b zmzAbX4Q%QywutZog3~o#Db5Mr@MXCKR#4DgBr?e1m1*+`ZloHaoZKe19A~5*Eqg6G zZN7qSJ@|7TlxYwZlBQ0+Y1U%1geD`=&3KJ|3+yjRBP#{A-z4mHd%x+g(quBX_Q1@> zzyT!vL0^*&FrK4!k4Ram2HM}XMh0o^!!*R4FrS8m{96-a(&Hk;948ED0q3m`vFG1- zunk%Y9eoM^_fs;`g4L=vHW6pA$tzdS#zWwLEth!I1U8V5S%)&^ zaR;SM^TI}Zwd53khU|}m>~}?3l=!FN?+$3qGA)Y1Xknv?rr4eoYardNep|QU8G=XX zvHAhu#%YPKWJ9+AQpfyv2Y1W5=J zPGN!BeUzLCR5hL~@T}Pu)DL-M%axd~Jzl)un>mRGQ&NtKjaPbR5-l2t5ttYM+OGIF zKib=*cLvSUmmZ(2qy;~Kr2mZQ$FCg9_R4fWq^!aboS7!OBymwnS_^2R$F}(HAo@@J zCAUG&9!N?;LfP0L-w0`zXXdZZaM+#r9hvZ5!VwJ?zcOf9(0OiOT1C&aT-Q_Rw5{GG z@%;IQ{%2QqjF0gZN!8Fxi#3;FVQ{)=X_%t;Ty|Z}oMtB14}F%QvV$Yg1ks&1DmDs? zF1C^9l%d(Zin>d=g3v8tnj{qyitYdKYZwL`%AxCt9ubJCgBLHMB#92jE<8-Xb^|INdq>=vZ36!# zn#j|N7J%psJi*-MB126oa;xI}q9WNM2bRCMc|D}|)ak#6B(vOa_~`Nyq;j^4hqXJ^${N}5V%-j-8mCnz!-14PKIB$ubo~UV>tM!<2 zh#T{2YP5YzYiyY7vk#-{S^dI+mGv}hPn6F6mrfc1iS?P64(+hw{MK~GvGUX=r+eO4 z2RDS54S|Gy)t`Ovf?xbUp8t!6lsEvBIM92>p zhu9p~!03y-;k*V-eXTqV}rm{{OKedW)nZxGHEuD=5S6-vRGIJR6h{XPZiVJzazBzil0{h@V2{JALIgm}{A0zJ!0`gP-Au?&kXL z^Lca0oVl_b4t4@(uw*E$9pB+!3zLc|#vZaDy>lsi6GK3OH7?P1WGtye+ka101k&1Iu*Cf_8{a~sig zXFXz3Hdm_CZ}@(Dns11nMIHE6P=O1=f?Og=y#IdnrhJ&f#$c)7Mz?j#JZ+rkq_ZhV zn%wSf+UTpbitbOWmhc<BN}}JUr0hK5|P*hYr%o2Ga+I>hUKdE zvMgdTaxIFF7@1wQnl<*sWJd4N{=*>6r!hQV7KyYDE@#0%LIk_@X~Y43VI``n#}t*d z)*8r3RxS93XKK2+X-@En!o{VH9~z!Nb$e2ET!@9t)NY{D{MEIoC&nV=+=D&jSwAZz z3c01(;H=+(_<{+`CVx^TCV}OL*EzFUsJYfE`0^_Kx^!PcG>65=teCB|=Q(gYFii!` z@}j;v?_Y+%*}|B(b5-jFJgTR82+cBj!#r4jy$~%qhL+GFt%K{bk+e6!8w2>bN3r>~GR= z?AQGe*0xat-Y=Lo4?Y}X;G;!d^g)Tny&R+R3*yMUx{mlG*i-z3C=-kjs0;f#rpl*A3`v@s zSKhoF5BqMx2hT`WU{)Qy1^gVaoGCIXiW^vAo#V zMHHzW-1Tl<0-h0iT$x+2-%w1Qe|8FoaMDpC5P~Ctel4s=^ty?ueX7V+LosRXhZfWG zkxO+V4#bXbtEq1H!#DZXEvDmSmZ|DVR8em-m0*gLTmfI}bMOG=F9EOBJlWKRix^`D zd37w#4L%zEk;G$uC^y8qV_wSsdCGw5Z6DvMD`TaYMOX>YRQ}6R+=6n}^j@Fn}s@<8xFj%U|w)va!g1K;UOmS zJz+3|vTHdBF)$5ZrdofCK@SDdk;;=$p2~0fXwFLYtos(iZKmVM*WmO|9zD%}0$42H zpPd?*^ofymay;!ajrDhV262JH^nx&VFajmk+}+K5a>13 z@NxG>f{;(BY=1&#OUdIKxXVDMUX<&=?ae2T5#Pm^mfORq9+aLq5L_iP2IKM>J^i`c z*tibUjtwrDd6Km$k_9U(nf!JyVm62ZX3|8EZ97H%p=1XA1zz7%PyZW9svFgw`6tlk z!G5R+_C66WbH!;Z)xKg^BT)=eU7Ren{*=>aLvDwY3QH{<0*TF{@#iCMtxKf)4N#$k z9w)JcQe87%->H}EKF%{uO7~PZ51eTJ0JGgGu^2e_2`}-NY5TEDwtNL8PI+e0J!lQ7 z(PT~famUw*ES0tnDIwA3SRYhomnQT@S%AZhw(CMKte3X07lqKWjOPGe-OxK>oggxV zD609OQ#?x0t+E5uq8^`M8I^grtQWgoAxB1jc%YB(A20FyFWw{Yd@rwJ81Of}G6D)3 zH~9MgER9G4f+xit`EK4P?Qh+vM?-1R!k<|RZLR_@JZ6?f$Mu<7+I=XQ!yn`-M2gg4 z+G1toLE92w*!+C8={v^~iJpGk%#a(mWf^b)9A6}K0^{JJljcDMlwQTju!HGA#A%L_ z3T?YdHN@ap1QR6qdo{EM=WL~!Ku6=1H$Bm9;=F`ncDSmFp|s(GNMaLejM>#4f>e1J zrK+lM24GpTpuFY_3uD!%>+UE;4&2k%--7Sh6V$%Cm63bB;w%1M6~hN>GNC=j8+!Ta z9l`?J8R{w`@=Wa5u?&-%mD(~(S|NW@lf3yBbt%a=2~m$MjMt+#xr`##imvFx!0E2e zSsR%#MQL;c*L1L(@&|o%*~HhZC-NPQBq5$gj6}g>tvO%*5IDXhUPe6NLu4jKqCw^x z^nr)?rr-$o+Onow{&xScg_klbFgRZ*D6QQZin}aWWpR3roDva4gbaVqS?YIEBjS=Y zhCEcf%z1E$2gtekNy=S&JlEDc#>t2n7c3UM zG!v3bXO@6IY9J}(`l0Om4HP#GB?w2)dqYm5o9@UoV`@MmAS1p8L?N0$d%iI_VA!_< zF6!PfT!HFyv*s;l$&+hvFwH=NXQ}?=GDYqSr(c=PHZ$p@fK)yBMepu?Z?>am91?2e zSiNGn*UV3wRZTItG;)qGB48(Ya_|kKq3GG8CNenFz(ew{xTz&W5chOMn5veaie`+o z*8+UTVTs5Q3HvV6Vasas@U&{94zLwUGz8d)my!01DsIckU7J$@g z0)VCONJBYJLoQR=yrH(O`IkQ-uOuHg)2eC-^n|&p^k61I!xK6QTWNUR!{(rMD3)Oo zx6FRghquglECc#098dCWCh3?k)UPufGhH z@Yfr*=e@{Uwk_$sHvgsJL{(>o0?Zejq}i@yK7GkfIl2dD(sm6|ZJ$*c&xSF2-lL^V zbq`YLN2nWda3c|QAt3ziRY%v5T>*zUEEo~lKXWO%IjkahrkM=XCS)ZYiENx>xzRCp z_oaReSLYoC3PYTz)&)zkx>4kb;te<{A+Hu5K8u5jwj!D_xSbVV4=N3VyO=ZcjPOxA z^GTe2>du-_eBxjVqk#w3H8n`d)zUioW_r^DyULXc{DdOVOTQu#SExGzD}_~Jw(M(x z;24B+B4|S|IURaYFRH@}{hs=0Q6x8}11betspXoz^RKgd#GAOpm(fIts#z4BAuPU& z(KXW~Sn))de;wor^fnkB6u+H(ZVN3uZwOJ>(2oM!ZjeGig?`}$FBqX2zbZ(*hMCjb zF;4#4N<&;vYCC>3pOz&B&@s5Z=A*Q=U*CF7Evu-H# zK7~HW*%P=RvKT}~WYdOQy>9E|?t&D0wE8IcMmdYd%U=uN^XpUf7v1T%Qk8sLz}j*; zy{~6rb#95gu zT$6ajQZdqxpZ!EoJMur3C!8u(jUvSuan1X7V({5EhJeP29jtLecBAdEpb!&s83E(R z*Z0c2%x;p2yE5fSHDf}?p82L2u78{pxS`F>`0TSAdgsv^rDow_(-(!+C10oBa{(a`Q4{s-uK;jNyUq1gDh@t$iZ{u0+h#zCnsy zqn%+~oi`J|o|%(yK<(FySdu3uYQyJD*cA(%YnXM3p%*AVy9!3~XHQIdJr)9Z6ED8k z`oLJ(&ZiEiBwk9sSpj(5Yw@_o3~=|V9a0OsD&W{54_oGsWdl6z(XtLzP;D63kcfg5 zIcq`+=Yj%CH^gfpS?ks{_jLwlP|yyURHJ=CBwWAwGzK74!keHm8=w5h*6DA&m3-f< zRWQ(UmHCkOJI$Fv_^g+`4OA8yL^hVv_ZA+WQQchEV~$wm_XylT>UWGlYSWtP0U_To z%AniLfYd^UZHe(jhxZ!q0{{R5aNy(FLzChMM8am!(GoA zTrw=WNzR2Td~1A`j3k`RaR~jwsDhWmj404c@!c}Z86z&Ph;6(9%QrKcW$Th`3#qUxfw^r-mgv> zHrL1E52xc+&H1BR&y|y+EHi)9W80PVu6w;$U|KA>$jTyu6<-HIe{;%h5c}KcFnbXy$Xc``1qJ zsm88*);B^BUu=);UM@LfV)#~#&e87+HrDe_NyC@Q{-hS?9<83RmfF4o-}~wWU6JC2 zue`E|oC20zT1=w8YgzD$l_9=1dzj5C7< z%7!NvQe+QZjG|)?+nI0?K<2L~_+*ZDk7+v85@XX)hI_paJ(Q<6C7S&ukj*|dlAnjx z^ty2p>jE_|Vr#8Vm!Rhxo`+oK$|Am#y8qVH!=iA=sPw}%iR9A$G;;`9f=nw`hoBk) z?bZhP*Z}S->dr;$IH8=r;ey@&$;&)omll*ijlJGR|E3A@L*Nc?9gn*Mb60PW=y5ex z3g}k)@pXP6u%oC+)o6S9H4r-@iVlHX?PuRf!O$H6Vg*O7uvrW@P z9Tr&OAEJ7?<-R*QQUxFFDX=xoM~M%45HY> zh>@y)|N4IrB!i}~zN6FX&HMl1hyQV%oN+wN7JhS_`{~U0HGh+NG-A(f8H=e%*79 zesDJt&buk@|8!wE40Y-b+0Xe0E+#;Y0Yn>2Gl~89#}5ZL*%x6eMI(6jQdczQA0B&j z+Yi{|+@pX0@Q;rZ78#I8SQ1754^K%3hQg__zdrq^Z~;qZ*yF4=_GSI2r?dxqoL(s3 z{Qv1f1=!=vd$w=(kB<`uZ%rzU-u+J#(SO`vfSx>fCs<$VQYrjT6U%=L7lpy`z`l416u#n1qQGsRA>9nDIf?+({j0g? zHGU{L3w$;NtcOztV6?g%@B--1U9_=RyaKvP_q8K{{&$H-`TyT}2&6mU54wUlsa${{ zv5QPxtq~(-YiJV2n;nu+kfN zI&tIp8-Kb_6RgA12vBayl<@MaAGpa02(~O%78N(D6H>;%kNPg$zo8D#%Nw&X!fKH` zqq{YtU9byQ0M_`hT%&|>02c7$Cr~|J_3Q!>iUL@o%t2Jb&ro@re1nBX+VK9zf5n9T zS7buQzy0_dG(~NJSN-nnh~EXoMBI=aEahBfp#5am2<~g3*$((;Sp&0wmS<3f^@eqZzbeUl(nR$Vb%d>!6t98jn0LcCi*PE zvi$7oS3|Au&u3JCyvD8&{pUJ%SXU6HAQOep-*z{({7qtc{Z@kx3AC5-Y=8%^UI|1~ zRxq4v-UY1EU8Tu95QL9^WlaCgw+r)(+y$``EbCdLfKpjb{TD>pj2SWj1R|=?Spb2I zgCyWN?uP~E*e7EZJ(TZWcp-Ui!T1b{%|j4$kXqWfS9DTYH7rjzcEI=&v$0PJz#xfYkyFhLdvC*?AyQ9zFC)7jb?*j`3<@#$-3(%M| zz`~g+-T^R4mc9<`(*+G&hIp)NrnFgZl~p|ZHToYe0Rcnbs|C3w6T9e(-US)x5(qHpeCbd+8OH?L(X-((klJ?zNL;7R(OV zf^$F=Wzzj&II7Bun*=-Dwujs}NtG6)wYeVTdw?ml0QV2eIu1mjS&Mal}8#e&# zoc$cc7S6^t0Cpx&YLu+%uoBMq44Z8p1bXIyUKfMhBdn)Y;3dkddld7wf_GQrIe(1H zlA^`!!%je@mrU_`1cN06U0N<1-GcRu;e;VXYV7^AIEnh3m-hXkI z+c!dMvwo6){B6<=z@y*M3uKkb!pB|xy>l1(Rq3T@WQeL|9^EXca|C_kbf2;_coBm< zlzA6jC)T1RF;SpHnxu%oA$raopB})znjI#k)Q=4~;Aa8q_KyjiDQQ*HR(hc(L91my zyOiZ&=Lg~7Wv}~isPsbBdqTEAh4Gb|HcnqU?7wg1j@_F9@;Q7y__ShWa4HZW>Wgzl@V91Fs4qyy&?c} z_j_@qa*^X|MKBH08zzi@!)#!{p3S|!lOm8$$Z*lnaw=My66NKW?e=v5R+NnQE)NL% zP(hmYzTjb3Vb<4hUgyxFSZ4=PNDUq%M^NkiunJz{k z7k=4I9`W1qNQ?s(h1s7EB-Of49OlmjmJymIQRKZ5htKfh#V~1u<;97~(!sPW-ORNE z<3V)oA{_6)hlK|<57z89Ta1?|`D@y({XIeNA$7HD^Oqiv`LRA2B|M2k8;m9lgKDfF z)lHj0tH|o}`uhUppH0Jp`}n|7cqC9q!0w@gnjK-Ca7cU}#PrHB0HTmuBwJKe8l#pPdKe8NLmN-6nPr`0kjjZDV2`7y-B zveoq`H_QqJT27~uk2^V#q3)p^Zv)PE8&MWleOzuH&uMll1gH;J%OwhWI8dMO|+-jJyyz zEB0P{P?R^*s!4me0r59M2dEi&RUP6Pe!zATgY=pbt4ID4a@HnJE_Q>?#Pp&E)-BzQQEHh)^&nslUyj3 zi0KHXcSg%2QY^Tw7JjvP2_Q*BCL?D)j!#FXz1jK#vT1D%=qRdFdXd&%Z3;)fpW(~o z)s50h8%)&vNrzP{jk&YFd7lmI?fbD~M409RoCgqReA1o!nbpGDGp7H>BvxLSZXe6H zb5)N?kZ`xUh>&kCC2Kk>#c#&NI&Bek!X3!&_SLnySaOxngs;O`b)C&&clK5h*3MK` z^ofV7)0f>xb`T!lMZSBS1GIBN!o8=M)Ya#}1gb37dd&=a--Jfy@I`!eeo`jxl6T~i zV}$P9XOcJNBHk*Y)jPR0$@>;CP5AxJOBqehJZe+W4Sc>yf*XCPI&4ixD$3`p?~!3s zkV~ucW3=yKjqvTj3&Pwjw1Eix(OtIEyYal*n62~vR7X^H$E<^NZ7}5y3Hs>MpR7)= zzh%UsPr_KQp5zE`@~%I+yUJmAe=jfXsiYmN(kWyAWNC~uYuHG^avc3=&9JWCIAJ0V zf&`WEym~0>{0#5C`H^31*>u9e4_^W^?8!8?-eoArRFXF0US3v;z@zQQVd{w#MlTu2Cwk$hV#iSgZJ$hylx z6Q*R=#g81m_HpTC1+uQHsdeF_NY3Pf&aUlo@rLMnOfDs-%X@(FFKksw2Q7>$kXDb3!QvQN+i;Fe(jUr+R2ybw3TFYYGlriI-$({c-UyjgEsNdDiaPm9mh1dcZHZO3Nf zkFZev8(wo9Bv*vp=3L#0O2C=XcSWY@ND^M!USbOwb5fT3HhgSG&(O%rqLd#;*!jH6 zUKq!~`%#Xh^2?5~ThbeSWnv?BnF^YLC_C$?E+?QM;3ucLq;EKz6BU35^0)I(qHXcd zmgF+H&E4LJH3(%qvbenrSG#?&)iAAnDnfte#=8{0%~^OhH%;|KY*^eV9xt0)SHM*9 zcrAj(%{Nd?X1R=U^xTvD+*AFr_#2eM zMPmdrFEjapjV;kEXFXE(-)blA!<-UtPV;Mv2I3m*cx~5}%_j5@IwI#zcy=SJ1Z`q^uibKsV8*SWUHR}nD?*V9-Dgxni+_yF+sWhw9ImMR^gEOGI`@^5RW-}7Y-CGVr5LCu&)oZ z7eCuO35DJPgMLdUk#{M6=SP+H%dU=h!0&Vy zLr?qY-tRH(ej>Ki_O2+pzNO~Y)~ofiFQv&V`R{|rb~o)OB3aKVuw5+MI%isMxvGy& z-y65-K~Ih%;vh-7$8WCY8e2W!G>5RXKg_y0`?7uJ_Q=PKA?zU|+n%jBeczicQhryk z_r)qM{Nu_zxsftYZ_*~)p0==4&PcF-fAkb0?rM9h1~o!Z<`Q)F-U69oV|#;WIY!r9ry($gdrDMW(DV4ZZhiT@)cslK6m0ieF+@_yM27#hQ0&ps zk=>&(*_ib>>|=k@SqUjDy&m!6I>2-OE|y9|*vW|Ln<}EIlu7hj+0eu99kZie+GTPp zThI+)pM8n$#QS5B>wRZ`V%i^c`ZHwH!yk@Iu?Lbch8vcEs=?~}=bN)jsCztXFxB&oT%G!I z_Vg7jZ-@MyTATHIEG+!I2LmGkNy4lYb+-(=_^7m`lVh&oyAxYIzXs7GdGVBx9+6P1 zIk*9$j13FjW4mzwB@RdV6NE-mo4b@b;z}l+IkpbnF~4Ig6WjZ$E_um)7Gm+={gRPU z`z0ft{J-2Sf=-5KL^A4XM=s03CdsbSXjd^Kcw5+)8qY*qW7ehrzB^0TcV0nooI6y! zApmEV+j`%8Wmf4_=qS3}`!fzEzuz;a+j=nwoRior!}3|D-!r-ci^Y}SWT7p5)tgR~ z7>aM9*X_vEs1!q|{5R0`KvtrAackr=_#Mih8K;6N z{B!Bz?6mx&$N8@02PjR#-Cn`u2O=Y)b?-F;{YeZDGvlw_Mo9@D?7B(i(kIlq`@>bV z4WMf+xjr64YKKF7MZ`YNaY7n9oBd+L#oHw8s}$~jde`q+Q7ATAT>BX1)2S5MXM_0L&7zL`(A`K-35 z_g2iW!tSE4-tp~CxgS%+BdgMXH5rbRvPwDp;0>T;>mw1R9IQ|F+0~DFNu^=)Ace_J&-TvT);~7ycSlQK5$-1+;7RkMnriQ4?p*Y< zAR-w^WOsiGy&no=99aHA`DZD*2Bb&QS4~b&(24SId+uULx)n1ul2$g;yuf58-hO#L z-$M+YSje^WI|yili-~LG@Ts8L2W^9&{Q}x^#&;Z@aPrn&l=W9PhA0OZF``k^$bUJ! zi>nNthR7&$%<1_;B3v5GY@0C3Jul{vFYRVJjJb+_H!wBD8~?6(HCe_M+r(G41I12` zV+&=SVLI>I+Hkm?Mhj&FV1hzRwnnvcGuMQ)4v@si0 zgw>j_c)J`f6)H6-({j%bJcg|UTBlB8)njjLBG;R&CZ?k;4tTCFBFXJCj`Be<3L7xQ z7!llf%c;LVP{)%S4n`AP;w3`sMvgy?jSub>7)4k@9Pf<(qVkcG_{?@t8z$f`ziY-# zI8`rYigd|b zjLE{PD~{jkVk2*a)iykAxaHn4E>#2^mz&q3faO!Ef9^`TDL*P|=14f8)VAqD%W}cq zx%xDcbMVRYIQmcPp9HLLKQ6R#UqbtM=>d|zBTfYg!ZM?Dr5$WnEbMI{o`I0k&?kJ( ztNgk9|J7qnnjiX4TBF$3BQ}*j)E+HuIWS9Ve^1>$qnh6C@;Y>HUZ((~=GQQjyGUx$ zPbvt7%mHy3H_6nJ>Gq)Sg=w!TPuL=UMj-(yap~Xu8rLq=^lFdwi63~c{Yxjq$PO{e ze4C6~3lP?k`cazWj~Qgz`(isLot8K>w`&dvGB>J}@0wiZO4aG4$mBI%h!iLO6|cFw zA&{~=**6l84CH1sd_qO$tH>!Y8F_`o#AfAB`HtBN_O z>^|&N=P-%0~$K&|7(1bF$z0BW2)HefIF0 ztZu>TPPm%Wyqm>1;=7SRze~mkf1mm9rg-C&jm(d~rC@i*TI&9sM zp|Sj^&~{bBY0k-(f2iI=qA~B$)lYw++=VmTUq9SFQZ}dn)O`;2=XQk1|23HW8vk=J zZPm}7c?H1|c|H(*+d95(8cR1p=73i5rl|Cz7D&2cRJ5}29M)E__T;CsrBKZajmk}E z2k;NJ4Bm47dkXR_qv;nzXoT=d*?t83BJ-Llf5*D~+wk*WONQb`!6sBfQXeAiCYu*O z=_Ka5YS^y(+-04T6r7p%o@D*qNp+F9lWMA9kd`?l&#}E^AIw*k#c}A(rp@j8+!cb#64C6zW^G8w<4y9NGGSS)1xVn@5|UFqq9s69N1THZ9BDZSlTm2DnB1xFw?1FSc> zIF+Z}m%%a5j;yOn-!)!Uff7+raClWG>Se!fI7VWWU|we4v0I=9D3^EzG8Ol`@MV=o z>UDgkTg-j+uk#{Fau_yiME(zZZyi-twDt>&pwb;%LPBW-Hb^%}ODiE=0@4lA-Jx`Y zfQTS1<)&fNA>Aq6-EikV?>+b4@s9J|bH4xY7<(}IW3yOmt~sAMpXd37Yc8Y$hBR8z zTQD}m|E&)lrYv`ekyJ8@Dxw_Z8Y2k%7 zuuWBW&x<;P#*_*4ddOvd(KMLlaTs6uSzY1FE((#KyWWyKrtj&w=R+k9$0^!`x598V z$jAs82s~Jq-+gwmnbDsx)0)=0JehB8mmduCQF$Ctlf^`6P79b8dII|K}8n19L_2A0JU>vo52oiJ9J}KR=Cw zRYX9>=@=!5qkf+4FnUJ&D`L(-1e)vV*UGsZJ+0~dM|C((+?&vQD-q_j9oqV; zmP)2Op;YIha^>{*GXszcMNL0Ta?~Pb3+ZeeTEDED4EME^u^_KBKN4fb8}umdqE5&4 zKWE*zh$C0+O|Zf%&%TJNirJd;1v}WKbsQ# ztOTb*D2+HnB3P_MHWf&Se7<}DAbP9_mmdM02l*rt#6HMQR{Md(_;W_w zlPu$uzQ64r4=+8AGHU`?-Y_bLbf@9G8(*7~5iAYt&)(}?6935pjy+O!HQeDgnj^_y zL`e7I4Th=){9Vsgd>lFUZc^>b6_GZh9<>#BeI@tOoZuYbeLh>FG0+=9uYpYLe-0Zv zuP%z-u>x_zTzxODzDmSv&Z-R_BL0_lQ}2HrSOdkYD-M`{0_piZ{h_@KiNp%( z2%;ePx4n$m1L(fi@W^@EVoM~wQ=Hfw8M0?6H?wxH^vocCj79HEm9SS{Kz;cS9yczb z0Po^VvKqy0Z?nsK?mGF?xZ>i1y=AL+^z(J$LjiQcs;~FF!t#FEvpWB_=`&g%TX#Ym z9ZTUg1LqEv{|W-4YFd%g8kb$Dg=LP3oF3=u6JxzDFyWBri`Tj$QnDHSaEfw=*m6oQ zZZC<9$y#>d#=Yd#>e7E6HfBGL4#O6}8YU9ALldKnf2}zd+-$zb>Gjck|6>=ceg!Kcd#Y@gNBX%^(|p zI`z8@f)#d?41kS)_pGd#I}Gdl)2a?O<#kQU>cS#WN8U9o%5 zLp{AUzeg=N&nlXt>^T#f@*95^mBWuA;MD!wa23uJrxnj?<=JPl$J|%|7C*z_3IoWA zWrK~S6`@I$PE8=u}~0y{2Q7)E=YJ#&CA0@yF}amLh90>qVF8mP(W; zuu6rw9u4wOKgo?H;)SlArDUd^1Tqu0{(Ce)vUF4(P(OC2?-i)w9#DE(A@#=x91XP4 z*h$X$#XV{&YK+Mp`=E)QAG5&w?rM07ypilBL-Cu_= znV|C1@z~WTBE?@qgGh>}92}1(YQWP5RZ|G2P)9uJD>b3nXJ~79GQu9rqVK9ni{E>> zo~4g%@$G;%%Xm*rFF>CzPgct>(*G9 z^+oKVZX{QLG;82`+r_mPEUB-$@+&Q^TZblT=2-3SDunP^mC(%}eTDba6^A)2bp;5= z3KjL4f2Ol0#BlXz<#^Gz#ZFbXhbFDjy5Yerdvi6$nAK{_87g+TN6Q-MS_}qFS^I)G zjW16Ik#C9MS@j~!Wb@RfoAvzP+ucbgY;B#a-*2u^Xq&e=-jZ53M1>e2Dai^vbe4O# z5OQQQUW>Di9x3TnYc5h^CbF9|Hu&}}#{R*whNFG43HLw3MCX4G6Mxn<=N3~6*pB>- z=9)ilp;M zVe{y7Pi~iZ^LZo^eV#nZ;G{N4_RVE58(xdjv{o07I{wz+7Y;KTGA#2`jUM7=SvZCh z4Y$MV3_*!6iE)~bjagT@wRzt$>w32NLDH>v>kQpn%JuX8#UnYaNb~W@oT28pB-61% z5q-mkhv*UylZLaqnU+-rir ztXP#?{kafGUSk2FJ=~bjyP9h}Z!!{y6dqrq zK{KJn-}3cM7w!IjmR2|I3c+Rv%>1H{$rt@W7I*z z8I$PKiEm2FbvZF#V>)tLGC6x_^kjdJVri8hMOC~O>ugRQ1c-85U!0stm>Uio@9;YN z<81HGiDP5liK=d?Iwi1}%!yc2tev$X4i z{7Oh0C3P!u&^oSSykOPipM{q8>X^px@&=*(NGw(W9*U(Nh#JY;mG5Uh=9|bAMqS)l zzm{ul;*|cTw|L`+-7Ju*tbXYBc>NW#gz!_0l?}Nx#ikuB`p6Lblf*nkO^=)l(WB?- z*VA4P5luRT-^`93^%kBA6%uLbJ^AyFC!N#8U5Im&1a=NYJm__b1f4cTpL zi8puMkg&@%YqUB2{A$GouH~0bB-D){rG14>O;^qVwMDjh@Ph>58f0B>zt2MCFKN;I z5!Uz8#y;Y!XN~INcWI%IRJ)QSx^Lt4)C@s9(h@w$!J(KOj8Vl?i%)oI1&o7NA{2P$ zUU}Yw{;~27z-lddZ0YkcvI3bjB^LgYSI>y@Iq0p}7T&#!Wu-NJ#N2~qH`(DezpZzU zzQAX;8I{g&@~8TUnkf@d5JGlM=_yOzkqQv!8wU*T0|n8ZuR2=z!@v63!g|ZpBnzEz zI`i)mdT%3C3&~s(vN z`PBPowoM}PfS*@>>n{c(oDLv&%TYFWzgQo?kB|#*;`21S7JEYR*3D(KaF|mh-8DCg z8-4ReC;=X|%s1$+r~-ec>mL37C|yI`~Gx9l2P zE0>mzgYimyQ>dK^G)?(S#cnq%lv({`rwL}U4$pNjpT%!wZZ!_ECWH^g=A}mx<>S3d zfxQ(ES?}H=+mvCe%Y0zn3<$8~-UqXZ@b+YdD;vqiW>+XtA9tZ|$Wt^Noa^6uFDuN}ap!Eo9{0B9)O zT8X7;(4SDB(CWDb%de=H1IQDhsT)sI(N!Zb^CZqk5;(!gr!JTlX48idAV8%3)yA_= ztX&I}*$~)C$(nbhC=tWZZ)a}UMv>i|wXNvPEd0`QkGEsM60KjM|7DBNRFqp!u;YU} zAauT0@-65g8poJiXuH^64zmS36gBDRGnL(reGm>}I1$1mzKxfAe%*&N#z43Q@;s6I znEuKS+!l-Q+a|MEQMHUiEmn%Y@DRbj)P`zx@3|uHZ|^St*E$VGbi#K}?}7gO_`uwr zU5Ef-zJDV?3UVE#jAfSNG9=g=&dc*npgHB z{<`vO>C;2&%ueA<`qQQ#_Q!9sd)8SQvSyuq)k6HQsQ)-7WBw%+B21OHbZJn{K0Kf-87 zDA!-66!(-@(Pz4S6`5;xwf#k#MW-;o;bJ3vNOk0|@dkDk?gZ0;dDi~$yey-)KDM0T z10vfbL9=ZD;XtbsO>L6le!qIIO0=`l{9rNeVXJ*&LH*#x4d_(5zxZpmb?MJm^qdjr zXqKCtVb$_n=67pPF#qAwXI^Y22LKM+sEjn#NlJsLc)JuU$!nX<7~3jlMen=WS8y=D z^Fe#p89r|jW@dc8B2ws)f5;SQXwhRfv+gmu{)dzFn?m)9mOb(L+$`rVHZxS!Xwz}O zeG#JlLu9bX^kBKwv3COLI{Jy08b$V;AKocEx>89!sp6xTX8G~xsa9L&@6r`Hv!JcU zm4gWD60uujE8G~ei+)W*LT~Gmrne)LmEE4y^i_@!%NfuEpZ0g7{rT%@pKj{B;jAaU z926aCO}7|R35ey-M2-Lg+Tm%aUL1esU9!FE$l>8-;>v#YpfM!^61b|_mIxPeeQL`R zC7_q;YWV3{E9y5B&D$r&=x`E7A)e0yLjj$~Qfo9Pj?OAns1cTUW>z;w6Dgw}$sW3& zSNlKaR|pkTF$ioebJx@<9B9_f?b@uz_4T=`zb;$$6I?j?#)IYFaGbHPGrJ+hL+bIY zw?$P2TQdUTRZzUKxn(MNhABnD@C2j9lH7J)U~W^f!Q(U$$3}&rR8r%zqq>vT~!vzP3&@YSicjFe7>f3J@GzMhXP!<%WGHs#%q4L+*7?6=iR6P^!a^&QiVd}`6VNia&;8Gm&tgg6h> zWH}6Jy%t4*OJ^6|!_#JBDa2t%_=Mr*_gt6EKcdG*4hhX-2)ln>}VH`YMB)p9%-|4^e%3H5# zs2=`{54-(C4ig3PI@Vfp)J6Bzg*a8#DV9ATXd3j6u()hlh?)*k$L=Bv?htX45~9{e zZnay*|KdX6>g;HUYx!4{l(V44AC|X89F){9hwt3(1N5}yMK|5jb;ZN4X++?PevJ6a zX@Zg$^;M{$zW?HbDJvRWOx?1{j`-*Q=HqvWA4_cs^K^Fkci%+(V`{XnSgsem6mUyemF=_18X$0qoSf?TcHKd=XSZZ^PhZK0`anC?D*dxruV=37{E}NB*p>mLtvhH^FO*ejUBQd zFk)GYVgb1dhA;-{@*Q!{=ZTFKPJro&xw(YCNYj+of&~1)z0-i+@+EAQV3w%!}afu zk~)A%2Z9V&4bjdF<~u~#lP@EPP~4LS0Y-q!^>mGp)3pzgvfdnoh_KkWRHhNdc+9SR zVD&l9%t4SmmWXXUSIBtgdl0PmWx#R4+`L=&xhtpzr<@75zC8Tcni%hs+>R(4r`%<& z!B6V;z1s5*s!7jgoEg#(ezYB}t>X3z*Yh>CGjn^~)s~y>w|JW2yLLWKQX(VW^jSJL zfUm{EJ!{u?z_a!Aon9_JxnN_j+s6Z$TYSGuy6k$9h1G)Dh~_+0_L6(x=A>4Y;-SE# zR3DAn$LXhXy})`E>%Nu;i||G5BlZKCntn$IiM|f7XQAytxVS8rxHecG2GR=}=DiuH zX!yDZ{|4~Whax~VG@L+@H|GN3t8jB#>Vp;Jdvy9a#?$1mEJUc7b&z1;9O3Z9i^$na zJy)4K09US(sR_re896+PXP)=(<#vcW zFV@{L>%Lp3KzMG9aD8pZOB)qWz&|6e%>eAUZx@$?K6=GKh^E6a^Eh33{W1RN4KMJ& z3P;#xwaEC2IgCes;nJeCAQ@yj@2|WHpuEYhMfCM9pIr@w5CevxY&np{2yPmcl(`=* z5a!z92SB#)Cx91GXread0$SRHh)Qbh83H2@TYSWpEGK*g+dyQw%VId6lzCTyV2+v< z;P*ro+ylx#UQa?GVv0n78Sv~^XLfsyixsdOf0+r)7skKovkIi(BH$}xe}>jF-C5a- zspQKkY}&HrNw0)8{ZkUkA`L*b1w0t|eEwSnBSd0l9^B zBH*fa9RbW3JUS4+{o~jS@fq^|gU!Xb4dEGP^8)jZZ9wGyX7NRfW@Lr;jxi+}xz?S> zuQl67-RGzG*FN{Qbk`R^@3=NQ!shDc+H8@gJk#d6?04`M ziQ(u6#xK&1lF6umBb0nOoYc|;2SJ3SfIl16UK1Ymf~!<%Dcss_19>a?#x_;yo?1R+!2dWRr8 z#&04Do&Nh@x^d0b))M2Yy&jrItXRj*9!9&e09H=HHjD1xzXu>S2u$?eY_>e^dvo!V^nATRmpc4e;cJ+*$2NIb;mYoC`SQW7S z*l{Jj1L;2#q(!C*&OdeC1iPYZ0Xa#`L1^uuEC-zJWxTsJ-6@edf0g0 zz&1RJPtTnn!O z-?szQvf{ZOr#lVHb^yL$hJYXgt@*+}sB@T%Tu?A%KyZ@PaIUX7K3#2}9A+f+Gc?c0 z?d16lUn43a=;ROw|3gb)cgLo#WtyOaZ{6hgKngj22!R`9rYlqEeP?t}m@{4q zqp?A)2EcZ#n9j?MKxpOZAgbXxDCJZ<%#U@(WjKdWDXJ~^h)WuFOalIsy1|@Oj%6_3 zQCP(;h`}eKiDq?1?4Q5-X#*|R2zszGkS{YQD8Y;pY`6{F*Iu|%|FedaosNiJ8ZwCi z)Da09#nt6RnzMp~OqtS8Xj)6&o+H<4ahqtFw;-%rmKqcNhixDd#E&kT zfVoKr-~>Z@@M#@c`xxLz^( zgl)Twf@J5yDPS56Ik~nHi9MFd_^><3cGhJOJXNN7#!X@_v_n~mV6*x&eM1pHMT!XvX)v* zBMMtVaWsw0()!|E(8JTf0Jo7rKOy@Mojn00+Z>|EmK zBP8YTdFdfJn*ICNOg)4J&=EHFees&+kAhdc@MZOC1_qU8`78RNUwyyN&|=*Cj9V)e zE;(%a(QX6#8yqYWXUEp%Tz zEo}?m3I){cYKOqD`$9)u&nw|&OvqO>4~^-XVW_v*cnYwt*j0BgO@`AOM$#UCCu za~Kl9UM=S*yOFGTwRk1tD*DR)=!U6d_TIz1QolxWgi2t*LDPm5^Gzy>X)F$2IXR{l zo5xhIcrE@Ud)K#2!|yGaN_UU?R+w zB}eDyY!z_Ap7z`|-ckrPU(1C+QM-NO(q>=Gq6GgyxXLXFJl+B^#w%Oj#1wS}ms8RB z!6E$1B@J|L7N0-fY#!wa`W4#?(EaCoo3ugc;vE@XEGWh9vIsL=9pEPSfhtZXQ|N|G z2*gTpkovKj3ctw`-LtX^cNJ0`{^EjCA0L(RXWQOr1soarZ2Ng5za*bed$ec<<OGdnNJhb zZY1m(VqW(&SI~E@wI!^nGIgt--eMkRhgJ@um4EIXXt@}U)CUF>S7sOqAK~sE(ycJk ziW|2_^c3mNZ(v8Ar8{M}o_B!x^!+P(^W)VRlh_B2gz;UdQuK6RAQSz@!lqqEx0W`- zet(w0UiFc8AE?VNEdyyn{fo@2kP*ri)_DBl9a>mUe}YMu9A4U>wVNtu$ZOQ|O%1}y z*)Lw;Ze73ZYaX3Ozk4#?Xdjs;gv80q-f4JB=t#0{<^Zq)PZh}Iw5QdRMUL*Q1KGPF zEMeq$5&Jl4Yk7y|S95Q&sAvbr3d8S#%u2djq2`WspGhJNr;~%Ul{iV6uIYf zpmiMgC_Q{SO8R_%b!Q)XnsHu>o7#;r(XtT`@{5E}&xxNqx!Mb~Y&PA?1p=%AcI5a5 zc<7cgaRaf9>>JY=kO2`8wtCt=*DI9&bgzTk+M>>?s>m>|PJt z+zw@FWSx5=+KW)C44gN;F_9UGSL3SNpP zl}t4sJa`~4q2U85lP+|~+8w1e-5}NJLo=bR2>q%(>d2fvW_!C7bYQ)xd1@~VpZ$p3?KkMY4(S(tMmd-~xK`4Kw0xjq|LJy5>c18Y zdr8$L)gaqoc^(Cb-)&Df9skTw@dY$atf;U?wkZq=Xw}a1uR`k;#NT)Fa|y%=u6G>s?QTG2%BHA*_>cz5hnEWndl?Bo*4Me11-?k5&W#C_)d@XTFIrBCk3{jfT} z`aI*>!uH>oC4pd0!qfTW*ZDaPRBvt6CWEk6ML11V%9VSlNpzi5F8*enr!5_fex9fUAUU$gwJ926FKV*tp= zTkPwnSivGBmtrlSf2>r8_Z6HkK(AEcsPpb1m9qIY!(8$sY7b+=bn(M8ZySxHA`RXz zzV#DT07?62yeucX{9Q2Mi2&HglYE=4uYd7qVLmo8Jbeu+--s2%&w25}J0bunPCouR zR73FcT=QZ@bpfiW^_}I6^Qpy=L1$SPr}Z#*Q{H-p%^NGnY{id4E31BjwZYtK&nl|=*rNLS3?t;>G<%+c z`(a=55;HmBZA292&Ie3C8i9!r_Dh3~mjNw}66mj|wr>kGhK;*&*`zez$^Dj~ENgj8 z#zLy;+K+n6Z_JuEfEM6F5?ltSMhzjls&nTyaeq=AkJnS#9cAg2He&K-^F6mbtx(ZC zLK295yvpMvpo!XaK>fTEE+rNFMk-1ggh6ra2hlTLBxtnZ4XcED^p>X$eLLrBL|&wF z@5`SDjm!ll~ikk#nUz-@*tC$T{Zrp=?vHbf!$=~G%0p$|) z`}&T->?LjKQ|c|b1*Gr9diiK+*fEjTYWu8V_F}=shB5-&*urW>=;RnE2Lq(~LszaI= zR*@Rh4<4}55x#*&68hR(9|!md&Zg*KGpj>)++cjDue(Q}Hi6~F8e^t&>#!k(KaPC{rd;po8?ox~cq2Xzyk48v7{hMc* z2dKfCECccHth$W+#r67*Gb;?<;p%`K#rAEcP&{&^jdPLq6s;W&)Ib?MO}mJ#Rx7-hW?2Y z%hI}C33i_cw7q6wa^A3>_`$)7LSYW{|-+lz(b7Mw1^tImSbKW9k8Bi zIgDiYGShly{mHQPF&T5+VYOkbxSQ%gOwP!F(tHTi#6JBe_mThwNdT3g;^*27G_q4R zTu2RdM1=CX>TYe8V0kNE0LJbEWJ;JByDtsjs`Dh@%Ka5UpBbA%WlJQ7@$7G z-J8V=8}gxvEL2`L9I6nG+dY{op7u{+M4FnHGB?vgGGm5pDw#Vv1{gQd40=eU8QDDp zMrxutFGatb@rJm!K%dZ~JIXZ|K3rwVv3S~Mu<R?l- z!PrIcyjXMa=X(=Re+|0%8-^J-B@)9<*w>et((MyN`&0DImEi<5yU%aEVWbfP2CU+w zSM1X+b#{3F)B;dk8#%f6q_5)h5<%Aq>_XzTlDVRO;IZ~s=A&eyev3VjRzi6c+dsh< z%+V@~AHa2N&lX>>Np6nQS2$jq~X(4cu-61?mUy`MvZOh{uIuh)o}lm+N&MA2S!+{&w@yT1$V-8>4?2 zWD{Wj{WZ#mAm1p7Uo@8La(Q0D5Q636(Bw8|4Cot5rI`<(tY}~jM1Dm8_gKl3VPjv}YWgN~R@ZDn^0rc1e&z;0ax%TG>T(4y>un~`XOl94U^T|#7ZeW`eg`39gu5J^*b{_^h;Kl4ljHO4-L zCB5opoA2NAGzWTaMYr|Tw{Bu&C!BVb3B5X{*6z^HToOE;f27MRlDN6|vTm5{t{AU8 z*zEZR(x@YBFJ>M0)4Z>OxVdfcZ@hMp7D1ZC}Ssl9#H3w!Vb+mJ9e@77;Xnj`M79G+~KXHSaGdOPL*b3;d0hQ2pJ;y(MI7!n|9F_A8fWkx17&hzHHH z7G7WWI*%*H4lZBVh3pnUK!bm{o zewWc%B5;e7-(-Gtxxq0uqCOwI)V7%Vu#7sfFvw&DL{Z2*a7sjkqo1|ZhPochz$LN^ z`&Nps%y{<*WITpt>4P?SJWGq7Z`iTRlvYoFe39@1^;H5EwlA-L3M;*Rcv-h}^pOey zEP%c~V0Lo2G_{$%+fAS5VWoKUS`~*$zg!q*63z%&!5Tr|VQ3fphh3x;uejS`sk@Pt zIrg_asE4Fg*Kz`s_b(O8z(E9b$dX;EWAo##Lr&CYy=+8F6!+)5m-!1)gMOUPIKr2F zz#8C04q+rsqNLW?$z zG&0#QQ0m)&HfLidPj=axx#AQ(q;MV*Jzd6b#=oi-?hMK{$q6J9Z6H5Ae>=QWt#V{J zFQ1<7YOqvCfuXlsE%eT1KDL92fwRO2VmxGY!^@WQaQ9Z4o2uL)T*G84@433$)y>o| zuP4h6ADcA{rSau|)-3>?m0_ZP$d(*>k(=FNA4XIg`LeGP z0~>N~m)sq8TIemZB1O)wS%}+tPe&p3Xj=90oKBKu74*}UW#6)^2=xJT-!g2uBP%?3 zabyYR51kiayu7_?O!lShY?qVYB8C3Nqx|Z0sP$N){e3N?{CdL5YqRVq=x6F)nZA4a z%$iY>$dM|azF(@=PhG>!n2pu;&6m&?eFM)~9Fu1GJU@BaN@vU3@2#t(VmGJcjYAGE*8%cP359E$0NSXVt3N$UyT!07w6B%c?QesR#NH~3fMK#iO4FgmC_ zlfgBz8qqxjC=15nl?KHO>Z1OHmq*Ad8GWK1_8&?^Pa}o-u@s4{_1*;p6y`@zNWF~V zy_noORypYh#h`K+R(=}Vf$v7pfleUNi`SSO;XwmgHQTPtZ=xgU65@tQW_uBm6W)?|UUjKI1xhpqWF8i zv%~2;?2KP{rKe(tviaLLU*ARAwu78JRgfhBah3;y?GL%`24vW zSX*iu^w>lv)i0h0ei-1JQ z(+Hoo&Dy0Ivt+Vbc$tl_P4X&~7+pAF8mNeOCWOCovo`KJkP`JN_8J6&3U5|nsgCG; z0tH?icGW%V1v2!vwMUjkUk=z`vC;Ns-JfLc9)9HKbXYa?LPr2E$jaX6FfPd9;JkI1cKI8_EH72>y%4XQN*0c!2vav4Ig8fu_BC1<3 z|9t2=di=vJU((Z-n>q`STJ-<)2)QF0c*d*=X`78k#4PcUtE}ZXiQa6Pd0!uR9R83w zd+zMKdPP`yzTknOogue695Vc(nm?S__9B?l7%da+#aa@7LA9<@wycE(!Qi*<)F9FD zwxDa5=ccpZgZG?M9K-q=$Z82B}S8nd=iNo z6Rbtic-^6LAN=ClI;G>>N^=@gy%pk*G|K!MOP~`egI0aPEUkp z3Gw{%MvX|BJX6(=ta@km-(6}8*aFqdReE;6^q7? ze;|k{yx&%nQ}8CpV5ol%H>5@Qi(3tD2AO7Sg*mh&%cIyQmic*S|H7H;wk@xn&=%iz zOk&ACWqIl9ZAH0VS+znB4JCJ)k z{V;;O^^{#xal2vJdfzp3#QEhhkfPEi{;-ptC|qnUT#!;yXa)0QI#2)Xqm9=;OR;W1 zV4K}qo_@zg>n)k&)2Yt%6Evpt&?W@|Z&fyK|NIl^5|bMwx0OF=Me*XTml?sDF^UDD za-c)}`PZHUMfHez!-?#Eld&HS4qe+n70#_KZt3e8zotM3*G?tcy4pcIRf=OUC;mLy zbRHbg5|L)0UOrc}_ReerXA`ElvdLuPJQpN_sbBGEY2?b|K~Yt>@x|juMvS?!#{`;` z!JW+K({4v`G3nw1M>5}~)nBv($jQ<7o5{g5^hIzB)k-FR6jcYaBx+52{Ca~Fm6PBR zEz}BjdE1N4E$He(TL(Hc-i#4yOLOKfk;o`L8U6L#5XWP(`C|^Z;WaNe@x8DB;d=!( zug;=AQiZMW?o?OZE*cW!W#{7s)EJTv1JnaA{sIckZR`1{h`xvlOPRx;kh?><4M1|# z?_^J0ZPlnC9Tq3G|l=-*LDb`dg8Kwsz+=;C&gEW^1Sa$qsBmPYM?g<;?bDQ8ipzO3k!!OpA*#I zGrnjGNU4q$v$P=>7jL@iC8Oz=iLL)I$KAuS4%Ky?eQ|av^TgSUYoE&al+S1-UFYsb zQ)t3=JzBS9CM`aZc-8MyuO{QE=XLpapO|kxF7*c#V|azVzv5k-9#&cN(m;cMYDaY> zz>xLSx}Z%)1OZ)uTO|1DL(=8bRKoS*;5G~2g!8~pePK9kou2)rvHcV)H8X-1gti}i zguMm{%$bfay1oe`dEpIwd5x>-Joav-t!RPRrCu@QEv?=+{nQ?K(j&clK{iI{txECr z;pdNA^8zA0g0u;g)&zdS0H!-e-@lnrG!VNQwfCC1ZQiJqy^MTa&ukQ~XSi-496c9s zSRncMK=F&-{cN7dFGuWaJ}3xd)3!jM`_cR$;~{|!-F!7ZZGwx=j52?&m7%!btA^KO z9?qS5oYqWIJ+YFH%1!^)MUN2L{k10tv*l5RAnuYQf?Ktqb`NF}-8}+4Q~PQkuk8zY zga%3#mx(?0akxacub6)eX6p~ZG<|Ar_3G%n z{}PnH_qommFNqwiHB9WWnj)Xc^I^SXO@FrO)kSkZ{~6>fKt4e~m<3j-{GCRD&73}J z3~6gf%Piv;!^u;rU<#K>zq}0ar;$N~MVxyhiaCMOi0%R1neBV@t#M&5`^ZUrtI(54WGjGXcp9FT>KK&{n@j zY_s@D(Jrq`;m_5h9v#zePiWbm8P=UQT+}bO!CAS0qJyQ?(ig*Id|Q;$A}>`7EoihK zdDsF)s$8jcKXU~{LaR$2o;E+#e{p$7*2$%<`%)qi@BMUEBaT5nmRv#l1@u#i@KpI) za_7EbB(2Lcue&7CxaHZkffHv0_Nx*uK{j!H$>c>_RI!N>_SguU>b~LXbm3-lFr+zV zWYbjyaIgg%!+A0<>_hW8^Bt^SmEU?^oiiT^Yivt+l6tYV)($&y72^I7KC!X6X`5V7 zUqT2Vm&}`D=x0if_$-aS&db!K7uPCxAoYsI?ER5YDbqYn{uB6o3|nLs@JkRoK52bG zv%zf@6rV9C0C&~>Y`T7s6{+DeYv0LUZSwo&tHVz^>4oNMowOh~u}6k2W*+Fs^zgR- z4%7VUtAfG@jI08ZoGzeCKEjR91Nwj-DC1^=ydfTXXX{PJv*I?;Zly}#UB`UepyRd_ zgv{&)s*#554|-!@(BVXLTrSo*oqvZ0Y=dZ zN5pnHilDL7(h(tNK$nS$iAPt#D@Ro~L>nVtK#ixPN#%6&mqEpmWi47c&3Cfixn4g`+&BZ*pA(V>;{$KlBjV36H zyhiAoXOoCp1YH%1%DY~AEpldoWo)0W!7iv)YIL~b$f#k=-**4ar%)~ZdT5rnJ-?Oj znj7E=ayPNMa&T;I`MxzSZdfjX^}_N~_?jA#jJ)%_1TRH6TzYHl;l?XSO&X`Y?)N_z z)4jW{YEwfYM!41MLGRXy`5vo7en5DpuT5^`yp6llQypHbI@(dTbd-LtUIBQD5jsEw zsFKgl+Mak?0V@gIc(D-#yNLAxWNQQ>7Q>1nlKPJ%{Maypx5XJ&86VVPswknie2Kd6 zYvPB_Z$@llmtokxcH!JvqU`b=<3%kX)Nc=S?H$K);Ul9+a_uvT;`JKLMgqRL|0_Ma zKoloW1`bqc2)nGPy)S+&Z=6%8(a} zFLG5*UgpDZdfgAU=s@1(E54v~9?`PaNV!ykc=ESfvVW5E*R-ghC@) zpz~#JbHi)e0$mT>Cv;tDEi5h6G~Q=fMWQu?*v|==c@eucF%B5gL4G|oo?m5kWd!2x zErinK+s3LZF9g)H0#H)p6Z6i(*$L4hnaS5W00Z|w+B@s0s=jvLOGty#Al)E{gl5ll<1C+WFGB-^H{jW~&hnxeEH$qcpDs$D zvtU6&PGd-66- z*#DW&wfd((I-5lZTUsO4jBxt!l$p&Vg;s73)Ae>%S`E^NfjW{zkDU(lqr>2G@C=Mp1feUMs34a; zA=$!+R@Ir>zW>y&%CP&;&vYI55fG2?0t*2;c03`lk069b%r0#U*7j>&*S@ZtxU>55 z5+>9^aAzV!@CqrQ!`4}!wmOfzJ^>08if0#Ox+HC6<@_|K0&COelgrbkv2Hv@|UKb36cEZ-dbw{gh=PGVaSDL{ZmHg8=`i!E$M5-9#u3bcsN^OEZ@qG6eOL) z>V_arIeQl}yIJ3=iQUJ9l1&mnIr~A<v5o|%bu_<%m{naJs*3Yab)XW8foGy*nUe~{c%-i+1%lb$m{gbWy%1; ztH&ejpK1xiY$oYeSD_CN>;;Y z4o>HP_+Z5y=aYbVK>OX?;}gi0Y-cfoNU)Fl>5L!$2;fJ#{!NqEU^WhHDg%`prn@8| z+;)Mj5n~e(pGWUI#MPDIM2?UJfAD%LLJ+*T*PNF*czG+o$YRq?FA=Bh3_S>WEx-Bh zvh`Pkd}pFfarmt&mhkffdpR~mR^cCa`Z;oJ^3IE}^M_K|d-D)iysR2UH@{z>+h{vA zXmMa<{H}H-Y)lRILlj6Drv9Yw-ZZd?V$nxW3C&mKiI5lRf8ZuJ^{Pedlx#3vZpAqD zCrz6@1*@Q7VwwPs<%S|2gU{H}J##{N{Slv|a3qyExK^p4UZ=R`Um4`gO4YlhIK1)T z8V1k1j+0%CxxX&Ipl<@_zt zdX@bk&su>^?~u0bS{P6K*RdjbnP3vn(F*+t7ni9dy?&j4B*e)${`kx-WM4zYFd^4+ z_WkJt8B zs_xHQ5kR_5*ylBs|Lj`10)0DlX{}{2dub#W(B>P8dm{qdPZOKvsb;U#Ru4dnDh4nPoAWOl6T`P|7^$p1MUW#tXVO)bDx+`TYh3l>pP&-yySb+qugQ3VdUl zAN6$p2{-3nqkPE`0dr^qu4k*%Hc=}X3f)ZVH0~eMhmtgsd7#>W!oJ-*ppq2a$2V+| zrII06Bc&SCM>00{VLKBk3pXf zFkfkv4HuknadYVj9~s0$VzwB=#mWB8hQo7J>|NndeY@7~(Xl~EepNrS%sX6dSo4x= zIUCL7#0}@>A(0+g*9Me~tRgCbJNDGu39V?XQF-VosGIq4Gh*lw_ zX5m=)Sj(0j>5$-3Lc({M#!wBaZc-1M>Hz~%I`}Ns z5N}tOLq>?1?J&AU@i;tv1b~zwN%!hr&ah^Bo#wMhH;1QU-YS!Npu#A+GWk07^>Sj> zx9*F-(IrT81=#k*%d02IeWCYyUwe3yl*#Tknsv5%%n&3-#9|Ws#TxtGVO(&D6rF|Y zM$9b6j+h!a;p~UJJg5>p`6+?U+##0?H8#&Ma9HXaF`ZQZ*K~5cH(-nhyveul*5mK( zUoWQse!CnpZ4|AS-zgM)pu*fv!{nL#>Mzt)?-_E@rtxR?r;L?Kx@3{>kVSFn+8<}A z;|r%?bOgn*P|JOSe^+bz6uhi09sLeh2p6v|_g_P)i2bYB%uH)~E;?k6+tn1TcJ)C_ z*+tB!X8JkyzrSn$^>yh~z6gA4=@tc^81bSZyj(UAUkIp{{y@jCf3)GB{BR}l8SF{V zkhb*fzs8vLCJ@_XmQ0y99J0u)+z4}<1L`US`PU(DV;}uc8V0=bFwlyiFqgS<4vu*M zgSX5Pw&*#Y3`+-`I~*zanF_CvFQqt)HgHo?xVByc6pwNtLxR#`dH+}e_X@f3wR>Vt zmL-7o{)vVKzsf}e1bH;O(B>Y&eN=>7&r%q5SY^P`)JpZtx3BLIx8P7><;&#n9a zv5Uo6B-GD1{#GB;BCF7r{&>lzQd7*y_BXnj7P%6T-~}C7DSziK8l^`pjJvn%{?5Vw zbK?KgOaIdq|5v-BLg5#}?z9XDOcC+b4TqXQ3K6(T@B^=dj~$`5o-2Q;dHqFnbGY4h z@-@8$4y4Vj2o;)T#Qwy);*98^f0$^#vmAtb-((@=WoBv8WBUAkwmPjyPlfw=O_S2? zS^j#=e!lvp`{zpI5G-JuYk;$S@FIY5J3zppZ3SXU`GD~~-QK!ZkQ7GnlfjB!N70qE zw+bM+hd^4uFHGBnTJPF8fd@|8Il= z=*=&5QH;($Al)fMhM{sP2z+pY5s~E8~zKpgtLPpzXwx8dcV?08N6xSM6Wipf)Rw_(jB?WWMv#NBaA$u1<`lQ z=^ybPD=9I!G;V7L17Fqrvj$;y-RqgjAZ*T>^qPYo*uemvoEWb-itiFOvBKl3Oc)uA z#6W=P9AUyo-CXLopms`Q?yRc%3H5;wIuT;@gMp>1h>)`Yz>I@O6@y>Lub6ed>x~DGFkt{e}w4VX3FNkwaq5ZWl`dq^wA6?4LH^ zwd)9i+5qf?Y&(V`Y*c>@tpZru#t!US&3kQCm?CR;i8KQppC2~~O{6KzSjN4Hm%o6+ zZ1-}QOM~H%z}W9sdXC6h{ozv0b%Q@ykk+vgc%y`n(MHZKz|%zx0lKX9Z#NFBJmTt| zk#7Jw$yk)V=@>x(0uGM%$OwbmGVmWU!f5cxM-)8%bk~{DRP`j2C`5k9g#?kGZeucM z`K?-q^>GJCzHw6eBfF*mKHD)UXAXLq^a9lP1p)4y-%9XwOeGi>GS#wQ+@yleg}n zAPEL#KA+U$b*tuo;t5c9>`qcypO8Dk`uALcuMf$i z69yK^=Fg{1)AW3X);^5N^F1N!n*;ZdZcKZ)9m#G8MDK!f^6{Tdt-j>`A5uk@BtrBZ0*h1=@%7<}9C zggE$Hpf4R^{F^ItJV6}B^Lk*%TUZ?-zQ|{fDBj1~?l$hWXwN~2z3@G3QCaFy^s@Jz z(S0YUG|!rK>|Mh=@6Dll-1;s4`{^r z7Y8jUg{ufHjkaEiPTc;4cV)f@luPXrU-5xE&Vj#NZ)h3$3bOh_IwOlC&W{V?jRgK@ z5Ox;m=Qr2gSQE(~C>8h-I$QQK-y2Plv*5VYCntu?{1gao9>)oVA%SgF_q0HRxPGs$ zkZE+|6Go{;Z%L{#n|t^V z6Xy;#WIKJ#k#sRocF*RB(fPczKVIT>+Fo3U4c2`9qP`lRC^WOOvuNOKIEPF&QWem&t)7a3Qw-pA5QE?57ZtoEU zXKd$H3SuOJN8p7w1&+p3U6R)U(|VKaZ5wzFHDUO^wajAmve$hM2cX`;eF^L+Q`XR& z#i1>1)v}}SpkElr+Pg1P+a^b-aXxUD#@)*1XF6 zF-Xg+!}1!VL&ERD2%S=|)crFw)1R(;6f1IS?5R4Xn64vm#ms&xr?Izcm$3>Q!Dp6j zR`{$b&kBz+Mq2#+v3*Q#$x!xkf`oozM^|gzhBPH=|oM?ViBw|8Lru21_ zWp~+a)p*HY>i$~M&JeIlV5#qRAtYaR^mTqmKZ=?0T@&IU@wysK%z#hCH-HuQ2Ovef7VDJ65C~!XKuwsOr)8(O7 z68|AL8|S$400_bej}gZx$s0^sT6X_nm4zxXvl+f0Jykx<`AxP<>T8G3x=5$|LWPa# z$>c4#Z`5+W#EO#@?~p%Uv?WfpRpA8g8^hGX+t4Abjda+r%hKb4I=aH}hS9<4Tll`I z7OTXvYO`_oBMmS^HyC3q1v%A@e8|AjAo2oW{mHSqx^et~)jC&D3$;cNv-xC9Ki$o25D`IYlihVQs^jWNFa1t#udL4C{UEQu$k6 z7oHR_)!P4HSki8RlcxZ$u&;9YJJ=<5%ekLw_E-92^JR+%_XOJtS^VYTn-@|gHV27+5w@Ir&b_D2p;@XH`!>jgz!9# zX_ALQ9U~xtEuS#cle=7h=nitpQrHYh7m4X*`bWhb$H&uWgaYAlxzS>J=WkFxqtL{2 zo||F6GCw*z3LYcqlSlg-AShiaNS)r&p&(pkjxaVW*osePxL?cJhU@QmVi`5fjwnwW$**lUVXa7rf6mj?P7b7hxBvAlk8^4HkJu zN{9Bu@l|iqyt<8^(ve?AZ-orZ6xrxU&^Ogbvm_oaJoidr`HH=dAyZZPPvJD6B9Mbn zF9$w;T{FMrBJ2uTk&6ZeLZxg`+%IP>`i)XT!md?9A4YLm8@4 zhxqcOtM-DMBdRj&vFAOG7$KUFrYD{GoGk17EFErPN`Gb(XlABb1Dqc()_xZY6sIIu z;dayJY>0k&N!=cy>R%*ikhM!OIfag{VHv*SHfJUJZd2!t#sd-m_A&WQ@@W<(pNrhl?|hOmXfP`TQsiGd1dsSRIPq#_&vhivxu}RsWRH zAZ9MU(c1pSrJu{S>>1h7R40v&8FQ`h2s9y&ZbPr*!5iWb@uGWN4^+6>dqkbQsx;xRVtQZdmnU&DpMnTwyI(4I)3nuN6cvEA=~xyzc!ztGuU)A?$Jpu(3T z@>f(Kww*X;)US_$BCHtnfq_WR#l@L6C6wM8>V&l7O5$q}c@hz;9zA@E`nLaL5WRuc z!}j~HJZp4sue1CGqZ4X3=6E;eYI{nS77}=66^BYpbL$HW3QvUJ|2$qMkIS(X{gL5C z3@!dXag86EdjtDjPH#C?F5TAz922U)Ua|`B1~}Q#v%?4jd4Dfk&G!&N>slthy$?PY zAM#0ke`=|!VUgFEGRjM7>Ugfrk4H&?)baGxHgv|Jwl0dcrP`cLzn1983HIca_{Oz1 zkKTP;k5;nKHlv8l?6c4l@5`PH(%Gy6^5jfVViN)h9=@5)C^-8cE^;pcf-vDj6_9Ey$EfRU7YvT*hx{f;AVA=6%6SG*AWb1_!Io1+E z<97mzUeywMQ+0>7jvGgm&)6dv+^+{AK+6qh8Yf21Y?)t8fC)6{dU!(Js<2YF$!FesAA5F_;IiM9L9 zVV8&Ecip#P=iLn9Cp#t4xKa#)=k!Y~y_>4MN=}XC*MIVRONI9#aSR$plB9~tW|6e_ zkT7s0#NZ#wKNLQCXO=|7ZET6Et;o54ll=92pz#-9T_-C!CtPu1DH_idKH^C~Aj0;( z-=gGHp9o5@aMk$de9_xu?HlewdOlrfh0o&$d)=L8w>uwU&(FL{hVSRRFKmalVrB1j z;qj=@X|C1vMFh4V7bzR{7$@#O#h!f;enHvk5g1g|lS(=i%!Hbo6W47HAVftLTYOXyYP^BF#|iRe4eo*b+l^5EoMSEL}Tp;H>S)J_^h)rVH%C$?A@}Z=&)@Dl$iEfTA#5**k}eU6ff zTV1K5O7U$IBaw!(7TF@nePK?$C|Yq6&o?@{Yf_^7IDfoACW_;=vP22FX8Fvqr7E=Z zqUo_D92MH+JzHfR>y&YIXgKEY9`x71hz9&MFsQihe@<}WN)zbX?w#X@>NtKqQE4iq z43=GCR2@QDE=9>Vu&qqQ)F|7sj6w?=?ZuQ-Pk8OooTYaSgw1BySHK|Iyvj6HJ!LJF zeUV4t2fya4A(_Lr%Y(Rzwl(uzt*bGeQryIs&%!sQG#3X8h!%hE&Q@zs{iwo!NLFQ| z1^s3xrQ&`(z!$v1&^c}`7yxU-XJs!xp$aRWec>{YedJQz`|)#?xwo6i_d)5%HKJK) z)%^J|nd0IHSoCwR=xxgq%kjKae8emWMhDD-vTg;K7yQK*QtFpDt>p`&bXUsHG{%TDD5U+66NaAnGtH?p9NEl*zM5sn|2YdDVChVyD zjuzzi6ZdQsZ0;vT<&L;hAJ)|Q(^9VlMcOkEPz3X{h6Z73w71_ZU#Sv9m1kU6HqW0= zE8|Yf$#8B)t2IQ`pD1;4GTWVBs~52kOdTu|V8xE=df&!A$usmM5EbD>S2L2V{=}VD zcS^S75wVfuDSG2Crh#EbqE6I4!Io#>mV{=(6LoZ+XS3eCzn@577+P-G8L+4ui~rK- zMR)&bciL$q=2zUq>7_*6O*^IK$Y{3m04)!C)0qB(4rIAiwS6~8IZaHYXRGA!rb*jI3!OQb#fJ288hFt9j5ReCKU zstlDoHOEBNBWQ7}fM1ifJXuw!&+Y`4Icqwz=)4uGHN*7tnTW;tn^njgWX{4u6{8-H zw&NkuG+Z&5oKhvnIcCyo+;9ZXI#X5>sg5gMz19n3ClPl}(fzJMc- zd&ui1j}yk7MI?65Ka=XzOinW&$U)j5)lW%y+4eQwX9_9>o!XYEDr=`q;ml<|aoeli zh+05bKF9c~o4%izHx+PV-4tSIM3|W;;IMvC+yr+&JHdG@zjo8a`9oZVUP*%wBJnh# zOl`0tf3Tm6z45SIjI?<$&Rs_j_^yQmD4<$);j(t#IsKd(Rfu(tjBB_o`25|{$DQUh-?0+NR zT{wf5#lqH?lvoZn_&z;{91HOF{D`Ap`wahWsP zi_jGmB-r)9fYsE6*f5caJJn zc{x<)c)eTALJnb@#wmK{vNzXY!)aK~wF0&u3Z6R-`))PA&d+DeC}x*FM!O6B)Kcn{ zb*0|O-t)EPI&$qtoj5#ZrMQTLrfvTWv;AzU0V@ZQe?Oup*s)${5ZG4{l`?F)*waW= ztr!kHuyC%Z^%tXnrGYty;kWJC&NP%(%ORi0IC0B1fz-E7Vj9o4jL_^6O=u&R)`wnH zPhncafO2|SG&{)X;|&>%%yTAi-HNB8aFKa!}MdPh?U2H~~L*Dd;;8Q(U zLz=yP{{=?(<{PwUBpSTu)%{G4;gKP6uXx+XA1I}%IMJBS&jIB=nF?0p?@SCyV5u~w zxDC|-69DvQ`U)^@3QmRHkm363Z%dsTalUd`D!59W$q>oA3VU;OMEd|a`|#m`U25?} zwpjy^<&Cbz@BdtW!DxPYw8mQAsUq2-Hlt)@uC=ow>ShtCT=uwUG-yGR$!+?es%Gn5 zmAKvkI zffCe8$wgHd)Q#~Gh%KB_GOD5C?-jX8zT8Yk{S}tWiut++Y#O47#=#|WS0+so@5 zTSYY+Tim=UxKgm>%DnzSozs) zvcq`AuN@b!LYkYFq2t`=y9@Bm_j|Krcy32EiS8oS{>8R?(5-nTN+xfA@aEo4y}j8a zK}yQe7#jzRYKi(@5H(o+8T*a-}r_k0!(4Np{V ze;Q=H^;f#Cte)c^WzFAP@=|S5A z(p`mE9G+g*KAvvD>GahC6D3}K8+(UqHO_4EVWP

m^m=cWXDZ7b!E{*~tMYDl7vA zp)oGU6v$w9Z$V{jIP|TD6C7+Ca(<2(_>rh2rv7~Skjy$<28jsPEV4j=7B4!5S@AU~ zu+`%>-qnV(rC?NV51GZvzyA{&owCV=fF_(G5<&z8v$cmE?)@P37Rmc8VWGj^9!|F0 zY^;sY2;z-Jr%7!>hyxoo@4!n(dWia3%L;c|aj~Ts+2tfAs@fgv!!ZFh^w6$yeIO4_c#AMjQR%c~H1iEKRxYS*Rs{?WZCX2%f|$tK3|u=GUoFoon&EjekxZA- z-g(7!TLm1|F^!@;dc0(NOeUUVc9DsAcF!%|zFawJdpgI08wM$=k!B`d#Jc8xPi_H%E(1y{o* zuVT}@+lXLzFlQZW!U5}kQ1N~FzzEnxZ;H6u$n)?QXH(g>dTyGw6x`*G0~9tj`X%L} zhwX&s^L0+OujNMr{(ueB>>Ad+=c=qITEZ`_vF)17+c7ipz~=!}NcqD?)r9 z69Y9Ix5VR9)<4S7Z(w7@n^2%OzY&A)zxZNr}VZQ%)?3-c-;thvZzx(n5WhE%6C#m@ZwpVvOIeIvm zRKeC#Z!WLdNIyHt+^VJ6{s^y=m=Zxoq|Q2dwy+;pDQl0ozJ$FBWqZBClpZg0MJ?@; zrpCm@P{nwj`jN5LnUBs$BxkVWmiE%liWf1;S5kdy+2j$=Omew%m)~#I%le7MVc-af zY>hPT3>ll-KVhBp9Vr~El#McFQ?Q6djI0@&vcR6Yil~acI83_lP&kji?UCesRdIJv zSy|$zFIhF@N1woJ8-@&#U_uv&N&lX#g`Yz^B~9g|>gICyO+3hR0FEBHg~3{rd6R0p z*z|&if*3lZHxVz;QWf?E8SB2zSc@Zw?t4sJa_oK9H;)&ztaFIj)mLwb`Q1TG>!xjd zgru;bbU37S$CM%1>F<$a<=jj(v%UR|+fNUiqPP(rpCL)oEWe=xX5jwJaPLikU;a0j28*UFhKP(EKY5YRd%{W zp_YA#g?(nG(sC5MsN?QSyPKlP7vs#-*cnFgSTg76_Z6o%S`fzeUx`wA9<=iQJ~MTM z633|=3T)39H47Z;MD{(L9bTiTILjFZ{(;VCbW=$I}ov)ai&)3{X) zy*Bff*3eiLc0z6o3(xg39<++~=_}hZQVGH=DNC*`+2i(%wmVI6SxD%Y)XTvsmw}k^ zwjT4(3#*>4<~u74#Tsb^g{s98&ALA%S68EMv$52Ud~W5-PXk^AVxp)pcua9HvXw^Yx`P49gRA5ty%$GF>DmVp&og zl@nihK#WYdkWP{MojydBD=&&V+>U&0dAwNgi8S#(y@zQ{;B?Sn)8+(zX(cxk8! zIy6U&YVsJV=Z=F&5>gz8qipyY64h|mt*4`tfW~c+ra(>-R0Od|6{fxC3-cFi4l07J z``6-Qh=Ig_&plK`K=mJgq+$kn-L_bXVekisgkq7;z1Fqx|BGY6fKVY3L7j}I@Q^Te z;!u^0cH_nea;z8J@-dqG7C>e!#bQs6&d+&}(-SY!w9f4cgAJOFXVpEAg-<$};BL|Ff2 z7ySG8NRkZw`;Y(Sv0^Nk>D^mKl}fDt&6WP!l@LLNQU19#|EaktjG#lK$6sdr_m>LJ o2s#hve>(3!+?fB*citbA9?S~|bmwJBx3rQj2ZvX%Q literal 0 HcmV?d00001 diff --git a/docs/source/index.rst b/docs/source/index.rst index b0103a336..34eb23b28 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -43,27 +43,13 @@ Example .. ipython:: python - import datafusion - from datafusion import col - import pyarrow + from datafusion import SessionContext - # create a context - ctx = datafusion.SessionContext() + ctx = SessionContext() - # create a RecordBatch and a new DataFrame from it - batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], - names=["a", "b"], - ) - df = ctx.create_dataframe([[batch]], name="batch_array") + df = ctx.read_csv("pokemon.csv") - # create a new statement - df = df.select( - col("a") + col("b"), - col("a") - col("b"), - ) - - df + df.show() .. _toc.links: @@ -85,9 +71,10 @@ Example user-guide/introduction user-guide/basics - user-guide/configuration + user-guide/data-sources user-guide/common-operations/index user-guide/io/index + user-guide/configuration user-guide/sql diff --git a/docs/source/user-guide/basics.rst b/docs/source/user-guide/basics.rst index 3c97d1ef9..f37378a41 100644 --- a/docs/source/user-guide/basics.rst +++ b/docs/source/user-guide/basics.rst @@ -20,72 +20,76 @@ Concepts ======== -In this section, we will cover a basic example to introduce a few key concepts. +In this section, we will cover a basic example to introduce a few key concepts. We will use the same +source file as described in the :ref:`Introduction `, the Pokemon data set. -.. code-block:: python +.. ipython:: python - import datafusion - from datafusion import col - import pyarrow + from datafusion import SessionContext, col, lit, functions as f - # create a context - ctx = datafusion.SessionContext() + ctx = SessionContext() - # create a RecordBatch and a new DataFrame from it - batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], - names=["a", "b"], - ) - df = ctx.create_dataframe([[batch]]) + df = ctx.read_parquet("yellow_tripdata_2021-01.parquet") - # create a new statement df = df.select( - col("a") + col("b"), - col("a") - col("b"), + "trip_distance", + col("total_amount").alias("total"), + (f.round(lit(100.0) * col("tip_amount") / col("total_amount"), lit(1))).alias("tip_percent"), ) - # execute and collect the first (and only) batch - result = df.collect()[0] + df.show() -The first statement group: +Session Context +--------------- + +The first statement group creates a :py:class:`~datafusion.context.SessionContext`. .. code-block:: python # create a context ctx = datafusion.SessionContext() -creates a :py:class:`~datafusion.context.SessionContext`, that is, the main interface for executing queries with DataFusion. It maintains the state -of the connection between a user and an instance of the DataFusion engine. Additionally it provides the following functionality: +A Session Context is the main interface for executing queries with DataFusion. It maintains the state +of the connection between a user and an instance of the DataFusion engine. Additionally it provides +the following functionality: -- Create a DataFrame from a CSV or Parquet data source. -- Register a CSV or Parquet data source as a table that can be referenced from a SQL query. -- Register a custom data source that can be referenced from a SQL query. +- Create a DataFrame from a data source. +- Register a data source as a table that can be referenced from a SQL query. - Execute a SQL query +DataFrame +--------- + The second statement group creates a :code:`DataFrame`, .. code-block:: python - # create a RecordBatch and a new DataFrame from it - batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], - names=["a", "b"], - ) - df = ctx.create_dataframe([[batch]]) + # Create a DataFrame from a file + df = ctx.read_parquet("yellow_tripdata_2021-01.parquet") A DataFrame refers to a (logical) set of rows that share the same column names, similar to a `Pandas DataFrame `_. DataFrames are typically created by calling a method on :py:class:`~datafusion.context.SessionContext`, such as :code:`read_csv`, and can then be modified by calling the transformation methods, such as :py:func:`~datafusion.dataframe.DataFrame.filter`, :py:func:`~datafusion.dataframe.DataFrame.select`, :py:func:`~datafusion.dataframe.DataFrame.aggregate`, and :py:func:`~datafusion.dataframe.DataFrame.limit` to build up a query definition. -The third statement uses :code:`Expressions` to build up a query definition. +Expressions +----------- + +The third statement uses :code:`Expressions` to build up a query definition. You can find +explanations for what the functions below do in the user documentation for +:py:func:`~datafusion.col`, :py:func:`~datafusion.lit`, :py:func:`~datafusion.functions.round`, +and :py:func:`~datafusion.expr.Expr.alias`. .. code-block:: python df = df.select( - col("a") + col("b"), - col("a") - col("b"), + "trip_distance", + col("total_amount").alias("total"), + (f.round(lit(100.0) * col("tip_amount") / col("total_amount"), lit(1))).alias("tip_percent"), ) -Finally the :py:func:`~datafusion.dataframe.DataFrame.collect` method converts the logical plan represented by the DataFrame into a physical plan and execute it, -collecting all results into a list of `RecordBatch `_. +Finally the :py:func:`~datafusion.dataframe.DataFrame.show` method converts the logical plan +represented by the DataFrame into a physical plan and execute it, collecting all results and +displaying them to the user. It is important to note that DataFusion performs lazy evaluation +of the DataFrame. Until you call a method such as :py:func:`~datafusion.dataframe.DataFrame.show` +or :py:func:`~datafusion.dataframe.DataFrame.collect`, DataFusion will not perform the query. diff --git a/docs/source/user-guide/common-operations/aggregations.rst b/docs/source/user-guide/common-operations/aggregations.rst index 8fee26a15..e458e5fcb 100644 --- a/docs/source/user-guide/common-operations/aggregations.rst +++ b/docs/source/user-guide/common-operations/aggregations.rst @@ -26,15 +26,7 @@ to form a single summary value. For performing an aggregation, DataFusion provid .. ipython:: python - import urllib.request - from datafusion import SessionContext - from datafusion import col, lit - from datafusion import functions as f - - urllib.request.urlretrieve( - "https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv", - "pokemon.csv", - ) + from datafusion import SessionContext, col, lit, functions as f ctx = SessionContext() df = ctx.read_csv("pokemon.csv") diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index a0b95c908..8d6a80855 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -25,14 +25,8 @@ We'll use the pokemon dataset in the following examples. .. ipython:: python - import urllib.request from datafusion import SessionContext - urllib.request.urlretrieve( - "https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv", - "pokemon.csv", - ) - ctx = SessionContext() ctx.register_csv("pokemon", "pokemon.csv") df = ctx.table("pokemon") diff --git a/docs/source/user-guide/common-operations/index.rst b/docs/source/user-guide/common-operations/index.rst index b15b04c62..d7c708c21 100644 --- a/docs/source/user-guide/common-operations/index.rst +++ b/docs/source/user-guide/common-operations/index.rst @@ -18,6 +18,8 @@ Common Operations ================= +The contents of this section are designed to guide a new user through how to use DataFusion. + .. toctree:: :maxdepth: 2 diff --git a/docs/source/user-guide/common-operations/select-and-filter.rst b/docs/source/user-guide/common-operations/select-and-filter.rst index 075909129..083bcbbd2 100644 --- a/docs/source/user-guide/common-operations/select-and-filter.rst +++ b/docs/source/user-guide/common-operations/select-and-filter.rst @@ -21,18 +21,15 @@ Column Selections Use :py:func:`~datafusion.dataframe.DataFrame.select` for basic column selection. DataFusion can work with several file types, to start simple we can use a subset of the -`TLC Trip Record Data `_ +`TLC Trip Record Data `_, +which you can download `here `_. .. ipython:: python - - import urllib.request - from datafusion import SessionContext - urllib.request.urlretrieve("https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet", - "yellow_trip_data.parquet") + from datafusion import SessionContext ctx = SessionContext() - df = ctx.read_parquet("yellow_trip_data.parquet") + df = ctx.read_parquet("yellow_tripdata_2021-01.parquet") df.select("trip_distance", "passenger_count") For mathematical or logical operations use :py:func:`~datafusion.col` to select columns, and give meaningful names to the resulting diff --git a/docs/source/user-guide/common-operations/windows.rst b/docs/source/user-guide/common-operations/windows.rst index 609176897..8225d125a 100644 --- a/docs/source/user-guide/common-operations/windows.rst +++ b/docs/source/user-guide/common-operations/windows.rst @@ -30,16 +30,10 @@ We'll use the pokemon dataset (from Ritchie Vink) in the following examples. .. ipython:: python - import urllib.request from datafusion import SessionContext from datafusion import col from datafusion import functions as f - urllib.request.urlretrieve( - "https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv", - "pokemon.csv", - ) - ctx = SessionContext() df = ctx.read_csv("pokemon.csv") diff --git a/docs/source/user-guide/data-sources.rst b/docs/source/user-guide/data-sources.rst new file mode 100644 index 000000000..ba5967c97 --- /dev/null +++ b/docs/source/user-guide/data-sources.rst @@ -0,0 +1,187 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +.. _user_guide_data_sources: + +Data Sources +============ + +DataFusion provides a wide variety of ways to get data into a DataFrame to perform operations. + +Local file +---------- + +DataFusion has the abilty to read from a variety of popular file formats, such as :ref:`Parquet `, +:ref:`CSV `, :ref:`JSON `, and :ref:`AVRO `. + +.. ipython:: python + + from datafusion import SessionContext + ctx = SessionContext() + df = ctx.read_csv("pokemon.csv") + df.show() + +Create in-memory +---------------- + +Sometimes it can be convenient to create a small DataFrame from a Python list or dictionary object. +To do this in DataFusion, you can use one of the three functions +:py:func:`~datafusion.context.SessionContext.from_pydict`, +:py:func:`~datafusion.context.SessionContext.from_pylist`, or +:py:func:`~datafusion.context.SessionContext.create_dataframe`. + +As their names suggest, ``from_pydict`` and ``from_pylist`` will create DataFrames from Python +dictionary and list objects, respectively. ``create_dataframe`` assumes you will pass in a list +of list of `PyArrow Record Batches `_. + +The following three examples all will create identical DataFrames: + +.. ipython:: python + + import pyarrow as pa + + ctx.from_pylist([ + { "a": 1, "b": 10.0, "c": "alpha" }, + { "a": 2, "b": 20.0, "c": "beta" }, + { "a": 3, "b": 30.0, "c": "gamma" }, + ]).show() + + ctx.from_pydict({ + "a": [1, 2, 3], + "b": [10.0, 20.0, 30.0], + "c": ["alpha", "beta", "gamma"], + }).show() + + batch = pa.RecordBatch.from_arrays( + [ + pa.array([1, 2, 3]), + pa.array([10.0, 20.0, 30.0]), + pa.array(["alpha", "beta", "gamma"]), + ], + names=["a", "b", "c"], + ) + + ctx.create_dataframe([[batch]]).show() + + +Object Store +------------ + +DataFusion has support for multiple storage options in addition to local files. +The example below requires an appropriate S3 account with access credentials. + +Supported Object Stores are + +- :py:class:`~datafusion.object_store.AmazonS3` +- :py:class:`~datafusion.object_store.GoogleCloud` +- :py:class:`~datafusion.object_store.Http` +- :py:class:`~datafusion.object_store.LocalFileSystem` +- :py:class:`~datafusion.object_store.MicrosoftAzure` + +.. code-block:: python + + from datafusion.object_store import AmazonS3 + + region = "us-east-1" + bucket_name = "yellow-trips" + + s3 = AmazonS3( + bucket_name=bucket_name, + region=region, + access_key_id=os.getenv("AWS_ACCESS_KEY_ID"), + secret_access_key=os.getenv("AWS_SECRET_ACCESS_KEY"), + ) + + path = f"s3://{bucket_name}/" + ctx.register_object_store("s3://", s3, None) + + ctx.register_parquet("trips", path) + + ctx.table("trips").show() + +Other DataFrame Libraries +------------------------- + +DataFusion can import DataFrames directly from other libraries, such as +`Polars `_ and `Pandas `_. +Since DataFusion version 42.0.0, any DataFrame library that supports the Arrow FFI PyCapsule +interface can be imported to DataFusion using the +:py:func:`~datafusion.context.SessionContext.from_arrow` function. Older verions of Polars may +not support the arrow interface. In those cases, you can still import via the +:py:func:`~datafusion.context.SessionContext.from_polars` function. + +.. code-block:: python + + import pandas as pd + + data = { "a": [1, 2, 3], "b": [10.0, 20.0, 30.0], "c": ["alpha", "beta", "gamma"] } + pandas_df = pd.DataFrame(data) + + datafusion_df = ctx.from_arrow(pandas_df) + datafusion_df.show() + +.. code-block:: python + + import polars as pl + polars_df = pl.DataFrame(data) + + datafusion_df = ctx.from_arrow(polars_df) + datafusion_df.show() + +Delta Lake +---------- + +DataFusion 43.0.0 and later support the ability to register table providers from sources such +as Delta Lake. This will require a recent version of +`deltalake `_ to provide the required interfaces. + +.. code-block:: python + + from deltalake import DeltaTable + + delta_table = DeltaTable("path_to_table") + ctx.register_table_provider("my_delta_table", delta_table) + df = ctx.table("my_delta_table") + df.show() + +On older versions of ``deltalake`` (prior to 0.22) you can use the +`Arrow DataSet `_ +interface to import to DataFusion, but this does not support features such as filter push down +which can lead to a significant performance difference. + +.. code-block:: python + + from deltalake import DeltaTable + + delta_table = DeltaTable("path_to_table") + ctx.register_dataset("my_delta_table", delta_table.to_pyarrow_dataset()) + df = ctx.table("my_delta_table") + df.show() + +Iceberg +------- + +Coming soon! + +Custom Table Provider +--------------------- + +You can implement a custom Data Provider in Rust and expose it to DataFusion through the +the interface as describe in the :ref:`Custom Table Provider ` +section. This is an advanced topic, but a +`user example `_ +is provided in the DataFusion repository. diff --git a/docs/source/user-guide/introduction.rst b/docs/source/user-guide/introduction.rst index 8abb9113e..7b30ef2b2 100644 --- a/docs/source/user-guide/introduction.rst +++ b/docs/source/user-guide/introduction.rst @@ -39,5 +39,39 @@ You can verify the installation by running: import datafusion datafusion.__version__ +In this documentation we will also show some examples for how DataFusion integrates +with Jupyter notebooks. To install and start a Jupyter labs session use +.. code-block:: shell + + pip install jupyterlab + jupyter lab + +To demonstrate working with DataFusion, we need a data source. Later in the tutorial we will show +options for data sources. For our first example, we demonstrate using a Pokemon dataset that you +can download +`here `_. + +With that file in place you can use the following python example to view the DataFrame in +DataFusion. + +.. ipython:: python + + from datafusion import SessionContext + + ctx = SessionContext() + + df = ctx.read_csv("pokemon.csv") + + df.show() + +If you are working in a Jupyter notebook, you can also use the following to give you a table +display that may be easier to read. + +.. code-block:: shell + + display(df) +.. image:: ../images/jupyter_lab_df_view.png + :width: 800 + :alt: Rendered table showing Pokemon DataFrame diff --git a/docs/source/user-guide/io/avro.rst b/docs/source/user-guide/io/avro.rst index 5f1ff728e..66398ac7f 100644 --- a/docs/source/user-guide/io/avro.rst +++ b/docs/source/user-guide/io/avro.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _io_avro: + Avro ==== diff --git a/docs/source/user-guide/io/csv.rst b/docs/source/user-guide/io/csv.rst index d2a62bfec..144b6615c 100644 --- a/docs/source/user-guide/io/csv.rst +++ b/docs/source/user-guide/io/csv.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _io_csv: + CSV === diff --git a/docs/source/user-guide/io/json.rst b/docs/source/user-guide/io/json.rst index f9da3755a..39030db7f 100644 --- a/docs/source/user-guide/io/json.rst +++ b/docs/source/user-guide/io/json.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _io_json: + JSON ==== `JSON `_ (JavaScript Object Notation) is a lightweight data-interchange format. diff --git a/docs/source/user-guide/io/parquet.rst b/docs/source/user-guide/io/parquet.rst index 75bc981cc..c5b9ca3d4 100644 --- a/docs/source/user-guide/io/parquet.rst +++ b/docs/source/user-guide/io/parquet.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _io_parquet: + Parquet ======= @@ -22,7 +24,6 @@ It is quite simple to read a parquet file using the :py:func:`~datafusion.contex .. code-block:: python - from datafusion import SessionContext ctx = SessionContext() diff --git a/docs/source/user-guide/io/table_provider.rst b/docs/source/user-guide/io/table_provider.rst index 2ff9ae46f..bd1d6b80f 100644 --- a/docs/source/user-guide/io/table_provider.rst +++ b/docs/source/user-guide/io/table_provider.rst @@ -15,6 +15,8 @@ .. specific language governing permissions and limitations .. under the License. +.. _io_custom_table_provider: + Custom Table Provider ===================== diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index efd4038ae..e283f590e 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -473,7 +473,7 @@ def join_on( *on_exprs: Expr, how: Literal["inner", "left", "right", "full", "semi", "anti"] = "inner", ) -> DataFrame: - """Join two :py:class:`DataFrame`using the specified expressions. + """Join two :py:class:`DataFrame` using the specified expressions. On expressions are used to support in-equality predicates. Equality predicates are correctly optimized diff --git a/python/datafusion/plan.py b/python/datafusion/plan.py index 3836edec6..a71965f41 100644 --- a/python/datafusion/plan.py +++ b/python/datafusion/plan.py @@ -42,7 +42,7 @@ class LogicalPlan: (table) with a potentially different schema. Plans form a dataflow tree where data flows from leaves up to the root to produce the query result. - `LogicalPlan`s can be created by the SQL query planner, the DataFrame API, + A `LogicalPlan` can be created by the SQL query planner, the DataFrame API, or programmatically (for example custom query languages). """ @@ -107,7 +107,7 @@ def __init__(self, plan: df_internal.ExecutionPlan) -> None: self._raw_plan = plan def children(self) -> List[ExecutionPlan]: - """Get a list of children `ExecutionPlan`s that act as inputs to this plan. + """Get a list of children `ExecutionPlan` that act as inputs to this plan. The returned list will be empty for leaf nodes such as scans, will contain a single value for unary nodes, or two values for binary nodes (such as joins). From 2690e61b360a6224f1cd5b5bf29d8c082b87991d Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 30 Nov 2024 23:25:04 +0800 Subject: [PATCH 244/413] Add datafusion.extract (#959) * feat: add extract function as an alias for date_part * docs: update user guide to include examples for date_part and extract functions * fix: update examples in user guide to use f.to_timestamp for date extraction --- docs/source/user-guide/common-operations/functions.rst | 9 +++++++++ python/datafusion/functions.py | 9 +++++++++ python/tests/test_functions.py | 2 ++ 3 files changed, 20 insertions(+) diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index 8d6a80855..ad71c72ac 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -72,6 +72,15 @@ Convert to timestamps using :py:func:`~datafusion.functions.to_timestamp` df.select(f.to_timestamp(col('"Total"')).alias("timestamp")) +Extracting parts of a date using :py:func:`~datafusion.functions.date_part` (alias :py:func:`~datafusion.functions.extract`) + +.. ipython:: python + + df.select( + f.date_part(literal("month"), f.to_timestamp(col('"Total"'))).alias("month"), + f.extract(literal("day"), f.to_timestamp(col('"Total"'))).alias("day") + ) + String ------ diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 6ad4c50c2..15ad8822f 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -128,6 +128,7 @@ "empty", "encode", "ends_with", + "extract", "exp", "factorial", "find_in_set", @@ -994,6 +995,14 @@ def date_part(part: Expr, date: Expr) -> Expr: return Expr(f.date_part(part.expr, date.expr)) +def extract(part: Expr, date: Expr) -> Expr: + """Extracts a subfield from the date. + + This is an alias for :py:func:`date_part`. + """ + return date_part(part, date) + + def date_trunc(part: Expr, date: Expr) -> Expr: """Truncates the date to a specified level of precision.""" return Expr(f.date_trunc(part.expr, date.expr)) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 0d40032bb..0d2fa8f94 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -866,6 +866,7 @@ def test_temporal_functions(df): f.to_timestamp_seconds(literal("2023-09-07 05:06:14.523952")), f.to_timestamp_millis(literal("2023-09-07 05:06:14.523952")), f.to_timestamp_micros(literal("2023-09-07 05:06:14.523952")), + f.extract(literal("day"), column("d")), ) result = df.collect() assert len(result) == 1 @@ -903,6 +904,7 @@ def test_temporal_functions(df): assert result.column(9) == pa.array( [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") ) + assert result.column(10) == pa.array([31, 26, 2], type=pa.float64()) def test_case(df): From 5c834934dec89bd96ff70df3b278e9d6fe78f7ec Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 2 Dec 2024 08:05:52 -0700 Subject: [PATCH 245/413] chore: Prepare 43.0.0 Release (#960) * Generate changelog * cargo update --- CHANGELOG.md | 695 +---------------------------------- Cargo.lock | 410 +++++++++++---------- Cargo.toml | 2 +- dev/changelog/43.0.0.md | 73 ++++ dev/changelog/pre-43.0.0.md | 715 ++++++++++++++++++++++++++++++++++++ 5 files changed, 1014 insertions(+), 881 deletions(-) create mode 100644 dev/changelog/43.0.0.md create mode 100644 dev/changelog/pre-43.0.0.md diff --git a/CHANGELOG.md b/CHANGELOG.md index ae3a2348a..ae40911d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,697 +19,4 @@ # DataFusion Python Changelog -## [42.0.0](https://github.com/apache/datafusion-python/tree/42.0.0) (2024-10-06) - -This release consists of 20 commits from 6 contributors. See credits at the end of this changelog for more information. - -**Implemented enhancements:** - -- feat: expose between [#868](https://github.com/apache/datafusion-python/pull/868) (mesejo) -- feat: make register_csv accept a list of paths [#883](https://github.com/apache/datafusion-python/pull/883) (mesejo) -- feat: expose http object store [#885](https://github.com/apache/datafusion-python/pull/885) (mesejo) - -**Fixed bugs:** - -- fix: Calling `count` on a pyarrow dataset results in an error [#843](https://github.com/apache/datafusion-python/pull/843) (Michael-J-Ward) - -**Other:** - -- Upgrade datafusion [#867](https://github.com/apache/datafusion-python/pull/867) (emgeee) -- Feature/aggregates as windows [#871](https://github.com/apache/datafusion-python/pull/871) (timsaucer) -- Fix regression on register_udaf [#878](https://github.com/apache/datafusion-python/pull/878) (timsaucer) -- build(deps): upgrade setup-protoc action and protoc version number [#873](https://github.com/apache/datafusion-python/pull/873) (Michael-J-Ward) -- build(deps): bump prost-types from 0.13.2 to 0.13.3 [#881](https://github.com/apache/datafusion-python/pull/881) (dependabot[bot]) -- build(deps): bump prost from 0.13.2 to 0.13.3 [#882](https://github.com/apache/datafusion-python/pull/882) (dependabot[bot]) -- chore: remove XFAIL from passing tests [#884](https://github.com/apache/datafusion-python/pull/884) (Michael-J-Ward) -- Add user defined window function support [#880](https://github.com/apache/datafusion-python/pull/880) (timsaucer) -- build(deps): bump syn from 2.0.77 to 2.0.79 [#886](https://github.com/apache/datafusion-python/pull/886) (dependabot[bot]) -- fix example of reading parquet from s3 [#896](https://github.com/apache/datafusion-python/pull/896) (sir-sigurd) -- release-testing [#889](https://github.com/apache/datafusion-python/pull/889) (Michael-J-Ward) -- chore(bench): fix create_tables.sql for tpch benchmark [#897](https://github.com/apache/datafusion-python/pull/897) (Michael-J-Ward) -- Add physical and logical plan conversion to and from protobuf [#892](https://github.com/apache/datafusion-python/pull/892) (timsaucer) -- Feature/instance udfs [#890](https://github.com/apache/datafusion-python/pull/890) (timsaucer) -- chore(ci): remove Mambaforge variant from CI [#894](https://github.com/apache/datafusion-python/pull/894) (Michael-J-Ward) -- Use OnceLock to store TokioRuntime [#895](https://github.com/apache/datafusion-python/pull/895) (Michael-J-Ward) - -## Credits - -Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. - -``` - 7 Michael J Ward - 5 Tim Saucer - 3 Daniel Mesejo - 3 dependabot[bot] - 1 Matt Green - 1 Sergey Fedoseev -``` - -Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. - -## [41.0.0](https://github.com/apache/datafusion-python/tree/41.0.0) (2024-09-09) - -This release consists of 19 commits from 6 contributors. See credits at the end of this changelog for more information. - -**Implemented enhancements:** - -- feat: enable list of paths for read_csv [#824](https://github.com/apache/datafusion-python/pull/824) (mesejo) -- feat: better exception and message for table not found [#851](https://github.com/apache/datafusion-python/pull/851) (mesejo) -- feat: make cast accept built-in Python types [#858](https://github.com/apache/datafusion-python/pull/858) (mesejo) - -**Other:** - -- chore: Prepare for 40.0.0 release [#801](https://github.com/apache/datafusion-python/pull/801) (andygrove) -- Add typing-extensions dependency to pyproject [#805](https://github.com/apache/datafusion-python/pull/805) (timsaucer) -- Upgrade deps to datafusion 41 [#802](https://github.com/apache/datafusion-python/pull/802) (Michael-J-Ward) -- Fix SessionContext init with only SessionConfig [#827](https://github.com/apache/datafusion-python/pull/827) (jcrist) -- build(deps): upgrade actions/{upload,download}-artifact@v3 to v4 [#829](https://github.com/apache/datafusion-python/pull/829) (Michael-J-Ward) -- Run ruff format in CI [#837](https://github.com/apache/datafusion-python/pull/837) (timsaucer) -- Add PyCapsule support for Arrow import and export [#825](https://github.com/apache/datafusion-python/pull/825) (timsaucer) -- Feature/expose when function [#836](https://github.com/apache/datafusion-python/pull/836) (timsaucer) -- Add Window Functions for use with function builder [#808](https://github.com/apache/datafusion-python/pull/808) (timsaucer) -- chore: fix typos [#844](https://github.com/apache/datafusion-python/pull/844) (mesejo) -- build(ci): use proper mac runners [#841](https://github.com/apache/datafusion-python/pull/841) (Michael-J-Ward) -- Set of small features [#839](https://github.com/apache/datafusion-python/pull/839) (timsaucer) -- chore: fix docstrings, typos [#852](https://github.com/apache/datafusion-python/pull/852) (mesejo) -- chore: Use datafusion re-exported dependencies [#856](https://github.com/apache/datafusion-python/pull/856) (emgeee) -- add guidelines on separating python and rust code [#860](https://github.com/apache/datafusion-python/pull/860) (Michael-J-Ward) -- Update Aggregate functions to take builder parameters [#859](https://github.com/apache/datafusion-python/pull/859) (timsaucer) - -## Credits - -Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. - -``` - 7 Tim Saucer - 5 Daniel Mesejo - 4 Michael J Ward - 1 Andy Grove - 1 Jim Crist-Harif - 1 Matt Green -``` - -Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. - -## [40.0.0](https://github.com/apache/datafusion-python/tree/40.0.0) (2024-08-09) - -This release consists of 18 commits from 4 contributors. See credits at the end of this changelog for more information. - -- Update changelog for 39.0.0 [#742](https://github.com/apache/datafusion-python/pull/742) (andygrove) -- build(deps): bump uuid from 1.8.0 to 1.9.1 [#744](https://github.com/apache/datafusion-python/pull/744) (dependabot[bot]) -- build(deps): bump mimalloc from 0.1.42 to 0.1.43 [#745](https://github.com/apache/datafusion-python/pull/745) (dependabot[bot]) -- build(deps): bump syn from 2.0.67 to 2.0.68 [#746](https://github.com/apache/datafusion-python/pull/746) (dependabot[bot]) -- Tsaucer/find window fn [#747](https://github.com/apache/datafusion-python/pull/747) (timsaucer) -- Python wrapper classes for all user interfaces [#750](https://github.com/apache/datafusion-python/pull/750) (timsaucer) -- Expose array sort [#764](https://github.com/apache/datafusion-python/pull/764) (timsaucer) -- Upgrade protobuf and remove GH Action googletest-installer [#773](https://github.com/apache/datafusion-python/pull/773) (Michael-J-Ward) -- Upgrade Datafusion 40 [#771](https://github.com/apache/datafusion-python/pull/771) (Michael-J-Ward) -- Bugfix: Calling count with None arguments [#768](https://github.com/apache/datafusion-python/pull/768) (timsaucer) -- Add in user example that compares a two different approaches to UDFs [#770](https://github.com/apache/datafusion-python/pull/770) (timsaucer) -- Add missing exports for wrapper modules [#782](https://github.com/apache/datafusion-python/pull/782) (timsaucer) -- Add PyExpr to_variant conversions [#793](https://github.com/apache/datafusion-python/pull/793) (Michael-J-Ward) -- Add missing expressions to wrapper export [#795](https://github.com/apache/datafusion-python/pull/795) (timsaucer) -- Doc/cross reference [#791](https://github.com/apache/datafusion-python/pull/791) (timsaucer) -- Re-Enable `num_centroids` to `approx_percentile_cont` [#798](https://github.com/apache/datafusion-python/pull/798) (Michael-J-Ward) -- UDAF process all state variables [#799](https://github.com/apache/datafusion-python/pull/799) (timsaucer) - -## Credits - -Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. - -``` - 9 Tim Saucer - 4 Michael J Ward - 3 dependabot[bot] - 2 Andy Grove -``` - -Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. - -## [39.0.0](https://github.com/apache/datafusion-python/tree/39.0.0) (2024-06-25) - -**Merged pull requests:** - -- ci: add substrait feature to linux builds [#720](https://github.com/apache/datafusion-python/pull/720) (Michael-J-Ward) -- Docs deploy action [#721](https://github.com/apache/datafusion-python/pull/721) (Michael-J-Ward) -- update deps [#723](https://github.com/apache/datafusion-python/pull/723) (Michael-J-Ward) -- Upgrade maturin [#725](https://github.com/apache/datafusion-python/pull/725) (Michael-J-Ward) -- Upgrade datafusion 39 [#728](https://github.com/apache/datafusion-python/pull/728) (Michael-J-Ward) -- use ScalarValue::to_pyarrow to convert to python object [#731](https://github.com/apache/datafusion-python/pull/731) (Michael-J-Ward) -- Pyo3 `Bound<'py, T>` api [#734](https://github.com/apache/datafusion-python/pull/734) (Michael-J-Ward) -- github test action: drop python 3.7, add python 3.12 [#736](https://github.com/apache/datafusion-python/pull/736) (Michael-J-Ward) -- Pyarrow filter pushdowns [#735](https://github.com/apache/datafusion-python/pull/735) (Michael-J-Ward) -- build(deps): bump syn from 2.0.66 to 2.0.67 [#738](https://github.com/apache/datafusion-python/pull/738) (dependabot[bot]) -- Pyo3 refactorings [#740](https://github.com/apache/datafusion-python/pull/740) (Michael-J-Ward) -- UDAF `sum` workaround [#741](https://github.com/apache/datafusion-python/pull/741) (Michael-J-Ward) - -## [38.0.1](https://github.com/apache/datafusion-python/tree/38.0.1) (2024-05-25) - -**Implemented enhancements:** - -- feat: add python bindings for ends_with function [#693](https://github.com/apache/datafusion-python/pull/693) (richtia) -- feat: expose `named_struct` in python [#700](https://github.com/apache/datafusion-python/pull/700) (Michael-J-Ward) - -**Merged pull requests:** - -- Add document about basics of working with expressions [#668](https://github.com/apache/datafusion-python/pull/668) (timsaucer) -- chore: Update Python release process now that DataFusion is TLP [#674](https://github.com/apache/datafusion-python/pull/674) (andygrove) -- Fix Docs [#676](https://github.com/apache/datafusion-python/pull/676) (Michael-J-Ward) -- Add examples from TPC-H [#666](https://github.com/apache/datafusion-python/pull/666) (timsaucer) -- fix conda nightly builds, attempt 2 [#689](https://github.com/apache/datafusion-python/pull/689) (Michael-J-Ward) -- Upgrade to datafusion 38 [#691](https://github.com/apache/datafusion-python/pull/691) (Michael-J-Ward) -- chore: update to maturin's recommended project layout for rust/python… [#695](https://github.com/apache/datafusion-python/pull/695) (Michael-J-Ward) -- chore: update cargo deps [#698](https://github.com/apache/datafusion-python/pull/698) (Michael-J-Ward) -- feat: add python bindings for ends_with function [#693](https://github.com/apache/datafusion-python/pull/693) (richtia) -- feat: expose `named_struct` in python [#700](https://github.com/apache/datafusion-python/pull/700) (Michael-J-Ward) -- Website fixes [#702](https://github.com/apache/datafusion-python/pull/702) (Michael-J-Ward) - -## [37.1.0](https://github.com/apache/datafusion-python/tree/37.1.0) (2024-05-08) - -**Implemented enhancements:** - -- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/datafusion-python/pull/610) (mesejo) - -**Documentation updates:** - -- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/datafusion-python/pull/661) (Michael-J-Ward) - -**Merged pull requests:** - -- Switch to Ruff for Python linting [#529](https://github.com/apache/datafusion-python/pull/529) (andygrove) -- Remove sql-on-pandas/polars/cudf examples [#602](https://github.com/apache/datafusion-python/pull/602) (andygrove) -- build(deps): bump object_store from 0.9.0 to 0.9.1 [#611](https://github.com/apache/datafusion-python/pull/611) (dependabot[bot]) -- More missing array funcs [#605](https://github.com/apache/datafusion-python/pull/605) (judahrand) -- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/datafusion-python/pull/610) (mesejo) -- build(deps): bump uuid from 1.7.0 to 1.8.0 [#615](https://github.com/apache/datafusion-python/pull/615) (dependabot[bot]) -- Bind SQLOptions and relative ctx method #567 [#588](https://github.com/apache/datafusion-python/pull/588) (giacomorebecchi) -- bugfix: no panic on empty table [#613](https://github.com/apache/datafusion-python/pull/613) (mesejo) -- Expose `register_listing_table` [#618](https://github.com/apache/datafusion-python/pull/618) (henrifroese) -- Expose unnest feature [#641](https://github.com/apache/datafusion-python/pull/641) (timsaucer) -- Update domain names and paths in asf yaml [#643](https://github.com/apache/datafusion-python/pull/643) (andygrove) -- use python 3.11 to publish docs [#645](https://github.com/apache/datafusion-python/pull/645) (andygrove) -- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/datafusion-python/pull/661) (Michael-J-Ward) -- Upgrade Datafusion to v37.1.0 [#669](https://github.com/apache/datafusion-python/pull/669) (Michael-J-Ward) - -## [36.0.0](https://github.com/apache/datafusion-python/tree/36.0.0) (2024-03-02) - -**Implemented enhancements:** - -- feat: Add `flatten` array function [#562](https://github.com/apache/datafusion-python/pull/562) (mobley-trent) - -**Documentation updates:** - -- docs: Add ASF attribution [#580](https://github.com/apache/datafusion-python/pull/580) (simicd) - -**Merged pull requests:** - -- Allow PyDataFrame to be used from other projects [#582](https://github.com/apache/datafusion-python/pull/582) (andygrove) -- docs: Add ASF attribution [#580](https://github.com/apache/datafusion-python/pull/580) (simicd) -- Add array functions [#560](https://github.com/apache/datafusion-python/pull/560) (ongchi) -- feat: Add `flatten` array function [#562](https://github.com/apache/datafusion-python/pull/562) (mobley-trent) - -## [35.0.0](https://github.com/apache/datafusion-python/tree/35.0.0) (2024-01-20) - -**Merged pull requests:** - -- build(deps): bump syn from 2.0.41 to 2.0.43 [#559](https://github.com/apache/datafusion-python/pull/559) (dependabot[bot]) -- build(deps): bump tokio from 1.35.0 to 1.35.1 [#558](https://github.com/apache/datafusion-python/pull/558) (dependabot[bot]) -- build(deps): bump async-trait from 0.1.74 to 0.1.77 [#556](https://github.com/apache/datafusion-python/pull/556) (dependabot[bot]) -- build(deps): bump pyo3 from 0.20.0 to 0.20.2 [#557](https://github.com/apache/datafusion-python/pull/557) (dependabot[bot]) - -## [34.0.0](https://github.com/apache/datafusion-python/tree/34.0.0) (2023-12-28) - -**Merged pull requests:** - -- Adjust visibility of crate private members & Functions [#537](https://github.com/apache/datafusion-python/pull/537) (jdye64) -- Update json.rst [#538](https://github.com/apache/datafusion-python/pull/538) (ray-andrew) -- Enable mimalloc local_dynamic_tls feature [#540](https://github.com/apache/datafusion-python/pull/540) (jdye64) -- Enable substrait feature to be built by default in CI, for nightlies … [#544](https://github.com/apache/datafusion-python/pull/544) (jdye64) - -## [33.0.0](https://github.com/apache/datafusion-python/tree/33.0.0) (2023-11-16) - -**Merged pull requests:** - -- First pass at getting architectured builds working [#350](https://github.com/apache/datafusion-python/pull/350) (charlesbluca) -- Remove libprotobuf dep [#527](https://github.com/apache/datafusion-python/pull/527) (jdye64) - -## [32.0.0](https://github.com/apache/datafusion-python/tree/32.0.0) (2023-10-21) - -**Implemented enhancements:** - -- feat: expose PyWindowFrame [#509](https://github.com/apache/datafusion-python/pull/509) (dlovell) -- add Binary String Functions;encode,decode [#494](https://github.com/apache/datafusion-python/pull/494) (jiangzhx) -- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/datafusion-python/pull/496) (jiangzhx) -- add first_value last_value [#498](https://github.com/apache/datafusion-python/pull/498) (jiangzhx) -- add regr\_\* functions [#499](https://github.com/apache/datafusion-python/pull/499) (jiangzhx) -- Add random missing bindings [#522](https://github.com/apache/datafusion-python/pull/522) (jdye64) -- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/datafusion-python/pull/519) (jdye64) -- Add support for window function bindings [#521](https://github.com/apache/datafusion-python/pull/521) (jdye64) - -**Merged pull requests:** - -- Prepare 31.0.0 release [#500](https://github.com/apache/datafusion-python/pull/500) (andygrove) -- Improve release process documentation [#505](https://github.com/apache/datafusion-python/pull/505) (andygrove) -- add Binary String Functions;encode,decode [#494](https://github.com/apache/datafusion-python/pull/494) (jiangzhx) -- build(deps): bump mimalloc from 0.1.38 to 0.1.39 [#502](https://github.com/apache/datafusion-python/pull/502) (dependabot[bot]) -- build(deps): bump syn from 2.0.32 to 2.0.35 [#503](https://github.com/apache/datafusion-python/pull/503) (dependabot[bot]) -- build(deps): bump syn from 2.0.35 to 2.0.37 [#506](https://github.com/apache/datafusion-python/pull/506) (dependabot[bot]) -- Use latest DataFusion [#511](https://github.com/apache/datafusion-python/pull/511) (andygrove) -- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/datafusion-python/pull/496) (jiangzhx) -- use DataFusion 32 [#515](https://github.com/apache/datafusion-python/pull/515) (andygrove) -- add first_value last_value [#498](https://github.com/apache/datafusion-python/pull/498) (jiangzhx) -- build(deps): bump regex-syntax from 0.7.5 to 0.8.1 [#517](https://github.com/apache/datafusion-python/pull/517) (dependabot[bot]) -- build(deps): bump pyo3-build-config from 0.19.2 to 0.20.0 [#516](https://github.com/apache/datafusion-python/pull/516) (dependabot[bot]) -- add regr\_\* functions [#499](https://github.com/apache/datafusion-python/pull/499) (jiangzhx) -- Add random missing bindings [#522](https://github.com/apache/datafusion-python/pull/522) (jdye64) -- build(deps): bump rustix from 0.38.18 to 0.38.19 [#523](https://github.com/apache/datafusion-python/pull/523) (dependabot[bot]) -- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/datafusion-python/pull/519) (jdye64) -- Add support for window function bindings [#521](https://github.com/apache/datafusion-python/pull/521) (jdye64) -- Small clippy fix [#524](https://github.com/apache/datafusion-python/pull/524) (andygrove) - -## [31.0.0](https://github.com/apache/datafusion-python/tree/31.0.0) (2023-09-12) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/28.0.0...31.0.0) - -**Implemented enhancements:** - -- feat: add case function (#447) [#448](https://github.com/apache/datafusion-python/pull/448) (mesejo) -- feat: add compression options [#456](https://github.com/apache/datafusion-python/pull/456) (mesejo) -- feat: add register_json [#458](https://github.com/apache/datafusion-python/pull/458) (mesejo) -- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/datafusion-python/pull/459) (mesejo) -- feat: add example of reading parquet from s3 [#460](https://github.com/apache/datafusion-python/pull/460) (mesejo) -- feat: add register_avro and read_table [#461](https://github.com/apache/datafusion-python/pull/461) (mesejo) -- feat: add missing scalar math functions [#465](https://github.com/apache/datafusion-python/pull/465) (mesejo) - -**Documentation updates:** - -- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/datafusion-python/pull/455) (mesejo) - -**Merged pull requests:** - -- Build Linux aarch64 wheel [#443](https://github.com/apache/datafusion-python/pull/443) (gokselk) -- feat: add case function (#447) [#448](https://github.com/apache/datafusion-python/pull/448) (mesejo) -- enhancement(docs): Add user guide (#432) [#445](https://github.com/apache/datafusion-python/pull/445) (mesejo) -- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/datafusion-python/pull/455) (mesejo) -- feat: add compression options [#456](https://github.com/apache/datafusion-python/pull/456) (mesejo) -- Upgrade to DF 28.0.0-rc1 [#457](https://github.com/apache/datafusion-python/pull/457) (andygrove) -- feat: add register_json [#458](https://github.com/apache/datafusion-python/pull/458) (mesejo) -- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/datafusion-python/pull/459) (mesejo) -- feat: add example of reading parquet from s3 [#460](https://github.com/apache/datafusion-python/pull/460) (mesejo) -- feat: add register_avro and read_table [#461](https://github.com/apache/datafusion-python/pull/461) (mesejo) -- feat: add missing scalar math functions [#465](https://github.com/apache/datafusion-python/pull/465) (mesejo) -- build(deps): bump arduino/setup-protoc from 1 to 2 [#452](https://github.com/apache/datafusion-python/pull/452) (dependabot[bot]) -- Revert "build(deps): bump arduino/setup-protoc from 1 to 2 (#452)" [#474](https://github.com/apache/datafusion-python/pull/474) (viirya) -- Minor: fix wrongly copied function description [#497](https://github.com/apache/datafusion-python/pull/497) (viirya) -- Upgrade to Datafusion 31.0.0 [#491](https://github.com/apache/datafusion-python/pull/491) (judahrand) -- Add `isnan` and `iszero` [#495](https://github.com/apache/datafusion-python/pull/495) (judahrand) - -## 30.0.0 - -- Skipped due to a breaking change in DataFusion - -## 29.0.0 - -- Skipped - -## [28.0.0](https://github.com/apache/datafusion-python/tree/28.0.0) (2023-07-25) - -**Implemented enhancements:** - -- feat: expose offset in python API [#437](https://github.com/apache/datafusion-python/pull/437) (cpcloud) - -**Merged pull requests:** - -- File based input utils [#433](https://github.com/apache/datafusion-python/pull/433) (jdye64) -- Upgrade to 28.0.0-rc1 [#434](https://github.com/apache/datafusion-python/pull/434) (andygrove) -- Introduces utility for obtaining SqlTable information from a file like location [#398](https://github.com/apache/datafusion-python/pull/398) (jdye64) -- feat: expose offset in python API [#437](https://github.com/apache/datafusion-python/pull/437) (cpcloud) -- Use DataFusion 28 [#439](https://github.com/apache/datafusion-python/pull/439) (andygrove) - -## [27.0.0](https://github.com/apache/datafusion-python/tree/27.0.0) (2023-07-03) - -**Merged pull requests:** - -- LogicalPlan.to_variant() make public [#412](https://github.com/apache/datafusion-python/pull/412) (jdye64) -- Prepare 27.0.0 release [#423](https://github.com/apache/datafusion-python/pull/423) (andygrove) - -## [26.0.0](https://github.com/apache/datafusion-python/tree/26.0.0) (2023-06-11) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/25.0.0...26.0.0) - -**Merged pull requests:** - -- Add Expr::Case when_then_else support to rex_call_operands function [#388](https://github.com/apache/datafusion-python/pull/388) (jdye64) -- Introduce BaseSessionContext abstract class [#390](https://github.com/apache/datafusion-python/pull/390) (jdye64) -- CRUD Schema support for `BaseSessionContext` [#392](https://github.com/apache/datafusion-python/pull/392) (jdye64) -- CRUD Table support for `BaseSessionContext` [#394](https://github.com/apache/datafusion-python/pull/394) (jdye64) - -## [25.0.0](https://github.com/apache/datafusion-python/tree/25.0.0) (2023-05-23) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/24.0.0...25.0.0) - -**Merged pull requests:** - -- Prepare 24.0.0 Release [#376](https://github.com/apache/datafusion-python/pull/376) (andygrove) -- build(deps): bump uuid from 1.3.1 to 1.3.2 [#359](https://github.com/apache/datafusion-python/pull/359) (dependabot[bot]) -- build(deps): bump mimalloc from 0.1.36 to 0.1.37 [#361](https://github.com/apache/datafusion-python/pull/361) (dependabot[bot]) -- build(deps): bump regex-syntax from 0.6.29 to 0.7.1 [#334](https://github.com/apache/datafusion-python/pull/334) (dependabot[bot]) -- upgrade maturin to 0.15.1 [#379](https://github.com/apache/datafusion-python/pull/379) (Jimexist) -- Expand Expr to include RexType basic support [#378](https://github.com/apache/datafusion-python/pull/378) (jdye64) -- Add Python script for generating changelog [#383](https://github.com/apache/datafusion-python/pull/383) (andygrove) - -## [24.0.0](https://github.com/apache/datafusion-python/tree/24.0.0) (2023-05-09) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/23.0.0...24.0.0) - -**Documentation updates:** - -- Fix link to user guide [#354](https://github.com/apache/datafusion-python/pull/354) (andygrove) - -**Merged pull requests:** - -- Add interface to serialize Substrait plans to Python Bytes. [#344](https://github.com/apache/datafusion-python/pull/344) (kylebrooks-8451) -- Add partition_count property to ExecutionPlan. [#346](https://github.com/apache/datafusion-python/pull/346) (kylebrooks-8451) -- Remove unsendable from all Rust pyclass types. [#348](https://github.com/apache/datafusion-python/pull/348) (kylebrooks-8451) -- Fix link to user guide [#354](https://github.com/apache/datafusion-python/pull/354) (andygrove) -- Fix SessionContext execute. [#353](https://github.com/apache/datafusion-python/pull/353) (kylebrooks-8451) -- Pub mod expr in lib.rs [#357](https://github.com/apache/datafusion-python/pull/357) (jdye64) -- Add benchmark derived from TPC-H [#355](https://github.com/apache/datafusion-python/pull/355) (andygrove) -- Add db-benchmark [#365](https://github.com/apache/datafusion-python/pull/365) (andygrove) -- First pass of documentation in mdBook [#364](https://github.com/apache/datafusion-python/pull/364) (MrPowers) -- Add 'pub' and '#[pyo3(get, set)]' to DataTypeMap [#371](https://github.com/apache/datafusion-python/pull/371) (jdye64) -- Fix db-benchmark [#369](https://github.com/apache/datafusion-python/pull/369) (andygrove) -- Docs explaining how to view query plans [#373](https://github.com/apache/datafusion-python/pull/373) (andygrove) -- Improve db-benchmark [#372](https://github.com/apache/datafusion-python/pull/372) (andygrove) -- Make expr member of PyExpr public [#375](https://github.com/apache/datafusion-python/pull/375) (jdye64) - -## [23.0.0](https://github.com/apache/datafusion-python/tree/23.0.0) (2023-04-23) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/22.0.0...23.0.0) - -**Merged pull requests:** - -- Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/datafusion-python/pull/321) (andygrove) -- Osx build linker args [#330](https://github.com/apache/datafusion-python/pull/330) (jdye64) -- Add requirements file for python 3.11 [#332](https://github.com/apache/datafusion-python/pull/332) (r4ntix) -- mac arm64 build [#338](https://github.com/apache/datafusion-python/pull/338) (andygrove) -- Add conda.yaml baseline workflow file [#281](https://github.com/apache/datafusion-python/pull/281) (jdye64) -- Prepare for 23.0.0 release [#335](https://github.com/apache/datafusion-python/pull/335) (andygrove) -- Reuse the Tokio Runtime [#341](https://github.com/apache/datafusion-python/pull/341) (kylebrooks-8451) - -## [22.0.0](https://github.com/apache/datafusion-python/tree/22.0.0) (2023-04-10) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/21.0.0...22.0.0) - -**Merged pull requests:** - -- Fix invalid build yaml [#308](https://github.com/apache/datafusion-python/pull/308) (andygrove) -- Try fix release build [#309](https://github.com/apache/datafusion-python/pull/309) (andygrove) -- Fix release build [#310](https://github.com/apache/datafusion-python/pull/310) (andygrove) -- Enable datafusion-substrait protoc feature, to remove compile-time dependency on protoc [#312](https://github.com/apache/datafusion-python/pull/312) (andygrove) -- Fix Mac/Win release builds in CI [#313](https://github.com/apache/datafusion-python/pull/313) (andygrove) -- install protoc in docs workflow [#314](https://github.com/apache/datafusion-python/pull/314) (andygrove) -- Fix documentation generation in CI [#315](https://github.com/apache/datafusion-python/pull/315) (andygrove) -- Source wheel fix [#319](https://github.com/apache/datafusion-python/pull/319) (andygrove) - -## [21.0.0](https://github.com/apache/datafusion-python/tree/21.0.0) (2023-03-30) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/20.0.0...21.0.0) - -**Merged pull requests:** - -- minor: Fix minor warning on unused import [#289](https://github.com/apache/datafusion-python/pull/289) (viirya) -- feature: Implement `describe()` method [#293](https://github.com/apache/datafusion-python/pull/293) (simicd) -- fix: Printed results not visible in debugger & notebooks [#296](https://github.com/apache/datafusion-python/pull/296) (simicd) -- add package.include and remove wildcard dependency [#295](https://github.com/apache/datafusion-python/pull/295) (andygrove) -- Update main branch name in docs workflow [#303](https://github.com/apache/datafusion-python/pull/303) (andygrove) -- Upgrade to DF 21 [#301](https://github.com/apache/datafusion-python/pull/301) (andygrove) - -## [20.0.0](https://github.com/apache/datafusion-python/tree/20.0.0) (2023-03-17) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/0.8.0...20.0.0) - -**Implemented enhancements:** - -- Empty relation bindings [#208](https://github.com/apache/datafusion-python/pull/208) (jdye64) -- wrap display_name and canonical_name functions [#214](https://github.com/apache/datafusion-python/pull/214) (jdye64) -- Add PyAlias bindings [#216](https://github.com/apache/datafusion-python/pull/216) (jdye64) -- Add bindings for scalar_variable [#218](https://github.com/apache/datafusion-python/pull/218) (jdye64) -- Bindings for LIKE type expressions [#220](https://github.com/apache/datafusion-python/pull/220) (jdye64) -- Bool expr bindings [#223](https://github.com/apache/datafusion-python/pull/223) (jdye64) -- Between bindings [#229](https://github.com/apache/datafusion-python/pull/229) (jdye64) -- Add bindings for GetIndexedField [#227](https://github.com/apache/datafusion-python/pull/227) (jdye64) -- Add bindings for case, cast, and trycast [#232](https://github.com/apache/datafusion-python/pull/232) (jdye64) -- add remaining expr bindings [#233](https://github.com/apache/datafusion-python/pull/233) (jdye64) -- feature: Additional export methods [#236](https://github.com/apache/datafusion-python/pull/236) (simicd) -- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/datafusion-python/pull/240) (iajoiner) -- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/datafusion-python/pull/242) (simicd) -- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/datafusion-python/pull/246) (iajoiner) -- feature: Set table name from ctx functions [#260](https://github.com/apache/datafusion-python/pull/260) (simicd) -- Explain bindings [#264](https://github.com/apache/datafusion-python/pull/264) (jdye64) -- Extension bindings [#266](https://github.com/apache/datafusion-python/pull/266) (jdye64) -- Subquery alias bindings [#269](https://github.com/apache/datafusion-python/pull/269) (jdye64) -- Create memory table [#271](https://github.com/apache/datafusion-python/pull/271) (jdye64) -- Create view bindings [#273](https://github.com/apache/datafusion-python/pull/273) (jdye64) -- Re-export Datafusion dependencies [#277](https://github.com/apache/datafusion-python/pull/277) (jdye64) -- Distinct bindings [#275](https://github.com/apache/datafusion-python/pull/275) (jdye64) -- Drop table bindings [#283](https://github.com/apache/datafusion-python/pull/283) (jdye64) -- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/datafusion-python/pull/285) (jdye64) -- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/datafusion-python/pull/287) (jdye64) - -**Documentation updates:** - -- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/datafusion-python/pull/258) (simicd) - -**Merged pull requests:** - -- Minor docs updates [#210](https://github.com/apache/datafusion-python/pull/210) (andygrove) -- Empty relation bindings [#208](https://github.com/apache/datafusion-python/pull/208) (jdye64) -- wrap display_name and canonical_name functions [#214](https://github.com/apache/datafusion-python/pull/214) (jdye64) -- Add PyAlias bindings [#216](https://github.com/apache/datafusion-python/pull/216) (jdye64) -- Add bindings for scalar_variable [#218](https://github.com/apache/datafusion-python/pull/218) (jdye64) -- Bindings for LIKE type expressions [#220](https://github.com/apache/datafusion-python/pull/220) (jdye64) -- Bool expr bindings [#223](https://github.com/apache/datafusion-python/pull/223) (jdye64) -- Between bindings [#229](https://github.com/apache/datafusion-python/pull/229) (jdye64) -- Add bindings for GetIndexedField [#227](https://github.com/apache/datafusion-python/pull/227) (jdye64) -- Add bindings for case, cast, and trycast [#232](https://github.com/apache/datafusion-python/pull/232) (jdye64) -- add remaining expr bindings [#233](https://github.com/apache/datafusion-python/pull/233) (jdye64) -- Pre-commit hooks [#228](https://github.com/apache/datafusion-python/pull/228) (jdye64) -- Implement new release process [#149](https://github.com/apache/datafusion-python/pull/149) (andygrove) -- feature: Additional export methods [#236](https://github.com/apache/datafusion-python/pull/236) (simicd) -- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/datafusion-python/pull/240) (iajoiner) -- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/datafusion-python/pull/242) (simicd) -- Fix release instructions [#238](https://github.com/apache/datafusion-python/pull/238) (andygrove) -- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/datafusion-python/pull/246) (iajoiner) -- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/datafusion-python/pull/258) (simicd) -- feature: Set table name from ctx functions [#260](https://github.com/apache/datafusion-python/pull/260) (simicd) -- Upgrade to DataFusion 19 [#262](https://github.com/apache/datafusion-python/pull/262) (andygrove) -- Explain bindings [#264](https://github.com/apache/datafusion-python/pull/264) (jdye64) -- Extension bindings [#266](https://github.com/apache/datafusion-python/pull/266) (jdye64) -- Subquery alias bindings [#269](https://github.com/apache/datafusion-python/pull/269) (jdye64) -- Create memory table [#271](https://github.com/apache/datafusion-python/pull/271) (jdye64) -- Create view bindings [#273](https://github.com/apache/datafusion-python/pull/273) (jdye64) -- Re-export Datafusion dependencies [#277](https://github.com/apache/datafusion-python/pull/277) (jdye64) -- Distinct bindings [#275](https://github.com/apache/datafusion-python/pull/275) (jdye64) -- build(deps): bump actions/checkout from 2 to 3 [#244](https://github.com/apache/datafusion-python/pull/244) (dependabot[bot]) -- build(deps): bump actions/upload-artifact from 2 to 3 [#245](https://github.com/apache/datafusion-python/pull/245) (dependabot[bot]) -- build(deps): bump actions/download-artifact from 2 to 3 [#243](https://github.com/apache/datafusion-python/pull/243) (dependabot[bot]) -- Use DataFusion 20 [#278](https://github.com/apache/datafusion-python/pull/278) (andygrove) -- Drop table bindings [#283](https://github.com/apache/datafusion-python/pull/283) (jdye64) -- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/datafusion-python/pull/285) (jdye64) -- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/datafusion-python/pull/287) (jdye64) - -## [0.8.0](https://github.com/apache/datafusion-python/tree/0.8.0) (2023-02-22) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/0.8.0-rc1...0.8.0) - -**Implemented enhancements:** - -- Add support for cuDF physical execution engine [\#202](https://github.com/apache/datafusion-python/issues/202) -- Make it easier to create a Pandas dataframe from DataFusion query results [\#139](https://github.com/apache/datafusion-python/issues/139) - -**Fixed bugs:** - -- Build error: could not compile `thiserror` due to 2 previous errors [\#69](https://github.com/apache/datafusion-python/issues/69) - -**Closed issues:** - -- Integrate with the new `object_store` crate [\#22](https://github.com/apache/datafusion-python/issues/22) - -**Merged pull requests:** - -- Update README in preparation for 0.8 release [\#206](https://github.com/apache/datafusion-python/pull/206) ([andygrove](https://github.com/andygrove)) -- Add support for cudf as a physical execution engine [\#205](https://github.com/apache/datafusion-python/pull/205) ([jdye64](https://github.com/jdye64)) -- Run `maturin develop` instead of `cargo build` in verification script [\#200](https://github.com/apache/datafusion-python/pull/200) ([andygrove](https://github.com/andygrove)) -- Add tests for recently added functionality [\#199](https://github.com/apache/datafusion-python/pull/199) ([andygrove](https://github.com/andygrove)) -- Implement `to_pandas()` [\#197](https://github.com/apache/datafusion-python/pull/197) ([simicd](https://github.com/simicd)) -- Add Python wrapper for LogicalPlan::Sort [\#196](https://github.com/apache/datafusion-python/pull/196) ([andygrove](https://github.com/andygrove)) -- Add Python wrapper for LogicalPlan::Aggregate [\#195](https://github.com/apache/datafusion-python/pull/195) ([andygrove](https://github.com/andygrove)) -- Add Python wrapper for LogicalPlan::Limit [\#193](https://github.com/apache/datafusion-python/pull/193) ([andygrove](https://github.com/andygrove)) -- Add Python wrapper for LogicalPlan::Filter [\#192](https://github.com/apache/datafusion-python/pull/192) ([andygrove](https://github.com/andygrove)) -- Add experimental support for executing SQL with Polars and Pandas [\#190](https://github.com/apache/datafusion-python/pull/190) ([andygrove](https://github.com/andygrove)) -- Update changelog for 0.8 release [\#188](https://github.com/apache/datafusion-python/pull/188) ([andygrove](https://github.com/andygrove)) -- Add ability to execute ExecutionPlan and get a stream of RecordBatch [\#186](https://github.com/apache/datafusion-python/pull/186) ([andygrove](https://github.com/andygrove)) -- Dffield bindings [\#185](https://github.com/apache/datafusion-python/pull/185) ([jdye64](https://github.com/jdye64)) -- Add bindings for DFSchema [\#183](https://github.com/apache/datafusion-python/pull/183) ([jdye64](https://github.com/jdye64)) -- test: Window functions [\#182](https://github.com/apache/datafusion-python/pull/182) ([simicd](https://github.com/simicd)) -- Add bindings for Projection [\#180](https://github.com/apache/datafusion-python/pull/180) ([jdye64](https://github.com/jdye64)) -- Table scan bindings [\#178](https://github.com/apache/datafusion-python/pull/178) ([jdye64](https://github.com/jdye64)) -- Make session configurable [\#176](https://github.com/apache/datafusion-python/pull/176) ([andygrove](https://github.com/andygrove)) -- Upgrade to DataFusion 18.0.0 [\#175](https://github.com/apache/datafusion-python/pull/175) ([andygrove](https://github.com/andygrove)) -- Use latest DataFusion rev in preparation for DF 18 release [\#174](https://github.com/apache/datafusion-python/pull/174) ([andygrove](https://github.com/andygrove)) -- Arrow type bindings [\#173](https://github.com/apache/datafusion-python/pull/173) ([jdye64](https://github.com/jdye64)) -- Pyo3 bump [\#171](https://github.com/apache/datafusion-python/pull/171) ([jdye64](https://github.com/jdye64)) -- feature: Add additional aggregation functions [\#170](https://github.com/apache/datafusion-python/pull/170) ([simicd](https://github.com/simicd)) -- Make from_substrait_plan return DataFrame instead of LogicalPlan [\#164](https://github.com/apache/datafusion-python/pull/164) ([andygrove](https://github.com/andygrove)) -- feature: Implement count method [\#163](https://github.com/apache/datafusion-python/pull/163) ([simicd](https://github.com/simicd)) -- CI Fixes [\#162](https://github.com/apache/datafusion-python/pull/162) ([jdye64](https://github.com/jdye64)) -- Upgrade to DataFusion 17 [\#160](https://github.com/apache/datafusion-python/pull/160) ([andygrove](https://github.com/andygrove)) -- feature: Improve string representation of datafusion classes [\#159](https://github.com/apache/datafusion-python/pull/159) ([simicd](https://github.com/simicd)) -- Make PyExecutionPlan.plan public [\#156](https://github.com/apache/datafusion-python/pull/156) ([andygrove](https://github.com/andygrove)) -- Expose methods on logical and execution plans [\#155](https://github.com/apache/datafusion-python/pull/155) ([andygrove](https://github.com/andygrove)) -- Fix clippy for new Rust version [\#154](https://github.com/apache/datafusion-python/pull/154) ([andygrove](https://github.com/andygrove)) -- Add DataFrame methods for accessing plans [\#153](https://github.com/apache/datafusion-python/pull/153) ([andygrove](https://github.com/andygrove)) -- Use DataFusion rev 5238e8c97f998b4d2cb9fab85fb182f325a1a7fb [\#150](https://github.com/apache/datafusion-python/pull/150) ([andygrove](https://github.com/andygrove)) -- build\(deps\): bump async-trait from 0.1.61 to 0.1.62 [\#148](https://github.com/apache/datafusion-python/pull/148) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Rename default branch from master to main [\#147](https://github.com/apache/datafusion-python/pull/147) ([andygrove](https://github.com/andygrove)) -- Substrait bindings [\#145](https://github.com/apache/datafusion-python/pull/145) ([jdye64](https://github.com/jdye64)) -- build\(deps\): bump uuid from 0.8.2 to 1.2.2 [\#143](https://github.com/apache/datafusion-python/pull/143) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Prepare for 0.8.0 release [\#141](https://github.com/apache/datafusion-python/pull/141) ([andygrove](https://github.com/andygrove)) -- Improve README and add more examples [\#137](https://github.com/apache/datafusion-python/pull/137) ([andygrove](https://github.com/andygrove)) -- test: Expand tests for built-in functions [\#129](https://github.com/apache/datafusion-python/pull/129) ([simicd](https://github.com/simicd)) -- build\(deps\): bump object_store from 0.5.2 to 0.5.3 [\#126](https://github.com/apache/datafusion-python/pull/126) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump mimalloc from 0.1.32 to 0.1.34 [\#125](https://github.com/apache/datafusion-python/pull/125) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Introduce conda directory containing datafusion-dev.yaml conda enviro… [\#124](https://github.com/apache/datafusion-python/pull/124) ([jdye64](https://github.com/jdye64)) -- build\(deps\): bump bzip2 from 0.4.3 to 0.4.4 [\#121](https://github.com/apache/datafusion-python/pull/121) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump tokio from 1.23.0 to 1.24.1 [\#119](https://github.com/apache/datafusion-python/pull/119) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump async-trait from 0.1.60 to 0.1.61 [\#118](https://github.com/apache/datafusion-python/pull/118) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Upgrade to DataFusion 16.0.0 [\#115](https://github.com/apache/datafusion-python/pull/115) ([andygrove](https://github.com/andygrove)) -- Bump async-trait from 0.1.57 to 0.1.60 [\#114](https://github.com/apache/datafusion-python/pull/114) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump object_store from 0.5.1 to 0.5.2 [\#112](https://github.com/apache/datafusion-python/pull/112) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Bump tokio from 1.21.2 to 1.23.0 [\#109](https://github.com/apache/datafusion-python/pull/109) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Add entries for publishing production \(asf-site\) and staging docs [\#107](https://github.com/apache/datafusion-python/pull/107) ([martin-g](https://github.com/martin-g)) -- Add a workflow that builds the docs and deploys them at staged or production [\#104](https://github.com/apache/datafusion-python/pull/104) ([martin-g](https://github.com/martin-g)) -- Upgrade to DataFusion 15.0.0 [\#103](https://github.com/apache/datafusion-python/pull/103) ([andygrove](https://github.com/andygrove)) -- build\(deps\): bump futures from 0.3.24 to 0.3.25 [\#102](https://github.com/apache/datafusion-python/pull/102) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump pyo3 from 0.17.2 to 0.17.3 [\#101](https://github.com/apache/datafusion-python/pull/101) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump mimalloc from 0.1.30 to 0.1.32 [\#98](https://github.com/apache/datafusion-python/pull/98) ([dependabot[bot]](https://github.com/apps/dependabot)) -- build\(deps\): bump rand from 0.7.3 to 0.8.5 [\#97](https://github.com/apache/datafusion-python/pull/97) ([dependabot[bot]](https://github.com/apps/dependabot)) -- Fix GitHub actions warnings [\#95](https://github.com/apache/datafusion-python/pull/95) ([martin-g](https://github.com/martin-g)) -- Fixes \#81 - Add CI workflow for source distribution [\#93](https://github.com/apache/datafusion-python/pull/93) ([martin-g](https://github.com/martin-g)) -- post-release updates [\#91](https://github.com/apache/datafusion-python/pull/91) ([andygrove](https://github.com/andygrove)) -- Build for manylinux 2014 [\#88](https://github.com/apache/datafusion-python/pull/88) ([martin-g](https://github.com/martin-g)) -- update release readme tag [\#86](https://github.com/apache/datafusion-python/pull/86) ([Jimexist](https://github.com/Jimexist)) -- Upgrade Maturin to 0.14.2 [\#85](https://github.com/apache/datafusion-python/pull/85) ([martin-g](https://github.com/martin-g)) -- Update release instructions [\#83](https://github.com/apache/datafusion-python/pull/83) ([andygrove](https://github.com/andygrove)) -- \[Functions\] - Add python function binding to `functions` [\#73](https://github.com/apache/datafusion-python/pull/73) ([francis-du](https://github.com/francis-du)) - -## [0.8.0-rc1](https://github.com/apache/datafusion-python/tree/0.8.0-rc1) (2023-02-17) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/0.7.0-rc2...0.8.0-rc1) - -**Implemented enhancements:** - -- Add bindings for datafusion_common::DFField [\#184](https://github.com/apache/datafusion-python/issues/184) -- Add bindings for DFSchema/DFSchemaRef [\#181](https://github.com/apache/datafusion-python/issues/181) -- Add bindings for datafusion_expr Projection [\#179](https://github.com/apache/datafusion-python/issues/179) -- Add bindings for `TableScan` struct from `datafusion_expr::TableScan` [\#177](https://github.com/apache/datafusion-python/issues/177) -- Add a "mapping" struct for types [\#172](https://github.com/apache/datafusion-python/issues/172) -- Improve string representation of datafusion classes \(dataframe, context, expression, ...\) [\#158](https://github.com/apache/datafusion-python/issues/158) -- Add DataFrame count method [\#151](https://github.com/apache/datafusion-python/issues/151) -- \[REQUEST\] Github Actions Improvements [\#146](https://github.com/apache/datafusion-python/issues/146) -- Change default branch name from master to main [\#144](https://github.com/apache/datafusion-python/issues/144) -- Bump pyo3 to 0.18.0 [\#140](https://github.com/apache/datafusion-python/issues/140) -- Add script for Python linting [\#134](https://github.com/apache/datafusion-python/issues/134) -- Add Python bindings for substrait module [\#132](https://github.com/apache/datafusion-python/issues/132) -- Expand unit tests for built-in functions [\#128](https://github.com/apache/datafusion-python/issues/128) -- support creating arrow-datafusion-python conda environment [\#122](https://github.com/apache/datafusion-python/issues/122) -- Build Python source distribution in GitHub workflow [\#81](https://github.com/apache/datafusion-python/issues/81) -- EPIC: Add all functions to python binding `functions` [\#72](https://github.com/apache/datafusion-python/issues/72) - -**Fixed bugs:** - -- Build is broken [\#161](https://github.com/apache/datafusion-python/issues/161) -- Out of memory when sorting [\#157](https://github.com/apache/datafusion-python/issues/157) -- window_lead test appears to be non-deterministic [\#135](https://github.com/apache/datafusion-python/issues/135) -- Reading csv does not work [\#130](https://github.com/apache/datafusion-python/issues/130) -- Github actions produce a lot of warnings [\#94](https://github.com/apache/datafusion-python/issues/94) -- ASF source release tarball has wrong directory name [\#90](https://github.com/apache/datafusion-python/issues/90) -- Python Release Build failing after upgrading to maturin 14.2 [\#87](https://github.com/apache/datafusion-python/issues/87) -- Maturin build hangs on Linux ARM64 [\#84](https://github.com/apache/datafusion-python/issues/84) -- Cannot install on Mac M1 from source tarball from testpypi [\#82](https://github.com/apache/datafusion-python/issues/82) -- ImportPathMismatchError when running pytest locally [\#77](https://github.com/apache/datafusion-python/issues/77) - -**Closed issues:** - -- Publish documentation for Python bindings [\#39](https://github.com/apache/datafusion-python/issues/39) -- Add Python binding for `approx_median` [\#32](https://github.com/apache/datafusion-python/issues/32) -- Release version 0.7.0 [\#7](https://github.com/apache/datafusion-python/issues/7) - -## [0.7.0-rc2](https://github.com/apache/datafusion-python/tree/0.7.0-rc2) (2022-11-26) - -[Full Changelog](https://github.com/apache/datafusion-python/compare/0.7.0...0.7.0-rc2) - -## [Unreleased](https://github.com/datafusion-contrib/datafusion-python/tree/HEAD) - -[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.1...HEAD) - -**Merged pull requests:** - -- use \_\_getitem\_\_ for df column selection [\#41](https://github.com/datafusion-contrib/datafusion-python/pull/41) ([Jimexist](https://github.com/Jimexist)) -- fix demo in readme [\#40](https://github.com/datafusion-contrib/datafusion-python/pull/40) ([Jimexist](https://github.com/Jimexist)) -- Implement select_columns [\#39](https://github.com/datafusion-contrib/datafusion-python/pull/39) ([andygrove](https://github.com/andygrove)) -- update readme and changelog [\#38](https://github.com/datafusion-contrib/datafusion-python/pull/38) ([Jimexist](https://github.com/Jimexist)) -- Add PyDataFrame.explain [\#36](https://github.com/datafusion-contrib/datafusion-python/pull/36) ([andygrove](https://github.com/andygrove)) -- Release 0.5.0 [\#34](https://github.com/datafusion-contrib/datafusion-python/pull/34) ([Jimexist](https://github.com/Jimexist)) -- disable nightly in workflow [\#33](https://github.com/datafusion-contrib/datafusion-python/pull/33) ([Jimexist](https://github.com/Jimexist)) -- update requirements to 37 and 310, update readme [\#32](https://github.com/datafusion-contrib/datafusion-python/pull/32) ([Jimexist](https://github.com/Jimexist)) -- Add custom global allocator [\#30](https://github.com/datafusion-contrib/datafusion-python/pull/30) ([matthewmturner](https://github.com/matthewmturner)) -- Remove pandas dependency [\#25](https://github.com/datafusion-contrib/datafusion-python/pull/25) ([matthewmturner](https://github.com/matthewmturner)) -- upgrade datafusion and pyo3 [\#20](https://github.com/datafusion-contrib/datafusion-python/pull/20) ([Jimexist](https://github.com/Jimexist)) -- update maturin 0.12+ [\#17](https://github.com/datafusion-contrib/datafusion-python/pull/17) ([Jimexist](https://github.com/Jimexist)) -- Update README.md [\#16](https://github.com/datafusion-contrib/datafusion-python/pull/16) ([Jimexist](https://github.com/Jimexist)) -- apply cargo clippy --fix [\#15](https://github.com/datafusion-contrib/datafusion-python/pull/15) ([Jimexist](https://github.com/Jimexist)) -- update test workflow to include rust clippy and check [\#14](https://github.com/datafusion-contrib/datafusion-python/pull/14) ([Jimexist](https://github.com/Jimexist)) -- use maturin 0.12.6 [\#13](https://github.com/datafusion-contrib/datafusion-python/pull/13) ([Jimexist](https://github.com/Jimexist)) -- apply cargo fmt [\#12](https://github.com/datafusion-contrib/datafusion-python/pull/12) ([Jimexist](https://github.com/Jimexist)) -- use stable not nightly [\#11](https://github.com/datafusion-contrib/datafusion-python/pull/11) ([Jimexist](https://github.com/Jimexist)) -- ci: test against more compilers, setup clippy and fix clippy lints [\#9](https://github.com/datafusion-contrib/datafusion-python/pull/9) ([cpcloud](https://github.com/cpcloud)) -- Fix use of importlib.metadata and unify requirements.txt [\#8](https://github.com/datafusion-contrib/datafusion-python/pull/8) ([cpcloud](https://github.com/cpcloud)) -- Ship the Cargo.lock file in the source distribution [\#7](https://github.com/datafusion-contrib/datafusion-python/pull/7) ([cpcloud](https://github.com/cpcloud)) -- add \_\_version\_\_ attribute to datafusion object [\#3](https://github.com/datafusion-contrib/datafusion-python/pull/3) ([tfeda](https://github.com/tfeda)) -- fix ci by fixing directories [\#2](https://github.com/datafusion-contrib/datafusion-python/pull/2) ([Jimexist](https://github.com/Jimexist)) -- setup workflow [\#1](https://github.com/datafusion-contrib/datafusion-python/pull/1) ([Jimexist](https://github.com/Jimexist)) - -## [0.5.1](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.1) (2022-03-15) - -[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.1-rc1...0.5.1) - -## [0.5.1-rc1](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.1-rc1) (2022-03-15) - -[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.0...0.5.1-rc1) - -## [0.5.0](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.0) (2022-03-10) - -[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.0-rc2...0.5.0) - -## [0.5.0-rc2](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.0-rc2) (2022-03-10) - -[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.0-rc1...0.5.0-rc2) - -**Closed issues:** - -- Add support for Ballista [\#37](https://github.com/datafusion-contrib/datafusion-python/issues/37) -- Implement DataFrame.explain [\#35](https://github.com/datafusion-contrib/datafusion-python/issues/35) - -## [0.5.0-rc1](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.0-rc1) (2022-03-09) - -[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/4c98b8e9c3c3f8e2e6a8f2d1ffcfefda344c4680...0.5.0-rc1) - -**Closed issues:** - -- Investigate exposing additional optimizations [\#28](https://github.com/datafusion-contrib/datafusion-python/issues/28) -- Use custom allocator in Python build [\#27](https://github.com/datafusion-contrib/datafusion-python/issues/27) -- Why is pandas a requirement? [\#24](https://github.com/datafusion-contrib/datafusion-python/issues/24) -- Unable to build [\#18](https://github.com/datafusion-contrib/datafusion-python/issues/18) -- Setup CI against multiple Python version [\#6](https://github.com/datafusion-contrib/datafusion-python/issues/6) +The changelogs have now moved [here](./dev/changelog). diff --git a/Cargo.lock b/Cargo.lock index 7b57b330a..d1f291be9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611cc2ae7d2e242c457e4be7f97036b8ad9ca152b499f53faf99b1ed8fc2553f" +checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" [[package]] name = "android-tzdata" @@ -157,7 +157,7 @@ dependencies = [ "snap", "strum 0.25.0", "strum_macros 0.25.3", - "thiserror", + "thiserror 1.0.69", "typed-builder", "uuid", "xz2", @@ -178,9 +178,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" +checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d" dependencies = [ "arrow-arith", "arrow-array", @@ -200,9 +200,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" +checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f" dependencies = [ "arrow-array", "arrow-buffer", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" +checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f" dependencies = [ "ahash", "arrow-buffer", @@ -226,15 +226,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "num", ] [[package]] name = "arrow-buffer" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" +checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16" dependencies = [ "bytes", "half", @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" +checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" +checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97" dependencies = [ "arrow-array", "arrow-buffer", @@ -283,9 +283,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" +checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623" dependencies = [ "arrow-buffer", "arrow-schema", @@ -295,9 +295,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" +checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8" dependencies = [ "arrow-array", "arrow-buffer", @@ -310,9 +310,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" +checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85" dependencies = [ "arrow-array", "arrow-buffer", @@ -330,9 +330,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" +checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" dependencies = [ "arrow-array", "arrow-buffer", @@ -345,9 +345,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" +checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff" dependencies = [ "ahash", "arrow-array", @@ -359,18 +359,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" +checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" +checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" dependencies = [ "ahash", "arrow-array", @@ -382,9 +382,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" +checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf" dependencies = [ "arrow-array", "arrow-buffer", @@ -411,9 +411,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "bzip2", "flate2", @@ -444,7 +444,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -455,7 +455,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -529,9 +529,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" +checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" dependencies = [ "arrayref", "arrayvec", @@ -584,9 +584,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "bzip2" @@ -611,9 +611,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.37" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" +checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" dependencies = [ "jobserver", "libc", @@ -668,18 +668,18 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" dependencies = [ "cc", ] [[package]] name = "comfy-table" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b34115915337defe99b2aff5c2ce6771e5fbc4079f4b506301f5cf394c8452f7" +checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" dependencies = [ "strum 0.26.3", "strum_macros 0.26.4", @@ -720,9 +720,9 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" dependencies = [ "core-foundation-sys", "libc", @@ -760,9 +760,9 @@ checksum = "69f3b219d28b6e3b4ac87bc1fc522e0803ab22e055da177bff0068c4150c61a6" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -809,9 +809,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" dependencies = [ "csv-core", "itoa", @@ -1288,7 +1288,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "42.0.0" +version = "43.0.0" dependencies = [ "arrow", "async-trait", @@ -1364,7 +1364,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1393,12 +1393,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1425,9 +1425,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -1504,7 +1504,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1563,8 +1563,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1581,9 +1583,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -1621,9 +1623,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "heck" @@ -1697,9 +1699,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -1890,7 +1892,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -1921,7 +1923,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", ] [[package]] @@ -1965,9 +1967,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -1980,10 +1982,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2059,9 +2062,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.162" +version = "0.2.167" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" [[package]] name = "libflate" @@ -2121,9 +2124,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "lock_api" @@ -2212,11 +2215,10 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "wasi", "windows-sys 0.52.0", @@ -2398,9 +2400,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.2.0" +version = "53.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e" +checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191" dependencies = [ "ahash", "arrow-array", @@ -2417,7 +2419,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.2", "lz4_flex", "num", "num-bigint", @@ -2558,9 +2560,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "portable-atomic" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" [[package]] name = "ppv-lite86" @@ -2578,14 +2580,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2617,7 +2619,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.87", + "syn 2.0.90", "tempfile", ] @@ -2631,7 +2633,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2699,7 +2701,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2712,7 +2714,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -2733,9 +2735,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", @@ -2744,26 +2746,29 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror", + "thiserror 2.0.3", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash", "rustls", + "rustls-pki-types", "slab", - "thiserror", + "thiserror 2.0.3", "tinyvec", "tracing", + "web-time", ] [[package]] @@ -2842,9 +2847,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -2956,9 +2961,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc_version" @@ -2971,9 +2976,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.39" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags 2.6.0", "errno", @@ -2984,9 +2989,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "once_cell", "ring", @@ -2998,12 +3003,11 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", "security-framework", @@ -3023,6 +3027,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time", +] [[package]] name = "rustls-webpki" @@ -3058,9 +3065,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -3086,7 +3093,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3097,9 +3104,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.11.1" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -3135,22 +3142,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3161,14 +3168,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -3185,7 +3192,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3269,7 +3276,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3280,9 +3287,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -3312,7 +3319,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3352,7 +3359,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3365,7 +3372,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3389,7 +3396,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.87", + "syn 2.0.90", "typify", "walkdir", ] @@ -3413,9 +3420,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -3424,9 +3431,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -3439,7 +3446,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3463,22 +3470,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.68" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.3", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", ] [[package]] name = "thiserror-impl" -version = "1.0.68" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3550,7 +3577,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3585,9 +3612,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -3596,20 +3623,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", ] @@ -3668,7 +3695,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] @@ -3702,8 +3729,8 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.87", - "thiserror", + "syn 2.0.90", + "thiserror 1.0.69", "unicode-ident", ] @@ -3720,15 +3747,15 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.87", + "syn 2.0.90", "typify-impl", ] [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-segmentation" @@ -3738,9 +3765,9 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unindent" @@ -3762,9 +3789,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -3826,9 +3853,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" dependencies = [ "cfg-if", "once_cell", @@ -3837,36 +3864,37 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3874,22 +3902,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" [[package]] name = "wasm-streams" @@ -3906,9 +3934,19 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", @@ -4089,9 +4127,9 @@ dependencies = [ [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -4101,13 +4139,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -4129,27 +4167,27 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", "synstructure", ] @@ -4178,7 +4216,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.90", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 02707b957..703fc5a26 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "42.0.0" +version = "43.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] diff --git a/dev/changelog/43.0.0.md b/dev/changelog/43.0.0.md new file mode 100644 index 000000000..bbb766910 --- /dev/null +++ b/dev/changelog/43.0.0.md @@ -0,0 +1,73 @@ + + +# Apache DataFusion Python 43.0.0 Changelog + +This release consists of 26 commits from 7 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: expose `drop` method [#913](https://github.com/apache/datafusion-python/pull/913) (ion-elgreco) +- feat: expose `join_on` [#914](https://github.com/apache/datafusion-python/pull/914) (ion-elgreco) +- feat: add fill_null/nan expressions [#919](https://github.com/apache/datafusion-python/pull/919) (ion-elgreco) +- feat: add `with_columns` [#909](https://github.com/apache/datafusion-python/pull/909) (ion-elgreco) +- feat: add `cast` to DataFrame [#916](https://github.com/apache/datafusion-python/pull/916) (ion-elgreco) +- feat: add `head`, `tail` methods [#915](https://github.com/apache/datafusion-python/pull/915) (ion-elgreco) + +**Fixed bugs:** + +- fix: remove use of deprecated `make_scalar_function` [#906](https://github.com/apache/datafusion-python/pull/906) (Michael-J-Ward) +- fix: udwf example [#948](https://github.com/apache/datafusion-python/pull/948) (mesejo) + +**Other:** + +- Ts/minor updates release process [#903](https://github.com/apache/datafusion-python/pull/903) (timsaucer) +- build(deps): bump pyo3 from 0.22.3 to 0.22.4 [#910](https://github.com/apache/datafusion-python/pull/910) (dependabot[bot]) +- refactor: `from_arrow` use protocol typehints [#917](https://github.com/apache/datafusion-python/pull/917) (ion-elgreco) +- Change requires-python version in pyproject.toml [#924](https://github.com/apache/datafusion-python/pull/924) (kosiew) +- chore: deprecate `select_columns` [#911](https://github.com/apache/datafusion-python/pull/911) (ion-elgreco) +- build(deps): bump uuid from 1.10.0 to 1.11.0 [#927](https://github.com/apache/datafusion-python/pull/927) (dependabot[bot]) +- Add array_empty scalar function [#931](https://github.com/apache/datafusion-python/pull/931) (kosiew) +- add `cardinality` function to calculate total distinct elements in an array [#937](https://github.com/apache/datafusion-python/pull/937) (kosiew) +- Add empty scalar function (alias of array_empty), fix a small typo [#938](https://github.com/apache/datafusion-python/pull/938) (kosiew) +- README How to develop section now also works on Apple M1 [#940](https://github.com/apache/datafusion-python/pull/940) (drauschenbach) +- refactor: dataframe `join` params [#912](https://github.com/apache/datafusion-python/pull/912) (ion-elgreco) +- Upgrade to Datafusion 43 [#905](https://github.com/apache/datafusion-python/pull/905) (Michael-J-Ward) +- build(deps): bump tokio from 1.40.0 to 1.41.1 [#946](https://github.com/apache/datafusion-python/pull/946) (dependabot[bot]) +- Add list_cat, list_concat, list_repeat [#942](https://github.com/apache/datafusion-python/pull/942) (kosiew) +- Add foreign table providers [#921](https://github.com/apache/datafusion-python/pull/921) (timsaucer) +- Add make_list and tests for make_list, make_array [#949](https://github.com/apache/datafusion-python/pull/949) (kosiew) +- Documentation updates: simplify examples and add section on data sources [#955](https://github.com/apache/datafusion-python/pull/955) (timsaucer) +- Add datafusion.extract [#959](https://github.com/apache/datafusion-python/pull/959) (kosiew) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 9 Ion Koutsouris + 7 kosiew + 3 Tim Saucer + 3 dependabot[bot] + 2 Michael J Ward + 1 Daniel Mesejo + 1 David Rauschenbach +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/dev/changelog/pre-43.0.0.md b/dev/changelog/pre-43.0.0.md new file mode 100644 index 000000000..ae3a2348a --- /dev/null +++ b/dev/changelog/pre-43.0.0.md @@ -0,0 +1,715 @@ + + +# DataFusion Python Changelog + +## [42.0.0](https://github.com/apache/datafusion-python/tree/42.0.0) (2024-10-06) + +This release consists of 20 commits from 6 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: expose between [#868](https://github.com/apache/datafusion-python/pull/868) (mesejo) +- feat: make register_csv accept a list of paths [#883](https://github.com/apache/datafusion-python/pull/883) (mesejo) +- feat: expose http object store [#885](https://github.com/apache/datafusion-python/pull/885) (mesejo) + +**Fixed bugs:** + +- fix: Calling `count` on a pyarrow dataset results in an error [#843](https://github.com/apache/datafusion-python/pull/843) (Michael-J-Ward) + +**Other:** + +- Upgrade datafusion [#867](https://github.com/apache/datafusion-python/pull/867) (emgeee) +- Feature/aggregates as windows [#871](https://github.com/apache/datafusion-python/pull/871) (timsaucer) +- Fix regression on register_udaf [#878](https://github.com/apache/datafusion-python/pull/878) (timsaucer) +- build(deps): upgrade setup-protoc action and protoc version number [#873](https://github.com/apache/datafusion-python/pull/873) (Michael-J-Ward) +- build(deps): bump prost-types from 0.13.2 to 0.13.3 [#881](https://github.com/apache/datafusion-python/pull/881) (dependabot[bot]) +- build(deps): bump prost from 0.13.2 to 0.13.3 [#882](https://github.com/apache/datafusion-python/pull/882) (dependabot[bot]) +- chore: remove XFAIL from passing tests [#884](https://github.com/apache/datafusion-python/pull/884) (Michael-J-Ward) +- Add user defined window function support [#880](https://github.com/apache/datafusion-python/pull/880) (timsaucer) +- build(deps): bump syn from 2.0.77 to 2.0.79 [#886](https://github.com/apache/datafusion-python/pull/886) (dependabot[bot]) +- fix example of reading parquet from s3 [#896](https://github.com/apache/datafusion-python/pull/896) (sir-sigurd) +- release-testing [#889](https://github.com/apache/datafusion-python/pull/889) (Michael-J-Ward) +- chore(bench): fix create_tables.sql for tpch benchmark [#897](https://github.com/apache/datafusion-python/pull/897) (Michael-J-Ward) +- Add physical and logical plan conversion to and from protobuf [#892](https://github.com/apache/datafusion-python/pull/892) (timsaucer) +- Feature/instance udfs [#890](https://github.com/apache/datafusion-python/pull/890) (timsaucer) +- chore(ci): remove Mambaforge variant from CI [#894](https://github.com/apache/datafusion-python/pull/894) (Michael-J-Ward) +- Use OnceLock to store TokioRuntime [#895](https://github.com/apache/datafusion-python/pull/895) (Michael-J-Ward) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 7 Michael J Ward + 5 Tim Saucer + 3 Daniel Mesejo + 3 dependabot[bot] + 1 Matt Green + 1 Sergey Fedoseev +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + +## [41.0.0](https://github.com/apache/datafusion-python/tree/41.0.0) (2024-09-09) + +This release consists of 19 commits from 6 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: enable list of paths for read_csv [#824](https://github.com/apache/datafusion-python/pull/824) (mesejo) +- feat: better exception and message for table not found [#851](https://github.com/apache/datafusion-python/pull/851) (mesejo) +- feat: make cast accept built-in Python types [#858](https://github.com/apache/datafusion-python/pull/858) (mesejo) + +**Other:** + +- chore: Prepare for 40.0.0 release [#801](https://github.com/apache/datafusion-python/pull/801) (andygrove) +- Add typing-extensions dependency to pyproject [#805](https://github.com/apache/datafusion-python/pull/805) (timsaucer) +- Upgrade deps to datafusion 41 [#802](https://github.com/apache/datafusion-python/pull/802) (Michael-J-Ward) +- Fix SessionContext init with only SessionConfig [#827](https://github.com/apache/datafusion-python/pull/827) (jcrist) +- build(deps): upgrade actions/{upload,download}-artifact@v3 to v4 [#829](https://github.com/apache/datafusion-python/pull/829) (Michael-J-Ward) +- Run ruff format in CI [#837](https://github.com/apache/datafusion-python/pull/837) (timsaucer) +- Add PyCapsule support for Arrow import and export [#825](https://github.com/apache/datafusion-python/pull/825) (timsaucer) +- Feature/expose when function [#836](https://github.com/apache/datafusion-python/pull/836) (timsaucer) +- Add Window Functions for use with function builder [#808](https://github.com/apache/datafusion-python/pull/808) (timsaucer) +- chore: fix typos [#844](https://github.com/apache/datafusion-python/pull/844) (mesejo) +- build(ci): use proper mac runners [#841](https://github.com/apache/datafusion-python/pull/841) (Michael-J-Ward) +- Set of small features [#839](https://github.com/apache/datafusion-python/pull/839) (timsaucer) +- chore: fix docstrings, typos [#852](https://github.com/apache/datafusion-python/pull/852) (mesejo) +- chore: Use datafusion re-exported dependencies [#856](https://github.com/apache/datafusion-python/pull/856) (emgeee) +- add guidelines on separating python and rust code [#860](https://github.com/apache/datafusion-python/pull/860) (Michael-J-Ward) +- Update Aggregate functions to take builder parameters [#859](https://github.com/apache/datafusion-python/pull/859) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 7 Tim Saucer + 5 Daniel Mesejo + 4 Michael J Ward + 1 Andy Grove + 1 Jim Crist-Harif + 1 Matt Green +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + +## [40.0.0](https://github.com/apache/datafusion-python/tree/40.0.0) (2024-08-09) + +This release consists of 18 commits from 4 contributors. See credits at the end of this changelog for more information. + +- Update changelog for 39.0.0 [#742](https://github.com/apache/datafusion-python/pull/742) (andygrove) +- build(deps): bump uuid from 1.8.0 to 1.9.1 [#744](https://github.com/apache/datafusion-python/pull/744) (dependabot[bot]) +- build(deps): bump mimalloc from 0.1.42 to 0.1.43 [#745](https://github.com/apache/datafusion-python/pull/745) (dependabot[bot]) +- build(deps): bump syn from 2.0.67 to 2.0.68 [#746](https://github.com/apache/datafusion-python/pull/746) (dependabot[bot]) +- Tsaucer/find window fn [#747](https://github.com/apache/datafusion-python/pull/747) (timsaucer) +- Python wrapper classes for all user interfaces [#750](https://github.com/apache/datafusion-python/pull/750) (timsaucer) +- Expose array sort [#764](https://github.com/apache/datafusion-python/pull/764) (timsaucer) +- Upgrade protobuf and remove GH Action googletest-installer [#773](https://github.com/apache/datafusion-python/pull/773) (Michael-J-Ward) +- Upgrade Datafusion 40 [#771](https://github.com/apache/datafusion-python/pull/771) (Michael-J-Ward) +- Bugfix: Calling count with None arguments [#768](https://github.com/apache/datafusion-python/pull/768) (timsaucer) +- Add in user example that compares a two different approaches to UDFs [#770](https://github.com/apache/datafusion-python/pull/770) (timsaucer) +- Add missing exports for wrapper modules [#782](https://github.com/apache/datafusion-python/pull/782) (timsaucer) +- Add PyExpr to_variant conversions [#793](https://github.com/apache/datafusion-python/pull/793) (Michael-J-Ward) +- Add missing expressions to wrapper export [#795](https://github.com/apache/datafusion-python/pull/795) (timsaucer) +- Doc/cross reference [#791](https://github.com/apache/datafusion-python/pull/791) (timsaucer) +- Re-Enable `num_centroids` to `approx_percentile_cont` [#798](https://github.com/apache/datafusion-python/pull/798) (Michael-J-Ward) +- UDAF process all state variables [#799](https://github.com/apache/datafusion-python/pull/799) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 9 Tim Saucer + 4 Michael J Ward + 3 dependabot[bot] + 2 Andy Grove +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + +## [39.0.0](https://github.com/apache/datafusion-python/tree/39.0.0) (2024-06-25) + +**Merged pull requests:** + +- ci: add substrait feature to linux builds [#720](https://github.com/apache/datafusion-python/pull/720) (Michael-J-Ward) +- Docs deploy action [#721](https://github.com/apache/datafusion-python/pull/721) (Michael-J-Ward) +- update deps [#723](https://github.com/apache/datafusion-python/pull/723) (Michael-J-Ward) +- Upgrade maturin [#725](https://github.com/apache/datafusion-python/pull/725) (Michael-J-Ward) +- Upgrade datafusion 39 [#728](https://github.com/apache/datafusion-python/pull/728) (Michael-J-Ward) +- use ScalarValue::to_pyarrow to convert to python object [#731](https://github.com/apache/datafusion-python/pull/731) (Michael-J-Ward) +- Pyo3 `Bound<'py, T>` api [#734](https://github.com/apache/datafusion-python/pull/734) (Michael-J-Ward) +- github test action: drop python 3.7, add python 3.12 [#736](https://github.com/apache/datafusion-python/pull/736) (Michael-J-Ward) +- Pyarrow filter pushdowns [#735](https://github.com/apache/datafusion-python/pull/735) (Michael-J-Ward) +- build(deps): bump syn from 2.0.66 to 2.0.67 [#738](https://github.com/apache/datafusion-python/pull/738) (dependabot[bot]) +- Pyo3 refactorings [#740](https://github.com/apache/datafusion-python/pull/740) (Michael-J-Ward) +- UDAF `sum` workaround [#741](https://github.com/apache/datafusion-python/pull/741) (Michael-J-Ward) + +## [38.0.1](https://github.com/apache/datafusion-python/tree/38.0.1) (2024-05-25) + +**Implemented enhancements:** + +- feat: add python bindings for ends_with function [#693](https://github.com/apache/datafusion-python/pull/693) (richtia) +- feat: expose `named_struct` in python [#700](https://github.com/apache/datafusion-python/pull/700) (Michael-J-Ward) + +**Merged pull requests:** + +- Add document about basics of working with expressions [#668](https://github.com/apache/datafusion-python/pull/668) (timsaucer) +- chore: Update Python release process now that DataFusion is TLP [#674](https://github.com/apache/datafusion-python/pull/674) (andygrove) +- Fix Docs [#676](https://github.com/apache/datafusion-python/pull/676) (Michael-J-Ward) +- Add examples from TPC-H [#666](https://github.com/apache/datafusion-python/pull/666) (timsaucer) +- fix conda nightly builds, attempt 2 [#689](https://github.com/apache/datafusion-python/pull/689) (Michael-J-Ward) +- Upgrade to datafusion 38 [#691](https://github.com/apache/datafusion-python/pull/691) (Michael-J-Ward) +- chore: update to maturin's recommended project layout for rust/python… [#695](https://github.com/apache/datafusion-python/pull/695) (Michael-J-Ward) +- chore: update cargo deps [#698](https://github.com/apache/datafusion-python/pull/698) (Michael-J-Ward) +- feat: add python bindings for ends_with function [#693](https://github.com/apache/datafusion-python/pull/693) (richtia) +- feat: expose `named_struct` in python [#700](https://github.com/apache/datafusion-python/pull/700) (Michael-J-Ward) +- Website fixes [#702](https://github.com/apache/datafusion-python/pull/702) (Michael-J-Ward) + +## [37.1.0](https://github.com/apache/datafusion-python/tree/37.1.0) (2024-05-08) + +**Implemented enhancements:** + +- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/datafusion-python/pull/610) (mesejo) + +**Documentation updates:** + +- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/datafusion-python/pull/661) (Michael-J-Ward) + +**Merged pull requests:** + +- Switch to Ruff for Python linting [#529](https://github.com/apache/datafusion-python/pull/529) (andygrove) +- Remove sql-on-pandas/polars/cudf examples [#602](https://github.com/apache/datafusion-python/pull/602) (andygrove) +- build(deps): bump object_store from 0.9.0 to 0.9.1 [#611](https://github.com/apache/datafusion-python/pull/611) (dependabot[bot]) +- More missing array funcs [#605](https://github.com/apache/datafusion-python/pull/605) (judahrand) +- feat: add execute_stream and execute_stream_partitioned [#610](https://github.com/apache/datafusion-python/pull/610) (mesejo) +- build(deps): bump uuid from 1.7.0 to 1.8.0 [#615](https://github.com/apache/datafusion-python/pull/615) (dependabot[bot]) +- Bind SQLOptions and relative ctx method #567 [#588](https://github.com/apache/datafusion-python/pull/588) (giacomorebecchi) +- bugfix: no panic on empty table [#613](https://github.com/apache/datafusion-python/pull/613) (mesejo) +- Expose `register_listing_table` [#618](https://github.com/apache/datafusion-python/pull/618) (henrifroese) +- Expose unnest feature [#641](https://github.com/apache/datafusion-python/pull/641) (timsaucer) +- Update domain names and paths in asf yaml [#643](https://github.com/apache/datafusion-python/pull/643) (andygrove) +- use python 3.11 to publish docs [#645](https://github.com/apache/datafusion-python/pull/645) (andygrove) +- docs: update docs CI to install python-311 requirements [#661](https://github.com/apache/datafusion-python/pull/661) (Michael-J-Ward) +- Upgrade Datafusion to v37.1.0 [#669](https://github.com/apache/datafusion-python/pull/669) (Michael-J-Ward) + +## [36.0.0](https://github.com/apache/datafusion-python/tree/36.0.0) (2024-03-02) + +**Implemented enhancements:** + +- feat: Add `flatten` array function [#562](https://github.com/apache/datafusion-python/pull/562) (mobley-trent) + +**Documentation updates:** + +- docs: Add ASF attribution [#580](https://github.com/apache/datafusion-python/pull/580) (simicd) + +**Merged pull requests:** + +- Allow PyDataFrame to be used from other projects [#582](https://github.com/apache/datafusion-python/pull/582) (andygrove) +- docs: Add ASF attribution [#580](https://github.com/apache/datafusion-python/pull/580) (simicd) +- Add array functions [#560](https://github.com/apache/datafusion-python/pull/560) (ongchi) +- feat: Add `flatten` array function [#562](https://github.com/apache/datafusion-python/pull/562) (mobley-trent) + +## [35.0.0](https://github.com/apache/datafusion-python/tree/35.0.0) (2024-01-20) + +**Merged pull requests:** + +- build(deps): bump syn from 2.0.41 to 2.0.43 [#559](https://github.com/apache/datafusion-python/pull/559) (dependabot[bot]) +- build(deps): bump tokio from 1.35.0 to 1.35.1 [#558](https://github.com/apache/datafusion-python/pull/558) (dependabot[bot]) +- build(deps): bump async-trait from 0.1.74 to 0.1.77 [#556](https://github.com/apache/datafusion-python/pull/556) (dependabot[bot]) +- build(deps): bump pyo3 from 0.20.0 to 0.20.2 [#557](https://github.com/apache/datafusion-python/pull/557) (dependabot[bot]) + +## [34.0.0](https://github.com/apache/datafusion-python/tree/34.0.0) (2023-12-28) + +**Merged pull requests:** + +- Adjust visibility of crate private members & Functions [#537](https://github.com/apache/datafusion-python/pull/537) (jdye64) +- Update json.rst [#538](https://github.com/apache/datafusion-python/pull/538) (ray-andrew) +- Enable mimalloc local_dynamic_tls feature [#540](https://github.com/apache/datafusion-python/pull/540) (jdye64) +- Enable substrait feature to be built by default in CI, for nightlies … [#544](https://github.com/apache/datafusion-python/pull/544) (jdye64) + +## [33.0.0](https://github.com/apache/datafusion-python/tree/33.0.0) (2023-11-16) + +**Merged pull requests:** + +- First pass at getting architectured builds working [#350](https://github.com/apache/datafusion-python/pull/350) (charlesbluca) +- Remove libprotobuf dep [#527](https://github.com/apache/datafusion-python/pull/527) (jdye64) + +## [32.0.0](https://github.com/apache/datafusion-python/tree/32.0.0) (2023-10-21) + +**Implemented enhancements:** + +- feat: expose PyWindowFrame [#509](https://github.com/apache/datafusion-python/pull/509) (dlovell) +- add Binary String Functions;encode,decode [#494](https://github.com/apache/datafusion-python/pull/494) (jiangzhx) +- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/datafusion-python/pull/496) (jiangzhx) +- add first_value last_value [#498](https://github.com/apache/datafusion-python/pull/498) (jiangzhx) +- add regr\_\* functions [#499](https://github.com/apache/datafusion-python/pull/499) (jiangzhx) +- Add random missing bindings [#522](https://github.com/apache/datafusion-python/pull/522) (jdye64) +- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/datafusion-python/pull/519) (jdye64) +- Add support for window function bindings [#521](https://github.com/apache/datafusion-python/pull/521) (jdye64) + +**Merged pull requests:** + +- Prepare 31.0.0 release [#500](https://github.com/apache/datafusion-python/pull/500) (andygrove) +- Improve release process documentation [#505](https://github.com/apache/datafusion-python/pull/505) (andygrove) +- add Binary String Functions;encode,decode [#494](https://github.com/apache/datafusion-python/pull/494) (jiangzhx) +- build(deps): bump mimalloc from 0.1.38 to 0.1.39 [#502](https://github.com/apache/datafusion-python/pull/502) (dependabot[bot]) +- build(deps): bump syn from 2.0.32 to 2.0.35 [#503](https://github.com/apache/datafusion-python/pull/503) (dependabot[bot]) +- build(deps): bump syn from 2.0.35 to 2.0.37 [#506](https://github.com/apache/datafusion-python/pull/506) (dependabot[bot]) +- Use latest DataFusion [#511](https://github.com/apache/datafusion-python/pull/511) (andygrove) +- add bit_and,bit_or,bit_xor,bool_add,bool_or [#496](https://github.com/apache/datafusion-python/pull/496) (jiangzhx) +- use DataFusion 32 [#515](https://github.com/apache/datafusion-python/pull/515) (andygrove) +- add first_value last_value [#498](https://github.com/apache/datafusion-python/pull/498) (jiangzhx) +- build(deps): bump regex-syntax from 0.7.5 to 0.8.1 [#517](https://github.com/apache/datafusion-python/pull/517) (dependabot[bot]) +- build(deps): bump pyo3-build-config from 0.19.2 to 0.20.0 [#516](https://github.com/apache/datafusion-python/pull/516) (dependabot[bot]) +- add regr\_\* functions [#499](https://github.com/apache/datafusion-python/pull/499) (jiangzhx) +- Add random missing bindings [#522](https://github.com/apache/datafusion-python/pull/522) (jdye64) +- build(deps): bump rustix from 0.38.18 to 0.38.19 [#523](https://github.com/apache/datafusion-python/pull/523) (dependabot[bot]) +- Allow for multiple input files per table instead of a single file [#519](https://github.com/apache/datafusion-python/pull/519) (jdye64) +- Add support for window function bindings [#521](https://github.com/apache/datafusion-python/pull/521) (jdye64) +- Small clippy fix [#524](https://github.com/apache/datafusion-python/pull/524) (andygrove) + +## [31.0.0](https://github.com/apache/datafusion-python/tree/31.0.0) (2023-09-12) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/28.0.0...31.0.0) + +**Implemented enhancements:** + +- feat: add case function (#447) [#448](https://github.com/apache/datafusion-python/pull/448) (mesejo) +- feat: add compression options [#456](https://github.com/apache/datafusion-python/pull/456) (mesejo) +- feat: add register_json [#458](https://github.com/apache/datafusion-python/pull/458) (mesejo) +- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/datafusion-python/pull/459) (mesejo) +- feat: add example of reading parquet from s3 [#460](https://github.com/apache/datafusion-python/pull/460) (mesejo) +- feat: add register_avro and read_table [#461](https://github.com/apache/datafusion-python/pull/461) (mesejo) +- feat: add missing scalar math functions [#465](https://github.com/apache/datafusion-python/pull/465) (mesejo) + +**Documentation updates:** + +- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/datafusion-python/pull/455) (mesejo) + +**Merged pull requests:** + +- Build Linux aarch64 wheel [#443](https://github.com/apache/datafusion-python/pull/443) (gokselk) +- feat: add case function (#447) [#448](https://github.com/apache/datafusion-python/pull/448) (mesejo) +- enhancement(docs): Add user guide (#432) [#445](https://github.com/apache/datafusion-python/pull/445) (mesejo) +- docs: include pre-commit hooks section in contributor guide [#455](https://github.com/apache/datafusion-python/pull/455) (mesejo) +- feat: add compression options [#456](https://github.com/apache/datafusion-python/pull/456) (mesejo) +- Upgrade to DF 28.0.0-rc1 [#457](https://github.com/apache/datafusion-python/pull/457) (andygrove) +- feat: add register_json [#458](https://github.com/apache/datafusion-python/pull/458) (mesejo) +- feat: add basic compression configuration to write_parquet [#459](https://github.com/apache/datafusion-python/pull/459) (mesejo) +- feat: add example of reading parquet from s3 [#460](https://github.com/apache/datafusion-python/pull/460) (mesejo) +- feat: add register_avro and read_table [#461](https://github.com/apache/datafusion-python/pull/461) (mesejo) +- feat: add missing scalar math functions [#465](https://github.com/apache/datafusion-python/pull/465) (mesejo) +- build(deps): bump arduino/setup-protoc from 1 to 2 [#452](https://github.com/apache/datafusion-python/pull/452) (dependabot[bot]) +- Revert "build(deps): bump arduino/setup-protoc from 1 to 2 (#452)" [#474](https://github.com/apache/datafusion-python/pull/474) (viirya) +- Minor: fix wrongly copied function description [#497](https://github.com/apache/datafusion-python/pull/497) (viirya) +- Upgrade to Datafusion 31.0.0 [#491](https://github.com/apache/datafusion-python/pull/491) (judahrand) +- Add `isnan` and `iszero` [#495](https://github.com/apache/datafusion-python/pull/495) (judahrand) + +## 30.0.0 + +- Skipped due to a breaking change in DataFusion + +## 29.0.0 + +- Skipped + +## [28.0.0](https://github.com/apache/datafusion-python/tree/28.0.0) (2023-07-25) + +**Implemented enhancements:** + +- feat: expose offset in python API [#437](https://github.com/apache/datafusion-python/pull/437) (cpcloud) + +**Merged pull requests:** + +- File based input utils [#433](https://github.com/apache/datafusion-python/pull/433) (jdye64) +- Upgrade to 28.0.0-rc1 [#434](https://github.com/apache/datafusion-python/pull/434) (andygrove) +- Introduces utility for obtaining SqlTable information from a file like location [#398](https://github.com/apache/datafusion-python/pull/398) (jdye64) +- feat: expose offset in python API [#437](https://github.com/apache/datafusion-python/pull/437) (cpcloud) +- Use DataFusion 28 [#439](https://github.com/apache/datafusion-python/pull/439) (andygrove) + +## [27.0.0](https://github.com/apache/datafusion-python/tree/27.0.0) (2023-07-03) + +**Merged pull requests:** + +- LogicalPlan.to_variant() make public [#412](https://github.com/apache/datafusion-python/pull/412) (jdye64) +- Prepare 27.0.0 release [#423](https://github.com/apache/datafusion-python/pull/423) (andygrove) + +## [26.0.0](https://github.com/apache/datafusion-python/tree/26.0.0) (2023-06-11) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/25.0.0...26.0.0) + +**Merged pull requests:** + +- Add Expr::Case when_then_else support to rex_call_operands function [#388](https://github.com/apache/datafusion-python/pull/388) (jdye64) +- Introduce BaseSessionContext abstract class [#390](https://github.com/apache/datafusion-python/pull/390) (jdye64) +- CRUD Schema support for `BaseSessionContext` [#392](https://github.com/apache/datafusion-python/pull/392) (jdye64) +- CRUD Table support for `BaseSessionContext` [#394](https://github.com/apache/datafusion-python/pull/394) (jdye64) + +## [25.0.0](https://github.com/apache/datafusion-python/tree/25.0.0) (2023-05-23) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/24.0.0...25.0.0) + +**Merged pull requests:** + +- Prepare 24.0.0 Release [#376](https://github.com/apache/datafusion-python/pull/376) (andygrove) +- build(deps): bump uuid from 1.3.1 to 1.3.2 [#359](https://github.com/apache/datafusion-python/pull/359) (dependabot[bot]) +- build(deps): bump mimalloc from 0.1.36 to 0.1.37 [#361](https://github.com/apache/datafusion-python/pull/361) (dependabot[bot]) +- build(deps): bump regex-syntax from 0.6.29 to 0.7.1 [#334](https://github.com/apache/datafusion-python/pull/334) (dependabot[bot]) +- upgrade maturin to 0.15.1 [#379](https://github.com/apache/datafusion-python/pull/379) (Jimexist) +- Expand Expr to include RexType basic support [#378](https://github.com/apache/datafusion-python/pull/378) (jdye64) +- Add Python script for generating changelog [#383](https://github.com/apache/datafusion-python/pull/383) (andygrove) + +## [24.0.0](https://github.com/apache/datafusion-python/tree/24.0.0) (2023-05-09) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/23.0.0...24.0.0) + +**Documentation updates:** + +- Fix link to user guide [#354](https://github.com/apache/datafusion-python/pull/354) (andygrove) + +**Merged pull requests:** + +- Add interface to serialize Substrait plans to Python Bytes. [#344](https://github.com/apache/datafusion-python/pull/344) (kylebrooks-8451) +- Add partition_count property to ExecutionPlan. [#346](https://github.com/apache/datafusion-python/pull/346) (kylebrooks-8451) +- Remove unsendable from all Rust pyclass types. [#348](https://github.com/apache/datafusion-python/pull/348) (kylebrooks-8451) +- Fix link to user guide [#354](https://github.com/apache/datafusion-python/pull/354) (andygrove) +- Fix SessionContext execute. [#353](https://github.com/apache/datafusion-python/pull/353) (kylebrooks-8451) +- Pub mod expr in lib.rs [#357](https://github.com/apache/datafusion-python/pull/357) (jdye64) +- Add benchmark derived from TPC-H [#355](https://github.com/apache/datafusion-python/pull/355) (andygrove) +- Add db-benchmark [#365](https://github.com/apache/datafusion-python/pull/365) (andygrove) +- First pass of documentation in mdBook [#364](https://github.com/apache/datafusion-python/pull/364) (MrPowers) +- Add 'pub' and '#[pyo3(get, set)]' to DataTypeMap [#371](https://github.com/apache/datafusion-python/pull/371) (jdye64) +- Fix db-benchmark [#369](https://github.com/apache/datafusion-python/pull/369) (andygrove) +- Docs explaining how to view query plans [#373](https://github.com/apache/datafusion-python/pull/373) (andygrove) +- Improve db-benchmark [#372](https://github.com/apache/datafusion-python/pull/372) (andygrove) +- Make expr member of PyExpr public [#375](https://github.com/apache/datafusion-python/pull/375) (jdye64) + +## [23.0.0](https://github.com/apache/datafusion-python/tree/23.0.0) (2023-04-23) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/22.0.0...23.0.0) + +**Merged pull requests:** + +- Improve API docs, README, and examples for configuring context [#321](https://github.com/apache/datafusion-python/pull/321) (andygrove) +- Osx build linker args [#330](https://github.com/apache/datafusion-python/pull/330) (jdye64) +- Add requirements file for python 3.11 [#332](https://github.com/apache/datafusion-python/pull/332) (r4ntix) +- mac arm64 build [#338](https://github.com/apache/datafusion-python/pull/338) (andygrove) +- Add conda.yaml baseline workflow file [#281](https://github.com/apache/datafusion-python/pull/281) (jdye64) +- Prepare for 23.0.0 release [#335](https://github.com/apache/datafusion-python/pull/335) (andygrove) +- Reuse the Tokio Runtime [#341](https://github.com/apache/datafusion-python/pull/341) (kylebrooks-8451) + +## [22.0.0](https://github.com/apache/datafusion-python/tree/22.0.0) (2023-04-10) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/21.0.0...22.0.0) + +**Merged pull requests:** + +- Fix invalid build yaml [#308](https://github.com/apache/datafusion-python/pull/308) (andygrove) +- Try fix release build [#309](https://github.com/apache/datafusion-python/pull/309) (andygrove) +- Fix release build [#310](https://github.com/apache/datafusion-python/pull/310) (andygrove) +- Enable datafusion-substrait protoc feature, to remove compile-time dependency on protoc [#312](https://github.com/apache/datafusion-python/pull/312) (andygrove) +- Fix Mac/Win release builds in CI [#313](https://github.com/apache/datafusion-python/pull/313) (andygrove) +- install protoc in docs workflow [#314](https://github.com/apache/datafusion-python/pull/314) (andygrove) +- Fix documentation generation in CI [#315](https://github.com/apache/datafusion-python/pull/315) (andygrove) +- Source wheel fix [#319](https://github.com/apache/datafusion-python/pull/319) (andygrove) + +## [21.0.0](https://github.com/apache/datafusion-python/tree/21.0.0) (2023-03-30) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/20.0.0...21.0.0) + +**Merged pull requests:** + +- minor: Fix minor warning on unused import [#289](https://github.com/apache/datafusion-python/pull/289) (viirya) +- feature: Implement `describe()` method [#293](https://github.com/apache/datafusion-python/pull/293) (simicd) +- fix: Printed results not visible in debugger & notebooks [#296](https://github.com/apache/datafusion-python/pull/296) (simicd) +- add package.include and remove wildcard dependency [#295](https://github.com/apache/datafusion-python/pull/295) (andygrove) +- Update main branch name in docs workflow [#303](https://github.com/apache/datafusion-python/pull/303) (andygrove) +- Upgrade to DF 21 [#301](https://github.com/apache/datafusion-python/pull/301) (andygrove) + +## [20.0.0](https://github.com/apache/datafusion-python/tree/20.0.0) (2023-03-17) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.8.0...20.0.0) + +**Implemented enhancements:** + +- Empty relation bindings [#208](https://github.com/apache/datafusion-python/pull/208) (jdye64) +- wrap display_name and canonical_name functions [#214](https://github.com/apache/datafusion-python/pull/214) (jdye64) +- Add PyAlias bindings [#216](https://github.com/apache/datafusion-python/pull/216) (jdye64) +- Add bindings for scalar_variable [#218](https://github.com/apache/datafusion-python/pull/218) (jdye64) +- Bindings for LIKE type expressions [#220](https://github.com/apache/datafusion-python/pull/220) (jdye64) +- Bool expr bindings [#223](https://github.com/apache/datafusion-python/pull/223) (jdye64) +- Between bindings [#229](https://github.com/apache/datafusion-python/pull/229) (jdye64) +- Add bindings for GetIndexedField [#227](https://github.com/apache/datafusion-python/pull/227) (jdye64) +- Add bindings for case, cast, and trycast [#232](https://github.com/apache/datafusion-python/pull/232) (jdye64) +- add remaining expr bindings [#233](https://github.com/apache/datafusion-python/pull/233) (jdye64) +- feature: Additional export methods [#236](https://github.com/apache/datafusion-python/pull/236) (simicd) +- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/datafusion-python/pull/240) (iajoiner) +- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/datafusion-python/pull/242) (simicd) +- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/datafusion-python/pull/246) (iajoiner) +- feature: Set table name from ctx functions [#260](https://github.com/apache/datafusion-python/pull/260) (simicd) +- Explain bindings [#264](https://github.com/apache/datafusion-python/pull/264) (jdye64) +- Extension bindings [#266](https://github.com/apache/datafusion-python/pull/266) (jdye64) +- Subquery alias bindings [#269](https://github.com/apache/datafusion-python/pull/269) (jdye64) +- Create memory table [#271](https://github.com/apache/datafusion-python/pull/271) (jdye64) +- Create view bindings [#273](https://github.com/apache/datafusion-python/pull/273) (jdye64) +- Re-export Datafusion dependencies [#277](https://github.com/apache/datafusion-python/pull/277) (jdye64) +- Distinct bindings [#275](https://github.com/apache/datafusion-python/pull/275) (jdye64) +- Drop table bindings [#283](https://github.com/apache/datafusion-python/pull/283) (jdye64) +- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/datafusion-python/pull/285) (jdye64) +- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/datafusion-python/pull/287) (jdye64) + +**Documentation updates:** + +- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/datafusion-python/pull/258) (simicd) + +**Merged pull requests:** + +- Minor docs updates [#210](https://github.com/apache/datafusion-python/pull/210) (andygrove) +- Empty relation bindings [#208](https://github.com/apache/datafusion-python/pull/208) (jdye64) +- wrap display_name and canonical_name functions [#214](https://github.com/apache/datafusion-python/pull/214) (jdye64) +- Add PyAlias bindings [#216](https://github.com/apache/datafusion-python/pull/216) (jdye64) +- Add bindings for scalar_variable [#218](https://github.com/apache/datafusion-python/pull/218) (jdye64) +- Bindings for LIKE type expressions [#220](https://github.com/apache/datafusion-python/pull/220) (jdye64) +- Bool expr bindings [#223](https://github.com/apache/datafusion-python/pull/223) (jdye64) +- Between bindings [#229](https://github.com/apache/datafusion-python/pull/229) (jdye64) +- Add bindings for GetIndexedField [#227](https://github.com/apache/datafusion-python/pull/227) (jdye64) +- Add bindings for case, cast, and trycast [#232](https://github.com/apache/datafusion-python/pull/232) (jdye64) +- add remaining expr bindings [#233](https://github.com/apache/datafusion-python/pull/233) (jdye64) +- Pre-commit hooks [#228](https://github.com/apache/datafusion-python/pull/228) (jdye64) +- Implement new release process [#149](https://github.com/apache/datafusion-python/pull/149) (andygrove) +- feature: Additional export methods [#236](https://github.com/apache/datafusion-python/pull/236) (simicd) +- Add Python wrapper for LogicalPlan::Union [#240](https://github.com/apache/datafusion-python/pull/240) (iajoiner) +- feature: Create dataframe from pandas, polars, dictionary, list or pyarrow Table [#242](https://github.com/apache/datafusion-python/pull/242) (simicd) +- Fix release instructions [#238](https://github.com/apache/datafusion-python/pull/238) (andygrove) +- Add Python wrappers for `LogicalPlan::Join` and `LogicalPlan::CrossJoin` [#246](https://github.com/apache/datafusion-python/pull/246) (iajoiner) +- docs: Example of calling Python UDF & UDAF in SQL [#258](https://github.com/apache/datafusion-python/pull/258) (simicd) +- feature: Set table name from ctx functions [#260](https://github.com/apache/datafusion-python/pull/260) (simicd) +- Upgrade to DataFusion 19 [#262](https://github.com/apache/datafusion-python/pull/262) (andygrove) +- Explain bindings [#264](https://github.com/apache/datafusion-python/pull/264) (jdye64) +- Extension bindings [#266](https://github.com/apache/datafusion-python/pull/266) (jdye64) +- Subquery alias bindings [#269](https://github.com/apache/datafusion-python/pull/269) (jdye64) +- Create memory table [#271](https://github.com/apache/datafusion-python/pull/271) (jdye64) +- Create view bindings [#273](https://github.com/apache/datafusion-python/pull/273) (jdye64) +- Re-export Datafusion dependencies [#277](https://github.com/apache/datafusion-python/pull/277) (jdye64) +- Distinct bindings [#275](https://github.com/apache/datafusion-python/pull/275) (jdye64) +- build(deps): bump actions/checkout from 2 to 3 [#244](https://github.com/apache/datafusion-python/pull/244) (dependabot[bot]) +- build(deps): bump actions/upload-artifact from 2 to 3 [#245](https://github.com/apache/datafusion-python/pull/245) (dependabot[bot]) +- build(deps): bump actions/download-artifact from 2 to 3 [#243](https://github.com/apache/datafusion-python/pull/243) (dependabot[bot]) +- Use DataFusion 20 [#278](https://github.com/apache/datafusion-python/pull/278) (andygrove) +- Drop table bindings [#283](https://github.com/apache/datafusion-python/pull/283) (jdye64) +- Bindings for LogicalPlan::Repartition [#285](https://github.com/apache/datafusion-python/pull/285) (jdye64) +- Expand Rust return type support for Arrow DataTypes in ScalarValue [#287](https://github.com/apache/datafusion-python/pull/287) (jdye64) + +## [0.8.0](https://github.com/apache/datafusion-python/tree/0.8.0) (2023-02-22) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.8.0-rc1...0.8.0) + +**Implemented enhancements:** + +- Add support for cuDF physical execution engine [\#202](https://github.com/apache/datafusion-python/issues/202) +- Make it easier to create a Pandas dataframe from DataFusion query results [\#139](https://github.com/apache/datafusion-python/issues/139) + +**Fixed bugs:** + +- Build error: could not compile `thiserror` due to 2 previous errors [\#69](https://github.com/apache/datafusion-python/issues/69) + +**Closed issues:** + +- Integrate with the new `object_store` crate [\#22](https://github.com/apache/datafusion-python/issues/22) + +**Merged pull requests:** + +- Update README in preparation for 0.8 release [\#206](https://github.com/apache/datafusion-python/pull/206) ([andygrove](https://github.com/andygrove)) +- Add support for cudf as a physical execution engine [\#205](https://github.com/apache/datafusion-python/pull/205) ([jdye64](https://github.com/jdye64)) +- Run `maturin develop` instead of `cargo build` in verification script [\#200](https://github.com/apache/datafusion-python/pull/200) ([andygrove](https://github.com/andygrove)) +- Add tests for recently added functionality [\#199](https://github.com/apache/datafusion-python/pull/199) ([andygrove](https://github.com/andygrove)) +- Implement `to_pandas()` [\#197](https://github.com/apache/datafusion-python/pull/197) ([simicd](https://github.com/simicd)) +- Add Python wrapper for LogicalPlan::Sort [\#196](https://github.com/apache/datafusion-python/pull/196) ([andygrove](https://github.com/andygrove)) +- Add Python wrapper for LogicalPlan::Aggregate [\#195](https://github.com/apache/datafusion-python/pull/195) ([andygrove](https://github.com/andygrove)) +- Add Python wrapper for LogicalPlan::Limit [\#193](https://github.com/apache/datafusion-python/pull/193) ([andygrove](https://github.com/andygrove)) +- Add Python wrapper for LogicalPlan::Filter [\#192](https://github.com/apache/datafusion-python/pull/192) ([andygrove](https://github.com/andygrove)) +- Add experimental support for executing SQL with Polars and Pandas [\#190](https://github.com/apache/datafusion-python/pull/190) ([andygrove](https://github.com/andygrove)) +- Update changelog for 0.8 release [\#188](https://github.com/apache/datafusion-python/pull/188) ([andygrove](https://github.com/andygrove)) +- Add ability to execute ExecutionPlan and get a stream of RecordBatch [\#186](https://github.com/apache/datafusion-python/pull/186) ([andygrove](https://github.com/andygrove)) +- Dffield bindings [\#185](https://github.com/apache/datafusion-python/pull/185) ([jdye64](https://github.com/jdye64)) +- Add bindings for DFSchema [\#183](https://github.com/apache/datafusion-python/pull/183) ([jdye64](https://github.com/jdye64)) +- test: Window functions [\#182](https://github.com/apache/datafusion-python/pull/182) ([simicd](https://github.com/simicd)) +- Add bindings for Projection [\#180](https://github.com/apache/datafusion-python/pull/180) ([jdye64](https://github.com/jdye64)) +- Table scan bindings [\#178](https://github.com/apache/datafusion-python/pull/178) ([jdye64](https://github.com/jdye64)) +- Make session configurable [\#176](https://github.com/apache/datafusion-python/pull/176) ([andygrove](https://github.com/andygrove)) +- Upgrade to DataFusion 18.0.0 [\#175](https://github.com/apache/datafusion-python/pull/175) ([andygrove](https://github.com/andygrove)) +- Use latest DataFusion rev in preparation for DF 18 release [\#174](https://github.com/apache/datafusion-python/pull/174) ([andygrove](https://github.com/andygrove)) +- Arrow type bindings [\#173](https://github.com/apache/datafusion-python/pull/173) ([jdye64](https://github.com/jdye64)) +- Pyo3 bump [\#171](https://github.com/apache/datafusion-python/pull/171) ([jdye64](https://github.com/jdye64)) +- feature: Add additional aggregation functions [\#170](https://github.com/apache/datafusion-python/pull/170) ([simicd](https://github.com/simicd)) +- Make from_substrait_plan return DataFrame instead of LogicalPlan [\#164](https://github.com/apache/datafusion-python/pull/164) ([andygrove](https://github.com/andygrove)) +- feature: Implement count method [\#163](https://github.com/apache/datafusion-python/pull/163) ([simicd](https://github.com/simicd)) +- CI Fixes [\#162](https://github.com/apache/datafusion-python/pull/162) ([jdye64](https://github.com/jdye64)) +- Upgrade to DataFusion 17 [\#160](https://github.com/apache/datafusion-python/pull/160) ([andygrove](https://github.com/andygrove)) +- feature: Improve string representation of datafusion classes [\#159](https://github.com/apache/datafusion-python/pull/159) ([simicd](https://github.com/simicd)) +- Make PyExecutionPlan.plan public [\#156](https://github.com/apache/datafusion-python/pull/156) ([andygrove](https://github.com/andygrove)) +- Expose methods on logical and execution plans [\#155](https://github.com/apache/datafusion-python/pull/155) ([andygrove](https://github.com/andygrove)) +- Fix clippy for new Rust version [\#154](https://github.com/apache/datafusion-python/pull/154) ([andygrove](https://github.com/andygrove)) +- Add DataFrame methods for accessing plans [\#153](https://github.com/apache/datafusion-python/pull/153) ([andygrove](https://github.com/andygrove)) +- Use DataFusion rev 5238e8c97f998b4d2cb9fab85fb182f325a1a7fb [\#150](https://github.com/apache/datafusion-python/pull/150) ([andygrove](https://github.com/andygrove)) +- build\(deps\): bump async-trait from 0.1.61 to 0.1.62 [\#148](https://github.com/apache/datafusion-python/pull/148) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Rename default branch from master to main [\#147](https://github.com/apache/datafusion-python/pull/147) ([andygrove](https://github.com/andygrove)) +- Substrait bindings [\#145](https://github.com/apache/datafusion-python/pull/145) ([jdye64](https://github.com/jdye64)) +- build\(deps\): bump uuid from 0.8.2 to 1.2.2 [\#143](https://github.com/apache/datafusion-python/pull/143) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Prepare for 0.8.0 release [\#141](https://github.com/apache/datafusion-python/pull/141) ([andygrove](https://github.com/andygrove)) +- Improve README and add more examples [\#137](https://github.com/apache/datafusion-python/pull/137) ([andygrove](https://github.com/andygrove)) +- test: Expand tests for built-in functions [\#129](https://github.com/apache/datafusion-python/pull/129) ([simicd](https://github.com/simicd)) +- build\(deps\): bump object_store from 0.5.2 to 0.5.3 [\#126](https://github.com/apache/datafusion-python/pull/126) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump mimalloc from 0.1.32 to 0.1.34 [\#125](https://github.com/apache/datafusion-python/pull/125) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Introduce conda directory containing datafusion-dev.yaml conda enviro… [\#124](https://github.com/apache/datafusion-python/pull/124) ([jdye64](https://github.com/jdye64)) +- build\(deps\): bump bzip2 from 0.4.3 to 0.4.4 [\#121](https://github.com/apache/datafusion-python/pull/121) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump tokio from 1.23.0 to 1.24.1 [\#119](https://github.com/apache/datafusion-python/pull/119) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump async-trait from 0.1.60 to 0.1.61 [\#118](https://github.com/apache/datafusion-python/pull/118) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Upgrade to DataFusion 16.0.0 [\#115](https://github.com/apache/datafusion-python/pull/115) ([andygrove](https://github.com/andygrove)) +- Bump async-trait from 0.1.57 to 0.1.60 [\#114](https://github.com/apache/datafusion-python/pull/114) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump object_store from 0.5.1 to 0.5.2 [\#112](https://github.com/apache/datafusion-python/pull/112) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Bump tokio from 1.21.2 to 1.23.0 [\#109](https://github.com/apache/datafusion-python/pull/109) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Add entries for publishing production \(asf-site\) and staging docs [\#107](https://github.com/apache/datafusion-python/pull/107) ([martin-g](https://github.com/martin-g)) +- Add a workflow that builds the docs and deploys them at staged or production [\#104](https://github.com/apache/datafusion-python/pull/104) ([martin-g](https://github.com/martin-g)) +- Upgrade to DataFusion 15.0.0 [\#103](https://github.com/apache/datafusion-python/pull/103) ([andygrove](https://github.com/andygrove)) +- build\(deps\): bump futures from 0.3.24 to 0.3.25 [\#102](https://github.com/apache/datafusion-python/pull/102) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump pyo3 from 0.17.2 to 0.17.3 [\#101](https://github.com/apache/datafusion-python/pull/101) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump mimalloc from 0.1.30 to 0.1.32 [\#98](https://github.com/apache/datafusion-python/pull/98) ([dependabot[bot]](https://github.com/apps/dependabot)) +- build\(deps\): bump rand from 0.7.3 to 0.8.5 [\#97](https://github.com/apache/datafusion-python/pull/97) ([dependabot[bot]](https://github.com/apps/dependabot)) +- Fix GitHub actions warnings [\#95](https://github.com/apache/datafusion-python/pull/95) ([martin-g](https://github.com/martin-g)) +- Fixes \#81 - Add CI workflow for source distribution [\#93](https://github.com/apache/datafusion-python/pull/93) ([martin-g](https://github.com/martin-g)) +- post-release updates [\#91](https://github.com/apache/datafusion-python/pull/91) ([andygrove](https://github.com/andygrove)) +- Build for manylinux 2014 [\#88](https://github.com/apache/datafusion-python/pull/88) ([martin-g](https://github.com/martin-g)) +- update release readme tag [\#86](https://github.com/apache/datafusion-python/pull/86) ([Jimexist](https://github.com/Jimexist)) +- Upgrade Maturin to 0.14.2 [\#85](https://github.com/apache/datafusion-python/pull/85) ([martin-g](https://github.com/martin-g)) +- Update release instructions [\#83](https://github.com/apache/datafusion-python/pull/83) ([andygrove](https://github.com/andygrove)) +- \[Functions\] - Add python function binding to `functions` [\#73](https://github.com/apache/datafusion-python/pull/73) ([francis-du](https://github.com/francis-du)) + +## [0.8.0-rc1](https://github.com/apache/datafusion-python/tree/0.8.0-rc1) (2023-02-17) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.7.0-rc2...0.8.0-rc1) + +**Implemented enhancements:** + +- Add bindings for datafusion_common::DFField [\#184](https://github.com/apache/datafusion-python/issues/184) +- Add bindings for DFSchema/DFSchemaRef [\#181](https://github.com/apache/datafusion-python/issues/181) +- Add bindings for datafusion_expr Projection [\#179](https://github.com/apache/datafusion-python/issues/179) +- Add bindings for `TableScan` struct from `datafusion_expr::TableScan` [\#177](https://github.com/apache/datafusion-python/issues/177) +- Add a "mapping" struct for types [\#172](https://github.com/apache/datafusion-python/issues/172) +- Improve string representation of datafusion classes \(dataframe, context, expression, ...\) [\#158](https://github.com/apache/datafusion-python/issues/158) +- Add DataFrame count method [\#151](https://github.com/apache/datafusion-python/issues/151) +- \[REQUEST\] Github Actions Improvements [\#146](https://github.com/apache/datafusion-python/issues/146) +- Change default branch name from master to main [\#144](https://github.com/apache/datafusion-python/issues/144) +- Bump pyo3 to 0.18.0 [\#140](https://github.com/apache/datafusion-python/issues/140) +- Add script for Python linting [\#134](https://github.com/apache/datafusion-python/issues/134) +- Add Python bindings for substrait module [\#132](https://github.com/apache/datafusion-python/issues/132) +- Expand unit tests for built-in functions [\#128](https://github.com/apache/datafusion-python/issues/128) +- support creating arrow-datafusion-python conda environment [\#122](https://github.com/apache/datafusion-python/issues/122) +- Build Python source distribution in GitHub workflow [\#81](https://github.com/apache/datafusion-python/issues/81) +- EPIC: Add all functions to python binding `functions` [\#72](https://github.com/apache/datafusion-python/issues/72) + +**Fixed bugs:** + +- Build is broken [\#161](https://github.com/apache/datafusion-python/issues/161) +- Out of memory when sorting [\#157](https://github.com/apache/datafusion-python/issues/157) +- window_lead test appears to be non-deterministic [\#135](https://github.com/apache/datafusion-python/issues/135) +- Reading csv does not work [\#130](https://github.com/apache/datafusion-python/issues/130) +- Github actions produce a lot of warnings [\#94](https://github.com/apache/datafusion-python/issues/94) +- ASF source release tarball has wrong directory name [\#90](https://github.com/apache/datafusion-python/issues/90) +- Python Release Build failing after upgrading to maturin 14.2 [\#87](https://github.com/apache/datafusion-python/issues/87) +- Maturin build hangs on Linux ARM64 [\#84](https://github.com/apache/datafusion-python/issues/84) +- Cannot install on Mac M1 from source tarball from testpypi [\#82](https://github.com/apache/datafusion-python/issues/82) +- ImportPathMismatchError when running pytest locally [\#77](https://github.com/apache/datafusion-python/issues/77) + +**Closed issues:** + +- Publish documentation for Python bindings [\#39](https://github.com/apache/datafusion-python/issues/39) +- Add Python binding for `approx_median` [\#32](https://github.com/apache/datafusion-python/issues/32) +- Release version 0.7.0 [\#7](https://github.com/apache/datafusion-python/issues/7) + +## [0.7.0-rc2](https://github.com/apache/datafusion-python/tree/0.7.0-rc2) (2022-11-26) + +[Full Changelog](https://github.com/apache/datafusion-python/compare/0.7.0...0.7.0-rc2) + +## [Unreleased](https://github.com/datafusion-contrib/datafusion-python/tree/HEAD) + +[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.1...HEAD) + +**Merged pull requests:** + +- use \_\_getitem\_\_ for df column selection [\#41](https://github.com/datafusion-contrib/datafusion-python/pull/41) ([Jimexist](https://github.com/Jimexist)) +- fix demo in readme [\#40](https://github.com/datafusion-contrib/datafusion-python/pull/40) ([Jimexist](https://github.com/Jimexist)) +- Implement select_columns [\#39](https://github.com/datafusion-contrib/datafusion-python/pull/39) ([andygrove](https://github.com/andygrove)) +- update readme and changelog [\#38](https://github.com/datafusion-contrib/datafusion-python/pull/38) ([Jimexist](https://github.com/Jimexist)) +- Add PyDataFrame.explain [\#36](https://github.com/datafusion-contrib/datafusion-python/pull/36) ([andygrove](https://github.com/andygrove)) +- Release 0.5.0 [\#34](https://github.com/datafusion-contrib/datafusion-python/pull/34) ([Jimexist](https://github.com/Jimexist)) +- disable nightly in workflow [\#33](https://github.com/datafusion-contrib/datafusion-python/pull/33) ([Jimexist](https://github.com/Jimexist)) +- update requirements to 37 and 310, update readme [\#32](https://github.com/datafusion-contrib/datafusion-python/pull/32) ([Jimexist](https://github.com/Jimexist)) +- Add custom global allocator [\#30](https://github.com/datafusion-contrib/datafusion-python/pull/30) ([matthewmturner](https://github.com/matthewmturner)) +- Remove pandas dependency [\#25](https://github.com/datafusion-contrib/datafusion-python/pull/25) ([matthewmturner](https://github.com/matthewmturner)) +- upgrade datafusion and pyo3 [\#20](https://github.com/datafusion-contrib/datafusion-python/pull/20) ([Jimexist](https://github.com/Jimexist)) +- update maturin 0.12+ [\#17](https://github.com/datafusion-contrib/datafusion-python/pull/17) ([Jimexist](https://github.com/Jimexist)) +- Update README.md [\#16](https://github.com/datafusion-contrib/datafusion-python/pull/16) ([Jimexist](https://github.com/Jimexist)) +- apply cargo clippy --fix [\#15](https://github.com/datafusion-contrib/datafusion-python/pull/15) ([Jimexist](https://github.com/Jimexist)) +- update test workflow to include rust clippy and check [\#14](https://github.com/datafusion-contrib/datafusion-python/pull/14) ([Jimexist](https://github.com/Jimexist)) +- use maturin 0.12.6 [\#13](https://github.com/datafusion-contrib/datafusion-python/pull/13) ([Jimexist](https://github.com/Jimexist)) +- apply cargo fmt [\#12](https://github.com/datafusion-contrib/datafusion-python/pull/12) ([Jimexist](https://github.com/Jimexist)) +- use stable not nightly [\#11](https://github.com/datafusion-contrib/datafusion-python/pull/11) ([Jimexist](https://github.com/Jimexist)) +- ci: test against more compilers, setup clippy and fix clippy lints [\#9](https://github.com/datafusion-contrib/datafusion-python/pull/9) ([cpcloud](https://github.com/cpcloud)) +- Fix use of importlib.metadata and unify requirements.txt [\#8](https://github.com/datafusion-contrib/datafusion-python/pull/8) ([cpcloud](https://github.com/cpcloud)) +- Ship the Cargo.lock file in the source distribution [\#7](https://github.com/datafusion-contrib/datafusion-python/pull/7) ([cpcloud](https://github.com/cpcloud)) +- add \_\_version\_\_ attribute to datafusion object [\#3](https://github.com/datafusion-contrib/datafusion-python/pull/3) ([tfeda](https://github.com/tfeda)) +- fix ci by fixing directories [\#2](https://github.com/datafusion-contrib/datafusion-python/pull/2) ([Jimexist](https://github.com/Jimexist)) +- setup workflow [\#1](https://github.com/datafusion-contrib/datafusion-python/pull/1) ([Jimexist](https://github.com/Jimexist)) + +## [0.5.1](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.1) (2022-03-15) + +[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.1-rc1...0.5.1) + +## [0.5.1-rc1](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.1-rc1) (2022-03-15) + +[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.0...0.5.1-rc1) + +## [0.5.0](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.0) (2022-03-10) + +[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.0-rc2...0.5.0) + +## [0.5.0-rc2](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.0-rc2) (2022-03-10) + +[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/0.5.0-rc1...0.5.0-rc2) + +**Closed issues:** + +- Add support for Ballista [\#37](https://github.com/datafusion-contrib/datafusion-python/issues/37) +- Implement DataFrame.explain [\#35](https://github.com/datafusion-contrib/datafusion-python/issues/35) + +## [0.5.0-rc1](https://github.com/datafusion-contrib/datafusion-python/tree/0.5.0-rc1) (2022-03-09) + +[Full Changelog](https://github.com/datafusion-contrib/datafusion-python/compare/4c98b8e9c3c3f8e2e6a8f2d1ffcfefda344c4680...0.5.0-rc1) + +**Closed issues:** + +- Investigate exposing additional optimizations [\#28](https://github.com/datafusion-contrib/datafusion-python/issues/28) +- Use custom allocator in Python build [\#27](https://github.com/datafusion-contrib/datafusion-python/issues/27) +- Why is pandas a requirement? [\#24](https://github.com/datafusion-contrib/datafusion-python/issues/24) +- Unable to build [\#18](https://github.com/datafusion-contrib/datafusion-python/issues/18) +- Setup CI against multiple Python version [\#6](https://github.com/datafusion-contrib/datafusion-python/issues/6) From 79c22d6d6c0809e7e93a0a23249baa516dbd8d6f Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 4 Dec 2024 05:56:11 -0500 Subject: [PATCH 246/413] Search default window functions if no session context was provided (#963) * Search default window functions if no session context was provided * Check if value is None because [] don't trigger the intended behavior --- python/datafusion/dataframe.py | 6 +++--- python/datafusion/functions.py | 1 + src/functions.rs | 11 +++++++++++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index e283f590e..0b38db924 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -446,14 +446,14 @@ def join( left_on = join_keys[0] right_on = join_keys[1] - if on: - if left_on or right_on: + if on is not None: + if left_on is not None or right_on is not None: raise ValueError( "`left_on` or `right_on` should not provided with `on`" ) left_on = on right_on = on - elif left_on or right_on: + elif left_on is not None or right_on is not None: if left_on is None or right_on is None: raise ValueError("`left_on` and `right_on` should both be provided.") else: diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 15ad8822f..f3ee5c092 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -431,6 +431,7 @@ def window( partition_by = expr_list_to_raw_expr_list(partition_by) order_by_raw = sort_list_to_raw_sort_list(order_by) window_frame = window_frame.window_frame if window_frame is not None else None + ctx = ctx.ctx if ctx is not None else None return Expr(f.window(name, args, partition_by, order_by_raw, window_frame, ctx)) diff --git a/src/functions.rs b/src/functions.rs index e29c57f9b..5c450286f 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -16,6 +16,7 @@ // under the License. use datafusion::functions_aggregate::all_default_aggregate_functions; +use datafusion::functions_window::all_default_window_functions; use datafusion::logical_expr::ExprFunctionExt; use datafusion::logical_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; @@ -282,6 +283,16 @@ fn find_window_fn(name: &str, ctx: Option) -> PyResult Date: Tue, 7 Jan 2025 21:28:25 +0800 Subject: [PATCH 247/413] Add arrow cast (#962) * feat: add data_type parameter to expr_fn macro for arrow_cast function * feat: add arrow_cast function to cast expressions to specified data types * docs: add casting section to user guide with examples for arrow_cast function * test: add unit test for arrow_cast function to validate casting to Float64 and Int32 * fix: update arrow_cast function to accept Expr type for data_type parameter * fix: update test_arrow_cast to use literal casting for data types * fix: update arrow_cast function to accept string type for data_type parameter * fix: update arrow_cast function to accept Expr type for data_type parameter * fix: update test_arrow_cast to use literal for data type parameters * fix: update arrow_cast function to use arg_1 for datatype parameter * fix: update arrow_cast function to accept string type for data_type parameter * Revert "fix: update arrow_cast function to accept string type for data_type parameter" This reverts commit eba0d320820e8f3f9688781f27b2a5579c0e9949. * fix: update test_arrow_cast to cast literals to string type for arrow_cast function * Revert "fix: update test_arrow_cast to cast literals to string type for arrow_cast function" This reverts commit 856ff8c4cad0075c282089b5368a7c3fd17f03d8. * fix: update arrow_cast function to accept string type for data_type parameter * Revert "fix: update arrow_cast function to accept string type for data_type parameter" This reverts commit 9e1ced7fb56c8aec47bc9f540ea5686c7246f022. * fix: add utf8_literal function to create UTF8 literal expressions in tests * Revert "fix: add utf8_literal function to create UTF8 literal expressions in tests" This reverts commit 11ed6749e02ab7b34d47fa105961f088f9fc9245. * feat: add utf8_literal function to create UTF8 literal expressions * fix: update test_arrow_cast to use column 'b' * fix: enhance utf8_literal function to handle non-string values * Add description for utf8_literal vs literal * docs: clarify utf8_literal function documentation to explain use case * docs: add clarification comments for utf8_literal usage in arrow_cast tests * docs: implement ruff recommendation * fix ruff errors * docs: update examples to use utf8_literal in arrow_cast function * docs: correct typo in comment for utf8_literal usage in test_arrow_cast * docs: remove redundant comment in test_arrow_cast for clarity * refactor: rename utf8_literal to string_literal and add alias str_lit * docs: improve docstring for string_literal function for clarity * docs: update import statement to include str_lit alias for string_literal --- .../user-guide/common-operations/functions.rst | 13 ++++++++++++- python/datafusion/__init__.py | 13 +++++++++++++ python/datafusion/expr.py | 16 ++++++++++++++++ python/datafusion/functions.py | 6 ++++++ python/tests/test_functions.py | 18 +++++++++++++++++- src/functions.rs | 3 ++- 6 files changed, 66 insertions(+), 3 deletions(-) diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index ad71c72ac..12097be8f 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -38,7 +38,7 @@ DataFusion offers mathematical functions such as :py:func:`~datafusion.functions .. ipython:: python - from datafusion import col, literal + from datafusion import col, literal, string_literal, str_lit from datafusion import functions as f df.select( @@ -104,6 +104,17 @@ This also includes the functions for regular expressions like :py:func:`~datafus f.regexp_replace(col('"Name"'), literal("saur"), literal("fleur")).alias("flowers") ) +Casting +------- + +Casting expressions to different data types using :py:func:`~datafusion.functions.arrow_cast` + +.. ipython:: python + + df.select( + f.arrow_cast(col('"Total"'), string_literal("Float64")).alias("total_as_float"), + f.arrow_cast(col('"Total"'), str_lit("Int32")).alias("total_as_int") + ) Other ----- diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index e0bc57f44..7367b0d3b 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -107,6 +107,19 @@ def literal(value): return Expr.literal(value) +def string_literal(value): + """Create a UTF8 literal expression. + + It differs from `literal` which creates a UTF8view literal. + """ + return Expr.string_literal(value) + + +def str_lit(value): + """Alias for `string_literal`.""" + return string_literal(value) + + def lit(value): """Create a literal expression.""" return Expr.literal(value) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index b10724381..16add16f4 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -380,6 +380,22 @@ def literal(value: Any) -> Expr: value = pa.scalar(value) return Expr(expr_internal.Expr.literal(value)) + @staticmethod + def string_literal(value: str) -> Expr: + """Creates a new expression representing a UTF8 literal value. + + It is different from `literal` because it is pa.string() instead of + pa.string_view() + + This is needed for cases where DataFusion is expecting a UTF8 instead of + UTF8View literal, like in: + https://github.com/apache/datafusion/blob/86740bfd3d9831d6b7c1d0e1bf4a21d91598a0ac/datafusion/functions/src/core/arrow_cast.rs#L179 + """ + if isinstance(value, str): + value = pa.scalar(value, type=pa.string()) + return Expr(expr_internal.Expr.literal(value)) + return Expr.literal(value) + @staticmethod def column(value: str) -> Expr: """Creates a new expression representing a column.""" diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index f3ee5c092..c0097c6ab 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -82,6 +82,7 @@ "array_to_string", "array_union", "arrow_typeof", + "arrow_cast", "ascii", "asin", "asinh", @@ -1109,6 +1110,11 @@ def arrow_typeof(arg: Expr) -> Expr: return Expr(f.arrow_typeof(arg.expr)) +def arrow_cast(expr: Expr, data_type: Expr) -> Expr: + """Casts an expression to a specified data type.""" + return Expr(f.arrow_cast(expr.expr, data_type.expr)) + + def random() -> Expr: """Returns a random value in the range ``0.0 <= x < 1.0``.""" return Expr(f.random()) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 0d2fa8f94..5dce188ed 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -23,7 +23,7 @@ from datafusion import SessionContext, column from datafusion import functions as f -from datafusion import literal +from datafusion import literal, string_literal np.seterr(invalid="ignore") @@ -907,6 +907,22 @@ def test_temporal_functions(df): assert result.column(10) == pa.array([31, 26, 2], type=pa.float64()) +def test_arrow_cast(df): + df = df.select( + # we use `string_literal` to return utf8 instead of `literal` which returns + # utf8view because datafusion.arrow_cast expects a utf8 instead of utf8view + # https://github.com/apache/datafusion/blob/86740bfd3d9831d6b7c1d0e1bf4a21d91598a0ac/datafusion/functions/src/core/arrow_cast.rs#L179 + f.arrow_cast(column("b"), string_literal("Float64")).alias("b_as_float"), + f.arrow_cast(column("b"), string_literal("Int32")).alias("b_as_int"), + ) + result = df.collect() + assert len(result) == 1 + result = result[0] + + assert result.column(0) == pa.array([4.0, 5.0, 6.0], type=pa.float64()) + assert result.column(1) == pa.array([4, 5, 6], type=pa.int32()) + + def test_case(df): df = df.select( f.case(column("b")).when(literal(4), literal(10)).otherwise(literal(8)), diff --git a/src/functions.rs b/src/functions.rs index 5c450286f..ccc1981bd 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -400,7 +400,6 @@ macro_rules! expr_fn { } }; } - /// Generates a [pyo3] wrapper for [datafusion::functions::expr_fn] /// /// These functions take a single `Vec` argument using `pyo3(signature = (*args))`. @@ -575,6 +574,7 @@ expr_fn_vec!(r#struct); // Use raw identifier since struct is a keyword expr_fn_vec!(named_struct); expr_fn!(from_unixtime, unixtime); expr_fn!(arrow_typeof, arg_1); +expr_fn!(arrow_cast, arg_1 datatype); expr_fn!(random); // Array Functions @@ -867,6 +867,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(range))?; m.add_wrapped(wrap_pyfunction!(array_agg))?; m.add_wrapped(wrap_pyfunction!(arrow_typeof))?; + m.add_wrapped(wrap_pyfunction!(arrow_cast))?; m.add_wrapped(wrap_pyfunction!(ascii))?; m.add_wrapped(wrap_pyfunction!(asin))?; m.add_wrapped(wrap_pyfunction!(asinh))?; From 85fe35cf433c2168fee40dc48bdaa80126bf4a42 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Tue, 7 Jan 2025 05:30:13 -0800 Subject: [PATCH 248/413] Fix small issues in pyproject.toml (#976) * Fix small issues in pyproject.toml * Update classifiers --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d327c0ec1..98bda5aae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ readme = "README.md" license = { file = "LICENSE.txt" } requires-python = ">=3.7" keywords = ["datafusion", "dataframe", "rust", "query-engine"] -classifier = [ +classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", @@ -39,10 +39,14 @@ classifier = [ "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python", "Programming Language :: Rust", ] dependencies = ["pyarrow>=11.0.0", "typing-extensions;python_version<'3.13'"] +dynamic = ["version"] [project.urls] homepage = "https://datafusion.apache.org/python" From 63b13da4bccd66cb474186ebc2c4a1f8ba82230f Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:34:44 +0100 Subject: [PATCH 249/413] chore: set validation and typehint (#983) --- python/datafusion/context.py | 13 ++++++++++++- src/context.rs | 4 ++-- src/dataframe.rs | 21 +-------------------- src/utils.rs | 21 +++++++++++++++++++++ 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index a07b5d175..3fa133346 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -63,6 +63,15 @@ def __arrow_c_array__( # noqa: D105 ) -> tuple[object, object]: ... +class TableProviderExportable(Protocol): + """Type hint for object that has __datafusion_table_provider__ PyCapsule. + + https://datafusion.apache.org/python/user-guide/io/table_provider.html + """ + + def __datafusion_table_provider__(self) -> object: ... # noqa: D105 + + class SessionConfig: """Session configuration options.""" @@ -685,7 +694,9 @@ def deregister_table(self, name: str) -> None: """Remove a table from the session.""" self.ctx.deregister_table(name) - def register_table_provider(self, name: str, provider: Any) -> None: + def register_table_provider( + self, name: str, provider: TableProviderExportable + ) -> None: """Register a table provider. This table provider must have a method called ``__datafusion_table_provider__`` diff --git a/src/context.rs b/src/context.rs index 8675e97df..0512285a7 100644 --- a/src/context.rs +++ b/src/context.rs @@ -43,7 +43,7 @@ use crate::store::StorageContexts; use crate::udaf::PyAggregateUDF; use crate::udf::PyScalarUDF; use crate::udwf::PyWindowUDF; -use crate::utils::{get_tokio_runtime, wait_for_future}; +use crate::utils::{get_tokio_runtime, validate_pycapsule, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; @@ -576,7 +576,7 @@ impl PySessionContext { if provider.hasattr("__datafusion_table_provider__")? { let capsule = provider.getattr("__datafusion_table_provider__")?.call0()?; let capsule = capsule.downcast::()?; - // validate_pycapsule(capsule, "arrow_array_stream")?; + validate_pycapsule(capsule, "datafusion_table_provider")?; let provider = unsafe { capsule.reference::() }; let provider: ForeignTableProvider = provider.into(); diff --git a/src/dataframe.rs b/src/dataframe.rs index e7d6ca6d6..fcb46a756 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -44,7 +44,7 @@ use crate::expr::sort_expr::to_sort_expressions; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; -use crate::utils::{get_tokio_runtime, wait_for_future}; +use crate::utils::{get_tokio_runtime, validate_pycapsule, wait_for_future}; use crate::{ errors::DataFusionError, expr::{sort_expr::PySortExpr, PyExpr}, @@ -724,22 +724,3 @@ fn record_batch_into_schema( RecordBatch::try_new(schema, data_arrays) } - -fn validate_pycapsule(capsule: &Bound, name: &str) -> PyResult<()> { - let capsule_name = capsule.name()?; - if capsule_name.is_none() { - return Err(PyValueError::new_err( - "Expected schema PyCapsule to have name set.", - )); - } - - let capsule_name = capsule_name.unwrap().to_str()?; - if capsule_name != name { - return Err(PyValueError::new_err(format!( - "Expected name '{}' in PyCapsule, instead got '{}'", - name, capsule_name - ))); - } - - Ok(()) -} diff --git a/src/utils.rs b/src/utils.rs index 7fb23cafe..795589752 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -18,7 +18,9 @@ use crate::errors::DataFusionError; use crate::TokioRuntime; use datafusion::logical_expr::Volatility; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use pyo3::types::PyCapsule; use std::future::Future; use std::sync::OnceLock; use tokio::runtime::Runtime; @@ -58,3 +60,22 @@ pub(crate) fn parse_volatility(value: &str) -> Result, name: &str) -> PyResult<()> { + let capsule_name = capsule.name()?; + if capsule_name.is_none() { + return Err(PyValueError::new_err( + "Expected schema PyCapsule to have name set.", + )); + } + + let capsule_name = capsule_name.unwrap().to_str()?; + if capsule_name != name { + return Err(PyValueError::new_err(format!( + "Expected name '{}' in PyCapsule, instead got '{}'", + name, capsule_name + ))); + } + + Ok(()) +} From 389164aa90c8dbe689f2e8eac0677ef2b80aaad9 Mon Sep 17 00:00:00 2001 From: Chongchen Chen Date: Thu, 9 Jan 2025 05:39:31 +0800 Subject: [PATCH 250/413] feat: support enable_url_table config (#980) * feat: support enable_url_table config * change enable_url_table as method * Remove whitespace --------- Co-authored-by: Tim Saucer --- examples/create-context.py | 3 +++ python/datafusion/context.py | 11 +++++++++++ src/context.rs | 6 ++++++ 3 files changed, 20 insertions(+) diff --git a/examples/create-context.py b/examples/create-context.py index 3184d4085..11525d8b8 100644 --- a/examples/create-context.py +++ b/examples/create-context.py @@ -37,3 +37,6 @@ ) ctx = SessionContext(config, runtime) print(ctx) + +ctx = ctx.enable_url_table() +print(ctx) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 3fa133346..6d7f574c4 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -481,6 +481,17 @@ def __init__( self.ctx = SessionContextInternal(config, runtime) + def enable_url_table(self) -> "SessionContext": + """Control if local files can be queried as tables. + + Returns: + A new :py:class:`SessionContext` object with url table enabled. + """ + klass = self.__class__ + obj = klass.__new__(klass) + obj.ctx = self.ctx.enable_url_table() + return obj + def register_object_store( self, schema: str, store: Any, host: str | None = None ) -> None: diff --git a/src/context.rs b/src/context.rs index 0512285a7..88c90e0fd 100644 --- a/src/context.rs +++ b/src/context.rs @@ -299,6 +299,12 @@ impl PySessionContext { }) } + pub fn enable_url_table(&self) -> PyResult { + Ok(PySessionContext { + ctx: self.ctx.clone().enable_url_table(), + }) + } + /// Register an object store with the given name #[pyo3(signature = (scheme, store, host=None))] pub fn register_object_store( From 4b262be15202f5efb9a963faf66452f7fb0bbad3 Mon Sep 17 00:00:00 2001 From: Kyle Barron Date: Thu, 9 Jan 2025 03:54:38 -0800 Subject: [PATCH 251/413] Support async iteration of RecordBatchStream (#975) * Support async iteration of RecordBatchStream * use __anext__ * use await * fix failing test * Since we are raising an error instead of returning a None, we can update the type hint. --------- Co-authored-by: Tim Saucer --- Cargo.lock | 14 +++++++++ Cargo.toml | 3 +- python/datafusion/record_batch.py | 16 ++++++---- python/tests/test_dataframe.py | 4 +-- src/record_batch.rs | 51 +++++++++++++++++++++++++------ 5 files changed, 69 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1f291be9..352771cdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1303,6 +1303,7 @@ dependencies = [ "prost", "prost-types", "pyo3", + "pyo3-async-runtimes", "pyo3-build-config", "tokio", "url", @@ -2672,6 +2673,19 @@ dependencies = [ "unindent", ] +[[package]] +name = "pyo3-async-runtimes" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2529f0be73ffd2be0cc43c013a640796558aa12d7ca0aab5cc14f375b4733031" +dependencies = [ + "futures", + "once_cell", + "pin-project-lite", + "pyo3", + "tokio", +] + [[package]] name = "pyo3-build-config" version = "0.22.6" diff --git a/Cargo.toml b/Cargo.toml index 703fc5a26..d28844685 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync"] } pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } +pyo3-async-runtimes = { version = "0.22", features = ["tokio-runtime"]} arrow = { version = "53", features = ["pyarrow"] } datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } datafusion-substrait = { version = "43.0.0", optional = true } @@ -60,4 +61,4 @@ crate-type = ["cdylib", "rlib"] [profile.release] lto = true -codegen-units = 1 \ No newline at end of file +codegen-units = 1 diff --git a/python/datafusion/record_batch.py b/python/datafusion/record_batch.py index 44936f7d8..75e58998f 100644 --- a/python/datafusion/record_batch.py +++ b/python/datafusion/record_batch.py @@ -57,20 +57,24 @@ def __init__(self, record_batch_stream: df_internal.RecordBatchStream) -> None: """This constructor is typically not called by the end user.""" self.rbs = record_batch_stream - def next(self) -> RecordBatch | None: + def next(self) -> RecordBatch: """See :py:func:`__next__` for the iterator function.""" - try: - next_batch = next(self) - except StopIteration: - return None + return next(self) - return next_batch + async def __anext__(self) -> RecordBatch: + """Async iterator function.""" + next_batch = await self.rbs.__anext__() + return RecordBatch(next_batch) def __next__(self) -> RecordBatch: """Iterator function.""" next_batch = next(self.rbs) return RecordBatch(next_batch) + def __aiter__(self) -> typing_extensions.Self: + """Async iterator function.""" + return self + def __iter__(self) -> typing_extensions.Self: """Iterator function.""" return self diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index b82f95e35..e3bd1b2a5 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -761,8 +761,8 @@ def test_execution_plan(aggregate_df): batch = stream.next() assert batch is not None # there should be no more batches - batch = stream.next() - assert batch is None + with pytest.raises(StopIteration): + stream.next() def test_repartition(df): diff --git a/src/record_batch.rs b/src/record_batch.rs index 427807f22..eacdb5867 100644 --- a/src/record_batch.rs +++ b/src/record_batch.rs @@ -15,13 +15,17 @@ // specific language governing permissions and limitations // under the License. +use std::sync::Arc; + use crate::utils::wait_for_future; use datafusion::arrow::pyarrow::ToPyArrow; use datafusion::arrow::record_batch::RecordBatch; use datafusion::physical_plan::SendableRecordBatchStream; use futures::StreamExt; +use pyo3::exceptions::{PyStopAsyncIteration, PyStopIteration}; use pyo3::prelude::*; use pyo3::{pyclass, pymethods, PyObject, PyResult, Python}; +use tokio::sync::Mutex; #[pyclass(name = "RecordBatch", module = "datafusion", subclass)] pub struct PyRecordBatch { @@ -43,31 +47,58 @@ impl From for PyRecordBatch { #[pyclass(name = "RecordBatchStream", module = "datafusion", subclass)] pub struct PyRecordBatchStream { - stream: SendableRecordBatchStream, + stream: Arc>, } impl PyRecordBatchStream { pub fn new(stream: SendableRecordBatchStream) -> Self { - Self { stream } + Self { + stream: Arc::new(Mutex::new(stream)), + } } } #[pymethods] impl PyRecordBatchStream { - fn next(&mut self, py: Python) -> PyResult> { - let result = self.stream.next(); - match wait_for_future(py, result) { - None => Ok(None), - Some(Ok(b)) => Ok(Some(b.into())), - Some(Err(e)) => Err(e.into()), - } + fn next(&mut self, py: Python) -> PyResult { + let stream = self.stream.clone(); + wait_for_future(py, next_stream(stream, true)) } - fn __next__(&mut self, py: Python) -> PyResult> { + fn __next__(&mut self, py: Python) -> PyResult { self.next(py) } + fn __anext__<'py>(&'py self, py: Python<'py>) -> PyResult> { + let stream = self.stream.clone(); + pyo3_async_runtimes::tokio::future_into_py(py, next_stream(stream, false)) + } + fn __iter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { slf } + + fn __aiter__(slf: PyRef<'_, Self>) -> PyRef<'_, Self> { + slf + } +} + +async fn next_stream( + stream: Arc>, + sync: bool, +) -> PyResult { + let mut stream = stream.lock().await; + match stream.next().await { + Some(Ok(batch)) => Ok(batch.into()), + Some(Err(e)) => Err(e.into()), + None => { + // Depending on whether the iteration is sync or not, we raise either a + // StopIteration or a StopAsyncIteration + if sync { + Err(PyStopIteration::new_err("stream exhausted")) + } else { + Err(PyStopAsyncIteration::new_err("stream exhausted")) + } + } + } } From db1bc62999f559d515a6a8a7f2194ab6d20b3035 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 9 Jan 2025 12:34:23 -0500 Subject: [PATCH 252/413] Chore/upgrade datafusion 44 (#973) * Bump DataFusion version to 44 * Trait definition for plan properties now returns LexOrdering * find_df_window_func was removed upstream * Prepare and Execute variants were removed from LogicalPlan * Substrait functions now take SessionState instead of SessionContext * Remove unused import * RuntimeConfig is now deprecated * Switch from RuntimeConfig to RuntimeEnvBuilder * Update return types on unit tests * DF 44 changes the execution plan properties to have boundedness and emission type * Initcap now returns stringview * Bump datafusion version in example --- Cargo.lock | 783 +++++++++--------- Cargo.toml | 9 +- README.md | 2 +- benchmarks/db-benchmark/groupby-datafusion.py | 6 +- benchmarks/tpch/tpch.py | 2 +- docs/source/user-guide/configuration.rst | 8 +- examples/create-context.py | 4 +- examples/ffi-table-provider/Cargo.toml | 4 +- python/datafusion/__init__.py | 4 +- python/datafusion/context.py | 55 +- python/tests/test_context.py | 10 +- python/tests/test_functions.py | 18 +- src/context.rs | 64 +- src/dataset_exec.rs | 20 +- src/functions.rs | 11 +- src/lib.rs | 2 +- src/sql/logical.rs | 2 - src/substrait.rs | 6 +- src/udwf.rs | 6 - 19 files changed, 529 insertions(+), 487 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 352771cdb..105cc30c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -132,20 +132,20 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "apache-avro" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb7c683b2f8f40970b70e39ff8be514c95b96fcb9c4af87e1ed2cb2e10801a0" +checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" dependencies = [ - "bzip2", + "bigdecimal", + "bzip2 0.4.4", "crc32fast", "digest", - "lazy_static", "libflate", "log", "num-bigint", @@ -153,15 +153,16 @@ dependencies = [ "rand", "regex-lite", "serde", + "serde_bytes", "serde_json", "snap", - "strum 0.25.0", - "strum_macros 0.25.3", + "strum", + "strum_macros", "thiserror 1.0.69", "typed-builder", "uuid", "xz2", - "zstd 0.12.4", + "zstd", ] [[package]] @@ -415,16 +416,15 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ - "bzip2", + "bzip2 0.4.4", "flate2", "futures-core", - "futures-io", "memchr", "pin-project-lite", "tokio", "xz2", - "zstd 0.13.2", - "zstd-safe 7.2.1", + "zstd", + "zstd-safe", ] [[package]] @@ -444,18 +444,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -506,6 +506,20 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -598,6 +612,16 @@ dependencies = [ "libc", ] +[[package]] +name = "bzip2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafdbf26611df8c14810e268ddceda071c297570a5fb360ceddf617fe417ef58" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2-sys" version = "0.1.11+1.0.8" @@ -611,9 +635,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.2" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "jobserver", "libc", @@ -634,9 +658,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -681,8 +705,8 @@ version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" dependencies = [ - "strum 0.26.3", - "strum_macros 0.26.4", + "strum", + "strum_macros", "unicode-width", ] @@ -708,9 +732,9 @@ dependencies = [ [[package]] name = "const_panic" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" +checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" [[package]] name = "constant_time_eq" @@ -778,18 +802,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -850,11 +874,10 @@ dependencies = [ [[package]] name = "datafusion" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" +checksum = "014fc8c384ecacedaabb3bc8359c2a6c6e9d8f7bea65be3434eccacfc37f52d9" dependencies = [ - "ahash", "apache-avro", "arrow", "arrow-array", @@ -863,7 +886,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.0", "chrono", "dashmap", "datafusion-catalog", @@ -874,6 +897,7 @@ dependencies = [ "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", + "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", @@ -884,19 +908,14 @@ dependencies = [ "flate2", "futures", "glob", - "half", - "hashbrown 0.14.5", - "indexmap", "itertools", "log", "num-traits", - "num_cpus", "object_store", "parking_lot", "parquet", - "paste", - "pin-project-lite", "rand", + "regex", "sqlparser", "tempfile", "tokio", @@ -904,14 +923,14 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.2", + "zstd", ] [[package]] name = "datafusion-catalog" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" +checksum = "ee60d33e210ef96070377ae667ece7caa0e959c8387496773d4a1a72f1a5012e" dependencies = [ "arrow-schema", "async-trait", @@ -924,9 +943,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" +checksum = "0b42b7d720fe21ed9cca2ebb635f3f13a12cfab786b41e0fba184fb2e620525b" dependencies = [ "ahash", "apache-avro", @@ -934,44 +953,48 @@ dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", - "chrono", "half", "hashbrown 0.14.5", "indexmap", - "instant", "libc", - "num_cpus", + "log", "object_store", "parquet", "paste", "pyo3", + "recursive", "sqlparser", "tokio", + "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" +checksum = "72fbf14d4079f7ce5306393084fe5057dddfdc2113577e0049310afa12e94281" dependencies = [ "log", "tokio", ] +[[package]] +name = "datafusion-doc" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278dbd64860ed0bb5240fc1f4cb6aeea437153910aea69bcf7d5a8d6d0454f3" + [[package]] name = "datafusion-execution" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" +checksum = "e22cb02af47e756468b3cbfee7a83e3d4f2278d452deb4b033ba933c75169486" dependencies = [ "arrow", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.5", "log", "object_store", "parking_lot", @@ -982,45 +1005,41 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" +checksum = "62298eadb1d15b525df1315e61a71519ffc563d41d5c3b2a30fda2d70f77b93c" dependencies = [ - "ahash", "arrow", - "arrow-array", - "arrow-buffer", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", "paste", + "recursive", "serde_json", "sqlparser", - "strum 0.26.3", - "strum_macros 0.26.4", ] [[package]] name = "datafusion-expr-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" +checksum = "dda7f73c5fc349251cd3dcb05773c5bf55d2505a698ef9d38dfc712161ea2f55" dependencies = [ "arrow", "datafusion-common", "itertools", - "paste", ] [[package]] name = "datafusion-ffi" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e923c459b53a26d92a8806d1f6a37fdf48bde51507a39eaed6f42a60f2bfd160" +checksum = "114e944790756b84c2cc5971eae24f5430980149345601939ac222885d4db5f7" dependencies = [ "abi_stable", "arrow", @@ -1028,7 +1047,6 @@ dependencies = [ "async-trait", "datafusion", "datafusion-proto", - "doc-comment", "futures", "log", "prost", @@ -1036,9 +1054,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" +checksum = "fd197f3b2975424d3a4898ea46651be855a46721a56727515dbd5c9e2fb597da" dependencies = [ "arrow", "arrow-buffer", @@ -1047,8 +1065,11 @@ dependencies = [ "blake3", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hashbrown 0.14.5", "hex", "itertools", @@ -1063,44 +1084,44 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" +checksum = "aabbe48fba18f9981b134124381bee9e46f93518b8ad2f9721ee296cef5affb9" dependencies = [ "ahash", "arrow", "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", "datafusion-functions-aggregate-common", + "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", "half", - "indexmap", "log", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" +checksum = "d7a3fefed9c8c11268d446d924baca8cabf52fe32f73fdaa20854bac6473590c" dependencies = [ "ahash", "arrow", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", - "rand", ] [[package]] name = "datafusion-functions-nested" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" +checksum = "6360f27464fab857bec698af39b2ae331dc07c8bf008fb4de387a19cdc6815a5" dependencies = [ "arrow", "arrow-array", @@ -1116,18 +1137,35 @@ dependencies = [ "itertools", "log", "paste", - "rand", +] + +[[package]] +name = "datafusion-functions-table" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c35c070eb705c12795dab399c3809f4dfbc290678c624d3989490ca9b8449c1" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", ] [[package]] name = "datafusion-functions-window" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" +checksum = "52229bca26b590b140900752226c829f15fc1a99840e1ca3ce1a9534690b82a8" dependencies = [ "datafusion-common", + "datafusion-doc", "datafusion-expr", "datafusion-functions-window-common", + "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", "log", @@ -1136,48 +1174,54 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" +checksum = "367befc303b64a668a10ae6988a064a9289e1999e71a7f8e526b6e14d6bdd9d6" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", ] +[[package]] +name = "datafusion-macros" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5de3c8f386ea991696553afe241a326ecbc3c98a12c562867e4be754d3a060c" +dependencies = [ + "quote", + "syn 2.0.95", +] + [[package]] name = "datafusion-optimizer" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" +checksum = "53b520413906f755910422b016fb73884ae6e9e1b376de4f9584b6c0e031da75" dependencies = [ "arrow", - "async-trait", "chrono", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.5", "indexmap", "itertools", "log", - "paste", + "recursive", + "regex", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" +checksum = "acd6ddc378f6ad19af95ccd6790dec8f8e1264bc4c70e99ddc1830c1a1c78ccd" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", - "arrow-ord", "arrow-schema", - "arrow-string", - "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1194,39 +1238,40 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" +checksum = "06e6c05458eccd74b4c77ed6a1fe63d52434240711de7f6960034794dad1caf5" dependencies = [ "ahash", "arrow", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "rand", + "itertools", ] [[package]] name = "datafusion-physical-optimizer" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" +checksum = "9dc3a82190f49c37d377f31317e07ab5d7588b837adadba8ac367baad5dc2351" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr-common", "datafusion-physical-expr", "datafusion-physical-plan", "itertools", + "log", + "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" +checksum = "6a6608bc9844b4ddb5ed4e687d173e6c88700b1d0482f43894617d18a1fe75da" dependencies = [ "ahash", "arrow", @@ -1240,7 +1285,6 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -1250,18 +1294,16 @@ dependencies = [ "indexmap", "itertools", "log", - "once_cell", "parking_lot", "pin-project-lite", - "rand", "tokio", ] [[package]] name = "datafusion-proto" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f730f7fc5a20134d4e5ecdf7bbf392002ac58163d58423ea28a702dc077b06e1" +checksum = "8e23b0998195e495bfa7b37cdceb317129a6c40522219f6872d2e0c9ae9f4fcb" dependencies = [ "arrow", "chrono", @@ -1275,14 +1317,12 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12c225fe49e4f943e35446b263613ada7a9e9f8d647544e6b07037b9803567df" +checksum = "cfc59992a29eed2d2c1dd779deac99083b217774ebcf90ee121840607a4d866f" dependencies = [ "arrow", - "chrono", "datafusion-common", - "object_store", "prost", ] @@ -1294,7 +1334,6 @@ dependencies = [ "async-trait", "datafusion", "datafusion-ffi", - "datafusion-functions-window-common", "datafusion-proto", "datafusion-substrait", "futures", @@ -1312,30 +1351,32 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" +checksum = "6a884061c79b33d0c8e84a6f4f4be8bdc12c0f53f5af28ddf5d6d95ac0b15fdc" dependencies = [ "arrow", "arrow-array", "arrow-schema", + "bigdecimal", "datafusion-common", "datafusion-expr", "indexmap", "log", + "recursive", "regex", "sqlparser", - "strum 0.26.3", ] [[package]] name = "datafusion-substrait" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9c768d2b4c4485c43afbaeeb86dd1f2ac3fb34a9e6e8c8b06180d2a223d5ba" +checksum = "d2ec36dd38512b1ecc7a3bb92e72046b944611b2f0d709445c1e51b0143bffd4" dependencies = [ "arrow-buffer", "async-recursion", + "async-trait", "chrono", "datafusion", "itertools", @@ -1365,15 +1406,9 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dyn-clone" version = "1.0.17" @@ -1404,9 +1439,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fixedbitset" @@ -1416,9 +1451,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "24.3.25" +version = "24.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1505,7 +1540,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -1578,9 +1613,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -1628,24 +1663,12 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hex" version = "0.4.3" @@ -1654,9 +1677,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1700,9 +1723,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -1720,9 +1743,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1893,7 +1916,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -1919,9 +1942,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1933,18 +1956,6 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "integer-encoding" version = "3.0.4" @@ -1983,9 +1994,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -1999,9 +2010,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2012,9 +2023,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -2023,9 +2034,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ "lexical-util", "static_assertions", @@ -2033,18 +2044,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" dependencies = [ "lexical-util", "lexical-write-integer", @@ -2053,9 +2064,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" dependencies = [ "lexical-util", "static_assertions", @@ -2063,9 +2074,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.167" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libflate" @@ -2119,9 +2130,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -2207,9 +2218,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -2253,6 +2264,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -2305,36 +2317,27 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", "futures", + "httparse", "humantime", "hyper", "itertools", @@ -2431,7 +2434,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.2", + "zstd", "zstd-sys", ] @@ -2466,7 +2469,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" dependencies = [ - "heck 0.5.0", + "heck", "itertools", "prost", "prost-types", @@ -2505,18 +2508,18 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -2524,9 +2527,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", "rand", @@ -2534,18 +2537,18 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2576,12 +2579,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" dependencies = [ "proc-macro2", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -2595,9 +2598,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", "prost-derive", @@ -2605,12 +2608,11 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" +checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "bytes", - "heck 0.5.0", + "heck", "itertools", "log", "multimap", @@ -2620,28 +2622,28 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.90", + "syn 2.0.95", "tempfile", ] [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ "prost", ] @@ -2655,6 +2657,15 @@ dependencies = [ "cmake", ] +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + [[package]] name = "pyo3" version = "0.22.6" @@ -2715,7 +2726,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -2724,11 +2735,11 @@ version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -2739,9 +2750,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" dependencies = [ "memchr", "serde", @@ -2760,7 +2771,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.10", "tokio", "tracing", ] @@ -2779,7 +2790,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.10", "tinyvec", "tracing", "web-time", @@ -2787,9 +2798,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -2801,9 +2812,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2838,11 +2849,31 @@ dependencies = [ "getrandom", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.95", +] + [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -2903,9 +2934,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "base64 0.22.1", "bytes", @@ -2937,6 +2968,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", + "tower", "tower-service", "url", "wasm-bindgen", @@ -2990,22 +3022,22 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring", @@ -3038,9 +3070,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -3058,9 +3090,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -3107,7 +3139,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3118,9 +3150,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -3131,9 +3163,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -3141,9 +3173,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" dependencies = [ "serde", ] @@ -3156,22 +3188,31 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3182,14 +3223,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -3206,7 +3247,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3253,9 +3294,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" @@ -3287,10 +3328,10 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3317,9 +3358,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.51.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" dependencies = [ "log", "sqlparser_derive", @@ -3327,13 +3368,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3343,38 +3384,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -name = "static_assertions" -version = "1.1.0" +name = "stacker" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] [[package]] -name = "strum" -version = "0.25.0" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.90", -] [[package]] name = "strum_macros" @@ -3382,20 +3414,20 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "substrait" -version = "0.45.5" +version = "0.50.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a127ae9d8e443cea5c2122eb2ffe5fe489e802a1e746a09c5a5cb59d074c0aeb" +checksum = "b1772d041c37cc7e6477733c76b2acf4ee36bd52b2ae4d9ea0ec9c87d003db32" dependencies = [ - "heck 0.5.0", + "heck", "pbjson", "pbjson-build", "pbjson-types", @@ -3410,7 +3442,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.90", + "syn 2.0.95", "typify", "walkdir", ] @@ -3434,9 +3466,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -3460,7 +3492,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3471,12 +3503,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3493,11 +3526,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.10", ] [[package]] @@ -3508,18 +3541,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3554,9 +3587,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -3569,9 +3602,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -3585,31 +3618,30 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -3618,6 +3650,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -3643,7 +3696,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3694,22 +3747,22 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" dependencies = [ "typed-builder-macro", ] [[package]] name = "typed-builder-macro" -version = "0.16.2" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" +checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3734,7 +3787,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ab345b6c0d8ae9500b9ff334a4c7c0d316c1c628dc55726b95887eb8dbd11" dependencies = [ - "heck 0.5.0", + "heck", "log", "proc-macro2", "quote", @@ -3743,7 +3796,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.90", + "syn 2.0.95", "thiserror 1.0.69", "unicode-ident", ] @@ -3761,7 +3814,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.90", + "syn 2.0.95", "typify-impl", ] @@ -3867,9 +3920,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -3878,24 +3931,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -3906,9 +3958,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3916,22 +3968,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" @@ -3948,9 +4000,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -4159,7 +4211,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "synstructure", ] @@ -4181,7 +4233,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -4201,7 +4253,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "synstructure", ] @@ -4230,16 +4282,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", -] - -[[package]] -name = "zstd" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" -dependencies = [ - "zstd-safe 6.0.6", + "syn 2.0.95", ] [[package]] @@ -4248,17 +4291,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.2.1", -] - -[[package]] -name = "zstd-safe" -version = "6.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d28844685..48219414a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,11 +38,10 @@ tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } pyo3-async-runtimes = { version = "0.22", features = ["tokio-runtime"]} arrow = { version = "53", features = ["pyarrow"] } -datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-substrait = { version = "43.0.0", optional = true } -datafusion-proto = { version = "43.0.0" } -datafusion-ffi = { version = "43.0.0" } -datafusion-functions-window-common = { version = "43.0.0" } +datafusion = { version = "44.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-substrait = { version = "44.0.0", optional = true } +datafusion-proto = { version = "44.0.0" } +datafusion-ffi = { version = "44.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.11", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } diff --git a/README.md b/README.md index 83b307e7a..ca612c1ab 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ It is possible to configure runtime (memory and disk settings) and configuration ```python runtime = ( - RuntimeConfig() + RuntimeEnvBuilder() .with_disk_manager_os() .with_fair_spill_pool(10000000) ) diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 3a4399f7d..960c8ba9a 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -22,7 +22,7 @@ from datafusion import ( col, functions as f, - RuntimeConfig, + RuntimeEnvBuilder, SessionConfig, SessionContext, ) @@ -85,7 +85,9 @@ def execute(df): # create a session context with explicit runtime and config settings runtime = ( - RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(64 * 1024 * 1024 * 1024) + RuntimeEnvBuilder() + .with_disk_manager_os() + .with_fair_spill_pool(64 * 1024 * 1024 * 1024) ) config = ( SessionConfig() diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py index 7f104a4cb..daa831b55 100644 --- a/benchmarks/tpch/tpch.py +++ b/benchmarks/tpch/tpch.py @@ -28,7 +28,7 @@ def bench(data_path, query_path): # create context # runtime = ( - # RuntimeConfig() + # RuntimeEnvBuilder() # .with_disk_manager_os() # .with_fair_spill_pool(10000000) # ) diff --git a/docs/source/user-guide/configuration.rst b/docs/source/user-guide/configuration.rst index 7d330019f..db200a46a 100644 --- a/docs/source/user-guide/configuration.rst +++ b/docs/source/user-guide/configuration.rst @@ -19,18 +19,18 @@ Configuration ============= Let's look at how we can configure DataFusion. When creating a :py:class:`~datafusion.context.SessionContext`, you can pass in -a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.context.RuntimeConfig` object. These two cover a wide range of options. +a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.context.RuntimeEnvBuilder` object. These two cover a wide range of options. .. code-block:: python - from datafusion import RuntimeConfig, SessionConfig, SessionContext + from datafusion import RuntimeEnvBuilder, SessionConfig, SessionContext # create a session context with default settings ctx = SessionContext() print(ctx) # create a session context with explicit runtime and config settings - runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) + runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) @@ -48,4 +48,4 @@ a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.conte You can read more about available :py:class:`~datafusion.context.SessionConfig` options in the `rust DataFusion Configuration guide `_, -and about :code:`RuntimeConfig` options in the rust `online API documentation `_. +and about :code:`RuntimeEnvBuilder` options in the rust `online API documentation `_. diff --git a/examples/create-context.py b/examples/create-context.py index 11525d8b8..760c8513e 100644 --- a/examples/create-context.py +++ b/examples/create-context.py @@ -15,14 +15,14 @@ # specific language governing permissions and limitations # under the License. -from datafusion import RuntimeConfig, SessionConfig, SessionContext +from datafusion import RuntimeEnvBuilder, SessionConfig, SessionContext # create a session context with default settings ctx = SessionContext() print(ctx) # create a session context with explicit runtime and config settings -runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) +runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) diff --git a/examples/ffi-table-provider/Cargo.toml b/examples/ffi-table-provider/Cargo.toml index 4e54eaf03..4e6f91f33 100644 --- a/examples/ffi-table-provider/Cargo.toml +++ b/examples/ffi-table-provider/Cargo.toml @@ -21,8 +21,8 @@ version = "0.1.0" edition = "2021" [dependencies] -datafusion = { version = "43.0.0" } -datafusion-ffi = { version = "43.0.0" } +datafusion = { version = "44.0.0" } +datafusion-ffi = { version = "44.0.0" } pyo3 = { version = "0.22.6", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "53.2.0" } arrow-array = { version = "53.2.0" } diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 7367b0d3b..2d8db42c8 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -29,7 +29,7 @@ from .context import ( SessionContext, SessionConfig, - RuntimeConfig, + RuntimeEnvBuilder, SQLOptions, ) @@ -66,7 +66,7 @@ "SessionContext", "SessionConfig", "SQLOptions", - "RuntimeConfig", + "RuntimeEnvBuilder", "Expr", "ScalarUDF", "WindowFrame", diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 6d7f574c4..3c284c9f9 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -20,7 +20,7 @@ from __future__ import annotations from ._internal import SessionConfig as SessionConfigInternal -from ._internal import RuntimeConfig as RuntimeConfigInternal +from ._internal import RuntimeEnvBuilder as RuntimeEnvBuilderInternal from ._internal import SQLOptions as SQLOptionsInternal from ._internal import SessionContext as SessionContextInternal @@ -265,39 +265,41 @@ def set(self, key: str, value: str) -> SessionConfig: return self -class RuntimeConfig: +class RuntimeEnvBuilder: """Runtime configuration options.""" def __init__(self) -> None: - """Create a new :py:class:`RuntimeConfig` with default values.""" - self.config_internal = RuntimeConfigInternal() + """Create a new :py:class:`RuntimeEnvBuilder` with default values.""" + self.config_internal = RuntimeEnvBuilderInternal() - def with_disk_manager_disabled(self) -> RuntimeConfig: + def with_disk_manager_disabled(self) -> RuntimeEnvBuilder: """Disable the disk manager, attempts to create temporary files will error. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ self.config_internal = self.config_internal.with_disk_manager_disabled() return self - def with_disk_manager_os(self) -> RuntimeConfig: + def with_disk_manager_os(self) -> RuntimeEnvBuilder: """Use the operating system's temporary directory for disk manager. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ self.config_internal = self.config_internal.with_disk_manager_os() return self - def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConfig: + def with_disk_manager_specified( + self, *paths: str | pathlib.Path + ) -> RuntimeEnvBuilder: """Use the specified paths for the disk manager's temporary files. Args: paths: Paths to use for the disk manager's temporary files. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ paths_list = [str(p) for p in paths] self.config_internal = self.config_internal.with_disk_manager_specified( @@ -305,16 +307,16 @@ def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConf ) return self - def with_unbounded_memory_pool(self) -> RuntimeConfig: + def with_unbounded_memory_pool(self) -> RuntimeEnvBuilder: """Use an unbounded memory pool. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ self.config_internal = self.config_internal.with_unbounded_memory_pool() return self - def with_fair_spill_pool(self, size: int) -> RuntimeConfig: + def with_fair_spill_pool(self, size: int) -> RuntimeEnvBuilder: """Use a fair spill pool with the specified size. This pool works best when you know beforehand the query has multiple spillable @@ -335,16 +337,16 @@ def with_fair_spill_pool(self, size: int) -> RuntimeConfig: size: Size of the memory pool in bytes. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. Examples usage:: - config = RuntimeConfig().with_fair_spill_pool(1024) + config = RuntimeEnvBuilder().with_fair_spill_pool(1024) """ self.config_internal = self.config_internal.with_fair_spill_pool(size) return self - def with_greedy_memory_pool(self, size: int) -> RuntimeConfig: + def with_greedy_memory_pool(self, size: int) -> RuntimeEnvBuilder: """Use a greedy memory pool with the specified size. This pool works well for queries that do not need to spill or have a single @@ -355,32 +357,39 @@ def with_greedy_memory_pool(self, size: int) -> RuntimeConfig: size: Size of the memory pool in bytes. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. Example usage:: - config = RuntimeConfig().with_greedy_memory_pool(1024) + config = RuntimeEnvBuilder().with_greedy_memory_pool(1024) """ self.config_internal = self.config_internal.with_greedy_memory_pool(size) return self - def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeConfig: + def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeEnvBuilder: """Use the specified path to create any needed temporary files. Args: path: Path to use for temporary files. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. Example usage:: - config = RuntimeConfig().with_temp_file_path("/tmp") + config = RuntimeEnvBuilder().with_temp_file_path("/tmp") """ self.config_internal = self.config_internal.with_temp_file_path(str(path)) return self +@deprecated("Use `RuntimeEnvBuilder` instead.") +class RuntimeConfig(RuntimeEnvBuilder): + """See `RuntimeEnvBuilder`.""" + + pass + + class SQLOptions: """Options to be used when performing SQL queries.""" @@ -454,7 +463,9 @@ class SessionContext: """ def __init__( - self, config: SessionConfig | None = None, runtime: RuntimeConfig | None = None + self, + config: SessionConfig | None = None, + runtime: RuntimeEnvBuilder | None = None, ) -> None: """Main interface for executing queries with DataFusion. diff --git a/python/tests/test_context.py b/python/tests/test_context.py index ab86faa9d..10e8ad0e9 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -25,7 +25,7 @@ from datafusion import ( DataFrame, - RuntimeConfig, + RuntimeEnvBuilder, SessionConfig, SessionContext, SQLOptions, @@ -43,7 +43,7 @@ def test_create_context_session_config_only(): def test_create_context_runtime_config_only(): - SessionContext(runtime=RuntimeConfig()) + SessionContext(runtime=RuntimeEnvBuilder()) @pytest.mark.parametrize("path_to_str", (True, False)) @@ -54,7 +54,7 @@ def test_runtime_configs(tmp_path, path_to_str): path1 = str(path1) if path_to_str else path1 path2 = str(path2) if path_to_str else path2 - runtime = RuntimeConfig().with_disk_manager_specified(path1, path2) + runtime = RuntimeEnvBuilder().with_disk_manager_specified(path1, path2) config = SessionConfig().with_default_catalog_and_schema("foo", "bar") ctx = SessionContext(config, runtime) assert ctx is not None @@ -67,7 +67,7 @@ def test_runtime_configs(tmp_path, path_to_str): def test_temporary_files(tmp_path, path_to_str): path = str(tmp_path) if path_to_str else tmp_path - runtime = RuntimeConfig().with_temp_file_path(path) + runtime = RuntimeEnvBuilder().with_temp_file_path(path) config = SessionConfig().with_default_catalog_and_schema("foo", "bar") ctx = SessionContext(config, runtime) assert ctx is not None @@ -77,7 +77,7 @@ def test_temporary_files(tmp_path, path_to_str): def test_create_context_with_all_valid_args(): - runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) + runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 5dce188ed..01c6c9cef 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -103,8 +103,11 @@ def test_lit_arith(df): result = df.collect() assert len(result) == 1 result = result[0] + assert result.column(0) == pa.array([5, 6, 7]) - assert result.column(1) == pa.array(["Hello!", "World!", "!!"]) + assert result.column(1) == pa.array( + ["Hello!", "World!", "!!"], type=pa.string_view() + ) def test_math_functions(): @@ -661,9 +664,12 @@ def test_array_function_obj_tests(stmt, py_expr): ), ( f.concat(column("a").cast(pa.string()), literal("?")), - pa.array(["Hello?", "World?", "!?"]), + pa.array(["Hello?", "World?", "!?"], type=pa.string_view()), + ), + ( + f.initcap(column("c")), + pa.array(["Hello ", " World ", " !"], type=pa.string_view()), ), - (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), (f.length(column("c")), pa.array([6, 7, 2], type=pa.int32())), (f.lower(column("a")), pa.array(["hello", "world", "!"])), @@ -871,8 +877,8 @@ def test_temporal_functions(df): result = df.collect() assert len(result) == 1 result = result[0] - assert result.column(0) == pa.array([12, 6, 7], type=pa.float64()) - assert result.column(1) == pa.array([2022, 2027, 2020], type=pa.float64()) + assert result.column(0) == pa.array([12, 6, 7], type=pa.int32()) + assert result.column(1) == pa.array([2022, 2027, 2020], type=pa.int32()) assert result.column(2) == pa.array( [datetime(2022, 12, 1), datetime(2027, 6, 1), datetime(2020, 7, 1)], type=pa.timestamp("us"), @@ -904,7 +910,7 @@ def test_temporal_functions(df): assert result.column(9) == pa.array( [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") ) - assert result.column(10) == pa.array([31, 26, 2], type=pa.float64()) + assert result.column(10) == pa.array([31, 26, 2], type=pa.int32()) def test_arrow_cast(df): diff --git a/src/context.rs b/src/context.rs index 88c90e0fd..bab7fd42a 100644 --- a/src/context.rs +++ b/src/context.rs @@ -62,7 +62,7 @@ use datafusion::execution::context::{ use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; use datafusion::execution::options::ReadOptions; -use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, @@ -165,62 +165,62 @@ impl PySessionConfig { } /// Runtime options for a SessionContext -#[pyclass(name = "RuntimeConfig", module = "datafusion", subclass)] +#[pyclass(name = "RuntimeEnvBuilder", module = "datafusion", subclass)] #[derive(Clone)] -pub struct PyRuntimeConfig { - pub config: RuntimeConfig, +pub struct PyRuntimeEnvBuilder { + pub builder: RuntimeEnvBuilder, } #[pymethods] -impl PyRuntimeConfig { +impl PyRuntimeEnvBuilder { #[new] fn new() -> Self { Self { - config: RuntimeConfig::default(), + builder: RuntimeEnvBuilder::default(), } } fn with_disk_manager_disabled(&self) -> Self { - let config = self.config.clone(); - let config = config.with_disk_manager(DiskManagerConfig::Disabled); - Self { config } + let mut builder = self.builder.clone(); + builder = builder.with_disk_manager(DiskManagerConfig::Disabled); + Self { builder } } fn with_disk_manager_os(&self) -> Self { - let config = self.config.clone(); - let config = config.with_disk_manager(DiskManagerConfig::NewOs); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_disk_manager(DiskManagerConfig::NewOs); + Self { builder } } fn with_disk_manager_specified(&self, paths: Vec) -> Self { - let config = self.config.clone(); + let builder = self.builder.clone(); let paths = paths.iter().map(|s| s.into()).collect(); - let config = config.with_disk_manager(DiskManagerConfig::NewSpecified(paths)); - Self { config } + let builder = builder.with_disk_manager(DiskManagerConfig::NewSpecified(paths)); + Self { builder } } fn with_unbounded_memory_pool(&self) -> Self { - let config = self.config.clone(); - let config = config.with_memory_pool(Arc::new(UnboundedMemoryPool::default())); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_memory_pool(Arc::new(UnboundedMemoryPool::default())); + Self { builder } } fn with_fair_spill_pool(&self, size: usize) -> Self { - let config = self.config.clone(); - let config = config.with_memory_pool(Arc::new(FairSpillPool::new(size))); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_memory_pool(Arc::new(FairSpillPool::new(size))); + Self { builder } } fn with_greedy_memory_pool(&self, size: usize) -> Self { - let config = self.config.clone(); - let config = config.with_memory_pool(Arc::new(GreedyMemoryPool::new(size))); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_memory_pool(Arc::new(GreedyMemoryPool::new(size))); + Self { builder } } fn with_temp_file_path(&self, path: &str) -> Self { - let config = self.config.clone(); - let config = config.with_temp_file_path(path); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_temp_file_path(path); + Self { builder } } } @@ -276,19 +276,19 @@ impl PySessionContext { #[new] pub fn new( config: Option, - runtime: Option, + runtime: Option, ) -> PyResult { let config = if let Some(c) = config { c.config } else { SessionConfig::default().with_information_schema(true) }; - let runtime_config = if let Some(c) = runtime { - c.config + let runtime_env_builder = if let Some(c) = runtime { + c.builder } else { - RuntimeConfig::default() + RuntimeEnvBuilder::default() }; - let runtime = Arc::new(RuntimeEnv::try_new(runtime_config)?); + let runtime = Arc::new(runtime_env_builder.build()?); let session_state = SessionStateBuilder::new() .with_config(config) .with_runtime_env(runtime) diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 2759aa678..9d2559429 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; /// Implements a Datafusion physical ExecutionPlan that delegates to a PyArrow Dataset /// This actually performs the projection, filtering and scanning of a Dataset use pyo3::prelude::*; @@ -34,11 +35,11 @@ use datafusion::error::{DataFusionError as InnerDataFusionError, Result as DFRes use datafusion::execution::context::TaskContext; use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::Expr; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalSortExpr}; +use datafusion::physical_expr::{EquivalenceProperties, LexOrdering}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, - Partitioning, SendableRecordBatchStream, Statistics, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, + SendableRecordBatchStream, Statistics, }; use crate::errors::DataFusionError; @@ -136,7 +137,8 @@ impl DatasetExec { let plan_properties = datafusion::physical_plan::PlanProperties::new( EquivalenceProperties::new(schema.clone()), Partitioning::UnknownPartitioning(fragments.len()), - ExecutionMode::Bounded, + EmissionType::Final, + Boundedness::Bounded, ); Ok(DatasetExec { @@ -251,12 +253,16 @@ impl ExecutionPlanProperties for DatasetExec { self.plan_properties.output_partitioning() } - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { + fn output_ordering(&self) -> Option<&LexOrdering> { None } - fn execution_mode(&self) -> datafusion::physical_plan::ExecutionMode { - self.plan_properties.execution_mode + fn boundedness(&self) -> Boundedness { + self.plan_properties.boundedness + } + + fn pipeline_behavior(&self) -> EmissionType { + self.plan_properties.emission_type } fn equivalence_properties(&self) -> &datafusion::physical_expr::EquivalenceProperties { diff --git a/src/functions.rs b/src/functions.rs index ccc1981bd..ae032d702 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -36,10 +36,7 @@ use datafusion::functions_aggregate; use datafusion::functions_window; use datafusion::logical_expr::expr::Alias; use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; -use datafusion::logical_expr::{ - expr::{find_df_window_func, WindowFunction}, - lit, Expr, WindowFunctionDefinition, -}; +use datafusion::logical_expr::{expr::WindowFunction, lit, Expr, WindowFunctionDefinition}; fn add_builder_fns_to_aggregate( agg_fn: Expr, @@ -232,12 +229,6 @@ fn when(when: PyExpr, then: PyExpr) -> PyResult { /// /// NOTE: we search the built-ins first because the `UDAF` versions currently do not have the same behavior. fn find_window_fn(name: &str, ctx: Option) -> PyResult { - // search built in window functions (soon to be deprecated) - let df_window_func = find_df_window_func(name); - if let Some(df_window_func) = df_window_func { - return Ok(df_window_func); - } - if let Some(ctx) = ctx { // search UDAFs let udaf = ctx diff --git a/src/lib.rs b/src/lib.rs index 0b57e0999..1111d5d06 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,7 +78,7 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 40f0a6a65..a541889c7 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -85,12 +85,10 @@ impl PyLogicalPlan { | LogicalPlan::Union(_) | LogicalPlan::Statement(_) | LogicalPlan::Values(_) - | LogicalPlan::Prepare(_) | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) | LogicalPlan::DescribeTable(_) - | LogicalPlan::Execute(_) | LogicalPlan::RecursiveQuery(_) => Err(py_unsupported_variant_err(format!( "Conversion of variant not implemented: {:?}", self.plan diff --git a/src/substrait.rs b/src/substrait.rs index f89b6b093..16e8c9507 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -114,7 +114,8 @@ impl PySubstraitProducer { /// Convert DataFusion LogicalPlan to Substrait Plan #[staticmethod] pub fn to_substrait_plan(plan: PyLogicalPlan, ctx: &PySessionContext) -> PyResult { - match producer::to_substrait_plan(&plan.plan, &ctx.ctx) { + let session_state = ctx.ctx.state(); + match producer::to_substrait_plan(&plan.plan, &session_state) { Ok(plan) => Ok(PyPlan { plan: *plan }), Err(e) => Err(py_datafusion_err(e)), } @@ -134,7 +135,8 @@ impl PySubstraitConsumer { plan: PyPlan, py: Python, ) -> PyResult { - let result = consumer::from_substrait_plan(&ctx.ctx, &plan.plan); + let session_state = ctx.ctx.state(); + let result = consumer::from_substrait_plan(&session_state, &plan.plan); let logical_plan = wait_for_future(py, result).map_err(DataFusionError::from)?; Ok(PyLogicalPlan::new(logical_plan)) } diff --git a/src/udwf.rs b/src/udwf.rs index 3f5ad0b1d..689eb79e3 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -22,9 +22,7 @@ use std::sync::Arc; use arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; use datafusion::logical_expr::window_state::WindowAggState; -use datafusion::physical_plan::PhysicalExpr; use datafusion::scalar::ScalarValue; -use datafusion_functions_window_common::expr::ExpressionArgs; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; @@ -319,8 +317,4 @@ impl WindowUDFImpl for MultiColumnWindowUDF { let _ = _partition_evaluator_args; (self.partition_evaluator_factory)() } - - fn expressions(&self, expr_args: ExpressionArgs) -> Vec> { - expr_args.input_exprs().into() - } } From 2d8b1d32f4941b2e02a29e9135025a32ba6ae471 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 11 Jan 2025 10:12:04 +0800 Subject: [PATCH 253/413] Default to ZSTD compression when writing Parquet (#981) * fix: update default compression to ZSTD and improve documentation for write_parquet method * fix: clarify compression level documentation for ZSTD in write_parquet method * fix: update default compression level for ZSTD to 4 in write_parquet method * fix: improve docstring formatting for DataFrame parquet writing method * feat: implement Compression enum and update write_parquet method to use it * add test * fix: remove unused import and update default compression to ZSTD in rs' write_parquet method * fix: update compression type strings to lowercase in DataFrame parquet writing method doc * test: update parquet compression tests to validate invalid and default compression levels * add comment on source of Compression * docs: enhance Compression enum documentation and add default level method * test: include gzip in default compression level tests for write_parquet * refactor: simplify Compression enum methods and improve type handling in DataFrame.write_parquet * docs: update Compression enum methods to include return type descriptions * move comment to within test * Ruff format --------- Co-authored-by: Tim Saucer --- python/datafusion/dataframe.py | 94 +++++++++++++++++++++++++++++++--- python/tests/test_dataframe.py | 14 ++++- src/dataframe.rs | 2 +- 3 files changed, 101 insertions(+), 9 deletions(-) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 0b38db924..f8aef0c91 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -21,7 +21,16 @@ from __future__ import annotations import warnings -from typing import Any, Iterable, List, TYPE_CHECKING, Literal, overload +from typing import ( + Any, + Iterable, + List, + TYPE_CHECKING, + Literal, + overload, + Optional, + Union, +) from datafusion.record_batch import RecordBatchStream from typing_extensions import deprecated from datafusion.plan import LogicalPlan, ExecutionPlan @@ -35,6 +44,60 @@ from datafusion._internal import DataFrame as DataFrameInternal from datafusion.expr import Expr, SortExpr, sort_or_default +from enum import Enum + + +# excerpt from deltalake +# https://github.com/apache/datafusion-python/pull/981#discussion_r1905619163 +class Compression(Enum): + """Enum representing the available compression types for Parquet files.""" + + UNCOMPRESSED = "uncompressed" + SNAPPY = "snappy" + GZIP = "gzip" + BROTLI = "brotli" + LZ4 = "lz4" + LZ0 = "lz0" + ZSTD = "zstd" + LZ4_RAW = "lz4_raw" + + @classmethod + def from_str(cls, value: str) -> "Compression": + """Convert a string to a Compression enum value. + + Args: + value: The string representation of the compression type. + + Returns: + The Compression enum lowercase value. + + Raises: + ValueError: If the string does not match any Compression enum value. + """ + try: + return cls(value.lower()) + except ValueError: + raise ValueError( + f"{value} is not a valid Compression. Valid values are: {[item.value for item in Compression]}" + ) + + def get_default_level(self) -> Optional[int]: + """Get the default compression level for the compression type. + + Returns: + The default compression level for the compression type. + """ + # GZIP, BROTLI default values from deltalake repo + # https://github.com/apache/datafusion-python/pull/981#discussion_r1905619163 + # ZSTD default value from delta-rs + # https://github.com/apache/datafusion-python/pull/981#discussion_r1904789223 + if self == Compression.GZIP: + return 6 + elif self == Compression.BROTLI: + return 1 + elif self == Compression.ZSTD: + return 4 + return None class DataFrame: @@ -620,17 +683,36 @@ def write_csv(self, path: str | pathlib.Path, with_header: bool = False) -> None def write_parquet( self, path: str | pathlib.Path, - compression: str = "uncompressed", + compression: Union[str, Compression] = Compression.ZSTD, compression_level: int | None = None, ) -> None: """Execute the :py:class:`DataFrame` and write the results to a Parquet file. Args: path: Path of the Parquet file to write. - compression: Compression type to use. - compression_level: Compression level to use. - """ - self.df.write_parquet(str(path), compression, compression_level) + compression: Compression type to use. Default is "ZSTD". + Available compression types are: + - "uncompressed": No compression. + - "snappy": Snappy compression. + - "gzip": Gzip compression. + - "brotli": Brotli compression. + - "lz0": LZ0 compression. + - "lz4": LZ4 compression. + - "lz4_raw": LZ4_RAW compression. + - "zstd": Zstandard compression. + compression_level: Compression level to use. For ZSTD, the + recommended range is 1 to 22, with the default being 4. Higher levels + provide better compression but slower speed. + """ + # Convert string to Compression enum if necessary + if isinstance(compression, str): + compression = Compression.from_str(compression) + + if compression in {Compression.GZIP, Compression.BROTLI, Compression.ZSTD}: + if compression_level is None: + compression_level = compression.get_default_level() + + self.df.write_parquet(str(path), compression.value, compression_level) def write_json(self, path: str | pathlib.Path) -> None: """Execute the :py:class:`DataFrame` and write the results to a JSON file. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index e3bd1b2a5..fa5f4e8c5 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -1107,14 +1107,24 @@ def test_write_compressed_parquet_wrong_compression_level( ) -@pytest.mark.parametrize("compression", ["brotli", "zstd", "wrong"]) -def test_write_compressed_parquet_missing_compression_level(df, tmp_path, compression): +@pytest.mark.parametrize("compression", ["wrong"]) +def test_write_compressed_parquet_invalid_compression(df, tmp_path, compression): path = tmp_path with pytest.raises(ValueError): df.write_parquet(str(path), compression=compression) +@pytest.mark.parametrize("compression", ["zstd", "brotli", "gzip"]) +def test_write_compressed_parquet_default_compression_level(df, tmp_path, compression): + # Test write_parquet with zstd, brotli, gzip default compression level, + # ie don't specify compression level + # should complete without error + path = tmp_path + + df.write_parquet(str(path), compression=compression) + + def test_dataframe_export(df) -> None: # Guarantees that we have the canonical implementation # reading our dataframe export diff --git a/src/dataframe.rs b/src/dataframe.rs index fcb46a756..71a6fe60f 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -463,7 +463,7 @@ impl PyDataFrame { /// Write a `DataFrame` to a Parquet file. #[pyo3(signature = ( path, - compression="uncompressed", + compression="zstd", compression_level=None ))] fn write_parquet( From 39fec53ca1182049700806a423f60d44f2a9676d Mon Sep 17 00:00:00 2001 From: kosiew Date: Tue, 14 Jan 2025 20:01:10 +0800 Subject: [PATCH 254/413] fix: correct LZ0 to LZO in compression options (#995) * fix: correct LZ0 to LZO in compression options * fix: disable LZO compression option and update tests to reflect its unavailability * fix: ruff format expected string in test_execution_plan * fix: update test for execution plan and add validation for invalid LZO compression * fix: remove LZO compression option and related test cases * ruff autoformat * fix: remove TODO comment regarding LZO compression implementation --- python/datafusion/dataframe.py | 6 ++++-- python/tests/test_dataframe.py | 2 ++ python/tests/test_functions.py | 18 +++++++++--------- src/dataframe.rs | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index f8aef0c91..b0c1abdad 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -57,7 +57,9 @@ class Compression(Enum): GZIP = "gzip" BROTLI = "brotli" LZ4 = "lz4" - LZ0 = "lz0" + # lzo is not implemented yet + # https://github.com/apache/arrow-rs/issues/6970 + # LZO = "lzo" ZSTD = "zstd" LZ4_RAW = "lz4_raw" @@ -696,10 +698,10 @@ def write_parquet( - "snappy": Snappy compression. - "gzip": Gzip compression. - "brotli": Brotli compression. - - "lz0": LZ0 compression. - "lz4": LZ4 compression. - "lz4_raw": LZ4_RAW compression. - "zstd": Zstandard compression. + Note: LZO is not yet implemented in arrow-rs and is therefore excluded. compression_level: Compression level to use. For ZSTD, the recommended range is 1 to 22, with the default being 4. Higher levels provide better compression but slower speed. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index fa5f4e8c5..a1a871e9a 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -1115,6 +1115,8 @@ def test_write_compressed_parquet_invalid_compression(df, tmp_path, compression) df.write_parquet(str(path), compression=compression) +# not testing lzo because it it not implemented yet +# https://github.com/apache/arrow-rs/issues/6970 @pytest.mark.parametrize("compression", ["zstd", "brotli", "gzip"]) def test_write_compressed_parquet_default_compression_level(df, tmp_path, compression): # Test write_parquet with zstd, brotli, gzip default compression level, diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 01c6c9cef..add170c17 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -790,9 +790,9 @@ def test_hash_functions(df): ) assert result.column(2) == pa.array( [ - b("185F8DB32271FE25F561A6FC938B2E26" "4306EC304EDA518007D1764826381969"), - b("78AE647DC5544D227130A0682A51E30B" "C7777FBB6D8A8F17007463A3ECD1D524"), - b("BB7208BC9B5D7C04F1236A82A0093A5E" "33F40423D5BA8D4266F7092C3BA43B62"), + b("185F8DB32271FE25F561A6FC938B2E264306EC304EDA518007D1764826381969"), + b("78AE647DC5544D227130A0682A51E30BC7777FBB6D8A8F17007463A3ECD1D524"), + b("BB7208BC9B5D7C04F1236A82A0093A5E33F40423D5BA8D4266F7092C3BA43B62"), ] ) assert result.column(3) == pa.array( @@ -838,16 +838,16 @@ def test_hash_functions(df): ) assert result.column(5) == pa.array( [ - b("F73A5FBF881F89B814871F46E26AD3FA" "37CB2921C5E8561618639015B3CCBB71"), - b("B792A0383FB9E7A189EC150686579532" "854E44B71AC394831DAED169BA85CCC5"), - b("27988A0E51812297C77A433F63523334" "6AEE29A829DCF4F46E0F58F402C6CFCB"), + b("F73A5FBF881F89B814871F46E26AD3FA37CB2921C5E8561618639015B3CCBB71"), + b("B792A0383FB9E7A189EC150686579532854E44B71AC394831DAED169BA85CCC5"), + b("27988A0E51812297C77A433F635233346AEE29A829DCF4F46E0F58F402C6CFCB"), ] ) assert result.column(6) == pa.array( [ - b("FBC2B0516EE8744D293B980779178A35" "08850FDCFE965985782C39601B65794F"), - b("BF73D18575A736E4037D45F9E316085B" "86C19BE6363DE6AA789E13DEAACC1C4E"), - b("C8D11B9F7237E4034ADBCD2005735F9B" "C4C597C75AD89F4492BEC8F77D15F7EB"), + b("FBC2B0516EE8744D293B980779178A3508850FDCFE965985782C39601B65794F"), + b("BF73D18575A736E4037D45F9E316085B86C19BE6363DE6AA789E13DEAACC1C4E"), + b("C8D11B9F7237E4034ADBCD2005735F9BC4C597C75AD89F4492BEC8F77D15F7EB"), ] ) assert result.column(7) == result.column(1) # SHA-224 diff --git a/src/dataframe.rs b/src/dataframe.rs index 71a6fe60f..b875480a7 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -491,7 +491,7 @@ impl PyDataFrame { ZstdLevel::try_new(verify_compression_level(compression_level)? as i32) .map_err(|e| PyValueError::new_err(format!("{e}")))?, ), - "lz0" => Compression::LZO, + "lzo" => Compression::LZO, "lz4" => Compression::LZ4, "lz4_raw" => Compression::LZ4_RAW, "uncompressed" => Compression::UNCOMPRESSED, From 31fee392d41f723179678ee7fdac2719ef1d40d1 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sun, 19 Jan 2025 07:51:14 -0500 Subject: [PATCH 255/413] Feat/use uv python management (#994) * Remove requirements files and add dependencies into pyproject.toml instead * Remove old conda files since we will use uv as our primary method for developers to set up environments * Working through CI changes to use uv instead of pip and conda * Add uv lock to exclude files * Revert "Remove old conda files since we will use uv as our primary method for developers to set up environments" This reverts commit 88aff7e39334cde8101ef6d313c6bb2bd1f3981c. * Windows workflows don't use source command * Add in extra include for ignoreing rat * Use uv commands in CI * Remove conda recipes and CI stages * Working on CI using uv * Install doc requirements * Remove caching uv * Set uv venv * Add requirements for building * Revert github action to allowed one * Call uv sync with verbose mode so users can see the build occuring in CI * Test setting specific hash on action * Test setting rust-toolchain github action with pinned version * Testing night rust toolchain against apache rejection criteria * Github action is fickle with the pattern matching * Switch all Ci to use nightly rust toolchain until infra team whitelists the stable toolchain * Speed up CI by preventing build during uv sync * Additional uv commands missing no-project option * Setting python versions of dependencies to match lowest supported python version, 3.8 * Update maturin and move to deps for dev * CI ordering was wrong and maturin needed uv option * Switch to stable toolchain * uv requires two dashes * Submodule init * change directories for unit tests * Add deps for build * Maturin build doesn't take uv as parameter * Update documentation for setting up with uv * Enable cache in CI * Update documentation to use uv * Small adjustment to CI config --- .github/workflows/build.yml | 62 +- .github/workflows/conda.yml | 107 - .github/workflows/docs.yaml | 23 +- .github/workflows/test.yaml | 59 +- .pre-commit-config.yaml | 4 +- README.md | 68 +- conda/environments/datafusion-cuda-dev.yaml | 44 - conda/environments/datafusion-dev.yaml | 41 - conda/recipes/bld.bat | 26 - conda/recipes/build.sh | 84 - conda/recipes/meta.yaml | 75 - dev/python_lint.sh | 2 +- dev/release/README.md | 27 +- dev/release/rat_exclude_files.txt | 4 +- dev/release/verify-release-candidate.sh | 6 +- docs/README.md | 32 +- docs/build.sh | 6 + docs/mdbook/src/installation.md | 53 +- docs/requirements.txt | 26 - .../source/contributor-guide/introduction.rst | 42 +- pyproject.toml | 25 +- requirements-310.txt | 195 -- requirements-311.txt | 175 -- requirements-312.txt | 184 -- requirements.in | 26 - uv.lock | 1842 +++++++++++++++++ 26 files changed, 2031 insertions(+), 1207 deletions(-) delete mode 100644 .github/workflows/conda.yml delete mode 100644 conda/environments/datafusion-cuda-dev.yaml delete mode 100644 conda/environments/datafusion-dev.yaml delete mode 100644 conda/recipes/bld.bat delete mode 100644 conda/recipes/build.sh delete mode 100644 conda/recipes/meta.yaml delete mode 100644 docs/requirements.txt delete mode 100644 requirements-310.txt delete mode 100644 requirements-311.txt delete mode 100644 requirements-312.txt delete mode 100644 requirements.in create mode 100644 uv.lock diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 084a96192..acabad3ca 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,28 +31,33 @@ jobs: - name: Install Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" + + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + # Use the --no-install-package to only install the dependencies + # but do not yet build the rust library - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install ruff + run: uv sync --dev --no-install-package datafusion + # Update output format to enable automatic inline annotations. - name: Run Ruff run: | - ruff check --output-format=github python/ - ruff format --check python/ + uv run --no-project ruff check --output-format=github python/ + uv run --no-project ruff format --check python/ generate-license: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions-rs/toolchain@v1 + - uses: astral-sh/setup-uv@v5 with: - profile: minimal - toolchain: stable - override: true + enable-cache: true + - name: Generate license file - run: python ./dev/create_license.py + run: uv run --no-project python ./dev/create_license.py - uses: actions/upload-artifact@v4 with: name: python-wheel-license @@ -74,15 +79,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Upgrade pip - run: python -m pip install --upgrade pip - - - name: Install maturin - run: pip install maturin==1.5.1 + - uses: dtolnay/rust-toolchain@stable - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -97,8 +94,14 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - name: Build Python package - run: maturin build --release --strip --features substrait + run: | + uv sync --dev --no-install-package datafusion + uv run --no-project maturin build --release --strip --features substrait - name: List Windows wheels if: matrix.os == 'windows-latest' @@ -132,15 +135,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Upgrade pip - run: python -m pip install --upgrade pip - - - name: Install maturin - run: pip install maturin==1.5.1 + - uses: dtolnay/rust-toolchain@stable - run: rm LICENSE.txt - name: Download LICENSE.txt @@ -155,9 +150,14 @@ jobs: version: "27.4" repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + - name: Build Python package run: | - maturin build --release --strip --features substrait + uv sync --dev --no-install-package datafusion + uv run --no-project maturin build --release --strip --features substrait - name: List Mac wheels run: find target/wheels/ diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml deleted file mode 100644 index c2b8fab02..000000000 --- a/.github/workflows/conda.yml +++ /dev/null @@ -1,107 +0,0 @@ -name: Build conda nightly -on: - push: - branches: - - main - pull_request: - paths: - - Cargo.toml - - Cargo.lock - - pyproject.toml - - conda/recipes/** - - .github/workflows/conda.yml - schedule: - - cron: '0 0 * * 0' - -# When this workflow is queued, automatically cancel any previous running -# or pending jobs from the same branch -concurrency: - group: conda-${{ github.head_ref }} - cancel-in-progress: true - -# Required shell entrypoint to have properly activated conda environments -defaults: - run: - shell: bash -l {0} - -jobs: - conda: - name: "Build conda nightlies (python: ${{ matrix.python }}, arch: ${{ matrix.arch }})" - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python: ["3.8", "3.9", "3.10", "3.11"] - arch: ["linux-64", "linux-aarch64"] - steps: - - name: Manage disk space - if: matrix.arch == 'linux-aarch64' - run: | - sudo mkdir -p /opt/empty_dir || true - for d in \ - /opt/ghc \ - /opt/hostedtoolcache \ - /usr/lib/jvm \ - /usr/local/.ghcup \ - /usr/local/lib/android \ - /usr/local/share/powershell \ - /usr/share/dotnet \ - /usr/share/swift \ - ; do - sudo rsync --stats -a --delete /opt/empty_dir/ $d || true - done - sudo apt-get purge -y -f firefox \ - google-chrome-stable \ - microsoft-edge-stable - sudo apt-get autoremove -y >& /dev/null - sudo apt-get autoclean -y >& /dev/null - sudo docker image prune --all --force - df -h - - name: Create swapfile - if: matrix.arch == 'linux-aarch64' - run: | - sudo fallocate -l 10GiB /swapfile || true - sudo chmod 600 /swapfile || true - sudo mkswap /swapfile || true - sudo swapon /swapfile || true - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Set up Python - uses: conda-incubator/setup-miniconda@v3.0.4 - with: - miniforge-variant: Miniforge3 - python-version: "3.8" - channel-priority: strict - - name: Install dependencies - run: | - conda install -c conda-forge conda-build conda-verify - - which python - pip list - conda list - # Clean the conda cache - - name: Clean Conda Cache - run: conda clean --all --yes - - name: Build conda packages - run: | - # suffix for nightly package versions - export VERSION_SUFFIX=a`date +%y%m%d` - - conda build conda/recipes \ - --python ${{ matrix.python }} \ - --variants "{target_platform: [${{ matrix.arch }}]}" \ - --error-overlinking \ - --no-test \ - --no-anaconda-upload \ - --output-folder packages - - name: Test conda packages - if: matrix.arch == 'linux-64' # can only test native platform packages - run: | - conda build --test packages/${{ matrix.arch }}/*.tar.bz2 - - name: Upload conda packages as artifacts - uses: actions/upload-artifact@v4 - with: - name: "conda nightlies (python - ${{ matrix.python }}, arch - ${{ matrix.arch }})" - # need to install all conda channel metadata to properly install locally - path: packages/ diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 86288e2d8..9037e0a5c 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -57,27 +57,24 @@ jobs: version: '27.4' repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Install dependencies - run: | - set -x - python3 -m venv venv - source venv/bin/activate - pip install -r requirements-311.txt - pip install -r docs/requirements.txt - - name: Build Datafusion + - name: Install dependencies and build + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Build repo run: | - set -x - source venv/bin/activate - maturin develop + uv venv + uv sync --dev --no-install-package datafusion --group docs + uv run --no-project maturin develop --uv - name: Build docs run: | set -x - source venv/bin/activate cd docs curl -O https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv curl -O https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet - make html + uv run --no-project make html - name: Copy & push the generated HTML if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref_type == 'tag') diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 21faedecd..c93d4c06f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,11 +43,10 @@ jobs: - uses: actions/checkout@v4 - name: Setup Rust Toolchain - uses: actions-rs/toolchain@v1 + uses: dtolnay/rust-toolchain@stable id: rust-toolchain with: - toolchain: ${{ matrix.toolchain }} - override: true + components: clippy,rustfmt - name: Install Protoc uses: arduino/setup-protoc@v3 @@ -64,60 +63,35 @@ jobs: uses: actions/cache@v4 with: path: ~/.cargo - key: cargo-cache-${{ steps.rust-toolchain.outputs.rustc_hash }}-${{ hashFiles('Cargo.lock') }} + key: cargo-cache-${{ steps.rust-toolchain.outputs.cachekey }}-${{ hashFiles('Cargo.lock') }} - name: Check Formatting - uses: actions-rs/cargo@v1 if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} - with: - command: fmt - args: -- --check + run: cargo fmt -- --check - name: Run Clippy - uses: actions-rs/cargo@v1 if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} - with: - command: clippy - args: --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure - - - name: Create Virtualenv (3.12) - if: ${{ matrix.python-version == '3.12' }} - run: | - python -m venv venv - source venv/bin/activate - pip install -r requirements-312.txt + run: cargo clippy --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure - - name: Create Virtualenv (3.10) - if: ${{ matrix.python-version == '3.10' }} - run: | - python -m venv venv - source venv/bin/activate - pip install -r requirements-310.txt - - - name: Create Virtualenv (3.11) - if: ${{ matrix.python-version == '3.11' }} - run: | - python -m venv venv - source venv/bin/activate - pip install -r requirements-311.txt + - name: Install dependencies and build + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true - name: Run tests env: RUST_BACKTRACE: 1 run: | git submodule update --init - source venv/bin/activate - pip install -e . -vv - pytest -v . + uv sync --dev --no-install-package datafusion + uv run --no-project maturin develop --uv + uv run --no-project pytest -v . - name: FFI unit tests run: | - source venv/bin/activate - pip install -e . -vv - pip install maturin==1.5.1 cd examples/ffi-table-provider - maturin develop --release --strip - pytest python/tests/_test_table_provider.py + uv run --no-project maturin develop --uv + uv run --no-project pytest python/tests/_test_table_provider.py - name: Cache the generated dataset id: cache-tpch-dataset @@ -134,7 +108,6 @@ jobs: - name: Run TPC-H examples run: | - source venv/bin/activate cd examples/tpch - python convert_data_to_parquet.py - pytest _tests.py + uv run --no-project python convert_data_to_parquet.py + uv run --no-project pytest _tests.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8509fae2c..e20fedf5c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,9 +17,9 @@ repos: - repo: https://github.com/rhysd/actionlint - rev: v1.6.23 + rev: v1.7.6 hooks: - - id: actionlint-docker + - id: actionlint-docker - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. rev: v0.3.0 diff --git a/README.md b/README.md index ca612c1ab..5aaf7f5f3 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,13 @@ See [examples](examples/README.md) for more information. - [Serialize query plans using Substrait](https://github.com/apache/datafusion-python/blob/main/examples/substrait.py) -## How to install (from pip) +## How to install + +### uv + +```bash +uv add datafusion +``` ### Pip @@ -164,61 +170,69 @@ You can verify the installation by running: ## How to develop -This assumes that you have rust and cargo installed. We use the workflow recommended by [pyo3](https://github.com/PyO3/pyo3) and [maturin](https://github.com/PyO3/maturin). +This assumes that you have rust and cargo installed. We use the workflow recommended by [pyo3](https://github.com/PyO3/pyo3) and [maturin](https://github.com/PyO3/maturin). The Maturin tools used in this workflow can be installed either via `uv` or `pip`. Both approaches should offer the same experience. It is recommended to use `uv` since it has significant performance improvements +over `pip`. -The Maturin tools used in this workflow can be installed either via Conda or Pip. Both approaches should offer the same experience. Multiple approaches are only offered to appease developer preference. Bootstrapping for both Conda and Pip are as follows. +Bootstrap (`uv`): -Bootstrap (Conda): +By default `uv` will attempt to build the datafusion python package. For our development we prefer to build manually. This means +that when creating your virtual environment using `uv sync` you need to pass in the additional `--no-install-package datafusion` +and for `uv run` commands the additional parameter `--no-project` ```bash # fetch this repo git clone git@github.com:apache/datafusion-python.git -# create the conda environment for dev -conda env create -f ./conda/environments/datafusion-dev.yaml -n datafusion-dev -# activate the conda environment -conda activate datafusion-dev +# create the virtual enviornment +uv sync --dev --no-install-package datafusion +# activate the environment +source .venv/bin/activate ``` -Or alternatively, if you are on an OS that supports CUDA Toolkit, you can use `-f ./conda/environments/datafusion-cuda-dev.yaml`. - -Bootstrap (Pip): +Bootstrap (`pip`): ```bash # fetch this repo git clone git@github.com:apache/datafusion-python.git # prepare development environment (used to build wheel / install in development) -python3 -m venv venv +python3 -m venv .venv # activate the venv -source venv/bin/activate +source .venv/bin/activate # update pip itself if necessary python -m pip install -U pip -# install dependencies (for Python 3.8+) -python -m pip install -r requirements.in +# install dependencies +python -m pip install -r pyproject.toml ``` The tests rely on test data in git submodules. ```bash -git submodule init -git submodule update +git submodule update --init ``` Whenever rust code changes (your changes or via `git pull`): ```bash # make sure you activate the venv using "source venv/bin/activate" first -maturin develop +maturin develop --uv python -m pytest ``` +Alternatively if you are using `uv` you can do the following without +needing to activate the virtual environment: + +```bash +uv run --no-project maturin develop --uv +uv --no-project pytest . +``` + ### Running & Installing pre-commit hooks -arrow-datafusion-python takes advantage of [pre-commit](https://pre-commit.com/) to assist developers with code linting to help reduce +`datafusion-python` takes advantage of [pre-commit](https://pre-commit.com/) to assist developers with code linting to help reduce the number of commits that ultimately fail in CI due to linter errors. Using the pre-commit hooks is optional for the developer but certainly helpful for keeping PRs clean and concise. Our pre-commit hooks can be installed by running `pre-commit install`, which will install the configurations in -your ARROW_DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit, failing to complete +your DATAFUSION_PYTHON_ROOT/.github directory and run each time you perform a commit, failing to complete the commit if an offending lint is found allowing you to make changes locally before pushing. The pre-commit hooks can also be run adhoc without installing them by simply running `pre-commit run --all-files` @@ -236,18 +250,8 @@ There are scripts in `ci/scripts` for running Rust and Python linters. ## How to update dependencies -To change test dependencies, change the `requirements.in` and run +To change test dependencies, change the `pyproject.toml` and run ```bash -# install pip-tools (this can be done only once), also consider running in venv -python -m pip install pip-tools -python -m piptools compile --generate-hashes -o requirements-310.txt +uv sync --dev --no-install-package datafusion ``` - -To update dependencies, run with `-U` - -```bash -python -m piptools compile -U --generate-hashes -o requirements-310.txt -``` - -More details [here](https://github.com/jazzband/pip-tools) diff --git a/conda/environments/datafusion-cuda-dev.yaml b/conda/environments/datafusion-cuda-dev.yaml deleted file mode 100644 index 1f6f23942..000000000 --- a/conda/environments/datafusion-cuda-dev.yaml +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -channels: - - conda-forge -dependencies: - - black - - flake8 - - isort - - maturin>=1.5.1 - - mypy - - numpy - - pyarrow>=11.0.0 - - pytest - - toml - - importlib_metadata - - python>=3.10 - # Packages useful for building distributions and releasing - - mamba - - conda-build - - anaconda-client - # Packages for documentation building - - sphinx - - pydata-sphinx-theme==0.8.0 - - myst-parser - - jinja2 - # GPU packages - - cudf - - cudatoolkit=11.8 -name: datafusion-dev diff --git a/conda/environments/datafusion-dev.yaml b/conda/environments/datafusion-dev.yaml deleted file mode 100644 index b4b503dc6..000000000 --- a/conda/environments/datafusion-dev.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -channels: - - conda-forge -dependencies: - - black - - flake8 - - isort - - maturin>=1.5.1 - - mypy - - numpy - - pyarrow>=11.0.0 - - pytest - - toml - - importlib_metadata - - python>=3.10 - # Packages useful for building distributions and releasing - - mamba - - conda-build - - anaconda-client - # Packages for documentation building - - sphinx - - pydata-sphinx-theme==0.8.0 - - myst-parser - - jinja2 -name: datafusion-dev diff --git a/conda/recipes/bld.bat b/conda/recipes/bld.bat deleted file mode 100644 index 90626a637..000000000 --- a/conda/recipes/bld.bat +++ /dev/null @@ -1,26 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -maturin build -vv -j %CPU_COUNT% --release --strip --features substrait --manylinux off --interpreter=%PYTHON% - -FOR /F "delims=" %%i IN ('dir /s /b target\wheels\*.whl') DO set datafusion_wheel=%%i - -%PYTHON% -m pip install --no-deps %datafusion_wheel% -vv - -cargo-bundle-licenses --format yaml --output THIRDPARTY.yml diff --git a/conda/recipes/build.sh b/conda/recipes/build.sh deleted file mode 100644 index 259894313..000000000 --- a/conda/recipes/build.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -ex - -# See https://github.com/conda-forge/rust-feedstock/blob/master/recipe/build.sh for cc env explanation -if [ "$c_compiler" = gcc ] ; then - case "$target_platform" in - linux-64) rust_env_arch=X86_64_UNKNOWN_LINUX_GNU ;; - linux-aarch64) rust_env_arch=AARCH64_UNKNOWN_LINUX_GNU ;; - linux-ppc64le) rust_env_arch=POWERPC64LE_UNKNOWN_LINUX_GNU ;; - *) echo "unknown target_platform $target_platform" ; exit 1 ;; - esac - - export CARGO_TARGET_${rust_env_arch}_LINKER=$CC -fi - -declare -a _xtra_maturin_args - -mkdir -p $SRC_DIR/.cargo - -if [ "$target_platform" = "osx-64" ] ; then - cat <> $SRC_DIR/.cargo/config -[target.x86_64-apple-darwin] -linker = "$CC" -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] - -EOF - - _xtra_maturin_args+=(--target=x86_64-apple-darwin) - -elif [ "$target_platform" = "osx-arm64" ] ; then - cat <> $SRC_DIR/.cargo/config -# Required for intermediate codegen stuff -[target.x86_64-apple-darwin] -linker = "$CC_FOR_BUILD" - -# Required for final binary artifacts for target -[target.aarch64-apple-darwin] -linker = "$CC" -rustflags = [ - "-C", "link-arg=-undefined", - "-C", "link-arg=dynamic_lookup", -] - -EOF - _xtra_maturin_args+=(--target=aarch64-apple-darwin) - - # This variable must be set to the directory containing the target's libpython DSO - export PYO3_CROSS_LIB_DIR=$PREFIX/lib - - # xref: https://github.com/PyO3/pyo3/commit/7beb2720 - export PYO3_PYTHON_VERSION=${PY_VER} - - # xref: https://github.com/conda-forge/python-feedstock/issues/621 - sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/os-patch.py - sed -i.bak 's,aarch64,arm64,g' $BUILD_PREFIX/venv/lib/platform-patch.py -fi - -maturin build -vv -j "${CPU_COUNT}" --release --strip --features substrait --manylinux off --interpreter="${PYTHON}" "${_xtra_maturin_args[@]}" - -"${PYTHON}" -m pip install $SRC_DIR/target/wheels/datafusion*.whl --no-deps -vv - -cargo-bundle-licenses --format yaml --output THIRDPARTY.yml diff --git a/conda/recipes/meta.yaml b/conda/recipes/meta.yaml deleted file mode 100644 index b0784253a..000000000 --- a/conda/recipes/meta.yaml +++ /dev/null @@ -1,75 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% set name = "datafusion" %} -{% set major_minor_patch = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').split('.') %} -{% set new_patch = major_minor_patch[2] | int + 1 %} -{% set version = (major_minor_patch[:2] + [new_patch]) | join('.') + environ.get('VERSION_SUFFIX', '') %} - - -package: - name: {{ name|lower }} - version: {{ version }} - -source: - git_url: ../.. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - string: py{{ python | replace(".", "") }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }} - -requirements: - build: - - python # [build_platform != target_platform] - - cross-python_{{ target_platform }} # [build_platform != target_platform] - - zlib # [build_platform != target_platform] - - {{ compiler('c') }} - - {{ compiler('rust') }} - - cargo-bundle-licenses - - maturin >=1.5.1,<1.6.0 - - libprotobuf =3 - host: - - python - - maturin >=1.5.1,<1.6.0 - - pip - - zlib - - xz # [linux64] - run: - - python - - pyarrow >=11.0.0 - - typing_extensions - -test: - imports: - - datafusion - commands: - - pip check - requires: - - pip - -about: - home: https://arrow.apache.org/datafusion - license: Apache-2.0 - license_family: APACHE - license_file: - - LICENSE.txt - - THIRDPARTY.yml - description: | - DataFusion is an extensible query execution framework, written in Rust, - that uses Apache Arrow as its in-memory format. - doc_url: https://arrow.apache.org/datafusion - dev_url: https://github.com/apache/arrow-datafusion diff --git a/dev/python_lint.sh b/dev/python_lint.sh index 29f0d4833..2d867f29d 100755 --- a/dev/python_lint.sh +++ b/dev/python_lint.sh @@ -21,6 +21,6 @@ # DataFusion CI does set -e -source venv/bin/activate +source .venv/bin/activate flake8 --exclude venv,benchmarks/db-benchmark --ignore=E501,W503 black --line-length 79 . diff --git a/dev/release/README.md b/dev/release/README.md index b2c015e1d..f0b333999 100644 --- a/dev/release/README.md +++ b/dev/release/README.md @@ -172,8 +172,8 @@ git checkout 40.0.0-rc1 git submodule update --init --recursive # create the env -python3 -m venv venv -source venv/bin/activate +python3 -m venv .venv +source .venv/bin/activate # install release candidate pip install --extra-index-url https://test.pypi.org/simple/ datafusion==40.0.0 @@ -218,28 +218,9 @@ uploading them using `twine`: twine upload --repository pypi dist-release/* ``` -### Publish Python Artifacts to Anaconda +### Publish Python Artifacts to conda-forge -Publishing artifacts to Anaconda is similar to PyPi. First, Download the source tarball created in the previous step and untar it. - -```bash -# Assuming you have an existing conda environment named `datafusion-dev` if not see root README for instructions -conda activate datafusion-dev -conda build . -``` - -This will setup a virtual conda environment and build the artifacts inside of that virtual env. This step can take a few minutes as the entire build, host, and runtime environments are setup. Once complete a local filesystem path will be emitted for the location of the resulting package. Observe that path and copy to your clipboard. - -Ex: `/home/conda/envs/datafusion/conda-bld/linux-64/datafusion-0.7.0.tar.bz2` - -Now you are ready to publish this resulting package to anaconda.org. This can be accomplished in a few simple steps. - -```bash -# First login to Anaconda with the datafusion credentials -anaconda login -# Upload the package -anaconda upload /home/conda/envs/datafusion/conda-bld/linux-64/datafusion-0.7.0.tar.bz2 -``` +Pypi packages auto upload to conda-forge via [datafusion feedstock](https://github.com/conda-forge/datafusion-feedstock) ### Push the Release Tag diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index f65ddd06e..dcd5d9aac 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -45,4 +45,6 @@ Cargo.lock .github/* benchmarks/tpch/queries/q*.sql benchmarks/tpch/create_tables.sql -.cargo/config.toml \ No newline at end of file +.cargo/config.toml +**/.cargo/config.toml +uv.lock \ No newline at end of file diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 3879a267f..1a9104b55 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -106,7 +106,7 @@ setup_tempdir() { } test_source_distribution() { - # install rust toolchain in a similar fashion like test-miniconda + # install rust toolchain export RUSTUP_HOME=$PWD/test-rustup export CARGO_HOME=$PWD/test-rustup @@ -125,8 +125,8 @@ test_source_distribution() { git clone https://github.com/apache/arrow-testing.git testing git clone https://github.com/apache/parquet-testing.git parquet-testing - python3 -m venv venv - source venv/bin/activate + python3 -m venv .venv + source .venv/bin/activate python3 -m pip install -U pip python3 -m pip install -r requirements-310.txt maturin develop diff --git a/docs/README.md b/docs/README.md index b4b94120e..2bffea9bd 100644 --- a/docs/README.md +++ b/docs/README.md @@ -26,42 +26,32 @@ when changes are merged to the main branch. ## Dependencies It's recommended to install build dependencies and build the documentation -inside a Python `venv`. +inside a Python `venv` using `uv`. To prepare building the documentation run the following on the root level of the project: -1. Set up virtual environment if it was not already created - ```bash - python3 -m venv venv - ``` -1. Activate virtual environment - ```bash - source venv/bin/activate - ``` -1. Install Datafusion's Python dependencies - ```bash - pip install -r requirements-310.txt - ``` -1. Install documentation dependencies - ```bash - pip install -r docs/requirements.txt - ``` +```bash +# Set up a virtual environment with the documentation dependencies +uv sync --dev --group docs --no-install-package datafusion +``` ## Build & Preview Run the provided script to build the HTML pages. ```bash -cd docs -./build.sh +# Build the repository +uv run --no-project maturin develop --uv +# Build the documentation +uv run --no-project docs/build.sh ``` -The HTML will be generated into a `build` directory. +The HTML will be generated into a `build` directory in `docs`. Preview the site on Linux by running this command. ```bash -firefox build/html/index.html +firefox docs/build/html/index.html ``` ## Release Process diff --git a/docs/build.sh b/docs/build.sh index 31398d195..f73330323 100755 --- a/docs/build.sh +++ b/docs/build.sh @@ -20,6 +20,10 @@ set -e +original_dir=$(pwd) +script_dir=$(dirname "$(realpath "$0")") +cd "$script_dir" || exit + if [ ! -f pokemon.csv ]; then curl -O https://gist.githubusercontent.com/ritchie46/cac6b337ea52281aa23c049250a4ff03/raw/89a957ff3919d90e6ef2d34235e6bf22304f3366/pokemon.csv fi @@ -33,3 +37,5 @@ rm -rf temp 2> /dev/null mkdir temp cp -rf source/* temp/ make SOURCEDIR=`pwd`/temp html + +cd "$original_dir" || exit diff --git a/docs/mdbook/src/installation.md b/docs/mdbook/src/installation.md index ba00c8b80..b29f3b66b 100644 --- a/docs/mdbook/src/installation.md +++ b/docs/mdbook/src/installation.md @@ -18,44 +18,45 @@ DataFusion is easy to install, just like any other Python library. -## Using pip +## Using uv -``` bash -pip install datafusion -``` +If you do not yet have a virtual environment, create one: -## Conda & JupyterLab setup +```bash +uv venv +``` -This section explains how to install DataFusion in a conda environment with other libraries that allow for a nice Jupyter workflow. This setup is completely optional. These steps are only needed if you'd like to run DataFusion in a Jupyter notebook and have an interface like this: +You can add datafusion to your virtual environment with the usual: -![DataFusion in Jupyter](https://github.com/MrPowers/datafusion-book/raw/main/src/images/datafusion-jupyterlab.png) +```bash +uv pip install datafusion +``` -Create a conda environment with DataFusion, Jupyter, and other useful dependencies in the `datafusion-env.yml` file: +Or, to add to a project: +```bash +uv add datafusion ``` -name: datafusion-env -channels: - - conda-forge - - defaults -dependencies: - - python=3.9 - - ipykernel - - nb_conda - - jupyterlab - - jupyterlab_code_formatter - - isort - - black - - pip - - pip: - - datafusion +## Using pip + +``` bash +pip install datafusion ``` -Create the environment with `conda env create -f datafusion-env.yml`. +## uv & JupyterLab setup -Activate the environment with `conda activate datafusion-env`. +This section explains how to install DataFusion in a uv environment with other libraries that allow for a nice Jupyter workflow. This setup is completely optional. These steps are only needed if you'd like to run DataFusion in a Jupyter notebook and have an interface like this: -Run `jupyter lab` or open the [JupyterLab Desktop application](https://github.com/jupyterlab/jupyterlab-desktop) to start running DataFusion in a Jupyter notebook. +![DataFusion in Jupyter](https://github.com/MrPowers/datafusion-book/raw/main/src/images/datafusion-jupyterlab.png) + +Create a virtual environment with DataFusion, Jupyter, and other useful dependencies and start the desktop application. + +```bash +uv venv +uv pip install datafusion jupyterlab jupyterlab_code_formatter +uv run jupyter lab +``` ## Examples diff --git a/docs/requirements.txt b/docs/requirements.txt deleted file mode 100644 index f5cece78e..000000000 --- a/docs/requirements.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -sphinx -pydata-sphinx-theme==0.8.0 -myst-parser -maturin -jinja2 -ipython -pandas -pickleshare -sphinx-autoapi diff --git a/docs/source/contributor-guide/introduction.rst b/docs/source/contributor-guide/introduction.rst index 4457a898f..fb98cfd1d 100644 --- a/docs/source/contributor-guide/introduction.rst +++ b/docs/source/contributor-guide/introduction.rst @@ -29,22 +29,24 @@ Doing so is a great way to help the community as well as get more familiar with How to develop -------------- -This assumes that you have rust and cargo installed. We use the workflow recommended by `pyo3 `_ and `maturin `_. +This assumes that you have rust and cargo installed. We use the workflow recommended by +`pyo3 `_ and `maturin `_. We recommend using +`uv `_ for python package management. + +By default `uv` will attempt to build the datafusion python package. For our development we prefer to build manually. This means +that when creating your virtual environment using `uv sync` you need to pass in the additional `--no-install-package datafusion` +and for `uv run` commands the additional parameter `--no-project` Bootstrap: .. code-block:: shell # fetch this repo - git clone git@github.com:apache/arrow-datafusion-python.git - # prepare development environment (used to build wheel / install in development) - python3 -m venv venv - # activate the venv - source venv/bin/activate - # update pip itself if necessary - python -m pip install -U pip - # install dependencies (for Python 3.8+) - python -m pip install -r requirements-310.txt + git clone git@github.com:apache/datafusion-python.git + # create the virtual enviornment + uv sync --dev --no-install-package datafusion + # activate the environment + source .venv/bin/activate The tests rely on test data in git submodules. @@ -58,8 +60,8 @@ Whenever rust code changes (your changes or via `git pull`): .. code-block:: shell - # make sure you activate the venv using "source venv/bin/activate" first - maturin develop + # make sure you activate the venv using "source .venv/bin/activate" first + maturin develop -uv python -m pytest Running & Installing pre-commit hooks @@ -86,20 +88,10 @@ Mostly, the ``python`` code is limited to pure wrappers with type hints and good Update Dependencies ------------------- -To change test dependencies, change the `requirements.in` and run - -.. code-block:: shell - - # install pip-tools (this can be done only once), also consider running in venv - python -m pip install pip-tools - python -m piptools compile --generate-hashes -o requirements-310.txt +To change test dependencies, change the ``pyproject.toml`` and run - -To update dependencies, run with `-U` +To update dependencies, run .. code-block:: shell - python -m piptools compile -U --generate-hashes -o requirements-310.txt - - -More details about pip-tools `here `_ + uv sync --dev --no-install-package datafusion diff --git a/pyproject.toml b/pyproject.toml index 98bda5aae..6e8acfe71 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ # under the License. [build-system] -requires = ["maturin>=1.5.1,<1.6.0"] +requires = ["maturin>=1.8.1"] build-backend = "maturin" [project] @@ -24,7 +24,7 @@ name = "datafusion" description = "Build and run queries against data" readme = "README.md" license = { file = "LICENSE.txt" } -requires-python = ">=3.7" +requires-python = ">=3.8" keywords = ["datafusion", "dataframe", "rust", "query-engine"] classifiers = [ "Development Status :: 2 - Pre-Alpha", @@ -35,7 +35,6 @@ classifiers = [ "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -82,3 +81,23 @@ max-doc-length = 88 "dev/*" = ["D"] "benchmarks/*" = ["D", "F"] "docs/*" = ["D"] + +[dependency-groups] +dev = [ + "maturin>=1.8.1", + "numpy>1.24.4 ; python_full_version >= '3.10'", + "pytest>=7.4.4", + "ruff>=0.9.1", + "toml>=0.10.2", +] +docs = [ + "sphinx>=7.1.2", + "pydata-sphinx-theme==0.8.0", + "myst-parser>=3.0.1", + "jinja2>=3.1.5", + "ipython>=8.12.3", + "pandas>=2.0.3", + "pickleshare>=0.7.5", + "sphinx-autoapi>=3.4.0", + "setuptools>=75.3.0", +] \ No newline at end of file diff --git a/requirements-310.txt b/requirements-310.txt deleted file mode 100644 index d7d25f3f1..000000000 --- a/requirements-310.txt +++ /dev/null @@ -1,195 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --generate-hashes --output-file=requirements-310.txt -# -exceptiongroup==1.2.1 \ - --hash=sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad \ - --hash=sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16 - # via pytest -iniconfig==2.0.0 \ - --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ - --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 - # via pytest -maturin==1.6.0 \ - --hash=sha256:16ef860df20028618b5a064da06b02c1c47acba064a4d25aaf84662a459ec599 \ - --hash=sha256:337899784955934dd67b30497d1dd5fab22da89f60bb079dbaf2eaa446b97a10 \ - --hash=sha256:4e931c92037128ade49cd26dd040d9c46ad8092d8170cc44f5c3a0b4a052d576 \ - --hash=sha256:50133965e52d8b5b969381fee3fde111ae2383905cdaba7650f256e08ccddcd4 \ - --hash=sha256:a2a2436628c36d98dabd79b52256df7e12fc4fd1b122984d9373fdf918fd4609 \ - --hash=sha256:aa4eb7dca7d246b466392f21016f67ff09a9aff2305fa714ca25a2344e4639e7 \ - --hash=sha256:b955025c24c8babc808db49e0ff90db8b4b1320dcc16b14eb26132841737230d \ - --hash=sha256:bd85edcb1b8e2bcddc1b7d16ce58ce00a66aa80c422745c8ad9e132ac40d4b48 \ - --hash=sha256:c87d1a7596c42b589099adb831343a56e02373588366e4cede96cbdf8bd68f9d \ - --hash=sha256:d67ca8dc7f3b2314bd3bf83c4de52645e220ee312fd526e53acc6a735f233fad \ - --hash=sha256:d8620970bd0b6a0acb99dbd0b1c2ebb7a69909d25f6023bdff9635a39001aa51 \ - --hash=sha256:d92b045e90ed919a8a2520dda64e3f384e5e746ea51e1498cc6ac3e9e5c76054 \ - --hash=sha256:dbbbf25dc3c207b0a7bd4f3aea1df33d4f22b8508592796a6f36f4d8ed216db0 - # via -r requirements.in -mypy==1.10.0 \ - --hash=sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061 \ - --hash=sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99 \ - --hash=sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de \ - --hash=sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a \ - --hash=sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9 \ - --hash=sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec \ - --hash=sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1 \ - --hash=sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131 \ - --hash=sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f \ - --hash=sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821 \ - --hash=sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5 \ - --hash=sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee \ - --hash=sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e \ - --hash=sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746 \ - --hash=sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2 \ - --hash=sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0 \ - --hash=sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b \ - --hash=sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53 \ - --hash=sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30 \ - --hash=sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda \ - --hash=sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051 \ - --hash=sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2 \ - --hash=sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7 \ - --hash=sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee \ - --hash=sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727 \ - --hash=sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976 \ - --hash=sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4 - # via -r requirements.in -mypy-extensions==1.0.0 \ - --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ - --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 - # via mypy -numpy==2.0.0 \ - --hash=sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f \ - --hash=sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238 \ - --hash=sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f \ - --hash=sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95 \ - --hash=sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a \ - --hash=sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a \ - --hash=sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2 \ - --hash=sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2 \ - --hash=sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f \ - --hash=sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609 \ - --hash=sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f \ - --hash=sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad \ - --hash=sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86 \ - --hash=sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65 \ - --hash=sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb \ - --hash=sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995 \ - --hash=sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a \ - --hash=sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85 \ - --hash=sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4 \ - --hash=sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275 \ - --hash=sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1 \ - --hash=sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196 \ - --hash=sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d \ - --hash=sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e \ - --hash=sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514 \ - --hash=sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f \ - --hash=sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6 \ - --hash=sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4 \ - --hash=sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44 \ - --hash=sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df \ - --hash=sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581 \ - --hash=sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787 \ - --hash=sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5 \ - --hash=sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc \ - --hash=sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871 \ - --hash=sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54 \ - --hash=sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2 \ - --hash=sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98 \ - --hash=sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9 \ - --hash=sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864 \ - --hash=sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de \ - --hash=sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289 \ - --hash=sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b \ - --hash=sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c \ - --hash=sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9 - # via - # -r requirements.in - # pyarrow -packaging==24.1 \ - --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ - --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 - # via pytest -pluggy==1.5.0 \ - --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ - --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 - # via pytest -pyarrow==16.1.0 \ - --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ - --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ - --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ - --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ - --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ - --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ - --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ - --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ - --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ - --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ - --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ - --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ - --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ - --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ - --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ - --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ - --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ - --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ - --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ - --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ - --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ - --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ - --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ - --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ - --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ - --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ - --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ - --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ - --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ - --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ - --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ - --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ - --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ - --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ - --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ - --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 - # via -r requirements.in -pytest==8.2.2 \ - --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ - --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 - # via -r requirements.in -ruff==0.4.9 \ - --hash=sha256:06b60f91bfa5514bb689b500a25ba48e897d18fea14dce14b48a0c40d1635893 \ - --hash=sha256:0e8e7b95673f22e0efd3571fb5b0cf71a5eaaa3cc8a776584f3b2cc878e46bff \ - --hash=sha256:2d45ddc6d82e1190ea737341326ecbc9a61447ba331b0a8962869fcada758505 \ - --hash=sha256:4555056049d46d8a381f746680db1c46e67ac3b00d714606304077682832998e \ - --hash=sha256:5d5460f789ccf4efd43f265a58538a2c24dbce15dbf560676e430375f20a8198 \ - --hash=sha256:673bddb893f21ab47a8334c8e0ea7fd6598ecc8e698da75bcd12a7b9d0a3206e \ - --hash=sha256:732dd550bfa5d85af8c3c6cbc47ba5b67c6aed8a89e2f011b908fc88f87649db \ - --hash=sha256:784d3ec9bd6493c3b720a0b76f741e6c2d7d44f6b2be87f5eef1ae8cc1d54c84 \ - --hash=sha256:78de3fdb95c4af084087628132336772b1c5044f6e710739d440fc0bccf4d321 \ - --hash=sha256:8064590fd1a50dcf4909c268b0e7c2498253273309ad3d97e4a752bb9df4f521 \ - --hash=sha256:88bffe9c6a454bf8529f9ab9091c99490578a593cc9f9822b7fc065ee0712a06 \ - --hash=sha256:8c1aff58c31948cc66d0b22951aa19edb5af0a3af40c936340cd32a8b1ab7438 \ - --hash=sha256:98ec2775fd2d856dc405635e5ee4ff177920f2141b8e2d9eb5bd6efd50e80317 \ - --hash=sha256:b262ed08d036ebe162123170b35703aaf9daffecb698cd367a8d585157732991 \ - --hash=sha256:e0a22c4157e53d006530c902107c7f550b9233e9706313ab57b892d7197d8e52 \ - --hash=sha256:e91175fbe48f8a2174c9aad70438fe9cb0a5732c4159b2a10a3565fea2d94cde \ - --hash=sha256:f1cb0828ac9533ba0135d148d214e284711ede33640465e706772645483427e3 - # via -r requirements.in -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via -r requirements.in -tomli==2.0.1 \ - --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ - --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f - # via - # maturin - # mypy - # pytest -typing-extensions==4.12.2 \ - --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ - --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 - # via mypy diff --git a/requirements-311.txt b/requirements-311.txt deleted file mode 100644 index 35b91133c..000000000 --- a/requirements-311.txt +++ /dev/null @@ -1,175 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --generate-hashes --output-file=requirements-311.txt -# -iniconfig==2.0.0 \ - --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ - --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 - # via pytest -maturin==1.6.0 \ - --hash=sha256:16ef860df20028618b5a064da06b02c1c47acba064a4d25aaf84662a459ec599 \ - --hash=sha256:337899784955934dd67b30497d1dd5fab22da89f60bb079dbaf2eaa446b97a10 \ - --hash=sha256:4e931c92037128ade49cd26dd040d9c46ad8092d8170cc44f5c3a0b4a052d576 \ - --hash=sha256:50133965e52d8b5b969381fee3fde111ae2383905cdaba7650f256e08ccddcd4 \ - --hash=sha256:a2a2436628c36d98dabd79b52256df7e12fc4fd1b122984d9373fdf918fd4609 \ - --hash=sha256:aa4eb7dca7d246b466392f21016f67ff09a9aff2305fa714ca25a2344e4639e7 \ - --hash=sha256:b955025c24c8babc808db49e0ff90db8b4b1320dcc16b14eb26132841737230d \ - --hash=sha256:bd85edcb1b8e2bcddc1b7d16ce58ce00a66aa80c422745c8ad9e132ac40d4b48 \ - --hash=sha256:c87d1a7596c42b589099adb831343a56e02373588366e4cede96cbdf8bd68f9d \ - --hash=sha256:d67ca8dc7f3b2314bd3bf83c4de52645e220ee312fd526e53acc6a735f233fad \ - --hash=sha256:d8620970bd0b6a0acb99dbd0b1c2ebb7a69909d25f6023bdff9635a39001aa51 \ - --hash=sha256:d92b045e90ed919a8a2520dda64e3f384e5e746ea51e1498cc6ac3e9e5c76054 \ - --hash=sha256:dbbbf25dc3c207b0a7bd4f3aea1df33d4f22b8508592796a6f36f4d8ed216db0 - # via -r requirements.in -mypy==1.10.0 \ - --hash=sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061 \ - --hash=sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99 \ - --hash=sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de \ - --hash=sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a \ - --hash=sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9 \ - --hash=sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec \ - --hash=sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1 \ - --hash=sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131 \ - --hash=sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f \ - --hash=sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821 \ - --hash=sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5 \ - --hash=sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee \ - --hash=sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e \ - --hash=sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746 \ - --hash=sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2 \ - --hash=sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0 \ - --hash=sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b \ - --hash=sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53 \ - --hash=sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30 \ - --hash=sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda \ - --hash=sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051 \ - --hash=sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2 \ - --hash=sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7 \ - --hash=sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee \ - --hash=sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727 \ - --hash=sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976 \ - --hash=sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4 - # via -r requirements.in -mypy-extensions==1.0.0 \ - --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ - --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 - # via mypy -numpy==1.26.4 \ - --hash=sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b \ - --hash=sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818 \ - --hash=sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20 \ - --hash=sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0 \ - --hash=sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010 \ - --hash=sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a \ - --hash=sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea \ - --hash=sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c \ - --hash=sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71 \ - --hash=sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110 \ - --hash=sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be \ - --hash=sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a \ - --hash=sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a \ - --hash=sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5 \ - --hash=sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed \ - --hash=sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd \ - --hash=sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c \ - --hash=sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e \ - --hash=sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0 \ - --hash=sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c \ - --hash=sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a \ - --hash=sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b \ - --hash=sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0 \ - --hash=sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6 \ - --hash=sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2 \ - --hash=sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a \ - --hash=sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30 \ - --hash=sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218 \ - --hash=sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5 \ - --hash=sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07 \ - --hash=sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2 \ - --hash=sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4 \ - --hash=sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764 \ - --hash=sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef \ - --hash=sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3 \ - --hash=sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f - # via - # -r requirements.in - # pyarrow -packaging==24.0 \ - --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \ - --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9 - # via pytest -pluggy==1.5.0 \ - --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ - --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 - # via pytest -pyarrow==16.1.0 \ - --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ - --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ - --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ - --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ - --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ - --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ - --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ - --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ - --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ - --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ - --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ - --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ - --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ - --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ - --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ - --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ - --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ - --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ - --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ - --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ - --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ - --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ - --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ - --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ - --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ - --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ - --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ - --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ - --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ - --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ - --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ - --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ - --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ - --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ - --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ - --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 - # via -r requirements.in -pytest==8.2.2 \ - --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ - --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 - # via -r requirements.in -ruff==0.4.8 \ - --hash=sha256:14019a06dbe29b608f6b7cbcec300e3170a8d86efaddb7b23405cb7f7dcaf780 \ - --hash=sha256:16d717b1d57b2e2fd68bd0bf80fb43931b79d05a7131aa477d66fc40fbd86268 \ - --hash=sha256:284c2e3f3396fb05f5f803c9fffb53ebbe09a3ebe7dda2929ed8d73ded736deb \ - --hash=sha256:384154a1c3f4bf537bac69f33720957ee49ac8d484bfc91720cc94172026ceed \ - --hash=sha256:6d795d7639212c2dfd01991259460101c22aabf420d9b943f153ab9d9706e6a9 \ - --hash=sha256:6ea874950daca5697309d976c9afba830d3bf0ed66887481d6bca1673fc5b66a \ - --hash=sha256:704977a658131651a22b5ebeb28b717ef42ac6ee3b11e91dc87b633b5d83142b \ - --hash=sha256:72584676164e15a68a15778fd1b17c28a519e7a0622161eb2debdcdabdc71883 \ - --hash=sha256:7663a6d78f6adb0eab270fa9cf1ff2d28618ca3a652b60f2a234d92b9ec89066 \ - --hash=sha256:9678d5c9b43315f323af2233a04d747409d1e3aa6789620083a82d1066a35199 \ - --hash=sha256:a7354f921e3fbe04d2a62d46707e569f9315e1a613307f7311a935743c51a764 \ - --hash=sha256:aad360893e92486662ef3be0a339c5ca3c1b109e0134fcd37d534d4be9fb8de3 \ - --hash=sha256:d05f8d6f0c3cce5026cecd83b7a143dcad503045857bc49662f736437380ad45 \ - --hash=sha256:e14a3a095d07560a9d6769a72f781d73259655919d9b396c650fc98a8157555d \ - --hash=sha256:e9d5ce97cacc99878aa0d084c626a15cd21e6b3d53fd6f9112b7fc485918e1fa \ - --hash=sha256:eeceb78da8afb6de0ddada93112869852d04f1cd0f6b80fe464fd4e35c330913 \ - --hash=sha256:fc95aac2943ddf360376be9aa3107c8cf9640083940a8c5bd824be692d2216dc - # via -r requirements.in -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via -r requirements.in -typing-extensions==4.12.1 \ - --hash=sha256:6024b58b69089e5a89c347397254e35f1bf02a907728ec7fee9bf0fe837d203a \ - --hash=sha256:915f5e35ff76f56588223f15fdd5938f9a1cf9195c0de25130c627e4d597f6d1 - # via mypy diff --git a/requirements-312.txt b/requirements-312.txt deleted file mode 100644 index e4de5a5d2..000000000 --- a/requirements-312.txt +++ /dev/null @@ -1,184 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --generate-hashes --output-file=requirements-312.txt -# -iniconfig==2.0.0 \ - --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \ - --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374 - # via pytest -maturin==1.6.0 \ - --hash=sha256:16ef860df20028618b5a064da06b02c1c47acba064a4d25aaf84662a459ec599 \ - --hash=sha256:337899784955934dd67b30497d1dd5fab22da89f60bb079dbaf2eaa446b97a10 \ - --hash=sha256:4e931c92037128ade49cd26dd040d9c46ad8092d8170cc44f5c3a0b4a052d576 \ - --hash=sha256:50133965e52d8b5b969381fee3fde111ae2383905cdaba7650f256e08ccddcd4 \ - --hash=sha256:a2a2436628c36d98dabd79b52256df7e12fc4fd1b122984d9373fdf918fd4609 \ - --hash=sha256:aa4eb7dca7d246b466392f21016f67ff09a9aff2305fa714ca25a2344e4639e7 \ - --hash=sha256:b955025c24c8babc808db49e0ff90db8b4b1320dcc16b14eb26132841737230d \ - --hash=sha256:bd85edcb1b8e2bcddc1b7d16ce58ce00a66aa80c422745c8ad9e132ac40d4b48 \ - --hash=sha256:c87d1a7596c42b589099adb831343a56e02373588366e4cede96cbdf8bd68f9d \ - --hash=sha256:d67ca8dc7f3b2314bd3bf83c4de52645e220ee312fd526e53acc6a735f233fad \ - --hash=sha256:d8620970bd0b6a0acb99dbd0b1c2ebb7a69909d25f6023bdff9635a39001aa51 \ - --hash=sha256:d92b045e90ed919a8a2520dda64e3f384e5e746ea51e1498cc6ac3e9e5c76054 \ - --hash=sha256:dbbbf25dc3c207b0a7bd4f3aea1df33d4f22b8508592796a6f36f4d8ed216db0 - # via -r requirements.in -mypy==1.10.0 \ - --hash=sha256:075cbf81f3e134eadaf247de187bd604748171d6b79736fa9b6c9685b4083061 \ - --hash=sha256:12b6bfc1b1a66095ab413160a6e520e1dc076a28f3e22f7fb25ba3b000b4ef99 \ - --hash=sha256:1ec404a7cbe9fc0e92cb0e67f55ce0c025014e26d33e54d9e506a0f2d07fe5de \ - --hash=sha256:28d0e038361b45f099cc086d9dd99c15ff14d0188f44ac883010e172ce86c38a \ - --hash=sha256:2b0695d605ddcd3eb2f736cd8b4e388288c21e7de85001e9f85df9187f2b50f9 \ - --hash=sha256:3236a4c8f535a0631f85f5fcdffba71c7feeef76a6002fcba7c1a8e57c8be1ec \ - --hash=sha256:3be66771aa5c97602f382230165b856c231d1277c511c9a8dd058be4784472e1 \ - --hash=sha256:3d087fcbec056c4ee34974da493a826ce316947485cef3901f511848e687c131 \ - --hash=sha256:3f298531bca95ff615b6e9f2fc0333aae27fa48052903a0ac90215021cdcfa4f \ - --hash=sha256:4a2b5cdbb5dd35aa08ea9114436e0d79aceb2f38e32c21684dcf8e24e1e92821 \ - --hash=sha256:4cf18f9d0efa1b16478c4c129eabec36148032575391095f73cae2e722fcf9d5 \ - --hash=sha256:8b2cbaca148d0754a54d44121b5825ae71868c7592a53b7292eeb0f3fdae95ee \ - --hash=sha256:8f55583b12156c399dce2df7d16f8a5095291354f1e839c252ec6c0611e86e2e \ - --hash=sha256:92f93b21c0fe73dc00abf91022234c79d793318b8a96faac147cd579c1671746 \ - --hash=sha256:9e36fb078cce9904c7989b9693e41cb9711e0600139ce3970c6ef814b6ebc2b2 \ - --hash=sha256:9fd50226364cd2737351c79807775136b0abe084433b55b2e29181a4c3c878c0 \ - --hash=sha256:a781f6ad4bab20eef8b65174a57e5203f4be627b46291f4589879bf4e257b97b \ - --hash=sha256:a87dbfa85971e8d59c9cc1fcf534efe664d8949e4c0b6b44e8ca548e746a8d53 \ - --hash=sha256:b808e12113505b97d9023b0b5e0c0705a90571c6feefc6f215c1df9381256e30 \ - --hash=sha256:bc6ac273b23c6b82da3bb25f4136c4fd42665f17f2cd850771cb600bdd2ebeda \ - --hash=sha256:cd777b780312ddb135bceb9bc8722a73ec95e042f911cc279e2ec3c667076051 \ - --hash=sha256:da1cbf08fb3b851ab3b9523a884c232774008267b1f83371ace57f412fe308c2 \ - --hash=sha256:e22e1527dc3d4aa94311d246b59e47f6455b8729f4968765ac1eacf9a4760bc7 \ - --hash=sha256:f8c083976eb530019175aabadb60921e73b4f45736760826aa1689dda8208aee \ - --hash=sha256:f90cff89eea89273727d8783fef5d4a934be2fdca11b47def50cf5d311aff727 \ - --hash=sha256:fa7ef5244615a2523b56c034becde4e9e3f9b034854c93639adb667ec9ec2976 \ - --hash=sha256:fcfc70599efde5c67862a07a1aaf50e55bce629ace26bb19dc17cece5dd31ca4 - # via -r requirements.in -mypy-extensions==1.0.0 \ - --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ - --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 - # via mypy -numpy==2.0.0 \ - --hash=sha256:04494f6ec467ccb5369d1808570ae55f6ed9b5809d7f035059000a37b8d7e86f \ - --hash=sha256:0a43f0974d501842866cc83471bdb0116ba0dffdbaac33ec05e6afed5b615238 \ - --hash=sha256:0e50842b2295ba8414c8c1d9d957083d5dfe9e16828b37de883f51fc53c4016f \ - --hash=sha256:0ec84b9ba0654f3b962802edc91424331f423dcf5d5f926676e0150789cb3d95 \ - --hash=sha256:17067d097ed036636fa79f6a869ac26df7db1ba22039d962422506640314933a \ - --hash=sha256:1cde1753efe513705a0c6d28f5884e22bdc30438bf0085c5c486cdaff40cd67a \ - --hash=sha256:1e72728e7501a450288fc8e1f9ebc73d90cfd4671ebbd631f3e7857c39bd16f2 \ - --hash=sha256:2635dbd200c2d6faf2ef9a0d04f0ecc6b13b3cad54f7c67c61155138835515d2 \ - --hash=sha256:2ce46fd0b8a0c947ae047d222f7136fc4d55538741373107574271bc00e20e8f \ - --hash=sha256:34f003cb88b1ba38cb9a9a4a3161c1604973d7f9d5552c38bc2f04f829536609 \ - --hash=sha256:354f373279768fa5a584bac997de6a6c9bc535c482592d7a813bb0c09be6c76f \ - --hash=sha256:38ecb5b0582cd125f67a629072fed6f83562d9dd04d7e03256c9829bdec027ad \ - --hash=sha256:3e8e01233d57639b2e30966c63d36fcea099d17c53bf424d77f088b0f4babd86 \ - --hash=sha256:3f6bed7f840d44c08ebdb73b1825282b801799e325bcbdfa6bc5c370e5aecc65 \ - --hash=sha256:4554eb96f0fd263041baf16cf0881b3f5dafae7a59b1049acb9540c4d57bc8cb \ - --hash=sha256:46e161722e0f619749d1cd892167039015b2c2817296104487cd03ed4a955995 \ - --hash=sha256:49d9f7d256fbc804391a7f72d4a617302b1afac1112fac19b6c6cec63fe7fe8a \ - --hash=sha256:4d2f62e55a4cd9c58c1d9a1c9edaedcd857a73cb6fda875bf79093f9d9086f85 \ - --hash=sha256:5f64641b42b2429f56ee08b4f427a4d2daf916ec59686061de751a55aafa22e4 \ - --hash=sha256:63b92c512d9dbcc37f9d81b123dec99fdb318ba38c8059afc78086fe73820275 \ - --hash=sha256:6d7696c615765091cc5093f76fd1fa069870304beaccfd58b5dcc69e55ef49c1 \ - --hash=sha256:79e843d186c8fb1b102bef3e2bc35ef81160ffef3194646a7fdd6a73c6b97196 \ - --hash=sha256:821eedb7165ead9eebdb569986968b541f9908979c2da8a4967ecac4439bae3d \ - --hash=sha256:84554fc53daa8f6abf8e8a66e076aff6ece62de68523d9f665f32d2fc50fd66e \ - --hash=sha256:8d83bb187fb647643bd56e1ae43f273c7f4dbcdf94550d7938cfc32566756514 \ - --hash=sha256:903703372d46bce88b6920a0cd86c3ad82dae2dbef157b5fc01b70ea1cfc430f \ - --hash=sha256:9416a5c2e92ace094e9f0082c5fd473502c91651fb896bc17690d6fc475128d6 \ - --hash=sha256:9a1712c015831da583b21c5bfe15e8684137097969c6d22e8316ba66b5baabe4 \ - --hash=sha256:9c27f0946a3536403efb0e1c28def1ae6730a72cd0d5878db38824855e3afc44 \ - --hash=sha256:a356364941fb0593bb899a1076b92dfa2029f6f5b8ba88a14fd0984aaf76d0df \ - --hash=sha256:a7039a136017eaa92c1848152827e1424701532ca8e8967fe480fe1569dae581 \ - --hash=sha256:acd3a644e4807e73b4e1867b769fbf1ce8c5d80e7caaef0d90dcdc640dfc9787 \ - --hash=sha256:ad0c86f3455fbd0de6c31a3056eb822fc939f81b1618f10ff3406971893b62a5 \ - --hash=sha256:b4c76e3d4c56f145d41b7b6751255feefae92edbc9a61e1758a98204200f30fc \ - --hash=sha256:b6f6a8f45d0313db07d6d1d37bd0b112f887e1369758a5419c0370ba915b3871 \ - --hash=sha256:c5a59996dc61835133b56a32ebe4ef3740ea5bc19b3983ac60cc32be5a665d54 \ - --hash=sha256:c73aafd1afca80afecb22718f8700b40ac7cab927b8abab3c3e337d70e10e5a2 \ - --hash=sha256:cee6cc0584f71adefe2c908856ccc98702baf95ff80092e4ca46061538a2ba98 \ - --hash=sha256:cef04d068f5fb0518a77857953193b6bb94809a806bd0a14983a8f12ada060c9 \ - --hash=sha256:cf5d1c9e6837f8af9f92b6bd3e86d513cdc11f60fd62185cc49ec7d1aba34864 \ - --hash=sha256:e61155fae27570692ad1d327e81c6cf27d535a5d7ef97648a17d922224b216de \ - --hash=sha256:e7f387600d424f91576af20518334df3d97bc76a300a755f9a8d6e4f5cadd289 \ - --hash=sha256:ed08d2703b5972ec736451b818c2eb9da80d66c3e84aed1deeb0c345fefe461b \ - --hash=sha256:fbd6acc766814ea6443628f4e6751d0da6593dae29c08c0b2606164db026970c \ - --hash=sha256:feff59f27338135776f6d4e2ec7aeeac5d5f7a08a83e80869121ef8164b74af9 - # via - # -r requirements.in - # pyarrow -packaging==24.1 \ - --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \ - --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124 - # via pytest -pluggy==1.5.0 \ - --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \ - --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669 - # via pytest -pyarrow==16.1.0 \ - --hash=sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a \ - --hash=sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2 \ - --hash=sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f \ - --hash=sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2 \ - --hash=sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315 \ - --hash=sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9 \ - --hash=sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b \ - --hash=sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55 \ - --hash=sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15 \ - --hash=sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e \ - --hash=sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f \ - --hash=sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c \ - --hash=sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a \ - --hash=sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa \ - --hash=sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a \ - --hash=sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd \ - --hash=sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628 \ - --hash=sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef \ - --hash=sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e \ - --hash=sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff \ - --hash=sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b \ - --hash=sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c \ - --hash=sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c \ - --hash=sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f \ - --hash=sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3 \ - --hash=sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6 \ - --hash=sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c \ - --hash=sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147 \ - --hash=sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5 \ - --hash=sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7 \ - --hash=sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710 \ - --hash=sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4 \ - --hash=sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed \ - --hash=sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848 \ - --hash=sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83 \ - --hash=sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444 - # via -r requirements.in -pytest==8.2.2 \ - --hash=sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343 \ - --hash=sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977 - # via -r requirements.in -ruff==0.4.9 \ - --hash=sha256:06b60f91bfa5514bb689b500a25ba48e897d18fea14dce14b48a0c40d1635893 \ - --hash=sha256:0e8e7b95673f22e0efd3571fb5b0cf71a5eaaa3cc8a776584f3b2cc878e46bff \ - --hash=sha256:2d45ddc6d82e1190ea737341326ecbc9a61447ba331b0a8962869fcada758505 \ - --hash=sha256:4555056049d46d8a381f746680db1c46e67ac3b00d714606304077682832998e \ - --hash=sha256:5d5460f789ccf4efd43f265a58538a2c24dbce15dbf560676e430375f20a8198 \ - --hash=sha256:673bddb893f21ab47a8334c8e0ea7fd6598ecc8e698da75bcd12a7b9d0a3206e \ - --hash=sha256:732dd550bfa5d85af8c3c6cbc47ba5b67c6aed8a89e2f011b908fc88f87649db \ - --hash=sha256:784d3ec9bd6493c3b720a0b76f741e6c2d7d44f6b2be87f5eef1ae8cc1d54c84 \ - --hash=sha256:78de3fdb95c4af084087628132336772b1c5044f6e710739d440fc0bccf4d321 \ - --hash=sha256:8064590fd1a50dcf4909c268b0e7c2498253273309ad3d97e4a752bb9df4f521 \ - --hash=sha256:88bffe9c6a454bf8529f9ab9091c99490578a593cc9f9822b7fc065ee0712a06 \ - --hash=sha256:8c1aff58c31948cc66d0b22951aa19edb5af0a3af40c936340cd32a8b1ab7438 \ - --hash=sha256:98ec2775fd2d856dc405635e5ee4ff177920f2141b8e2d9eb5bd6efd50e80317 \ - --hash=sha256:b262ed08d036ebe162123170b35703aaf9daffecb698cd367a8d585157732991 \ - --hash=sha256:e0a22c4157e53d006530c902107c7f550b9233e9706313ab57b892d7197d8e52 \ - --hash=sha256:e91175fbe48f8a2174c9aad70438fe9cb0a5732c4159b2a10a3565fea2d94cde \ - --hash=sha256:f1cb0828ac9533ba0135d148d214e284711ede33640465e706772645483427e3 - # via -r requirements.in -toml==0.10.2 \ - --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \ - --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f - # via -r requirements.in -typing-extensions==4.12.2 \ - --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ - --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 - # via mypy diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 1b7f62052..000000000 --- a/requirements.in +++ /dev/null @@ -1,26 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -maturin>=1.5.1 -mypy -numpy -pyarrow>=11.0.0 -pytest -ruff -toml -importlib_metadata; python_version < "3.8" -PyGitHub diff --git a/uv.lock b/uv.lock new file mode 100644 index 000000000..75d9ed018 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1842 @@ +version = 1 +requires-python = ">=3.8" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", + "python_full_version < '3.9'", +] + +[[package]] +name = "alabaster" +version = "0.7.13" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/94/71/a8ee96d1fd95ca04a0d2e2d9c4081dac4c2d2b12f7ddb899c8cb9bfd1532/alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2", size = 11454 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/88/c7083fc61120ab661c5d0b82cb77079fc1429d3f913a456c1c82cf4658f7/alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3", size = 13857 }, +] + +[[package]] +name = "alabaster" +version = "0.7.16" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/c9/3e/13dd8e5ed9094e734ac430b5d0eb4f2bb001708a8b7856cbf8e084e001ba/alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65", size = 23776 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/34/d4e1c02d3bee589efb5dfa17f88ea08bdb3e3eac12bc475462aec52ed223/alabaster-0.7.16-py3-none-any.whl", hash = "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92", size = 13511 }, +] + +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929 }, +] + +[[package]] +name = "appnope" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321 }, +] + +[[package]] +name = "astroid" +version = "3.2.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/53/1067e1113ecaf58312357f2cd93063674924119d80d173adc3f6f2387aa2/astroid-3.2.4.tar.gz", hash = "sha256:0e14202810b30da1b735827f78f5157be2bbd4a7a59b7707ca0bfc2fb4c0063a", size = 397576 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/80/96/b32bbbb46170a1c8b8b1f28c794202e25cfe743565e9d3469b8eb1e0cc05/astroid-3.2.4-py3-none-any.whl", hash = "sha256:413658a61eeca6202a59231abb473f932038fbcbf1666587f66d482083413a25", size = 276348 }, +] + +[[package]] +name = "astroid" +version = "3.3.8" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/c5/5c83c48bbf547f3dd8b587529db7cf5a265a3368b33e85e76af8ff6061d3/astroid-3.3.8.tar.gz", hash = "sha256:a88c7994f914a4ea8572fac479459f4955eeccc877be3f2d959a33273b0cf40b", size = 398196 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/28/0bc8a17d6cd4cc3c79ae41b7105a2b9a327c110e5ddd37a8a27b29a5c8a2/astroid-3.3.8-py3-none-any.whl", hash = "sha256:187ccc0c248bfbba564826c26f070494f7bc964fd286b6d9fff4420e55de828c", size = 275153 }, +] + +[[package]] +name = "asttokens" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/e7/82da0a03e7ba5141f05cce0d302e6eed121ae055e0456ca228bf693984bc/asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7", size = 61978 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/8a/c46dcc25341b5bce5472c718902eb3d38600a903b14fa6aeecef3f21a46f/asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2", size = 26918 }, +] + +[[package]] +name = "babel" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytz", marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2a/74/f1bc80f23eeba13393b7222b11d95ca3af2c1e28edca18af487137eefed9/babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316", size = 9348104 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/20/bc79bc575ba2e2a7f70e8a1155618bb1301eaa5132a8271373a6903f73f8/babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b", size = 9587599 }, +] + +[[package]] +name = "backcall" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/40/764a663805d84deee23043e1426a9175567db89c8b3287b5c2ad9f71aa93/backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", size = 18041 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255", size = 11157 }, +] + +[[package]] +name = "beautifulsoup4" +version = "4.12.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/ca/824b1195773ce6166d388573fc106ce56d4a805bd7427b624e063596ec58/beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051", size = 581181 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/fe/e8c672695b37eecc5cbf43e1d0638d88d66ba3a44c4d321c796f4e59167f/beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed", size = 147925 }, +] + +[[package]] +name = "certifi" +version = "2024.12.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/bd/1d41ee578ce09523c81a15426705dd20969f5abf006d1afe8aeff0dd776a/certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db", size = 166010 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/32/8f6669fc4798494966bf446c8c4a162e0b5d893dff088afddf76414f70e1/certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", size = 164927 }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/58/5580c1716040bc89206c77d8f74418caf82ce519aae06450393ca73475d1/charset_normalizer-3.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:91b36a978b5ae0ee86c394f5a54d6ef44db1de0815eb43de826d41d21e4af3de", size = 198013 }, + { url = "https://files.pythonhosted.org/packages/d0/11/00341177ae71c6f5159a08168bcb98c6e6d196d372c94511f9f6c9afe0c6/charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7461baadb4dc00fd9e0acbe254e3d7d2112e7f92ced2adc96e54ef6501c5f176", size = 141285 }, + { url = "https://files.pythonhosted.org/packages/01/09/11d684ea5819e5a8f5100fb0b38cf8d02b514746607934134d31233e02c8/charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e218488cd232553829be0664c2292d3af2eeeb94b32bea483cf79ac6a694e037", size = 151449 }, + { url = "https://files.pythonhosted.org/packages/08/06/9f5a12939db324d905dc1f70591ae7d7898d030d7662f0d426e2286f68c9/charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:80ed5e856eb7f30115aaf94e4a08114ccc8813e6ed1b5efa74f9f82e8509858f", size = 143892 }, + { url = "https://files.pythonhosted.org/packages/93/62/5e89cdfe04584cb7f4d36003ffa2936681b03ecc0754f8e969c2becb7e24/charset_normalizer-3.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b010a7a4fd316c3c484d482922d13044979e78d1861f0e0650423144c616a46a", size = 146123 }, + { url = "https://files.pythonhosted.org/packages/a9/ac/ab729a15c516da2ab70a05f8722ecfccc3f04ed7a18e45c75bbbaa347d61/charset_normalizer-3.4.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4532bff1b8421fd0a320463030c7520f56a79c9024a4e88f01c537316019005a", size = 147943 }, + { url = "https://files.pythonhosted.org/packages/03/d2/3f392f23f042615689456e9a274640c1d2e5dd1d52de36ab8f7955f8f050/charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d973f03c0cb71c5ed99037b870f2be986c3c05e63622c017ea9816881d2dd247", size = 142063 }, + { url = "https://files.pythonhosted.org/packages/f2/e3/e20aae5e1039a2cd9b08d9205f52142329f887f8cf70da3650326670bddf/charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3a3bd0dcd373514dcec91c411ddb9632c0d7d92aed7093b8c3bbb6d69ca74408", size = 150578 }, + { url = "https://files.pythonhosted.org/packages/8d/af/779ad72a4da0aed925e1139d458adc486e61076d7ecdcc09e610ea8678db/charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:d9c3cdf5390dcd29aa8056d13e8e99526cda0305acc038b96b30352aff5ff2bb", size = 153629 }, + { url = "https://files.pythonhosted.org/packages/c2/b6/7aa450b278e7aa92cf7732140bfd8be21f5f29d5bf334ae987c945276639/charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:2bdfe3ac2e1bbe5b59a1a63721eb3b95fc9b6817ae4a46debbb4e11f6232428d", size = 150778 }, + { url = "https://files.pythonhosted.org/packages/39/f4/d9f4f712d0951dcbfd42920d3db81b00dd23b6ab520419626f4023334056/charset_normalizer-3.4.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:eab677309cdb30d047996b36d34caeda1dc91149e4fdca0b1a039b3f79d9a807", size = 146453 }, + { url = "https://files.pythonhosted.org/packages/49/2b/999d0314e4ee0cff3cb83e6bc9aeddd397eeed693edb4facb901eb8fbb69/charset_normalizer-3.4.1-cp310-cp310-win32.whl", hash = "sha256:c0429126cf75e16c4f0ad00ee0eae4242dc652290f940152ca8c75c3a4b6ee8f", size = 95479 }, + { url = "https://files.pythonhosted.org/packages/2d/ce/3cbed41cff67e455a386fb5e5dd8906cdda2ed92fbc6297921f2e4419309/charset_normalizer-3.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:9f0b8b1c6d84c8034a44893aba5e767bf9c7a211e313a9605d9c617d7083829f", size = 102790 }, + { url = "https://files.pythonhosted.org/packages/72/80/41ef5d5a7935d2d3a773e3eaebf0a9350542f2cab4eac59a7a4741fbbbbe/charset_normalizer-3.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8bfa33f4f2672964266e940dd22a195989ba31669bd84629f05fab3ef4e2d125", size = 194995 }, + { url = "https://files.pythonhosted.org/packages/7a/28/0b9fefa7b8b080ec492110af6d88aa3dea91c464b17d53474b6e9ba5d2c5/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28bf57629c75e810b6ae989f03c0828d64d6b26a5e205535585f96093e405ed1", size = 139471 }, + { url = "https://files.pythonhosted.org/packages/71/64/d24ab1a997efb06402e3fc07317e94da358e2585165930d9d59ad45fcae2/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f08ff5e948271dc7e18a35641d2f11a4cd8dfd5634f55228b691e62b37125eb3", size = 149831 }, + { url = "https://files.pythonhosted.org/packages/37/ed/be39e5258e198655240db5e19e0b11379163ad7070962d6b0c87ed2c4d39/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:234ac59ea147c59ee4da87a0c0f098e9c8d169f4dc2a159ef720f1a61bbe27cd", size = 142335 }, + { url = "https://files.pythonhosted.org/packages/88/83/489e9504711fa05d8dde1574996408026bdbdbd938f23be67deebb5eca92/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd4ec41f914fa74ad1b8304bbc634b3de73d2a0889bd32076342a573e0779e00", size = 143862 }, + { url = "https://files.pythonhosted.org/packages/c6/c7/32da20821cf387b759ad24627a9aca289d2822de929b8a41b6241767b461/charset_normalizer-3.4.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eea6ee1db730b3483adf394ea72f808b6e18cf3cb6454b4d86e04fa8c4327a12", size = 145673 }, + { url = "https://files.pythonhosted.org/packages/68/85/f4288e96039abdd5aeb5c546fa20a37b50da71b5cf01e75e87f16cd43304/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c96836c97b1238e9c9e3fe90844c947d5afbf4f4c92762679acfe19927d81d77", size = 140211 }, + { url = "https://files.pythonhosted.org/packages/28/a3/a42e70d03cbdabc18997baf4f0227c73591a08041c149e710045c281f97b/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4d86f7aff21ee58f26dcf5ae81a9addbd914115cdebcbb2217e4f0ed8982e146", size = 148039 }, + { url = "https://files.pythonhosted.org/packages/85/e4/65699e8ab3014ecbe6f5c71d1a55d810fb716bbfd74f6283d5c2aa87febf/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:09b5e6733cbd160dcc09589227187e242a30a49ca5cefa5a7edd3f9d19ed53fd", size = 151939 }, + { url = "https://files.pythonhosted.org/packages/b1/82/8e9fe624cc5374193de6860aba3ea8070f584c8565ee77c168ec13274bd2/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5777ee0881f9499ed0f71cc82cf873d9a0ca8af166dfa0af8ec4e675b7df48e6", size = 149075 }, + { url = "https://files.pythonhosted.org/packages/3d/7b/82865ba54c765560c8433f65e8acb9217cb839a9e32b42af4aa8e945870f/charset_normalizer-3.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:237bdbe6159cff53b4f24f397d43c6336c6b0b42affbe857970cefbb620911c8", size = 144340 }, + { url = "https://files.pythonhosted.org/packages/b5/b6/9674a4b7d4d99a0d2df9b215da766ee682718f88055751e1e5e753c82db0/charset_normalizer-3.4.1-cp311-cp311-win32.whl", hash = "sha256:8417cb1f36cc0bc7eaba8ccb0e04d55f0ee52df06df3ad55259b9a323555fc8b", size = 95205 }, + { url = "https://files.pythonhosted.org/packages/1e/ab/45b180e175de4402dcf7547e4fb617283bae54ce35c27930a6f35b6bef15/charset_normalizer-3.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:d7f50a1f8c450f3925cb367d011448c39239bb3eb4117c36a6d354794de4ce76", size = 102441 }, + { url = "https://files.pythonhosted.org/packages/0a/9a/dd1e1cdceb841925b7798369a09279bd1cf183cef0f9ddf15a3a6502ee45/charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545", size = 196105 }, + { url = "https://files.pythonhosted.org/packages/d3/8c/90bfabf8c4809ecb648f39794cf2a84ff2e7d2a6cf159fe68d9a26160467/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7", size = 140404 }, + { url = "https://files.pythonhosted.org/packages/ad/8f/e410d57c721945ea3b4f1a04b74f70ce8fa800d393d72899f0a40526401f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757", size = 150423 }, + { url = "https://files.pythonhosted.org/packages/f0/b8/e6825e25deb691ff98cf5c9072ee0605dc2acfca98af70c2d1b1bc75190d/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa", size = 143184 }, + { url = "https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d", size = 145268 }, + { url = "https://files.pythonhosted.org/packages/74/94/8a5277664f27c3c438546f3eb53b33f5b19568eb7424736bdc440a88a31f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616", size = 147601 }, + { url = "https://files.pythonhosted.org/packages/7c/5f/6d352c51ee763623a98e31194823518e09bfa48be2a7e8383cf691bbb3d0/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b", size = 141098 }, + { url = "https://files.pythonhosted.org/packages/78/d4/f5704cb629ba5ab16d1d3d741396aec6dc3ca2b67757c45b0599bb010478/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d", size = 149520 }, + { url = "https://files.pythonhosted.org/packages/c5/96/64120b1d02b81785f222b976c0fb79a35875457fa9bb40827678e54d1bc8/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a", size = 152852 }, + { url = "https://files.pythonhosted.org/packages/84/c9/98e3732278a99f47d487fd3468bc60b882920cef29d1fa6ca460a1fdf4e6/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9", size = 150488 }, + { url = "https://files.pythonhosted.org/packages/13/0e/9c8d4cb99c98c1007cc11eda969ebfe837bbbd0acdb4736d228ccaabcd22/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1", size = 146192 }, + { url = "https://files.pythonhosted.org/packages/b2/21/2b6b5b860781a0b49427309cb8670785aa543fb2178de875b87b9cc97746/charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35", size = 95550 }, + { url = "https://files.pythonhosted.org/packages/21/5b/1b390b03b1d16c7e382b561c5329f83cc06623916aab983e8ab9239c7d5c/charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f", size = 102785 }, + { url = "https://files.pythonhosted.org/packages/38/94/ce8e6f63d18049672c76d07d119304e1e2d7c6098f0841b51c666e9f44a0/charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda", size = 195698 }, + { url = "https://files.pythonhosted.org/packages/24/2e/dfdd9770664aae179a96561cc6952ff08f9a8cd09a908f259a9dfa063568/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313", size = 140162 }, + { url = "https://files.pythonhosted.org/packages/24/4e/f646b9093cff8fc86f2d60af2de4dc17c759de9d554f130b140ea4738ca6/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9", size = 150263 }, + { url = "https://files.pythonhosted.org/packages/5e/67/2937f8d548c3ef6e2f9aab0f6e21001056f692d43282b165e7c56023e6dd/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b", size = 142966 }, + { url = "https://files.pythonhosted.org/packages/52/ed/b7f4f07de100bdb95c1756d3a4d17b90c1a3c53715c1a476f8738058e0fa/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11", size = 144992 }, + { url = "https://files.pythonhosted.org/packages/96/2c/d49710a6dbcd3776265f4c923bb73ebe83933dfbaa841c5da850fe0fd20b/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f", size = 147162 }, + { url = "https://files.pythonhosted.org/packages/b4/41/35ff1f9a6bd380303dea55e44c4933b4cc3c4850988927d4082ada230273/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd", size = 140972 }, + { url = "https://files.pythonhosted.org/packages/fb/43/c6a0b685fe6910d08ba971f62cd9c3e862a85770395ba5d9cad4fede33ab/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2", size = 149095 }, + { url = "https://files.pythonhosted.org/packages/4c/ff/a9a504662452e2d2878512115638966e75633519ec11f25fca3d2049a94a/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886", size = 152668 }, + { url = "https://files.pythonhosted.org/packages/6c/71/189996b6d9a4b932564701628af5cee6716733e9165af1d5e1b285c530ed/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601", size = 150073 }, + { url = "https://files.pythonhosted.org/packages/e4/93/946a86ce20790e11312c87c75ba68d5f6ad2208cfb52b2d6a2c32840d922/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd", size = 145732 }, + { url = "https://files.pythonhosted.org/packages/cd/e5/131d2fb1b0dddafc37be4f3a2fa79aa4c037368be9423061dccadfd90091/charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407", size = 95391 }, + { url = "https://files.pythonhosted.org/packages/27/f2/4f9a69cc7712b9b5ad8fdb87039fd89abba997ad5cbe690d1835d40405b0/charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971", size = 102702 }, + { url = "https://files.pythonhosted.org/packages/10/bd/6517ea94f2672e801011d50b5d06be2a0deaf566aea27bcdcd47e5195357/charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c", size = 195653 }, + { url = "https://files.pythonhosted.org/packages/e5/0d/815a2ba3f283b4eeaa5ece57acade365c5b4135f65a807a083c818716582/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9", size = 140701 }, + { url = "https://files.pythonhosted.org/packages/aa/17/c94be7ee0d142687e047fe1de72060f6d6837f40eedc26e87e6e124a3fc6/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8", size = 150495 }, + { url = "https://files.pythonhosted.org/packages/f7/33/557ac796c47165fc141e4fb71d7b0310f67e05cb420756f3a82e0a0068e0/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6", size = 142946 }, + { url = "https://files.pythonhosted.org/packages/1e/0d/38ef4ae41e9248d63fc4998d933cae22473b1b2ac4122cf908d0f5eb32aa/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c", size = 144737 }, + { url = "https://files.pythonhosted.org/packages/43/01/754cdb29dd0560f58290aaaa284d43eea343ad0512e6ad3b8b5c11f08592/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a", size = 147471 }, + { url = "https://files.pythonhosted.org/packages/ba/cd/861883ba5160c7a9bd242c30b2c71074cda2aefcc0addc91118e0d4e0765/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd", size = 140801 }, + { url = "https://files.pythonhosted.org/packages/6f/7f/0c0dad447819e90b93f8ed238cc8f11b91353c23c19e70fa80483a155bed/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd", size = 149312 }, + { url = "https://files.pythonhosted.org/packages/8e/09/9f8abcc6fff60fb727268b63c376c8c79cc37b833c2dfe1f535dfb59523b/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824", size = 152347 }, + { url = "https://files.pythonhosted.org/packages/be/e5/3f363dad2e24378f88ccf63ecc39e817c29f32e308ef21a7a6d9c1201165/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca", size = 149888 }, + { url = "https://files.pythonhosted.org/packages/e4/10/a78c0e91f487b4ad0ef7480ac765e15b774f83de2597f1b6ef0eaf7a2f99/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b", size = 145169 }, + { url = "https://files.pythonhosted.org/packages/d3/81/396e7d7f5d7420da8273c91175d2e9a3f569288e3611d521685e4b9ac9cc/charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e", size = 95094 }, + { url = "https://files.pythonhosted.org/packages/40/bb/20affbbd9ea29c71ea123769dc568a6d42052ff5089c5fe23e21e21084a6/charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4", size = 102139 }, + { url = "https://files.pythonhosted.org/packages/7f/c0/b913f8f02836ed9ab32ea643c6fe4d3325c3d8627cf6e78098671cafff86/charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41", size = 197867 }, + { url = "https://files.pythonhosted.org/packages/0f/6c/2bee440303d705b6fb1e2ec789543edec83d32d258299b16eed28aad48e0/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f", size = 141385 }, + { url = "https://files.pythonhosted.org/packages/3d/04/cb42585f07f6f9fd3219ffb6f37d5a39b4fd2db2355b23683060029c35f7/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2", size = 151367 }, + { url = "https://files.pythonhosted.org/packages/54/54/2412a5b093acb17f0222de007cc129ec0e0df198b5ad2ce5699355269dfe/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75832c08354f595c760a804588b9357d34ec00ba1c940c15e31e96d902093770", size = 143928 }, + { url = "https://files.pythonhosted.org/packages/5a/6d/e2773862b043dcf8a221342954f375392bb2ce6487bcd9f2c1b34e1d6781/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0af291f4fe114be0280cdd29d533696a77b5b49cfde5467176ecab32353395c4", size = 146203 }, + { url = "https://files.pythonhosted.org/packages/b9/f8/ca440ef60d8f8916022859885f231abb07ada3c347c03d63f283bec32ef5/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0167ddc8ab6508fe81860a57dd472b2ef4060e8d378f0cc555707126830f2537", size = 148082 }, + { url = "https://files.pythonhosted.org/packages/04/d2/42fd330901aaa4b805a1097856c2edf5095e260a597f65def493f4b8c833/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2a75d49014d118e4198bcee5ee0a6f25856b29b12dbf7cd012791f8a6cc5c496", size = 142053 }, + { url = "https://files.pythonhosted.org/packages/9e/af/3a97a4fa3c53586f1910dadfc916e9c4f35eeada36de4108f5096cb7215f/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363e2f92b0f0174b2f8238240a1a30142e3db7b957a5dd5689b0e75fb717cc78", size = 150625 }, + { url = "https://files.pythonhosted.org/packages/26/ae/23d6041322a3556e4da139663d02fb1b3c59a23ab2e2b56432bd2ad63ded/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ab36c8eb7e454e34e60eb55ca5d241a5d18b2c6244f6827a30e451c42410b5f7", size = 153549 }, + { url = "https://files.pythonhosted.org/packages/94/22/b8f2081c6a77cb20d97e57e0b385b481887aa08019d2459dc2858ed64871/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:4c0907b1928a36d5a998d72d64d8eaa7244989f7aaaf947500d3a800c83a3fd6", size = 150945 }, + { url = "https://files.pythonhosted.org/packages/c7/0b/c5ec5092747f801b8b093cdf5610e732b809d6cb11f4c51e35fc28d1d389/charset_normalizer-3.4.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:04432ad9479fa40ec0f387795ddad4437a2b50417c69fa275e212933519ff294", size = 146595 }, + { url = "https://files.pythonhosted.org/packages/0c/5a/0b59704c38470df6768aa154cc87b1ac7c9bb687990a1559dc8765e8627e/charset_normalizer-3.4.1-cp39-cp39-win32.whl", hash = "sha256:3bed14e9c89dcb10e8f3a29f9ccac4955aebe93c71ae803af79265c9ca5644c5", size = 95453 }, + { url = "https://files.pythonhosted.org/packages/85/2d/a9790237cb4d01a6d57afadc8573c8b73c609ade20b80f4cda30802009ee/charset_normalizer-3.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:49402233c892a461407c512a19435d1ce275543138294f7ef013f0b63d5d3765", size = 102811 }, + { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "datafusion" +version = "43.0.0" +source = { editable = "." } +dependencies = [ + { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "pyarrow", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] + +[package.dev-dependencies] +dev = [ + { name = "maturin" }, + { name = "numpy", version = "2.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest" }, + { name = "ruff" }, + { name = "toml" }, +] +docs = [ + { name = "ipython", version = "8.12.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "ipython", version = "8.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "ipython", version = "8.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "jinja2" }, + { name = "myst-parser", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "myst-parser", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pandas", version = "2.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "pickleshare" }, + { name = "pydata-sphinx-theme" }, + { name = "setuptools", version = "75.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "setuptools", version = "75.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinx-autoapi" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyarrow", specifier = ">=11.0.0" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "maturin", specifier = ">=1.8.1" }, + { name = "numpy", marker = "python_full_version >= '3.10'", specifier = ">1.24.4" }, + { name = "pytest", specifier = ">=7.4.4" }, + { name = "ruff", specifier = ">=0.9.1" }, + { name = "toml", specifier = ">=0.10.2" }, +] +docs = [ + { name = "ipython", specifier = ">=8.12.3" }, + { name = "jinja2", specifier = ">=3.1.5" }, + { name = "myst-parser", specifier = ">=3.0.1" }, + { name = "pandas", specifier = ">=2.0.3" }, + { name = "pickleshare", specifier = ">=0.7.5" }, + { name = "pydata-sphinx-theme", specifier = "==0.8.0" }, + { name = "setuptools", specifier = ">=75.3.0" }, + { name = "sphinx", specifier = ">=7.1.2" }, + { name = "sphinx-autoapi", specifier = ">=3.4.0" }, +] + +[[package]] +name = "decorator" +version = "5.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/0c/8d907af351aa16b42caae42f9d6aa37b900c67308052d10fdce809f8d952/decorator-5.1.1.tar.gz", hash = "sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330", size = 35016 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/50/83c593b07763e1161326b3b8c6686f0f4b0f24d5526546bee538c89837d6/decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186", size = 9073 }, +] + +[[package]] +name = "docutils" +version = "0.20.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/1f/53/a5da4f2c5739cf66290fac1431ee52aff6851c7c8ffd8264f13affd7bcdd/docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b", size = 2058365 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/87/f238c0670b94533ac0353a4e2a1a771a0cc73277b88bff23d3ae35a256c1/docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6", size = 572666 }, +] + +[[package]] +name = "docutils" +version = "0.21.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, +] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/09/35/2495c4ac46b980e4ca1f6ad6db102322ef3ad2410b79fdde159a4b0f3b92/exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc", size = 28883 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/cc/b7e31358aac6ed1ef2bb790a9746ac2c69bcb3c8588b41616914eb106eaf/exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", size = 16453 }, +] + +[[package]] +name = "executing" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/e3/7d45f492c2c4a0e8e0fad57d081a7c8a0286cdd86372b070cca1ec0caa1e/executing-2.1.0.tar.gz", hash = "sha256:8ea27ddd260da8150fa5a708269c4a10e76161e2496ec3e587da9e3c0fe4b9ab", size = 977485 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/fd/afcd0496feca3276f509df3dbd5dae726fcc756f1a08d9e25abe1733f962/executing-2.1.0-py2.py3-none-any.whl", hash = "sha256:8d63781349375b5ebccc3142f4b30350c0cd9c79f921cde38be2be4637e98eaf", size = 25805 }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, +] + +[[package]] +name = "imagesize" +version = "1.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/84/62473fb57d61e31fef6e36d64a179c8781605429fd927b5dd608c997be31/imagesize-1.4.1.tar.gz", hash = "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a", size = 1280026 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/62/85c4c919272577931d407be5ba5d71c20f0b616d31a0befe0ae45bb79abd/imagesize-1.4.1-py2.py3-none-any.whl", hash = "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", size = 8769 }, +] + +[[package]] +name = "importlib-metadata" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp", version = "3.20.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "zipp", version = "3.21.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514 }, +] + +[[package]] +name = "iniconfig" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/4b/cbd8e699e64a6f16ca3a8220661b5f83792b3017d0f79807cb8708d33913/iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", size = 4646 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, +] + +[[package]] +name = "ipython" +version = "8.12.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "appnope", marker = "python_full_version < '3.9' and sys_platform == 'darwin'" }, + { name = "backcall", marker = "python_full_version < '3.9'" }, + { name = "colorama", marker = "python_full_version < '3.9' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.9'" }, + { name = "jedi", marker = "python_full_version < '3.9'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.9'" }, + { name = "pexpect", marker = "python_full_version < '3.9' and sys_platform != 'win32'" }, + { name = "pickleshare", marker = "python_full_version < '3.9'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.9'" }, + { name = "pygments", marker = "python_full_version < '3.9'" }, + { name = "stack-data", marker = "python_full_version < '3.9'" }, + { name = "traitlets", marker = "python_full_version < '3.9'" }, + { name = "typing-extensions", marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/6a/44ef299b1762f5a73841e87fae8a73a8cc8aee538d6dc8c77a5afe1fd2ce/ipython-8.12.3.tar.gz", hash = "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363", size = 5470171 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/97/8fe103906cd81bc42d3b0175b5534a9f67dccae47d6451131cf8d0d70bb2/ipython-8.12.3-py3-none-any.whl", hash = "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c", size = 798307 }, +] + +[[package]] +name = "ipython" +version = "8.18.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version == '3.9.*' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version == '3.9.*'" }, + { name = "exceptiongroup", marker = "python_full_version == '3.9.*'" }, + { name = "jedi", marker = "python_full_version == '3.9.*'" }, + { name = "matplotlib-inline", marker = "python_full_version == '3.9.*'" }, + { name = "pexpect", marker = "python_full_version == '3.9.*' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version == '3.9.*'" }, + { name = "pygments", marker = "python_full_version == '3.9.*'" }, + { name = "stack-data", marker = "python_full_version == '3.9.*'" }, + { name = "traitlets", marker = "python_full_version == '3.9.*'" }, + { name = "typing-extensions", marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/b9/3ba6c45a6df813c09a48bac313c22ff83efa26cbb55011218d925a46e2ad/ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27", size = 5486330 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/6b/d9fdcdef2eb6a23f391251fde8781c38d42acd82abe84d054cb74f7863b0/ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397", size = 808161 }, +] + +[[package]] +name = "ipython" +version = "8.31.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version >= '3.10'" }, + { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, + { name = "jedi", marker = "python_full_version >= '3.10'" }, + { name = "matplotlib-inline", marker = "python_full_version >= '3.10'" }, + { name = "pexpect", marker = "python_full_version >= '3.10' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version >= '3.10'" }, + { name = "pygments", marker = "python_full_version >= '3.10'" }, + { name = "stack-data", marker = "python_full_version >= '3.10'" }, + { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.10' and python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/35/6f90fdddff7a08b7b715fccbd2427b5212c9525cd043d26fdc45bee0708d/ipython-8.31.0.tar.gz", hash = "sha256:b6a2274606bec6166405ff05e54932ed6e5cfecaca1fc05f2cacde7bb074d70b", size = 5501011 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/60/d0feb6b6d9fe4ab89fe8fe5b47cbf6cd936bfd9f1e7ffa9d0015425aeed6/ipython-8.31.0-py3-none-any.whl", hash = "sha256:46ec58f8d3d076a61d128fe517a51eb730e3aaf0c184ea8c17d16e366660c6a6", size = 821583 }, +] + +[[package]] +name = "jedi" +version = "0.19.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "parso" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/3a/79a912fbd4d8dd6fbb02bf69afd3bb72cf0c729bb3063c6f4498603db17a/jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0", size = 1231287 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/5a/9cac0c82afec3d09ccd97c8b6502d48f165f9124db81b4bcb90b4af974ee/jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9", size = 1572278 }, +] + +[[package]] +name = "jinja2" +version = "3.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe", version = "2.1.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "markupsafe", version = "3.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/92/b3130cbbf5591acf9ade8708c365f3238046ac7cb8ccba6e81abccb0ccff/jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb", size = 244674 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/0f/2ba5fbcd631e3e88689309dbe978c5769e883e4b84ebfe7da30b43275c5a/jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb", size = 134596 }, +] + +[[package]] +name = "markdown-it-py" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, +] + +[[package]] +name = "markupsafe" +version = "2.1.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/87/5b/aae44c6655f3801e81aa3eef09dbbf012431987ba564d7231722f68df02d/MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", size = 19384 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/54/ad5eb37bf9d51800010a74e4665425831a9db4e7c4e0fde4352e391e808e/MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", size = 18206 }, + { url = "https://files.pythonhosted.org/packages/6a/4a/a4d49415e600bacae038c67f9fecc1d5433b9d3c71a4de6f33537b89654c/MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", size = 14079 }, + { url = "https://files.pythonhosted.org/packages/0a/7b/85681ae3c33c385b10ac0f8dd025c30af83c78cec1c37a6aa3b55e67f5ec/MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", size = 26620 }, + { url = "https://files.pythonhosted.org/packages/7c/52/2b1b570f6b8b803cef5ac28fdf78c0da318916c7d2fe9402a84d591b394c/MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", size = 25818 }, + { url = "https://files.pythonhosted.org/packages/29/fe/a36ba8c7ca55621620b2d7c585313efd10729e63ef81e4e61f52330da781/MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", size = 25493 }, + { url = "https://files.pythonhosted.org/packages/60/ae/9c60231cdfda003434e8bd27282b1f4e197ad5a710c14bee8bea8a9ca4f0/MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", size = 30630 }, + { url = "https://files.pythonhosted.org/packages/65/dc/1510be4d179869f5dafe071aecb3f1f41b45d37c02329dfba01ff59e5ac5/MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", size = 29745 }, + { url = "https://files.pythonhosted.org/packages/30/39/8d845dd7d0b0613d86e0ef89549bfb5f61ed781f59af45fc96496e897f3a/MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", size = 30021 }, + { url = "https://files.pythonhosted.org/packages/c7/5c/356a6f62e4f3c5fbf2602b4771376af22a3b16efa74eb8716fb4e328e01e/MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", size = 16659 }, + { url = "https://files.pythonhosted.org/packages/69/48/acbf292615c65f0604a0c6fc402ce6d8c991276e16c80c46a8f758fbd30c/MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", size = 17213 }, + { url = "https://files.pythonhosted.org/packages/11/e7/291e55127bb2ae67c64d66cef01432b5933859dfb7d6949daa721b89d0b3/MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", size = 18219 }, + { url = "https://files.pythonhosted.org/packages/6b/cb/aed7a284c00dfa7c0682d14df85ad4955a350a21d2e3b06d8240497359bf/MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", size = 14098 }, + { url = "https://files.pythonhosted.org/packages/1c/cf/35fe557e53709e93feb65575c93927942087e9b97213eabc3fe9d5b25a55/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", size = 29014 }, + { url = "https://files.pythonhosted.org/packages/97/18/c30da5e7a0e7f4603abfc6780574131221d9148f323752c2755d48abad30/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", size = 28220 }, + { url = "https://files.pythonhosted.org/packages/0c/40/2e73e7d532d030b1e41180807a80d564eda53babaf04d65e15c1cf897e40/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", size = 27756 }, + { url = "https://files.pythonhosted.org/packages/18/46/5dca760547e8c59c5311b332f70605d24c99d1303dd9a6e1fc3ed0d73561/MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", size = 33988 }, + { url = "https://files.pythonhosted.org/packages/6d/c5/27febe918ac36397919cd4a67d5579cbbfa8da027fa1238af6285bb368ea/MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", size = 32718 }, + { url = "https://files.pythonhosted.org/packages/f8/81/56e567126a2c2bc2684d6391332e357589a96a76cb9f8e5052d85cb0ead8/MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", size = 33317 }, + { url = "https://files.pythonhosted.org/packages/00/0b/23f4b2470accb53285c613a3ab9ec19dc944eaf53592cb6d9e2af8aa24cc/MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", size = 16670 }, + { url = "https://files.pythonhosted.org/packages/b7/a2/c78a06a9ec6d04b3445a949615c4c7ed86a0b2eb68e44e7541b9d57067cc/MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", size = 17224 }, + { url = "https://files.pythonhosted.org/packages/53/bd/583bf3e4c8d6a321938c13f49d44024dbe5ed63e0a7ba127e454a66da974/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", size = 18215 }, + { url = "https://files.pythonhosted.org/packages/48/d6/e7cd795fc710292c3af3a06d80868ce4b02bfbbf370b7cee11d282815a2a/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", size = 14069 }, + { url = "https://files.pythonhosted.org/packages/51/b5/5d8ec796e2a08fc814a2c7d2584b55f889a55cf17dd1a90f2beb70744e5c/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", size = 29452 }, + { url = "https://files.pythonhosted.org/packages/0a/0d/2454f072fae3b5a137c119abf15465d1771319dfe9e4acbb31722a0fff91/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", size = 28462 }, + { url = "https://files.pythonhosted.org/packages/2d/75/fd6cb2e68780f72d47e6671840ca517bda5ef663d30ada7616b0462ad1e3/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", size = 27869 }, + { url = "https://files.pythonhosted.org/packages/b0/81/147c477391c2750e8fc7705829f7351cf1cd3be64406edcf900dc633feb2/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", size = 33906 }, + { url = "https://files.pythonhosted.org/packages/8b/ff/9a52b71839d7a256b563e85d11050e307121000dcebc97df120176b3ad93/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", size = 32296 }, + { url = "https://files.pythonhosted.org/packages/88/07/2dc76aa51b481eb96a4c3198894f38b480490e834479611a4053fbf08623/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", size = 33038 }, + { url = "https://files.pythonhosted.org/packages/96/0c/620c1fb3661858c0e37eb3cbffd8c6f732a67cd97296f725789679801b31/MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", size = 16572 }, + { url = "https://files.pythonhosted.org/packages/3f/14/c3554d512d5f9100a95e737502f4a2323a1959f6d0d01e0d0997b35f7b10/MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", size = 17127 }, + { url = "https://files.pythonhosted.org/packages/f8/ff/2c942a82c35a49df5de3a630ce0a8456ac2969691b230e530ac12314364c/MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", size = 18192 }, + { url = "https://files.pythonhosted.org/packages/4f/14/6f294b9c4f969d0c801a4615e221c1e084722ea6114ab2114189c5b8cbe0/MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", size = 14072 }, + { url = "https://files.pythonhosted.org/packages/81/d4/fd74714ed30a1dedd0b82427c02fa4deec64f173831ec716da11c51a50aa/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", size = 26928 }, + { url = "https://files.pythonhosted.org/packages/c7/bd/50319665ce81bb10e90d1cf76f9e1aa269ea6f7fa30ab4521f14d122a3df/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", size = 26106 }, + { url = "https://files.pythonhosted.org/packages/4c/6f/f2b0f675635b05f6afd5ea03c094557bdb8622fa8e673387444fe8d8e787/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68", size = 25781 }, + { url = "https://files.pythonhosted.org/packages/51/e0/393467cf899b34a9d3678e78961c2c8cdf49fb902a959ba54ece01273fb1/MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", size = 30518 }, + { url = "https://files.pythonhosted.org/packages/f6/02/5437e2ad33047290dafced9df741d9efc3e716b75583bbd73a9984f1b6f7/MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", size = 29669 }, + { url = "https://files.pythonhosted.org/packages/0e/7d/968284145ffd9d726183ed6237c77938c021abacde4e073020f920e060b2/MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", size = 29933 }, + { url = "https://files.pythonhosted.org/packages/bf/f3/ecb00fc8ab02b7beae8699f34db9357ae49d9f21d4d3de6f305f34fa949e/MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", size = 16656 }, + { url = "https://files.pythonhosted.org/packages/92/21/357205f03514a49b293e214ac39de01fadd0970a6e05e4bf1ddd0ffd0881/MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", size = 17206 }, + { url = "https://files.pythonhosted.org/packages/0f/31/780bb297db036ba7b7bbede5e1d7f1e14d704ad4beb3ce53fb495d22bc62/MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", size = 18193 }, + { url = "https://files.pythonhosted.org/packages/6c/77/d77701bbef72892affe060cdacb7a2ed7fd68dae3b477a8642f15ad3b132/MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", size = 14073 }, + { url = "https://files.pythonhosted.org/packages/d9/a7/1e558b4f78454c8a3a0199292d96159eb4d091f983bc35ef258314fe7269/MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", size = 26486 }, + { url = "https://files.pythonhosted.org/packages/5f/5a/360da85076688755ea0cceb92472923086993e86b5613bbae9fbc14136b0/MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", size = 25685 }, + { url = "https://files.pythonhosted.org/packages/6a/18/ae5a258e3401f9b8312f92b028c54d7026a97ec3ab20bfaddbdfa7d8cce8/MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", size = 25338 }, + { url = "https://files.pythonhosted.org/packages/0b/cc/48206bd61c5b9d0129f4d75243b156929b04c94c09041321456fd06a876d/MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", size = 30439 }, + { url = "https://files.pythonhosted.org/packages/d1/06/a41c112ab9ffdeeb5f77bc3e331fdadf97fa65e52e44ba31880f4e7f983c/MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", size = 29531 }, + { url = "https://files.pythonhosted.org/packages/02/8c/ab9a463301a50dab04d5472e998acbd4080597abc048166ded5c7aa768c8/MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", size = 29823 }, + { url = "https://files.pythonhosted.org/packages/bc/29/9bc18da763496b055d8e98ce476c8e718dcfd78157e17f555ce6dd7d0895/MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", size = 16658 }, + { url = "https://files.pythonhosted.org/packages/f6/f8/4da07de16f10551ca1f640c92b5f316f9394088b183c6a57183df6de5ae4/MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", size = 17211 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357 }, + { url = "https://files.pythonhosted.org/packages/04/e1/6e2194baeae0bca1fae6629dc0cbbb968d4d941469cbab11a3872edff374/MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158", size = 12393 }, + { url = "https://files.pythonhosted.org/packages/1d/69/35fa85a8ece0a437493dc61ce0bb6d459dcba482c34197e3efc829aa357f/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579", size = 21732 }, + { url = "https://files.pythonhosted.org/packages/22/35/137da042dfb4720b638d2937c38a9c2df83fe32d20e8c8f3185dbfef05f7/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d", size = 20866 }, + { url = "https://files.pythonhosted.org/packages/29/28/6d029a903727a1b62edb51863232152fd335d602def598dade38996887f0/MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb", size = 20964 }, + { url = "https://files.pythonhosted.org/packages/cc/cd/07438f95f83e8bc028279909d9c9bd39e24149b0d60053a97b2bc4f8aa51/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b", size = 21977 }, + { url = "https://files.pythonhosted.org/packages/29/01/84b57395b4cc062f9c4c55ce0df7d3108ca32397299d9df00fedd9117d3d/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c", size = 21366 }, + { url = "https://files.pythonhosted.org/packages/bd/6e/61ebf08d8940553afff20d1fb1ba7294b6f8d279df9fd0c0db911b4bbcfd/MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171", size = 21091 }, + { url = "https://files.pythonhosted.org/packages/11/23/ffbf53694e8c94ebd1e7e491de185124277964344733c45481f32ede2499/MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50", size = 15065 }, + { url = "https://files.pythonhosted.org/packages/44/06/e7175d06dd6e9172d4a69a72592cb3f7a996a9c396eee29082826449bbc3/MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a", size = 15514 }, + { url = "https://files.pythonhosted.org/packages/6b/28/bbf83e3f76936960b850435576dd5e67034e200469571be53f69174a2dfd/MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d", size = 14353 }, + { url = "https://files.pythonhosted.org/packages/6c/30/316d194b093cde57d448a4c3209f22e3046c5bb2fb0820b118292b334be7/MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93", size = 12392 }, + { url = "https://files.pythonhosted.org/packages/f2/96/9cdafba8445d3a53cae530aaf83c38ec64c4d5427d975c974084af5bc5d2/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832", size = 23984 }, + { url = "https://files.pythonhosted.org/packages/f1/a4/aefb044a2cd8d7334c8a47d3fb2c9f328ac48cb349468cc31c20b539305f/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84", size = 23120 }, + { url = "https://files.pythonhosted.org/packages/8d/21/5e4851379f88f3fad1de30361db501300d4f07bcad047d3cb0449fc51f8c/MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca", size = 23032 }, + { url = "https://files.pythonhosted.org/packages/00/7b/e92c64e079b2d0d7ddf69899c98842f3f9a60a1ae72657c89ce2655c999d/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798", size = 24057 }, + { url = "https://files.pythonhosted.org/packages/f9/ac/46f960ca323037caa0a10662ef97d0a4728e890334fc156b9f9e52bcc4ca/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e", size = 23359 }, + { url = "https://files.pythonhosted.org/packages/69/84/83439e16197337b8b14b6a5b9c2105fff81d42c2a7c5b58ac7b62ee2c3b1/MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4", size = 23306 }, + { url = "https://files.pythonhosted.org/packages/9a/34/a15aa69f01e2181ed8d2b685c0d2f6655d5cca2c4db0ddea775e631918cd/MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d", size = 15094 }, + { url = "https://files.pythonhosted.org/packages/da/b8/3a3bd761922d416f3dc5d00bfbed11f66b1ab89a0c2b6e887240a30b0f6b/MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b", size = 15521 }, + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348 }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149 }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118 }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993 }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178 }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319 }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352 }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097 }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601 }, + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352 }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122 }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085 }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978 }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208 }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357 }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344 }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101 }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603 }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510 }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486 }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480 }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914 }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796 }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473 }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114 }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098 }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208 }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, + { url = "https://files.pythonhosted.org/packages/a7/ea/9b1530c3fdeeca613faeb0fb5cbcf2389d816072fab72a71b45749ef6062/MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a", size = 14344 }, + { url = "https://files.pythonhosted.org/packages/4b/c2/fbdbfe48848e7112ab05e627e718e854d20192b674952d9042ebd8c9e5de/MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff", size = 12389 }, + { url = "https://files.pythonhosted.org/packages/f0/25/7a7c6e4dbd4f867d95d94ca15449e91e52856f6ed1905d58ef1de5e211d0/MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13", size = 21607 }, + { url = "https://files.pythonhosted.org/packages/53/8f/f339c98a178f3c1e545622206b40986a4c3307fe39f70ccd3d9df9a9e425/MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144", size = 20728 }, + { url = "https://files.pythonhosted.org/packages/1a/03/8496a1a78308456dbd50b23a385c69b41f2e9661c67ea1329849a598a8f9/MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29", size = 20826 }, + { url = "https://files.pythonhosted.org/packages/e6/cf/0a490a4bd363048c3022f2f475c8c05582179bb179defcee4766fb3dcc18/MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0", size = 21843 }, + { url = "https://files.pythonhosted.org/packages/19/a3/34187a78613920dfd3cdf68ef6ce5e99c4f3417f035694074beb8848cd77/MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0", size = 21219 }, + { url = "https://files.pythonhosted.org/packages/17/d8/5811082f85bb88410ad7e452263af048d685669bbbfb7b595e8689152498/MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178", size = 20946 }, + { url = "https://files.pythonhosted.org/packages/7c/31/bd635fb5989440d9365c5e3c47556cfea121c7803f5034ac843e8f37c2f2/MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f", size = 15063 }, + { url = "https://files.pythonhosted.org/packages/b3/73/085399401383ce949f727afec55ec3abd76648d04b9f22e1c0e99cb4bec3/MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a", size = 15506 }, +] + +[[package]] +name = "matplotlib-inline" +version = "0.1.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "traitlets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/5b/a36a337438a14116b16480db471ad061c36c3694df7c2084a0da7ba538b7/matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90", size = 8159 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/8e/9ad090d3553c280a8060fbf6e24dc1c0c29704ee7d1c372f0c174aa59285/matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca", size = 9899 }, +] + +[[package]] +name = "maturin" +version = "1.8.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9a/08/ccb0f917722a35ab0d758be9bb5edaf645c3a3d6170061f10d396ecd273f/maturin-1.8.1.tar.gz", hash = "sha256:49cd964aabf59f8b0a6969f9860d2cdf194ac331529caae14c884f5659568857", size = 197397 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/00/f34077315f34db8ad2ccf6bfe11b864ca27baab3a1320634da8e3cf89a48/maturin-1.8.1-py3-none-linux_armv6l.whl", hash = "sha256:7e590a23d9076b8a994f2e67bc63dc9a2d1c9a41b1e7b45ac354ba8275254e89", size = 7568415 }, + { url = "https://files.pythonhosted.org/packages/5c/07/9219976135ce0cb32d2fa6ea5c6d0ad709013d9a17967312e149b98153a6/maturin-1.8.1-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d8251a95682c83ea60988c804b620c181911cd824aa107b4a49ac5333c92968", size = 14527816 }, + { url = "https://files.pythonhosted.org/packages/e6/04/fa009a00903acdd1785d58322193140bfe358595347c39f315112dabdf9e/maturin-1.8.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b9fc1a4354cac5e32c190410208039812ea88c4a36bd2b6499268ec49ef5de00", size = 7580446 }, + { url = "https://files.pythonhosted.org/packages/9b/d4/414b2aab9bbfe88182b734d3aa1b4fef7d7701e50f6be48500378b8c8721/maturin-1.8.1-py3-none-manylinux_2_12_i686.manylinux2010_i686.musllinux_1_1_i686.whl", hash = "sha256:621e171c6b39f95f1d0df69a118416034fbd59c0f89dcaea8c2ea62019deecba", size = 7650535 }, + { url = "https://files.pythonhosted.org/packages/f0/64/879418a8a0196013ec1fb19eada0781c04a30e8d6d9227e80f91275a4f5b/maturin-1.8.1-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.musllinux_1_1_x86_64.whl", hash = "sha256:98f638739a5132962347871b85c91f525c9246ef4d99796ae98a2031e3df029f", size = 8006702 }, + { url = "https://files.pythonhosted.org/packages/39/c2/605829324f8371294f70303aca130682df75318958efed246873d3d604ab/maturin-1.8.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:f9f5c47521924b6e515cbc652a042fe5f17f8747445be9d931048e5d8ddb50a4", size = 7368164 }, + { url = "https://files.pythonhosted.org/packages/be/6c/30e136d397bb146b94b628c0ef7f17708281611b97849e2cf37847025ac7/maturin-1.8.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.musllinux_1_1_armv7l.whl", hash = "sha256:0f4407c7353c31bfbb8cdeb82bc2170e474cbfb97b5ba27568f440c9d6c1fdd4", size = 7450889 }, + { url = "https://files.pythonhosted.org/packages/1b/50/e1f5023512696d4e56096f702e2f68d6d9a30afe0a4eec82b0e27b8eb4e4/maturin-1.8.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.musllinux_1_1_ppc64le.whl", hash = "sha256:ec49cd70cad3c389946c6e2bc0bd50772a7fcb463040dd800720345897eec9bf", size = 9585819 }, + { url = "https://files.pythonhosted.org/packages/b7/80/b24b5248d89d2e5982553900237a337ea098ca9297b8369ca2aa95549e0f/maturin-1.8.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c08767d794de8f8a11c5c8b1b47a4ff9fb6ae2d2d97679e27030f2f509c8c2a0", size = 10920801 }, + { url = "https://files.pythonhosted.org/packages/6e/f4/8ede7a662fabf93456b44390a5ad22630e25fb5ddaecf787251071b2e143/maturin-1.8.1-py3-none-win32.whl", hash = "sha256:d678407713f3e10df33c5b3d7a343ec0551eb7f14d8ad9ba6febeb96f4e4c75c", size = 6873556 }, + { url = "https://files.pythonhosted.org/packages/9c/22/757f093ed0e319e9648155b8c9d716765442bea5bc98ebc58ad4ad5b0524/maturin-1.8.1-py3-none-win_amd64.whl", hash = "sha256:a526f90fe0e5cb59ffb81f4ff547ddc42e823bbdeae4a31012c0893ca6dcaf46", size = 7823153 }, + { url = "https://files.pythonhosted.org/packages/a4/f5/051413e04f6da25069db5e76759ecdb8cd2a8ab4a94045b5a3bf548c66fa/maturin-1.8.1-py3-none-win_arm64.whl", hash = "sha256:e95f077fd2ddd2f048182880eed458c308571a534be3eb2add4d3dac55bf57f4", size = 6552131 }, +] + +[[package]] +name = "mdit-py-plugins" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/03/a2ecab526543b152300717cf232bb4bb8605b6edb946c845016fa9c9c9fd/mdit_py_plugins-0.4.2.tar.gz", hash = "sha256:5f2cd1fdb606ddf152d37ec30e46101a60512bc0e5fa1a7002c36647b09e26b5", size = 43542 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/f7/7782a043553ee469c1ff49cfa1cdace2d6bf99a1f333cf38676b3ddf30da/mdit_py_plugins-0.4.2-py3-none-any.whl", hash = "sha256:0c673c3f889399a33b95e88d2f0d111b4447bdfea7f237dab2d488f459835636", size = 55316 }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, +] + +[[package]] +name = "myst-parser" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", + "python_full_version < '3.9'", +] +dependencies = [ + { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "jinja2", marker = "python_full_version < '3.10'" }, + { name = "markdown-it-py", marker = "python_full_version < '3.10'" }, + { name = "mdit-py-plugins", marker = "python_full_version < '3.10'" }, + { name = "pyyaml", marker = "python_full_version < '3.10'" }, + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/64/e2f13dac02f599980798c01156393b781aec983b52a6e4057ee58f07c43a/myst_parser-3.0.1.tar.gz", hash = "sha256:88f0cb406cb363b077d176b51c476f62d60604d68a8dcdf4832e080441301a87", size = 92392 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/de/21aa8394f16add8f7427f0a1326ccd2b3a2a8a3245c9252bc5ac034c6155/myst_parser-3.0.1-py3-none-any.whl", hash = "sha256:6457aaa33a5d474aca678b8ead9b3dc298e89c68e67012e73146ea6fd54babf1", size = 83163 }, +] + +[[package]] +name = "myst-parser" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "jinja2", marker = "python_full_version >= '3.10'" }, + { name = "markdown-it-py", marker = "python_full_version >= '3.10'" }, + { name = "mdit-py-plugins", marker = "python_full_version >= '3.10'" }, + { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/85/55/6d1741a1780e5e65038b74bce6689da15f620261c490c3511eb4c12bac4b/myst_parser-4.0.0.tar.gz", hash = "sha256:851c9dfb44e36e56d15d05e72f02b80da21a9e0d07cba96baf5e2d476bb91531", size = 93858 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/b4/b036f8fdb667587bb37df29dc6644681dd78b7a2a6321a34684b79412b28/myst_parser-4.0.0-py3-none-any.whl", hash = "sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d", size = 84563 }, +] + +[[package]] +name = "numpy" +version = "1.24.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/a4/9b/027bec52c633f6556dba6b722d9a0befb40498b9ceddd29cbe67a45a127c/numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463", size = 10911229 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/80/6cdfb3e275d95155a34659163b83c09e3a3ff9f1456880bec6cc63d71083/numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64", size = 19789140 }, + { url = "https://files.pythonhosted.org/packages/64/5f/3f01d753e2175cfade1013eea08db99ba1ee4bdb147ebcf3623b75d12aa7/numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1", size = 13854297 }, + { url = "https://files.pythonhosted.org/packages/5a/b3/2f9c21d799fa07053ffa151faccdceeb69beec5a010576b8991f614021f7/numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4", size = 13995611 }, + { url = "https://files.pythonhosted.org/packages/10/be/ae5bf4737cb79ba437879915791f6f26d92583c738d7d960ad94e5c36adf/numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6", size = 17282357 }, + { url = "https://files.pythonhosted.org/packages/c0/64/908c1087be6285f40e4b3e79454552a701664a079321cff519d8c7051d06/numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc", size = 12429222 }, + { url = "https://files.pythonhosted.org/packages/22/55/3d5a7c1142e0d9329ad27cece17933b0e2ab4e54ddc5c1861fbfeb3f7693/numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e", size = 14841514 }, + { url = "https://files.pythonhosted.org/packages/a9/cc/5ed2280a27e5dab12994c884f1f4d8c3bd4d885d02ae9e52a9d213a6a5e2/numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810", size = 19775508 }, + { url = "https://files.pythonhosted.org/packages/c0/bc/77635c657a3668cf652806210b8662e1aff84b818a55ba88257abf6637a8/numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254", size = 13840033 }, + { url = "https://files.pythonhosted.org/packages/a7/4c/96cdaa34f54c05e97c1c50f39f98d608f96f0677a6589e64e53104e22904/numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7", size = 13991951 }, + { url = "https://files.pythonhosted.org/packages/22/97/dfb1a31bb46686f09e68ea6ac5c63fdee0d22d7b23b8f3f7ea07712869ef/numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5", size = 17278923 }, + { url = "https://files.pythonhosted.org/packages/35/e2/76a11e54139654a324d107da1d98f99e7aa2a7ef97cfd7c631fba7dbde71/numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d", size = 12422446 }, + { url = "https://files.pythonhosted.org/packages/d8/ec/ebef2f7d7c28503f958f0f8b992e7ce606fb74f9e891199329d5f5f87404/numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694", size = 14834466 }, + { url = "https://files.pythonhosted.org/packages/11/10/943cfb579f1a02909ff96464c69893b1d25be3731b5d3652c2e0cf1281ea/numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61", size = 19780722 }, + { url = "https://files.pythonhosted.org/packages/a7/ae/f53b7b265fdc701e663fbb322a8e9d4b14d9cb7b2385f45ddfabfc4327e4/numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f", size = 13843102 }, + { url = "https://files.pythonhosted.org/packages/25/6f/2586a50ad72e8dbb1d8381f837008a0321a3516dfd7cb57fc8cf7e4bb06b/numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e", size = 14039616 }, + { url = "https://files.pythonhosted.org/packages/98/5d/5738903efe0ecb73e51eb44feafba32bdba2081263d40c5043568ff60faf/numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc", size = 17316263 }, + { url = "https://files.pythonhosted.org/packages/d1/57/8d328f0b91c733aa9aa7ee540dbc49b58796c862b4fbcb1146c701e888da/numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2", size = 12455660 }, + { url = "https://files.pythonhosted.org/packages/69/65/0d47953afa0ad569d12de5f65d964321c208492064c38fe3b0b9744f8d44/numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706", size = 14868112 }, + { url = "https://files.pythonhosted.org/packages/9a/cd/d5b0402b801c8a8b56b04c1e85c6165efab298d2f0ab741c2406516ede3a/numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400", size = 19816549 }, + { url = "https://files.pythonhosted.org/packages/14/27/638aaa446f39113a3ed38b37a66243e21b38110d021bfcb940c383e120f2/numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f", size = 13879950 }, + { url = "https://files.pythonhosted.org/packages/8f/27/91894916e50627476cff1a4e4363ab6179d01077d71b9afed41d9e1f18bf/numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9", size = 14030228 }, + { url = "https://files.pythonhosted.org/packages/7a/7c/d7b2a0417af6428440c0ad7cb9799073e507b1a465f827d058b826236964/numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d", size = 17311170 }, + { url = "https://files.pythonhosted.org/packages/18/9d/e02ace5d7dfccee796c37b995c63322674daf88ae2f4a4724c5dd0afcc91/numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835", size = 12454918 }, + { url = "https://files.pythonhosted.org/packages/63/38/6cc19d6b8bfa1d1a459daf2b3fe325453153ca7019976274b6f33d8b5663/numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8", size = 14867441 }, + { url = "https://files.pythonhosted.org/packages/a4/fd/8dff40e25e937c94257455c237b9b6bf5a30d42dd1cc11555533be099492/numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef", size = 19156590 }, + { url = "https://files.pythonhosted.org/packages/42/e7/4bf953c6e05df90c6d351af69966384fed8e988d0e8c54dad7103b59f3ba/numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a", size = 16705744 }, + { url = "https://files.pythonhosted.org/packages/fc/dd/9106005eb477d022b60b3817ed5937a43dad8fd1f20b0610ea8a32fcb407/numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2", size = 14734290 }, +] + +[[package]] +name = "numpy" +version = "2.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/91/3495b3237510f79f5d81f2508f9f13fea78ebfdf07538fc7444badda173d/numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece", size = 21165245 }, + { url = "https://files.pythonhosted.org/packages/05/33/26178c7d437a87082d11019292dce6d3fe6f0e9026b7b2309cbf3e489b1d/numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04", size = 13738540 }, + { url = "https://files.pythonhosted.org/packages/ec/31/cc46e13bf07644efc7a4bf68df2df5fb2a1a88d0cd0da9ddc84dc0033e51/numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66", size = 5300623 }, + { url = "https://files.pythonhosted.org/packages/6e/16/7bfcebf27bb4f9d7ec67332ffebee4d1bf085c84246552d52dbb548600e7/numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b", size = 6901774 }, + { url = "https://files.pythonhosted.org/packages/f9/a3/561c531c0e8bf082c5bef509d00d56f82e0ea7e1e3e3a7fc8fa78742a6e5/numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd", size = 13907081 }, + { url = "https://files.pythonhosted.org/packages/fa/66/f7177ab331876200ac7563a580140643d1179c8b4b6a6b0fc9838de2a9b8/numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318", size = 19523451 }, + { url = "https://files.pythonhosted.org/packages/25/7f/0b209498009ad6453e4efc2c65bcdf0ae08a182b2b7877d7ab38a92dc542/numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8", size = 19927572 }, + { url = "https://files.pythonhosted.org/packages/3e/df/2619393b1e1b565cd2d4c4403bdd979621e2c4dea1f8532754b2598ed63b/numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326", size = 14400722 }, + { url = "https://files.pythonhosted.org/packages/22/ad/77e921b9f256d5da36424ffb711ae79ca3f451ff8489eeca544d0701d74a/numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97", size = 6472170 }, + { url = "https://files.pythonhosted.org/packages/10/05/3442317535028bc29cf0c0dd4c191a4481e8376e9f0db6bcf29703cadae6/numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131", size = 15905558 }, + { url = "https://files.pythonhosted.org/packages/8b/cf/034500fb83041aa0286e0fb16e7c76e5c8b67c0711bb6e9e9737a717d5fe/numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448", size = 21169137 }, + { url = "https://files.pythonhosted.org/packages/4a/d9/32de45561811a4b87fbdee23b5797394e3d1504b4a7cf40c10199848893e/numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195", size = 13703552 }, + { url = "https://files.pythonhosted.org/packages/c1/ca/2f384720020c7b244d22508cb7ab23d95f179fcfff33c31a6eeba8d6c512/numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57", size = 5298957 }, + { url = "https://files.pythonhosted.org/packages/0e/78/a3e4f9fb6aa4e6fdca0c5428e8ba039408514388cf62d89651aade838269/numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a", size = 6905573 }, + { url = "https://files.pythonhosted.org/packages/a0/72/cfc3a1beb2caf4efc9d0b38a15fe34025230da27e1c08cc2eb9bfb1c7231/numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669", size = 13914330 }, + { url = "https://files.pythonhosted.org/packages/ba/a8/c17acf65a931ce551fee11b72e8de63bf7e8a6f0e21add4c937c83563538/numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951", size = 19534895 }, + { url = "https://files.pythonhosted.org/packages/ba/86/8767f3d54f6ae0165749f84648da9dcc8cd78ab65d415494962c86fac80f/numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9", size = 19937253 }, + { url = "https://files.pythonhosted.org/packages/df/87/f76450e6e1c14e5bb1eae6836478b1028e096fd02e85c1c37674606ab752/numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15", size = 14414074 }, + { url = "https://files.pythonhosted.org/packages/5c/ca/0f0f328e1e59f73754f06e1adfb909de43726d4f24c6a3f8805f34f2b0fa/numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4", size = 6470640 }, + { url = "https://files.pythonhosted.org/packages/eb/57/3a3f14d3a759dcf9bf6e9eda905794726b758819df4663f217d658a58695/numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc", size = 15910230 }, + { url = "https://files.pythonhosted.org/packages/45/40/2e117be60ec50d98fa08c2f8c48e09b3edea93cfcabd5a9ff6925d54b1c2/numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b", size = 20895803 }, + { url = "https://files.pythonhosted.org/packages/46/92/1b8b8dee833f53cef3e0a3f69b2374467789e0bb7399689582314df02651/numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e", size = 13471835 }, + { url = "https://files.pythonhosted.org/packages/7f/19/e2793bde475f1edaea6945be141aef6c8b4c669b90c90a300a8954d08f0a/numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c", size = 5038499 }, + { url = "https://files.pythonhosted.org/packages/e3/ff/ddf6dac2ff0dd50a7327bcdba45cb0264d0e96bb44d33324853f781a8f3c/numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c", size = 6633497 }, + { url = "https://files.pythonhosted.org/packages/72/21/67f36eac8e2d2cd652a2e69595a54128297cdcb1ff3931cfc87838874bd4/numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692", size = 13621158 }, + { url = "https://files.pythonhosted.org/packages/39/68/e9f1126d757653496dbc096cb429014347a36b228f5a991dae2c6b6cfd40/numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a", size = 19236173 }, + { url = "https://files.pythonhosted.org/packages/d1/e9/1f5333281e4ebf483ba1c888b1d61ba7e78d7e910fdd8e6499667041cc35/numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c", size = 19634174 }, + { url = "https://files.pythonhosted.org/packages/71/af/a469674070c8d8408384e3012e064299f7a2de540738a8e414dcfd639996/numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded", size = 14099701 }, + { url = "https://files.pythonhosted.org/packages/d0/3d/08ea9f239d0e0e939b6ca52ad403c84a2bce1bde301a8eb4888c1c1543f1/numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5", size = 6174313 }, + { url = "https://files.pythonhosted.org/packages/b2/b5/4ac39baebf1fdb2e72585c8352c56d063b6126be9fc95bd2bb5ef5770c20/numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a", size = 15606179 }, + { url = "https://files.pythonhosted.org/packages/43/c1/41c8f6df3162b0c6ffd4437d729115704bd43363de0090c7f913cfbc2d89/numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c", size = 21169942 }, + { url = "https://files.pythonhosted.org/packages/39/bc/fd298f308dcd232b56a4031fd6ddf11c43f9917fbc937e53762f7b5a3bb1/numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd", size = 13711512 }, + { url = "https://files.pythonhosted.org/packages/96/ff/06d1aa3eeb1c614eda245c1ba4fb88c483bee6520d361641331872ac4b82/numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b", size = 5306976 }, + { url = "https://files.pythonhosted.org/packages/2d/98/121996dcfb10a6087a05e54453e28e58694a7db62c5a5a29cee14c6e047b/numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729", size = 6906494 }, + { url = "https://files.pythonhosted.org/packages/15/31/9dffc70da6b9bbf7968f6551967fc21156207366272c2a40b4ed6008dc9b/numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1", size = 13912596 }, + { url = "https://files.pythonhosted.org/packages/b9/14/78635daab4b07c0930c919d451b8bf8c164774e6a3413aed04a6d95758ce/numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd", size = 19526099 }, + { url = "https://files.pythonhosted.org/packages/26/4c/0eeca4614003077f68bfe7aac8b7496f04221865b3a5e7cb230c9d055afd/numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d", size = 19932823 }, + { url = "https://files.pythonhosted.org/packages/f1/46/ea25b98b13dccaebddf1a803f8c748680d972e00507cd9bc6dcdb5aa2ac1/numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d", size = 14404424 }, + { url = "https://files.pythonhosted.org/packages/c8/a6/177dd88d95ecf07e722d21008b1b40e681a929eb9e329684d449c36586b2/numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa", size = 6476809 }, + { url = "https://files.pythonhosted.org/packages/ea/2b/7fc9f4e7ae5b507c1a3a21f0f15ed03e794c1242ea8a242ac158beb56034/numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73", size = 15911314 }, + { url = "https://files.pythonhosted.org/packages/8f/3b/df5a870ac6a3be3a86856ce195ef42eec7ae50d2a202be1f5a4b3b340e14/numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8", size = 21025288 }, + { url = "https://files.pythonhosted.org/packages/2c/97/51af92f18d6f6f2d9ad8b482a99fb74e142d71372da5d834b3a2747a446e/numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4", size = 6762793 }, + { url = "https://files.pythonhosted.org/packages/12/46/de1fbd0c1b5ccaa7f9a005b66761533e2f6a3e560096682683a223631fe9/numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c", size = 19334885 }, + { url = "https://files.pythonhosted.org/packages/cc/dc/d330a6faefd92b446ec0f0dfea4c3207bb1fef3c4771d19cf4543efd2c78/numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385", size = 15828784 }, +] + +[[package]] +name = "numpy" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/a5/fdbf6a7871703df6160b5cf3dd774074b086d278172285c52c2758b76305/numpy-2.2.1.tar.gz", hash = "sha256:45681fd7128c8ad1c379f0ca0776a8b0c6583d2f69889ddac01559dfe4390918", size = 20227662 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/c4/5588367dc9f91e1a813beb77de46ea8cab13f778e1b3a0e661ab031aba44/numpy-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5edb4e4caf751c1518e6a26a83501fda79bff41cc59dac48d70e6d65d4ec4440", size = 21213214 }, + { url = "https://files.pythonhosted.org/packages/d8/8b/32dd9f08419023a4cf856c5ad0b4eba9b830da85eafdef841a104c4fc05a/numpy-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa3017c40d513ccac9621a2364f939d39e550c542eb2a894b4c8da92b38896ab", size = 14352248 }, + { url = "https://files.pythonhosted.org/packages/84/2d/0e895d02940ba6e12389f0ab5cac5afcf8dc2dc0ade4e8cad33288a721bd/numpy-2.2.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:61048b4a49b1c93fe13426e04e04fdf5a03f456616f6e98c7576144677598675", size = 5391007 }, + { url = "https://files.pythonhosted.org/packages/11/b9/7f1e64a0d46d9c2af6d17966f641fb12d5b8ea3003f31b2308f3e3b9a6aa/numpy-2.2.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:7671dc19c7019103ca44e8d94917eba8534c76133523ca8406822efdd19c9308", size = 6926174 }, + { url = "https://files.pythonhosted.org/packages/2e/8c/043fa4418bc9364e364ab7aba8ff6ef5f6b9171ade22de8fbcf0e2fa4165/numpy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4250888bcb96617e00bfa28ac24850a83c9f3a16db471eca2ee1f1714df0f957", size = 14330914 }, + { url = "https://files.pythonhosted.org/packages/f7/b6/d8110985501ca8912dfc1c3bbef99d66e62d487f72e46b2337494df77364/numpy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7746f235c47abc72b102d3bce9977714c2444bdfaea7888d241b4c4bb6a78bf", size = 16379607 }, + { url = "https://files.pythonhosted.org/packages/e2/57/bdca9fb8bdaa810c3a4ff2eb3231379b77f618a7c0d24be9f7070db50775/numpy-2.2.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:059e6a747ae84fce488c3ee397cee7e5f905fd1bda5fb18c66bc41807ff119b2", size = 15541760 }, + { url = "https://files.pythonhosted.org/packages/97/55/3b9147b3cbc3b6b1abc2a411dec5337a46c873deca0dd0bf5bef9d0579cc/numpy-2.2.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f62aa6ee4eb43b024b0e5a01cf65a0bb078ef8c395e8713c6e8a12a697144528", size = 18168476 }, + { url = "https://files.pythonhosted.org/packages/00/e7/7c2cde16c9b87a8e14fdd262ca7849c4681cf48c8a774505f7e6f5e3b643/numpy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:48fd472630715e1c1c89bf1feab55c29098cb403cc184b4859f9c86d4fcb6a95", size = 6570985 }, + { url = "https://files.pythonhosted.org/packages/a1/a8/554b0e99fc4ac11ec481254781a10da180d0559c2ebf2c324232317349ee/numpy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:b541032178a718c165a49638d28272b771053f628382d5e9d1c93df23ff58dbf", size = 12913384 }, + { url = "https://files.pythonhosted.org/packages/59/14/645887347124e101d983e1daf95b48dc3e136bf8525cb4257bf9eab1b768/numpy-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:40f9e544c1c56ba8f1cf7686a8c9b5bb249e665d40d626a23899ba6d5d9e1484", size = 21217379 }, + { url = "https://files.pythonhosted.org/packages/9f/fd/2279000cf29f58ccfd3778cbf4670dfe3f7ce772df5e198c5abe9e88b7d7/numpy-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9b57eaa3b0cd8db52049ed0330747b0364e899e8a606a624813452b8203d5f7", size = 14388520 }, + { url = "https://files.pythonhosted.org/packages/58/b0/034eb5d5ba12d66ab658ff3455a31f20add0b78df8203c6a7451bd1bee21/numpy-2.2.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:bc8a37ad5b22c08e2dbd27df2b3ef7e5c0864235805b1e718a235bcb200cf1cb", size = 5389286 }, + { url = "https://files.pythonhosted.org/packages/5d/69/6f3cccde92e82e7835fdb475c2bf439761cbf8a1daa7c07338e1e132dfec/numpy-2.2.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:9036d6365d13b6cbe8f27a0eaf73ddcc070cae584e5ff94bb45e3e9d729feab5", size = 6930345 }, + { url = "https://files.pythonhosted.org/packages/d1/72/1cd38e91ab563e67f584293fcc6aca855c9ae46dba42e6b5ff4600022899/numpy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51faf345324db860b515d3f364eaa93d0e0551a88d6218a7d61286554d190d73", size = 14335748 }, + { url = "https://files.pythonhosted.org/packages/f2/d4/f999444e86986f3533e7151c272bd8186c55dda554284def18557e013a2a/numpy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38efc1e56b73cc9b182fe55e56e63b044dd26a72128fd2fbd502f75555d92591", size = 16391057 }, + { url = "https://files.pythonhosted.org/packages/99/7b/85cef6a3ae1b19542b7afd97d0b296526b6ef9e3c43ea0c4d9c4404fb2d0/numpy-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:31b89fa67a8042e96715c68e071a1200c4e172f93b0fbe01a14c0ff3ff820fc8", size = 15556943 }, + { url = "https://files.pythonhosted.org/packages/69/7e/b83cc884c3508e91af78760f6b17ab46ad649831b1fa35acb3eb26d9e6d2/numpy-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:4c86e2a209199ead7ee0af65e1d9992d1dce7e1f63c4b9a616500f93820658d0", size = 18180785 }, + { url = "https://files.pythonhosted.org/packages/b2/9f/eb4a9a38867de059dcd4b6e18d47c3867fbd3795d4c9557bb49278f94087/numpy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:b34d87e8a3090ea626003f87f9392b3929a7bbf4104a05b6667348b6bd4bf1cd", size = 6568983 }, + { url = "https://files.pythonhosted.org/packages/6d/1e/be3b9f3073da2f8c7fa361fcdc231b548266b0781029fdbaf75eeab997fd/numpy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:360137f8fb1b753c5cde3ac388597ad680eccbbbb3865ab65efea062c4a1fd16", size = 12917260 }, + { url = "https://files.pythonhosted.org/packages/62/12/b928871c570d4a87ab13d2cc19f8817f17e340d5481621930e76b80ffb7d/numpy-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:694f9e921a0c8f252980e85bce61ebbd07ed2b7d4fa72d0e4246f2f8aa6642ab", size = 20909861 }, + { url = "https://files.pythonhosted.org/packages/3d/c3/59df91ae1d8ad7c5e03efd63fd785dec62d96b0fe56d1f9ab600b55009af/numpy-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3683a8d166f2692664262fd4900f207791d005fb088d7fdb973cc8d663626faa", size = 14095776 }, + { url = "https://files.pythonhosted.org/packages/af/4e/8ed5868efc8e601fb69419644a280e9c482b75691466b73bfaab7d86922c/numpy-2.2.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:780077d95eafc2ccc3ced969db22377b3864e5b9a0ea5eb347cc93b3ea900315", size = 5126239 }, + { url = "https://files.pythonhosted.org/packages/1a/74/dd0bbe650d7bc0014b051f092f2de65e34a8155aabb1287698919d124d7f/numpy-2.2.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:55ba24ebe208344aa7a00e4482f65742969a039c2acfcb910bc6fcd776eb4355", size = 6659296 }, + { url = "https://files.pythonhosted.org/packages/7f/11/4ebd7a3f4a655764dc98481f97bd0a662fb340d1001be6050606be13e162/numpy-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b1d07b53b78bf84a96898c1bc139ad7f10fda7423f5fd158fd0f47ec5e01ac7", size = 14047121 }, + { url = "https://files.pythonhosted.org/packages/7f/a7/c1f1d978166eb6b98ad009503e4d93a8c1962d0eb14a885c352ee0276a54/numpy-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5062dc1a4e32a10dc2b8b13cedd58988261416e811c1dc4dbdea4f57eea61b0d", size = 16096599 }, + { url = "https://files.pythonhosted.org/packages/3d/6d/0e22afd5fcbb4d8d0091f3f46bf4e8906399c458d4293da23292c0ba5022/numpy-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fce4f615f8ca31b2e61aa0eb5865a21e14f5629515c9151850aa936c02a1ee51", size = 15243932 }, + { url = "https://files.pythonhosted.org/packages/03/39/e4e5832820131ba424092b9610d996b37e5557180f8e2d6aebb05c31ae54/numpy-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:67d4cda6fa6ffa073b08c8372aa5fa767ceb10c9a0587c707505a6d426f4e046", size = 17861032 }, + { url = "https://files.pythonhosted.org/packages/5f/8a/3794313acbf5e70df2d5c7d2aba8718676f8d054a05abe59e48417fb2981/numpy-2.2.1-cp312-cp312-win32.whl", hash = "sha256:32cb94448be47c500d2c7a95f93e2f21a01f1fd05dd2beea1ccd049bb6001cd2", size = 6274018 }, + { url = "https://files.pythonhosted.org/packages/17/c1/c31d3637f2641e25c7a19adf2ae822fdaf4ddd198b05d79a92a9ce7cb63e/numpy-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:ba5511d8f31c033a5fcbda22dd5c813630af98c70b2661f2d2c654ae3cdfcfc8", size = 12613843 }, + { url = "https://files.pythonhosted.org/packages/20/d6/91a26e671c396e0c10e327b763485ee295f5a5a7a48c553f18417e5a0ed5/numpy-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f1d09e520217618e76396377c81fba6f290d5f926f50c35f3a5f72b01a0da780", size = 20896464 }, + { url = "https://files.pythonhosted.org/packages/8c/40/5792ccccd91d45e87d9e00033abc4f6ca8a828467b193f711139ff1f1cd9/numpy-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3ecc47cd7f6ea0336042be87d9e7da378e5c7e9b3c8ad0f7c966f714fc10d821", size = 14111350 }, + { url = "https://files.pythonhosted.org/packages/c0/2a/fb0a27f846cb857cef0c4c92bef89f133a3a1abb4e16bba1c4dace2e9b49/numpy-2.2.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f419290bc8968a46c4933158c91a0012b7a99bb2e465d5ef5293879742f8797e", size = 5111629 }, + { url = "https://files.pythonhosted.org/packages/eb/e5/8e81bb9d84db88b047baf4e8b681a3e48d6390bc4d4e4453eca428ecbb49/numpy-2.2.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:5b6c390bfaef8c45a260554888966618328d30e72173697e5cabe6b285fb2348", size = 6645865 }, + { url = "https://files.pythonhosted.org/packages/7a/1a/a90ceb191dd2f9e2897c69dde93ccc2d57dd21ce2acbd7b0333e8eea4e8d/numpy-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:526fc406ab991a340744aad7e25251dd47a6720a685fa3331e5c59fef5282a59", size = 14043508 }, + { url = "https://files.pythonhosted.org/packages/f1/5a/e572284c86a59dec0871a49cd4e5351e20b9c751399d5f1d79628c0542cb/numpy-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74e6fdeb9a265624ec3a3918430205dff1df7e95a230779746a6af78bc615af", size = 16094100 }, + { url = "https://files.pythonhosted.org/packages/0c/2c/a79d24f364788386d85899dd280a94f30b0950be4b4a545f4fa4ed1d4ca7/numpy-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53c09385ff0b72ba79d8715683c1168c12e0b6e84fb0372e97553d1ea91efe51", size = 15239691 }, + { url = "https://files.pythonhosted.org/packages/cf/79/1e20fd1c9ce5a932111f964b544facc5bb9bde7865f5b42f00b4a6a9192b/numpy-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f3eac17d9ec51be534685ba877b6ab5edc3ab7ec95c8f163e5d7b39859524716", size = 17856571 }, + { url = "https://files.pythonhosted.org/packages/be/5b/cc155e107f75d694f562bdc84a26cc930569f3dfdfbccb3420b626065777/numpy-2.2.1-cp313-cp313-win32.whl", hash = "sha256:9ad014faa93dbb52c80d8f4d3dcf855865c876c9660cb9bd7553843dd03a4b1e", size = 6270841 }, + { url = "https://files.pythonhosted.org/packages/44/be/0e5cd009d2162e4138d79a5afb3b5d2341f0fe4777ab6e675aa3d4a42e21/numpy-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:164a829b6aacf79ca47ba4814b130c4020b202522a93d7bff2202bfb33b61c60", size = 12606618 }, + { url = "https://files.pythonhosted.org/packages/a8/87/04ddf02dd86fb17c7485a5f87b605c4437966d53de1e3745d450343a6f56/numpy-2.2.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4dfda918a13cc4f81e9118dea249e192ab167a0bb1966272d5503e39234d694e", size = 20921004 }, + { url = "https://files.pythonhosted.org/packages/6e/3e/d0e9e32ab14005425d180ef950badf31b862f3839c5b927796648b11f88a/numpy-2.2.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:733585f9f4b62e9b3528dd1070ec4f52b8acf64215b60a845fa13ebd73cd0712", size = 14119910 }, + { url = "https://files.pythonhosted.org/packages/b5/5b/aa2d1905b04a8fb681e08742bb79a7bddfc160c7ce8e1ff6d5c821be0236/numpy-2.2.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:89b16a18e7bba224ce5114db863e7029803c179979e1af6ad6a6b11f70545008", size = 5153612 }, + { url = "https://files.pythonhosted.org/packages/ce/35/6831808028df0648d9b43c5df7e1051129aa0d562525bacb70019c5f5030/numpy-2.2.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:676f4eebf6b2d430300f1f4f4c2461685f8269f94c89698d832cdf9277f30b84", size = 6668401 }, + { url = "https://files.pythonhosted.org/packages/b1/38/10ef509ad63a5946cc042f98d838daebfe7eaf45b9daaf13df2086b15ff9/numpy-2.2.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27f5cdf9f493b35f7e41e8368e7d7b4bbafaf9660cba53fb21d2cd174ec09631", size = 14014198 }, + { url = "https://files.pythonhosted.org/packages/df/f8/c80968ae01df23e249ee0a4487fae55a4c0fe2f838dfe9cc907aa8aea0fa/numpy-2.2.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1ad395cf254c4fbb5b2132fee391f361a6e8c1adbd28f2cd8e79308a615fe9d", size = 16076211 }, + { url = "https://files.pythonhosted.org/packages/09/69/05c169376016a0b614b432967ac46ff14269eaffab80040ec03ae1ae8e2c/numpy-2.2.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:08ef779aed40dbc52729d6ffe7dd51df85796a702afbf68a4f4e41fafdc8bda5", size = 15220266 }, + { url = "https://files.pythonhosted.org/packages/f1/ff/94a4ce67ea909f41cf7ea712aebbe832dc67decad22944a1020bb398a5ee/numpy-2.2.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:26c9c4382b19fcfbbed3238a14abf7ff223890ea1936b8890f058e7ba35e8d71", size = 17852844 }, + { url = "https://files.pythonhosted.org/packages/46/72/8a5dbce4020dfc595592333ef2fbb0a187d084ca243b67766d29d03e0096/numpy-2.2.1-cp313-cp313t-win32.whl", hash = "sha256:93cf4e045bae74c90ca833cba583c14b62cb4ba2cba0abd2b141ab52548247e2", size = 6326007 }, + { url = "https://files.pythonhosted.org/packages/7b/9c/4fce9cf39dde2562584e4cfd351a0140240f82c0e3569ce25a250f47037d/numpy-2.2.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bff7d8ec20f5f42607599f9994770fa65d76edca264a87b5e4ea5629bce12268", size = 12693107 }, + { url = "https://files.pythonhosted.org/packages/f1/65/d36a76b811ffe0a4515e290cb05cb0e22171b1b0f0db6bee9141cf023545/numpy-2.2.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7ba9cc93a91d86365a5d270dee221fdc04fb68d7478e6bf6af650de78a8339e3", size = 21044672 }, + { url = "https://files.pythonhosted.org/packages/aa/3f/b644199f165063154df486d95198d814578f13dd4d8c1651e075bf1cb8af/numpy-2.2.1-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:3d03883435a19794e41f147612a77a8f56d4e52822337844fff3d4040a142964", size = 6789873 }, + { url = "https://files.pythonhosted.org/packages/d7/df/2adb0bb98a3cbe8a6c3c6d1019aede1f1d8b83927ced228a46cc56c7a206/numpy-2.2.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4511d9e6071452b944207c8ce46ad2f897307910b402ea5fa975da32e0102800", size = 16194933 }, + { url = "https://files.pythonhosted.org/packages/13/3e/1959d5219a9e6d200638d924cedda6a606392f7186a4ed56478252e70d55/numpy-2.2.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5c5cc0cbabe9452038ed984d05ac87910f89370b9242371bd9079cb4af61811e", size = 12820057 }, +] + +[[package]] +name = "packaging" +version = "24.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d0/63/68dbb6eb2de9cb10ee4c9c14a0148804425e13c4fb20d61cce69f53106da/packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f", size = 163950 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, +] + +[[package]] +name = "pandas" +version = "2.0.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "python-dateutil", marker = "python_full_version < '3.9'" }, + { name = "pytz", marker = "python_full_version < '3.9'" }, + { name = "tzdata", marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/a7/824332581e258b5aa4f3763ecb2a797e5f9a54269044ba2e50ac19936b32/pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c", size = 5284455 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/b2/0d4a5729ce1ce11630c4fc5d5522a33b967b3ca146c210f58efde7c40e99/pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8", size = 11760908 }, + { url = "https://files.pythonhosted.org/packages/4a/f6/f620ca62365d83e663a255a41b08d2fc2eaf304e0b8b21bb6d62a7390fe3/pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f", size = 10823486 }, + { url = "https://files.pythonhosted.org/packages/c2/59/cb4234bc9b968c57e81861b306b10cd8170272c57b098b724d3de5eda124/pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183", size = 11571897 }, + { url = "https://files.pythonhosted.org/packages/e3/59/35a2892bf09ded9c1bf3804461efe772836a5261ef5dfb4e264ce813ff99/pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0", size = 12306421 }, + { url = "https://files.pythonhosted.org/packages/94/71/3a0c25433c54bb29b48e3155b959ac78f4c4f2f06f94d8318aac612cb80f/pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210", size = 9540792 }, + { url = "https://files.pythonhosted.org/packages/ed/30/b97456e7063edac0e5a405128065f0cd2033adfe3716fb2256c186bd41d0/pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e", size = 10664333 }, + { url = "https://files.pythonhosted.org/packages/b3/92/a5e5133421b49e901a12e02a6a7ef3a0130e10d13db8cb657fdd0cba3b90/pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8", size = 11645672 }, + { url = "https://files.pythonhosted.org/packages/8f/bb/aea1fbeed5b474cb8634364718abe9030d7cc7a30bf51f40bd494bbc89a2/pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26", size = 10693229 }, + { url = "https://files.pythonhosted.org/packages/d6/90/e7d387f1a416b14e59290baa7a454a90d719baebbf77433ff1bdcc727800/pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d", size = 11581591 }, + { url = "https://files.pythonhosted.org/packages/d0/28/88b81881c056376254618fad622a5e94b5126db8c61157ea1910cd1c040a/pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df", size = 12219370 }, + { url = "https://files.pythonhosted.org/packages/e4/a5/212b9039e25bf8ebb97e417a96660e3dc925dacd3f8653d531b8f7fd9be4/pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd", size = 9482935 }, + { url = "https://files.pythonhosted.org/packages/9e/71/756a1be6bee0209d8c0d8c5e3b9fc72c00373f384a4017095ec404aec3ad/pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b", size = 10607692 }, + { url = "https://files.pythonhosted.org/packages/78/a8/07dd10f90ca915ed914853cd57f79bfc22e1ef4384ab56cb4336d2fc1f2a/pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061", size = 11653303 }, + { url = "https://files.pythonhosted.org/packages/53/c3/f8e87361f7fdf42012def602bfa2a593423c729f5cb7c97aed7f51be66ac/pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5", size = 10710932 }, + { url = "https://files.pythonhosted.org/packages/a7/87/828d50c81ce0f434163bf70b925a0eec6076808e0bca312a79322b141f66/pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089", size = 11684018 }, + { url = "https://files.pythonhosted.org/packages/f8/7f/5b047effafbdd34e52c9e2d7e44f729a0655efafb22198c45cf692cdc157/pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0", size = 12353723 }, + { url = "https://files.pythonhosted.org/packages/ea/ae/26a2eda7fa581347d69e51f93892493b2074ef3352ac71033c9f32c52389/pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02", size = 9646403 }, + { url = "https://files.pythonhosted.org/packages/c3/6c/ea362eef61f05553aaf1a24b3e96b2d0603f5dc71a3bd35688a24ed88843/pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78", size = 10777638 }, + { url = "https://files.pythonhosted.org/packages/f8/c7/cfef920b7b457dff6928e824896cb82367650ea127d048ee0b820026db4f/pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b", size = 11834160 }, + { url = "https://files.pythonhosted.org/packages/6c/1c/689c9d99bc4e5d366a5fd871f0bcdee98a6581e240f96b78d2d08f103774/pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e", size = 10862752 }, + { url = "https://files.pythonhosted.org/packages/cc/b8/4d082f41c27c95bf90485d1447b647cc7e5680fea75e315669dc6e4cb398/pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b", size = 11715852 }, + { url = "https://files.pythonhosted.org/packages/9e/0d/91a9fd2c202f2b1d97a38ab591890f86480ecbb596cbc56d035f6f23fdcc/pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641", size = 12398496 }, + { url = "https://files.pythonhosted.org/packages/26/7d/d8aa0a2c4f3f5f8ea59fb946c8eafe8f508090ca73e2b08a9af853c1103e/pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682", size = 9630766 }, + { url = "https://files.pythonhosted.org/packages/9a/f2/0ad053856debbe90c83de1b4f05915f85fd2146f20faf9daa3b320d36df3/pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc", size = 10755902 }, +] + +[[package]] +name = "pandas" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +dependencies = [ + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "numpy", version = "2.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "python-dateutil", marker = "python_full_version >= '3.9'" }, + { name = "pytz", marker = "python_full_version >= '3.9'" }, + { name = "tzdata", marker = "python_full_version >= '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/aa/70/c853aec59839bceed032d52010ff5f1b8d87dc3114b762e4ba2727661a3b/pandas-2.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1948ddde24197a0f7add2bdc4ca83bf2b1ef84a1bc8ccffd95eda17fd836ecb5", size = 12580827 }, + { url = "https://files.pythonhosted.org/packages/99/f2/c4527768739ffa4469b2b4fff05aa3768a478aed89a2f271a79a40eee984/pandas-2.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:381175499d3802cde0eabbaf6324cce0c4f5d52ca6f8c377c29ad442f50f6348", size = 11303897 }, + { url = "https://files.pythonhosted.org/packages/ed/12/86c1747ea27989d7a4064f806ce2bae2c6d575b950be087837bdfcabacc9/pandas-2.2.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d9c45366def9a3dd85a6454c0e7908f2b3b8e9c138f5dc38fed7ce720d8453ed", size = 66480908 }, + { url = "https://files.pythonhosted.org/packages/44/50/7db2cd5e6373ae796f0ddad3675268c8d59fb6076e66f0c339d61cea886b/pandas-2.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:86976a1c5b25ae3f8ccae3a5306e443569ee3c3faf444dfd0f41cda24667ad57", size = 13064210 }, + { url = "https://files.pythonhosted.org/packages/61/61/a89015a6d5536cb0d6c3ba02cebed51a95538cf83472975275e28ebf7d0c/pandas-2.2.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:b8661b0238a69d7aafe156b7fa86c44b881387509653fdf857bebc5e4008ad42", size = 16754292 }, + { url = "https://files.pythonhosted.org/packages/ce/0d/4cc7b69ce37fac07645a94e1d4b0880b15999494372c1523508511b09e40/pandas-2.2.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:37e0aced3e8f539eccf2e099f65cdb9c8aa85109b0be6e93e2baff94264bdc6f", size = 14416379 }, + { url = "https://files.pythonhosted.org/packages/31/9e/6ebb433de864a6cd45716af52a4d7a8c3c9aaf3a98368e61db9e69e69a9c/pandas-2.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:56534ce0746a58afaf7942ba4863e0ef81c9c50d3f0ae93e9497d6a41a057645", size = 11598471 }, + { url = "https://files.pythonhosted.org/packages/a8/44/d9502bf0ed197ba9bf1103c9867d5904ddcaf869e52329787fc54ed70cc8/pandas-2.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:66108071e1b935240e74525006034333f98bcdb87ea116de573a6a0dccb6c039", size = 12602222 }, + { url = "https://files.pythonhosted.org/packages/52/11/9eac327a38834f162b8250aab32a6781339c69afe7574368fffe46387edf/pandas-2.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7c2875855b0ff77b2a64a0365e24455d9990730d6431b9e0ee18ad8acee13dbd", size = 11321274 }, + { url = "https://files.pythonhosted.org/packages/45/fb/c4beeb084718598ba19aa9f5abbc8aed8b42f90930da861fcb1acdb54c3a/pandas-2.2.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cd8d0c3be0515c12fed0bdbae072551c8b54b7192c7b1fda0ba56059a0179698", size = 15579836 }, + { url = "https://files.pythonhosted.org/packages/cd/5f/4dba1d39bb9c38d574a9a22548c540177f78ea47b32f99c0ff2ec499fac5/pandas-2.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c124333816c3a9b03fbeef3a9f230ba9a737e9e5bb4060aa2107a86cc0a497fc", size = 13058505 }, + { url = "https://files.pythonhosted.org/packages/b9/57/708135b90391995361636634df1f1130d03ba456e95bcf576fada459115a/pandas-2.2.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:63cc132e40a2e084cf01adf0775b15ac515ba905d7dcca47e9a251819c575ef3", size = 16744420 }, + { url = "https://files.pythonhosted.org/packages/86/4a/03ed6b7ee323cf30404265c284cee9c65c56a212e0a08d9ee06984ba2240/pandas-2.2.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29401dbfa9ad77319367d36940cd8a0b3a11aba16063e39632d98b0e931ddf32", size = 14440457 }, + { url = "https://files.pythonhosted.org/packages/ed/8c/87ddf1fcb55d11f9f847e3c69bb1c6f8e46e2f40ab1a2d2abadb2401b007/pandas-2.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:3fc6873a41186404dad67245896a6e440baacc92f5b716ccd1bc9ed2995ab2c5", size = 11617166 }, + { url = "https://files.pythonhosted.org/packages/17/a3/fb2734118db0af37ea7433f57f722c0a56687e14b14690edff0cdb4b7e58/pandas-2.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b1d432e8d08679a40e2a6d8b2f9770a5c21793a6f9f47fdd52c5ce1948a5a8a9", size = 12529893 }, + { url = "https://files.pythonhosted.org/packages/e1/0c/ad295fd74bfac85358fd579e271cded3ac969de81f62dd0142c426b9da91/pandas-2.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a5a1595fe639f5988ba6a8e5bc9649af3baf26df3998a0abe56c02609392e0a4", size = 11363475 }, + { url = "https://files.pythonhosted.org/packages/c6/2a/4bba3f03f7d07207481fed47f5b35f556c7441acddc368ec43d6643c5777/pandas-2.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5de54125a92bb4d1c051c0659e6fcb75256bf799a732a87184e5ea503965bce3", size = 15188645 }, + { url = "https://files.pythonhosted.org/packages/38/f8/d8fddee9ed0d0c0f4a2132c1dfcf0e3e53265055da8df952a53e7eaf178c/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319", size = 12739445 }, + { url = "https://files.pythonhosted.org/packages/20/e8/45a05d9c39d2cea61ab175dbe6a2de1d05b679e8de2011da4ee190d7e748/pandas-2.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6dfcb5ee8d4d50c06a51c2fffa6cff6272098ad6540aed1a76d15fb9318194d8", size = 16359235 }, + { url = "https://files.pythonhosted.org/packages/1d/99/617d07a6a5e429ff90c90da64d428516605a1ec7d7bea494235e1c3882de/pandas-2.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:062309c1b9ea12a50e8ce661145c6aab431b1e99530d3cd60640e255778bd43a", size = 14056756 }, + { url = "https://files.pythonhosted.org/packages/29/d4/1244ab8edf173a10fd601f7e13b9566c1b525c4f365d6bee918e68381889/pandas-2.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:59ef3764d0fe818125a5097d2ae867ca3fa64df032331b7e0917cf5d7bf66b13", size = 11504248 }, + { url = "https://files.pythonhosted.org/packages/64/22/3b8f4e0ed70644e85cfdcd57454686b9057c6c38d2f74fe4b8bc2527214a/pandas-2.2.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f00d1345d84d8c86a63e476bb4955e46458b304b9575dcf71102b5c705320015", size = 12477643 }, + { url = "https://files.pythonhosted.org/packages/e4/93/b3f5d1838500e22c8d793625da672f3eec046b1a99257666c94446969282/pandas-2.2.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3508d914817e153ad359d7e069d752cdd736a247c322d932eb89e6bc84217f28", size = 11281573 }, + { url = "https://files.pythonhosted.org/packages/f5/94/6c79b07f0e5aab1dcfa35a75f4817f5c4f677931d4234afcd75f0e6a66ca/pandas-2.2.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22a9d949bfc9a502d320aa04e5d02feab689d61da4e7764b62c30b991c42c5f0", size = 15196085 }, + { url = "https://files.pythonhosted.org/packages/e8/31/aa8da88ca0eadbabd0a639788a6da13bb2ff6edbbb9f29aa786450a30a91/pandas-2.2.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3a255b2c19987fbbe62a9dfd6cff7ff2aa9ccab3fc75218fd4b7530f01efa24", size = 12711809 }, + { url = "https://files.pythonhosted.org/packages/ee/7c/c6dbdb0cb2a4344cacfb8de1c5808ca885b2e4dcfde8008266608f9372af/pandas-2.2.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:800250ecdadb6d9c78eae4990da62743b857b470883fa27f652db8bdde7f6659", size = 16356316 }, + { url = "https://files.pythonhosted.org/packages/57/b7/8b757e7d92023b832869fa8881a992696a0bfe2e26f72c9ae9f255988d42/pandas-2.2.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6374c452ff3ec675a8f46fd9ab25c4ad0ba590b71cf0656f8b6daa5202bca3fb", size = 14022055 }, + { url = "https://files.pythonhosted.org/packages/3b/bc/4b18e2b8c002572c5a441a64826252ce5da2aa738855747247a971988043/pandas-2.2.3-cp313-cp313-win_amd64.whl", hash = "sha256:61c5ad4043f791b61dd4752191d9f07f0ae412515d59ba8f005832a532f8736d", size = 11481175 }, + { url = "https://files.pythonhosted.org/packages/76/a3/a5d88146815e972d40d19247b2c162e88213ef51c7c25993942c39dbf41d/pandas-2.2.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:3b71f27954685ee685317063bf13c7709a7ba74fc996b84fc6821c59b0f06468", size = 12615650 }, + { url = "https://files.pythonhosted.org/packages/9c/8c/f0fd18f6140ddafc0c24122c8a964e48294acc579d47def376fef12bcb4a/pandas-2.2.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:38cf8125c40dae9d5acc10fa66af8ea6fdf760b2714ee482ca691fc66e6fcb18", size = 11290177 }, + { url = "https://files.pythonhosted.org/packages/ed/f9/e995754eab9c0f14c6777401f7eece0943840b7a9fc932221c19d1abee9f/pandas-2.2.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba96630bc17c875161df3818780af30e43be9b166ce51c9a18c1feae342906c2", size = 14651526 }, + { url = "https://files.pythonhosted.org/packages/25/b0/98d6ae2e1abac4f35230aa756005e8654649d305df9a28b16b9ae4353bff/pandas-2.2.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1db71525a1538b30142094edb9adc10be3f3e176748cd7acc2240c2f2e5aa3a4", size = 11871013 }, + { url = "https://files.pythonhosted.org/packages/cc/57/0f72a10f9db6a4628744c8e8f0df4e6e21de01212c7c981d31e50ffc8328/pandas-2.2.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:15c0e1e02e93116177d29ff83e8b1619c93ddc9c49083f237d4312337a61165d", size = 15711620 }, + { url = "https://files.pythonhosted.org/packages/ab/5f/b38085618b950b79d2d9164a711c52b10aefc0ae6833b96f626b7021b2ed/pandas-2.2.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:ad5b65698ab28ed8d7f18790a0dc58005c7629f227be9ecc1072aa74c0c1d43a", size = 13098436 }, + { url = "https://files.pythonhosted.org/packages/ca/8c/8848a4c9b8fdf5a534fe2077af948bf53cd713d77ffbcd7bd15710348fd7/pandas-2.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bc6b93f9b966093cb0fd62ff1a7e4c09e6d546ad7c1de191767baffc57628f39", size = 12595535 }, + { url = "https://files.pythonhosted.org/packages/9c/b9/5cead4f63b6d31bdefeb21a679bc5a7f4aaf262ca7e07e2bc1c341b68470/pandas-2.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5dbca4c1acd72e8eeef4753eeca07de9b1db4f398669d5994086f788a5d7cc30", size = 11319822 }, + { url = "https://files.pythonhosted.org/packages/31/af/89e35619fb573366fa68dc26dad6ad2c08c17b8004aad6d98f1a31ce4bb3/pandas-2.2.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8cd6d7cc958a3910f934ea8dbdf17b2364827bb4dafc38ce6eef6bb3d65ff09c", size = 15625439 }, + { url = "https://files.pythonhosted.org/packages/3d/dd/bed19c2974296661493d7acc4407b1d2db4e2a482197df100f8f965b6225/pandas-2.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99df71520d25fade9db7c1076ac94eb994f4d2673ef2aa2e86ee039b6746d20c", size = 13068928 }, + { url = "https://files.pythonhosted.org/packages/31/a3/18508e10a31ea108d746c848b5a05c0711e0278fa0d6f1c52a8ec52b80a5/pandas-2.2.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:31d0ced62d4ea3e231a9f228366919a5ea0b07440d9d4dac345376fd8e1477ea", size = 16783266 }, + { url = "https://files.pythonhosted.org/packages/c4/a5/3429bd13d82bebc78f4d78c3945efedef63a7cd0c15c17b2eeb838d1121f/pandas-2.2.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7eee9e7cea6adf3e3d24e304ac6b8300646e2a5d1cd3a3c2abed9101b0846761", size = 14450871 }, + { url = "https://files.pythonhosted.org/packages/2f/49/5c30646e96c684570925b772eac4eb0a8cb0ca590fa978f56c5d3ae73ea1/pandas-2.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:4850ba03528b6dd51d6c5d273c46f183f39a9baf3f0143e566b89450965b105e", size = 11618011 }, +] + +[[package]] +name = "parso" +version = "0.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/94/68e2e17afaa9169cf6412ab0f28623903be73d1b32e208d9e8e541bb086d/parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d", size = 400609 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/ac/dac4a63f978e4dcb3c6d3a78c4d8e0192a113d288502a1216950c41b1027/parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18", size = 103650 }, +] + +[[package]] +name = "pexpect" +version = "4.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "ptyprocess" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/c3/059298687310d527a58bb01f3b1965787ee3b40dce76752eda8b44e9a2c5/pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523", size = 63772 }, +] + +[[package]] +name = "pickleshare" +version = "0.7.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/b6/df3c1c9b616e9c0edbc4fbab6ddd09df9535849c64ba51fcb6531c32d4d8/pickleshare-0.7.5.tar.gz", hash = "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca", size = 6161 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/41/220f49aaea88bc6fa6cba8d05ecf24676326156c23b991e80b3f2fc24c77/pickleshare-0.7.5-py2.py3-none-any.whl", hash = "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56", size = 6877 }, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 }, +] + +[[package]] +name = "prompt-toolkit" +version = "3.0.48" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/4f/feb5e137aff82f7c7f3248267b97451da3644f6cdc218edfe549fb354127/prompt_toolkit-3.0.48.tar.gz", hash = "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90", size = 424684 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/6a/fd08d94654f7e67c52ca30523a178b3f8ccc4237fce4be90d39c938a831a/prompt_toolkit-3.0.48-py3-none-any.whl", hash = "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e", size = 386595 }, +] + +[[package]] +name = "ptyprocess" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/e5/16ff212c1e452235a90aeb09066144d0c5a6a8c0834397e03f5224495c4e/ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220", size = 70762 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/a6/858897256d0deac81a172289110f31629fc4cee19b6f01283303e18c8db3/ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35", size = 13993 }, +] + +[[package]] +name = "pure-eval" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/05/0a34433a064256a578f1783a10da6df098ceaa4a57bbeaa96a6c0352786b/pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42", size = 19752 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, +] + +[[package]] +name = "pyarrow" +version = "17.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/27/4e/ea6d43f324169f8aec0e57569443a38bab4b398d09769ca64f7b4d467de3/pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28", size = 1112479 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/5d/78d4b040bc5ff2fc6c3d03e80fca396b742f6c125b8af06bcf7427f931bc/pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07", size = 28994846 }, + { url = "https://files.pythonhosted.org/packages/3b/73/8ed168db7642e91180330e4ea9f3ff8bab404678f00d32d7df0871a4933b/pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655", size = 27165908 }, + { url = "https://files.pythonhosted.org/packages/81/36/e78c24be99242063f6d0590ef68c857ea07bdea470242c361e9a15bd57a4/pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545", size = 39264209 }, + { url = "https://files.pythonhosted.org/packages/18/4c/3db637d7578f683b0a8fb8999b436bdbedd6e3517bd4f90c70853cf3ad20/pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2", size = 39862883 }, + { url = "https://files.pythonhosted.org/packages/81/3c/0580626896c842614a523e66b351181ed5bb14e5dfc263cd68cea2c46d90/pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8", size = 38723009 }, + { url = "https://files.pythonhosted.org/packages/ee/fb/c1b47f0ada36d856a352da261a44d7344d8f22e2f7db3945f8c3b81be5dd/pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047", size = 39855626 }, + { url = "https://files.pythonhosted.org/packages/19/09/b0a02908180a25d57312ab5919069c39fddf30602568980419f4b02393f6/pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087", size = 25147242 }, + { url = "https://files.pythonhosted.org/packages/f9/46/ce89f87c2936f5bb9d879473b9663ce7a4b1f4359acc2f0eb39865eaa1af/pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977", size = 29028748 }, + { url = "https://files.pythonhosted.org/packages/8d/8e/ce2e9b2146de422f6638333c01903140e9ada244a2a477918a368306c64c/pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3", size = 27190965 }, + { url = "https://files.pythonhosted.org/packages/3b/c8/5675719570eb1acd809481c6d64e2136ffb340bc387f4ca62dce79516cea/pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15", size = 39269081 }, + { url = "https://files.pythonhosted.org/packages/5e/78/3931194f16ab681ebb87ad252e7b8d2c8b23dad49706cadc865dff4a1dd3/pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597", size = 39864921 }, + { url = "https://files.pythonhosted.org/packages/d8/81/69b6606093363f55a2a574c018901c40952d4e902e670656d18213c71ad7/pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420", size = 38740798 }, + { url = "https://files.pythonhosted.org/packages/4c/21/9ca93b84b92ef927814cb7ba37f0774a484c849d58f0b692b16af8eebcfb/pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4", size = 39871877 }, + { url = "https://files.pythonhosted.org/packages/30/d1/63a7c248432c71c7d3ee803e706590a0b81ce1a8d2b2ae49677774b813bb/pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03", size = 25151089 }, + { url = "https://files.pythonhosted.org/packages/d4/62/ce6ac1275a432b4a27c55fe96c58147f111d8ba1ad800a112d31859fae2f/pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22", size = 29019418 }, + { url = "https://files.pythonhosted.org/packages/8e/0a/dbd0c134e7a0c30bea439675cc120012337202e5fac7163ba839aa3691d2/pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053", size = 27152197 }, + { url = "https://files.pythonhosted.org/packages/cb/05/3f4a16498349db79090767620d6dc23c1ec0c658a668d61d76b87706c65d/pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a", size = 39263026 }, + { url = "https://files.pythonhosted.org/packages/c2/0c/ea2107236740be8fa0e0d4a293a095c9f43546a2465bb7df34eee9126b09/pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc", size = 39880798 }, + { url = "https://files.pythonhosted.org/packages/f6/b0/b9164a8bc495083c10c281cc65064553ec87b7537d6f742a89d5953a2a3e/pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a", size = 38715172 }, + { url = "https://files.pythonhosted.org/packages/f1/c4/9625418a1413005e486c006e56675334929fad864347c5ae7c1b2e7fe639/pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b", size = 39874508 }, + { url = "https://files.pythonhosted.org/packages/ae/49/baafe2a964f663413be3bd1cf5c45ed98c5e42e804e2328e18f4570027c1/pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7", size = 25099235 }, + { url = "https://files.pythonhosted.org/packages/8d/bd/8f52c1d7b430260f80a349cffa2df351750a737b5336313d56dcadeb9ae1/pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204", size = 28999345 }, + { url = "https://files.pythonhosted.org/packages/64/d9/51e35550f2f18b8815a2ab25948f735434db32000c0e91eba3a32634782a/pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8", size = 27168441 }, + { url = "https://files.pythonhosted.org/packages/18/d8/7161d87d07ea51be70c49f615004c1446d5723622a18b2681f7e4b71bf6e/pyarrow-17.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155", size = 39363163 }, + { url = "https://files.pythonhosted.org/packages/3f/08/bc497130789833de09e345e3ce4647e3ce86517c4f70f2144f0367ca378b/pyarrow-17.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145", size = 39965253 }, + { url = "https://files.pythonhosted.org/packages/d3/2e/493dd7db889402b4c7871ca7dfdd20f2c5deedbff802d3eb8576359930f9/pyarrow-17.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c", size = 38805378 }, + { url = "https://files.pythonhosted.org/packages/e6/c1/4c6bcdf7a820034aa91a8b4d25fef38809be79b42ca7aaa16d4680b0bbac/pyarrow-17.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c", size = 39958364 }, + { url = "https://files.pythonhosted.org/packages/d1/db/42ac644453cfdfc60fe002b46d647fe7a6dfad753ef7b28e99b4c936ad5d/pyarrow-17.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca", size = 25229211 }, + { url = "https://files.pythonhosted.org/packages/43/e0/a898096d35be240aa61fb2d54db58b86d664b10e1e51256f9300f47565e8/pyarrow-17.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb", size = 29007881 }, + { url = "https://files.pythonhosted.org/packages/59/22/f7d14907ed0697b5dd488d393129f2738629fa5bcba863e00931b7975946/pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df", size = 27178117 }, + { url = "https://files.pythonhosted.org/packages/bf/ee/661211feac0ed48467b1d5c57298c91403809ec3ab78b1d175e1d6ad03cf/pyarrow-17.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687", size = 39273896 }, + { url = "https://files.pythonhosted.org/packages/af/61/bcd9b58e38ead6ad42b9ed00da33a3f862bc1d445e3d3164799c25550ac2/pyarrow-17.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b", size = 39875438 }, + { url = "https://files.pythonhosted.org/packages/75/63/29d1bfcc57af73cde3fc3baccab2f37548de512dbe0ab294b033cd203516/pyarrow-17.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5", size = 38735092 }, + { url = "https://files.pythonhosted.org/packages/39/f4/90258b4de753df7cc61cefb0312f8abcf226672e96cc64996e66afce817a/pyarrow-17.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda", size = 39867610 }, + { url = "https://files.pythonhosted.org/packages/e7/f6/b75d4816c32f1618ed31a005ee635dd1d91d8164495d94f2ea092f594661/pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204", size = 25148611 }, +] + +[[package]] +name = "pyarrow" +version = "18.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/bb/8d4a1573f66e0684f190dd2b55fd0b97a7214de8882d58a3867e777bf640/pyarrow-18.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c", size = 29531620 }, + { url = "https://files.pythonhosted.org/packages/30/90/893acfad917533b624a97b9e498c0e8393908508a0a72d624fe935e632bf/pyarrow-18.1.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:b516dad76f258a702f7ca0250885fc93d1fa5ac13ad51258e39d402bd9e2e1e4", size = 30836521 }, + { url = "https://files.pythonhosted.org/packages/a3/2a/526545a7464b5fb2fa6e2c4bad16ca90e59e1843025c534fd907b7f73e5a/pyarrow-18.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f443122c8e31f4c9199cb23dca29ab9427cef990f283f80fe15b8e124bcc49b", size = 39213905 }, + { url = "https://files.pythonhosted.org/packages/8a/77/4b3fab91a30e19e233e738d0c5eca5a8f6dd05758bc349a2ca262c65de79/pyarrow-18.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0a03da7f2758645d17b7b4f83c8bffeae5bbb7f974523fe901f36288d2eab71", size = 40128881 }, + { url = "https://files.pythonhosted.org/packages/aa/e2/a88e16c5e45e562449c52305bd3bc2f9d704295322d3434656e7ccac1444/pyarrow-18.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:ba17845efe3aa358ec266cf9cc2800fa73038211fb27968bfa88acd09261a470", size = 38627517 }, + { url = "https://files.pythonhosted.org/packages/6d/84/8037c20005ccc7b869726465be0957bd9c29cfc88612962030f08292ad06/pyarrow-18.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:3c35813c11a059056a22a3bef520461310f2f7eea5c8a11ef9de7062a23f8d56", size = 40060187 }, + { url = "https://files.pythonhosted.org/packages/2a/38/d6435c723ff73df8ae74626ea778262fbcc2b9b0d1a4f3db915b61711b05/pyarrow-18.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9736ba3c85129d72aefa21b4f3bd715bc4190fe4426715abfff90481e7d00812", size = 25118314 }, + { url = "https://files.pythonhosted.org/packages/9e/4d/a4988e7d82f4fbc797715db4185939a658eeffb07a25bab7262bed1ea076/pyarrow-18.1.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:eaeabf638408de2772ce3d7793b2668d4bb93807deed1725413b70e3156a7854", size = 29554860 }, + { url = "https://files.pythonhosted.org/packages/59/03/3a42c5c1e4bd4c900ab62aa1ff6b472bdb159ba8f1c3e5deadab7222244f/pyarrow-18.1.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:3b2e2239339c538f3464308fd345113f886ad031ef8266c6f004d49769bb074c", size = 30867076 }, + { url = "https://files.pythonhosted.org/packages/75/7e/332055ac913373e89256dce9d14b7708f55f7bd5be631456c897f0237738/pyarrow-18.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f39a2e0ed32a0970e4e46c262753417a60c43a3246972cfc2d3eb85aedd01b21", size = 39212135 }, + { url = "https://files.pythonhosted.org/packages/8c/64/5099cdb325828722ef7ffeba9a4696f238eb0cdeae227f831c2d77fcf1bd/pyarrow-18.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e31e9417ba9c42627574bdbfeada7217ad8a4cbbe45b9d6bdd4b62abbca4c6f6", size = 40125195 }, + { url = "https://files.pythonhosted.org/packages/83/88/1938d783727db1b178ff71bc6a6143d7939e406db83a9ec23cad3dad325c/pyarrow-18.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:01c034b576ce0eef554f7c3d8c341714954be9b3f5d5bc7117006b85fcf302fe", size = 38641884 }, + { url = "https://files.pythonhosted.org/packages/5e/b5/9e14e9f7590e0eaa435ecea84dabb137284a4dbba7b3c337b58b65b76d95/pyarrow-18.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f266a2c0fc31995a06ebd30bcfdb7f615d7278035ec5b1cd71c48d56daaf30b0", size = 40076877 }, + { url = "https://files.pythonhosted.org/packages/4d/a3/817ac7fe0891a2d66e247e223080f3a6a262d8aefd77e11e8c27e6acf4e1/pyarrow-18.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:d4f13eee18433f99adefaeb7e01d83b59f73360c231d4782d9ddfaf1c3fbde0a", size = 25119811 }, + { url = "https://files.pythonhosted.org/packages/6a/50/12829e7111b932581e51dda51d5cb39207a056c30fe31ef43f14c63c4d7e/pyarrow-18.1.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:9f3a76670b263dc41d0ae877f09124ab96ce10e4e48f3e3e4257273cee61ad0d", size = 29514620 }, + { url = "https://files.pythonhosted.org/packages/d1/41/468c944eab157702e96abab3d07b48b8424927d4933541ab43788bb6964d/pyarrow-18.1.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:da31fbca07c435be88a0c321402c4e31a2ba61593ec7473630769de8346b54ee", size = 30856494 }, + { url = "https://files.pythonhosted.org/packages/68/f9/29fb659b390312a7345aeb858a9d9c157552a8852522f2c8bad437c29c0a/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:543ad8459bc438efc46d29a759e1079436290bd583141384c6f7a1068ed6f992", size = 39203624 }, + { url = "https://files.pythonhosted.org/packages/6e/f6/19360dae44200e35753c5c2889dc478154cd78e61b1f738514c9f131734d/pyarrow-18.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0743e503c55be0fdb5c08e7d44853da27f19dc854531c0570f9f394ec9671d54", size = 40139341 }, + { url = "https://files.pythonhosted.org/packages/bb/e6/9b3afbbcf10cc724312e824af94a2e993d8ace22994d823f5c35324cebf5/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:d4b3d2a34780645bed6414e22dda55a92e0fcd1b8a637fba86800ad737057e33", size = 38618629 }, + { url = "https://files.pythonhosted.org/packages/3a/2e/3b99f8a3d9e0ccae0e961978a0d0089b25fb46ebbcfb5ebae3cca179a5b3/pyarrow-18.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:c52f81aa6f6575058d8e2c782bf79d4f9fdc89887f16825ec3a66607a5dd8e30", size = 40078661 }, + { url = "https://files.pythonhosted.org/packages/76/52/f8da04195000099d394012b8d42c503d7041b79f778d854f410e5f05049a/pyarrow-18.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:0ad4892617e1a6c7a551cfc827e072a633eaff758fa09f21c4ee548c30bcaf99", size = 25092330 }, + { url = "https://files.pythonhosted.org/packages/cb/87/aa4d249732edef6ad88899399047d7e49311a55749d3c373007d034ee471/pyarrow-18.1.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:84e314d22231357d473eabec709d0ba285fa706a72377f9cc8e1cb3c8013813b", size = 29497406 }, + { url = "https://files.pythonhosted.org/packages/3c/c7/ed6adb46d93a3177540e228b5ca30d99fc8ea3b13bdb88b6f8b6467e2cb7/pyarrow-18.1.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:f591704ac05dfd0477bb8f8e0bd4b5dc52c1cadf50503858dce3a15db6e46ff2", size = 30835095 }, + { url = "https://files.pythonhosted.org/packages/41/d7/ed85001edfb96200ff606943cff71d64f91926ab42828676c0fc0db98963/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acb7564204d3c40babf93a05624fc6a8ec1ab1def295c363afc40b0c9e66c191", size = 39194527 }, + { url = "https://files.pythonhosted.org/packages/59/16/35e28eab126342fa391593415d79477e89582de411bb95232f28b131a769/pyarrow-18.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74de649d1d2ccb778f7c3afff6085bd5092aed4c23df9feeb45dd6b16f3811aa", size = 40131443 }, + { url = "https://files.pythonhosted.org/packages/0c/95/e855880614c8da20f4cd74fa85d7268c725cf0013dc754048593a38896a0/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:f96bd502cb11abb08efea6dab09c003305161cb6c9eafd432e35e76e7fa9b90c", size = 38608750 }, + { url = "https://files.pythonhosted.org/packages/54/9d/f253554b1457d4fdb3831b7bd5f8f00f1795585a606eabf6fec0a58a9c38/pyarrow-18.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:36ac22d7782554754a3b50201b607d553a8d71b78cdf03b33c1125be4b52397c", size = 40066690 }, + { url = "https://files.pythonhosted.org/packages/2f/58/8912a2563e6b8273e8aa7b605a345bba5a06204549826f6493065575ebc0/pyarrow-18.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25dbacab8c5952df0ca6ca0af28f50d45bd31c1ff6fcf79e2d120b4a65ee7181", size = 25081054 }, + { url = "https://files.pythonhosted.org/packages/82/f9/d06ddc06cab1ada0c2f2fd205ac8c25c2701182de1b9c4bf7a0a44844431/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:6a276190309aba7bc9d5bd2933230458b3521a4317acfefe69a354f2fe59f2bc", size = 29525542 }, + { url = "https://files.pythonhosted.org/packages/ab/94/8917e3b961810587ecbdaa417f8ebac0abb25105ae667b7aa11c05876976/pyarrow-18.1.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:ad514dbfcffe30124ce655d72771ae070f30bf850b48bc4d9d3b25993ee0e386", size = 30829412 }, + { url = "https://files.pythonhosted.org/packages/5e/e3/3b16c3190f3d71d3b10f6758d2d5f7779ef008c4fd367cedab3ed178a9f7/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aebc13a11ed3032d8dd6e7171eb6e86d40d67a5639d96c35142bd568b9299324", size = 39119106 }, + { url = "https://files.pythonhosted.org/packages/1d/d6/5d704b0d25c3c79532f8c0639f253ec2803b897100f64bcb3f53ced236e5/pyarrow-18.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d6cf5c05f3cee251d80e98726b5c7cc9f21bab9e9783673bac58e6dfab57ecc8", size = 40090940 }, + { url = "https://files.pythonhosted.org/packages/37/29/366bc7e588220d74ec00e497ac6710c2833c9176f0372fe0286929b2d64c/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:11b676cd410cf162d3f6a70b43fb9e1e40affbc542a1e9ed3681895f2962d3d9", size = 38548177 }, + { url = "https://files.pythonhosted.org/packages/c8/11/fabf6ecabb1fe5b7d96889228ca2a9158c4c3bb732e3b8ee3f7f6d40b703/pyarrow-18.1.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:b76130d835261b38f14fc41fdfb39ad8d672afb84c447126b84d5472244cfaba", size = 40043567 }, + { url = "https://files.pythonhosted.org/packages/fd/9b/60516e3876ec6f25b0909afa70f90a15de83b48c7c0d8042fac4e64c4411/pyarrow-18.1.0-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:0b331e477e40f07238adc7ba7469c36b908f07c89b95dd4bd3a0ec84a3d1e21e", size = 29543752 }, + { url = "https://files.pythonhosted.org/packages/14/a7/bd08b6f1a2bd2e71dc6bb0451fc1872607e44c83daf1ee63c82764a2d233/pyarrow-18.1.0-cp39-cp39-macosx_12_0_x86_64.whl", hash = "sha256:2c4dd0c9010a25ba03e198fe743b1cc03cd33c08190afff371749c52ccbbaf76", size = 30850753 }, + { url = "https://files.pythonhosted.org/packages/84/c9/62ef9c6281c0e5b4ee1afa9d7bd556e72e06da6706b7906c32c15e69b3d6/pyarrow-18.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f97b31b4c4e21ff58c6f330235ff893cc81e23da081b1a4b1c982075e0ed4e9", size = 39226870 }, + { url = "https://files.pythonhosted.org/packages/b2/99/a6e89e71655a38475e76b060777c8bf69c078b772bec3b7daf7361440f05/pyarrow-18.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a4813cb8ecf1809871fd2d64a8eff740a1bd3691bbe55f01a3cf6c5ec869754", size = 40139114 }, + { url = "https://files.pythonhosted.org/packages/64/a9/06d79923890682e4fe7a16524abee307407008a413115354aaf3226b8410/pyarrow-18.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:05a5636ec3eb5cc2a36c6edb534a38ef57b2ab127292a716d00eabb887835f1e", size = 38639231 }, + { url = "https://files.pythonhosted.org/packages/3b/8c/4c3ed19026a00740b81fe1c87f3ff235b2763a0a1ddf5711a9d026b775ce/pyarrow-18.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:73eeed32e724ea3568bb06161cad5fa7751e45bc2228e33dcb10c614044165c7", size = 40070949 }, + { url = "https://files.pythonhosted.org/packages/87/d8/94161a7ca5c55199484e926165e9e33f318ea1d1b0d7cdbcbc3652b933ec/pyarrow-18.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052", size = 25301373 }, +] + +[[package]] +name = "pydata-sphinx-theme" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/d6/3921de802cf1ee771f0e76c9068b52498aeb8eeec6b830ff931c81c7ecf3/pydata_sphinx_theme-0.8.0.tar.gz", hash = "sha256:9f72015d9c572ea92e3007ab221a8325767c426783b6b9941813e65fa988dc90", size = 1123746 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/26/0694318d46c7d90ab602ae27b24431e939f1600f9a4c69d1e727ec57289f/pydata_sphinx_theme-0.8.0-py3-none-any.whl", hash = "sha256:fbcbb833a07d3ad8dd997dd40dc94da18d98b41c68123ab0182b58fe92271204", size = 3284997 }, +] + +[[package]] +name = "pygments" +version = "2.19.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, +] + +[[package]] +name = "pytest" +version = "8.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/35/30e0d83068951d90a01852cb1cef56e5d8a09d20c7f511634cc2f7e0372a/pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761", size = 1445919 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/92/76a1c94d3afee238333bc0a42b82935dd8f9cf8ce9e336ff87ee14d9e1cf/pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6", size = 343083 }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, +] + +[[package]] +name = "pytz" +version = "2024.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/31/3c70bf7603cc2dca0f19bdc53b4537a797747a58875b552c8c413d963a3f/pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a", size = 319692 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/11/c3/005fcca25ce078d2cc29fd559379817424e94885510568bc1bc53d7d5846/pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725", size = 508002 }, +] + +[[package]] +name = "pyyaml" +version = "6.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/ed/79a089b6be93607fa5cdaedf301d7dfb23af5f25c398d5ead2525b063e17/pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e", size = 130631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/95/a3fac87cb7158e231b5a6012e438c647e1a87f09f8e0d123acec8ab8bf71/PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086", size = 184199 }, + { url = "https://files.pythonhosted.org/packages/c7/7a/68bd47624dab8fd4afbfd3c48e3b79efe09098ae941de5b58abcbadff5cb/PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf", size = 171758 }, + { url = "https://files.pythonhosted.org/packages/49/ee/14c54df452143b9ee9f0f29074d7ca5516a36edb0b4cc40c3f280131656f/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237", size = 718463 }, + { url = "https://files.pythonhosted.org/packages/4d/61/de363a97476e766574650d742205be468921a7b532aa2499fcd886b62530/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b", size = 719280 }, + { url = "https://files.pythonhosted.org/packages/6b/4e/1523cb902fd98355e2e9ea5e5eb237cbc5f3ad5f3075fa65087aa0ecb669/PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed", size = 751239 }, + { url = "https://files.pythonhosted.org/packages/b7/33/5504b3a9a4464893c32f118a9cc045190a91637b119a9c881da1cf6b7a72/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180", size = 695802 }, + { url = "https://files.pythonhosted.org/packages/5c/20/8347dcabd41ef3a3cdc4f7b7a2aff3d06598c8779faa189cdbf878b626a4/PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68", size = 720527 }, + { url = "https://files.pythonhosted.org/packages/be/aa/5afe99233fb360d0ff37377145a949ae258aaab831bde4792b32650a4378/PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99", size = 144052 }, + { url = "https://files.pythonhosted.org/packages/b5/84/0fa4b06f6d6c958d207620fc60005e241ecedceee58931bb20138e1e5776/PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e", size = 161774 }, + { url = "https://files.pythonhosted.org/packages/f8/aa/7af4e81f7acba21a4c6be026da38fd2b872ca46226673c89a758ebdc4fd2/PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774", size = 184612 }, + { url = "https://files.pythonhosted.org/packages/8b/62/b9faa998fd185f65c1371643678e4d58254add437edb764a08c5a98fb986/PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee", size = 172040 }, + { url = "https://files.pythonhosted.org/packages/ad/0c/c804f5f922a9a6563bab712d8dcc70251e8af811fce4524d57c2c0fd49a4/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c", size = 736829 }, + { url = "https://files.pythonhosted.org/packages/51/16/6af8d6a6b210c8e54f1406a6b9481febf9c64a3109c541567e35a49aa2e7/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317", size = 764167 }, + { url = "https://files.pythonhosted.org/packages/75/e4/2c27590dfc9992f73aabbeb9241ae20220bd9452df27483b6e56d3975cc5/PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85", size = 762952 }, + { url = "https://files.pythonhosted.org/packages/9b/97/ecc1abf4a823f5ac61941a9c00fe501b02ac3ab0e373c3857f7d4b83e2b6/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4", size = 735301 }, + { url = "https://files.pythonhosted.org/packages/45/73/0f49dacd6e82c9430e46f4a027baa4ca205e8b0a9dce1397f44edc23559d/PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e", size = 756638 }, + { url = "https://files.pythonhosted.org/packages/22/5f/956f0f9fc65223a58fbc14459bf34b4cc48dec52e00535c79b8db361aabd/PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5", size = 143850 }, + { url = "https://files.pythonhosted.org/packages/ed/23/8da0bbe2ab9dcdd11f4f4557ccaf95c10b9811b13ecced089d43ce59c3c8/PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44", size = 161980 }, + { url = "https://files.pythonhosted.org/packages/86/0c/c581167fc46d6d6d7ddcfb8c843a4de25bdd27e4466938109ca68492292c/PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab", size = 183873 }, + { url = "https://files.pythonhosted.org/packages/a8/0c/38374f5bb272c051e2a69281d71cba6fdb983413e6758b84482905e29a5d/PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725", size = 173302 }, + { url = "https://files.pythonhosted.org/packages/c3/93/9916574aa8c00aa06bbac729972eb1071d002b8e158bd0e83a3b9a20a1f7/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5", size = 739154 }, + { url = "https://files.pythonhosted.org/packages/95/0f/b8938f1cbd09739c6da569d172531567dbcc9789e0029aa070856f123984/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425", size = 766223 }, + { url = "https://files.pythonhosted.org/packages/b9/2b/614b4752f2e127db5cc206abc23a8c19678e92b23c3db30fc86ab731d3bd/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476", size = 767542 }, + { url = "https://files.pythonhosted.org/packages/d4/00/dd137d5bcc7efea1836d6264f049359861cf548469d18da90cd8216cf05f/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48", size = 731164 }, + { url = "https://files.pythonhosted.org/packages/c9/1f/4f998c900485e5c0ef43838363ba4a9723ac0ad73a9dc42068b12aaba4e4/PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b", size = 756611 }, + { url = "https://files.pythonhosted.org/packages/df/d1/f5a275fdb252768b7a11ec63585bc38d0e87c9e05668a139fea92b80634c/PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4", size = 140591 }, + { url = "https://files.pythonhosted.org/packages/0c/e8/4f648c598b17c3d06e8753d7d13d57542b30d56e6c2dedf9c331ae56312e/PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8", size = 156338 }, + { url = "https://files.pythonhosted.org/packages/ef/e3/3af305b830494fa85d95f6d95ef7fa73f2ee1cc8ef5b495c7c3269fb835f/PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba", size = 181309 }, + { url = "https://files.pythonhosted.org/packages/45/9f/3b1c20a0b7a3200524eb0076cc027a970d320bd3a6592873c85c92a08731/PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1", size = 171679 }, + { url = "https://files.pythonhosted.org/packages/7c/9a/337322f27005c33bcb656c655fa78325b730324c78620e8328ae28b64d0c/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133", size = 733428 }, + { url = "https://files.pythonhosted.org/packages/a3/69/864fbe19e6c18ea3cc196cbe5d392175b4cf3d5d0ac1403ec3f2d237ebb5/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484", size = 763361 }, + { url = "https://files.pythonhosted.org/packages/04/24/b7721e4845c2f162d26f50521b825fb061bc0a5afcf9a386840f23ea19fa/PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5", size = 759523 }, + { url = "https://files.pythonhosted.org/packages/2b/b2/e3234f59ba06559c6ff63c4e10baea10e5e7df868092bf9ab40e5b9c56b6/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc", size = 726660 }, + { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, + { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, + { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, + { url = "https://files.pythonhosted.org/packages/74/d9/323a59d506f12f498c2097488d80d16f4cf965cee1791eab58b56b19f47a/PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a", size = 183218 }, + { url = "https://files.pythonhosted.org/packages/74/cc/20c34d00f04d785f2028737e2e2a8254e1425102e730fee1d6396f832577/PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5", size = 728067 }, + { url = "https://files.pythonhosted.org/packages/20/52/551c69ca1501d21c0de51ddafa8c23a0191ef296ff098e98358f69080577/PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d", size = 757812 }, + { url = "https://files.pythonhosted.org/packages/fd/7f/2c3697bba5d4aa5cc2afe81826d73dfae5f049458e44732c7a0938baa673/PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083", size = 746531 }, + { url = "https://files.pythonhosted.org/packages/8c/ab/6226d3df99900e580091bb44258fde77a8433511a86883bd4681ea19a858/PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706", size = 800820 }, + { url = "https://files.pythonhosted.org/packages/a0/99/a9eb0f3e710c06c5d922026f6736e920d431812ace24aae38228d0d64b04/PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a", size = 145514 }, + { url = "https://files.pythonhosted.org/packages/75/8a/ee831ad5fafa4431099aa4e078d4c8efd43cd5e48fbc774641d233b683a9/PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff", size = 162702 }, + { url = "https://files.pythonhosted.org/packages/65/d8/b7a1db13636d7fb7d4ff431593c510c8b8fca920ade06ca8ef20015493c5/PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", size = 184777 }, + { url = "https://files.pythonhosted.org/packages/0a/02/6ec546cd45143fdf9840b2c6be8d875116a64076218b61d68e12548e5839/PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", size = 172318 }, + { url = "https://files.pythonhosted.org/packages/0e/9a/8cc68be846c972bda34f6c2a93abb644fb2476f4dcc924d52175786932c9/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", size = 720891 }, + { url = "https://files.pythonhosted.org/packages/e9/6c/6e1b7f40181bc4805e2e07f4abc10a88ce4648e7e95ff1abe4ae4014a9b2/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12", size = 722614 }, + { url = "https://files.pythonhosted.org/packages/3d/32/e7bd8535d22ea2874cef6a81021ba019474ace0d13a4819c2a4bce79bd6a/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19", size = 737360 }, + { url = "https://files.pythonhosted.org/packages/d7/12/7322c1e30b9be969670b672573d45479edef72c9a0deac3bb2868f5d7469/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e", size = 699006 }, + { url = "https://files.pythonhosted.org/packages/82/72/04fcad41ca56491995076630c3ec1e834be241664c0c09a64c9a2589b507/PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725", size = 723577 }, + { url = "https://files.pythonhosted.org/packages/ed/5e/46168b1f2757f1fcd442bc3029cd8767d88a98c9c05770d8b420948743bb/PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631", size = 144593 }, + { url = "https://files.pythonhosted.org/packages/19/87/5124b1c1f2412bb95c59ec481eaf936cd32f0fe2a7b16b97b81c4c017a6a/PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8", size = 162312 }, +] + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "urllib3", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, +] + +[[package]] +name = "ruff" +version = "0.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/3e/e89f736f01aa9517a97e2e7e0ce8d34a4d8207087b3cfdec95133fee13b5/ruff-0.9.1.tar.gz", hash = "sha256:fd2b25ecaf907d6458fa842675382c8597b3c746a2dde6717fe3415425df0c17", size = 3498844 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/05/c3a2e0feb3d5d394cdfd552de01df9d3ec8a3a3771bbff247fab7e668653/ruff-0.9.1-py3-none-linux_armv6l.whl", hash = "sha256:84330dda7abcc270e6055551aca93fdde1b0685fc4fd358f26410f9349cf1743", size = 10645241 }, + { url = "https://files.pythonhosted.org/packages/dd/da/59f0a40e5f88ee5c054ad175caaa2319fc96571e1d29ab4730728f2aad4f/ruff-0.9.1-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:3cae39ba5d137054b0e5b472aee3b78a7c884e61591b100aeb544bcd1fc38d4f", size = 10391066 }, + { url = "https://files.pythonhosted.org/packages/b7/fe/85e1c1acf0ba04a3f2d54ae61073da030f7a5dc386194f96f3c6ca444a78/ruff-0.9.1-py3-none-macosx_11_0_arm64.whl", hash = "sha256:50c647ff96f4ba288db0ad87048257753733763b409b2faf2ea78b45c8bb7fcb", size = 10012308 }, + { url = "https://files.pythonhosted.org/packages/6f/9b/780aa5d4bdca8dcea4309264b8faa304bac30e1ce0bcc910422bfcadd203/ruff-0.9.1-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f0c8b149e9c7353cace7d698e1656ffcf1e36e50f8ea3b5d5f7f87ff9986a7ca", size = 10881960 }, + { url = "https://files.pythonhosted.org/packages/12/f4/dac4361afbfe520afa7186439e8094e4884ae3b15c8fc75fb2e759c1f267/ruff-0.9.1-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:beb3298604540c884d8b282fe7625651378e1986c25df51dec5b2f60cafc31ce", size = 10414803 }, + { url = "https://files.pythonhosted.org/packages/f0/a2/057a3cb7999513cb78d6cb33a7d1cc6401c82d7332583786e4dad9e38e44/ruff-0.9.1-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:39d0174ccc45c439093971cc06ed3ac4dc545f5e8bdacf9f067adf879544d969", size = 11464929 }, + { url = "https://files.pythonhosted.org/packages/eb/c6/1ccfcc209bee465ced4874dcfeaadc88aafcc1ea9c9f31ef66f063c187f0/ruff-0.9.1-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:69572926c0f0c9912288915214ca9b2809525ea263603370b9e00bed2ba56dbd", size = 12170717 }, + { url = "https://files.pythonhosted.org/packages/84/97/4a524027518525c7cf6931e9fd3b2382be5e4b75b2b61bec02681a7685a5/ruff-0.9.1-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:937267afce0c9170d6d29f01fcd1f4378172dec6760a9f4dface48cdabf9610a", size = 11708921 }, + { url = "https://files.pythonhosted.org/packages/a6/a4/4e77cf6065c700d5593b25fca6cf725b1ab6d70674904f876254d0112ed0/ruff-0.9.1-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:186c2313de946f2c22bdf5954b8dd083e124bcfb685732cfb0beae0c47233d9b", size = 13058074 }, + { url = "https://files.pythonhosted.org/packages/f9/d6/fcb78e0531e863d0a952c4c5600cc5cd317437f0e5f031cd2288b117bb37/ruff-0.9.1-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3f94942a3bb767675d9a051867c036655fe9f6c8a491539156a6f7e6b5f31831", size = 11281093 }, + { url = "https://files.pythonhosted.org/packages/e4/3b/7235bbeff00c95dc2d073cfdbf2b871b5bbf476754c5d277815d286b4328/ruff-0.9.1-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:728d791b769cc28c05f12c280f99e8896932e9833fef1dd8756a6af2261fd1ab", size = 10882610 }, + { url = "https://files.pythonhosted.org/packages/2a/66/5599d23257c61cf038137f82999ca8f9d0080d9d5134440a461bef85b461/ruff-0.9.1-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2f312c86fb40c5c02b44a29a750ee3b21002bd813b5233facdaf63a51d9a85e1", size = 10489273 }, + { url = "https://files.pythonhosted.org/packages/78/85/de4aa057e2532db0f9761e2c2c13834991e087787b93e4aeb5f1cb10d2df/ruff-0.9.1-py3-none-musllinux_1_2_i686.whl", hash = "sha256:ae017c3a29bee341ba584f3823f805abbe5fe9cd97f87ed07ecbf533c4c88366", size = 11003314 }, + { url = "https://files.pythonhosted.org/packages/00/42/afedcaa089116d81447347f76041ff46025849fedb0ed2b187d24cf70fca/ruff-0.9.1-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5dc40a378a0e21b4cfe2b8a0f1812a6572fc7b230ef12cd9fac9161aa91d807f", size = 11342982 }, + { url = "https://files.pythonhosted.org/packages/39/c6/fe45f3eb27e3948b41a305d8b768e949bf6a39310e9df73f6c576d7f1d9f/ruff-0.9.1-py3-none-win32.whl", hash = "sha256:46ebf5cc106cf7e7378ca3c28ce4293b61b449cd121b98699be727d40b79ba72", size = 8819750 }, + { url = "https://files.pythonhosted.org/packages/38/8d/580db77c3b9d5c3d9479e55b0b832d279c30c8f00ab0190d4cd8fc67831c/ruff-0.9.1-py3-none-win_amd64.whl", hash = "sha256:342a824b46ddbcdddd3abfbb332fa7fcaac5488bf18073e841236aadf4ad5c19", size = 9701331 }, + { url = "https://files.pythonhosted.org/packages/b2/94/0498cdb7316ed67a1928300dd87d659c933479f44dec51b4f62bfd1f8028/ruff-0.9.1-py3-none-win_arm64.whl", hash = "sha256:1cd76c7f9c679e6e8f2af8f778367dca82b95009bc7b1a85a47f1521ae524fa7", size = 9145708 }, +] + +[[package]] +name = "setuptools" +version = "75.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/22/a438e0caa4576f8c383fa4d35f1cc01655a46c75be358960d815bfbb12bd/setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686", size = 1351577 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/12/282ee9bce8b58130cb762fbc9beabd531549952cac11fc56add11dcb7ea0/setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd", size = 1251070 }, +] + +[[package]] +name = "setuptools" +version = "75.8.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/92/ec/089608b791d210aec4e7f97488e67ab0d33add3efccb83a056cbafe3a2a6/setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", size = 1343222 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/8a/b9dc7678803429e4a3bc9ba462fa3dd9066824d3c607490235c6a796be5a/setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3", size = 1228782 }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, +] + +[[package]] +name = "snowballstemmer" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/44/7b/af302bebf22c749c56c9c3e8ae13190b5b5db37a33d9068652e8f73b7089/snowballstemmer-2.2.0.tar.gz", hash = "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1", size = 86699 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ed/dc/c02e01294f7265e63a7315fe086dd1df7dacb9f840a804da846b96d01b96/snowballstemmer-2.2.0-py2.py3-none-any.whl", hash = "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a", size = 93002 }, +] + +[[package]] +name = "soupsieve" +version = "2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/ce/fbaeed4f9fb8b2daa961f90591662df6a86c1abf25c548329a86920aedfb/soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb", size = 101569 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/c2/fe97d779f3ef3b15f05c94a2f1e3d21732574ed441687474db9d342a7315/soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9", size = 36186 }, +] + +[[package]] +name = "sphinx" +version = "7.1.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +dependencies = [ + { name = "alabaster", version = "0.7.13", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "babel", marker = "python_full_version < '3.9'" }, + { name = "colorama", marker = "python_full_version < '3.9' and sys_platform == 'win32'" }, + { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "imagesize", marker = "python_full_version < '3.9'" }, + { name = "importlib-metadata", marker = "python_full_version < '3.9'" }, + { name = "jinja2", marker = "python_full_version < '3.9'" }, + { name = "packaging", marker = "python_full_version < '3.9'" }, + { name = "pygments", marker = "python_full_version < '3.9'" }, + { name = "requests", marker = "python_full_version < '3.9'" }, + { name = "snowballstemmer", marker = "python_full_version < '3.9'" }, + { name = "sphinxcontrib-applehelp", version = "1.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinxcontrib-devhelp", version = "1.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinxcontrib-htmlhelp", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.9'" }, + { name = "sphinxcontrib-qthelp", version = "1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinxcontrib-serializinghtml", version = "1.1.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/dc/01/688bdf9282241dca09fe6e3a1110eda399fa9b10d0672db609e37c2e7a39/sphinx-7.1.2.tar.gz", hash = "sha256:780f4d32f1d7d1126576e0e5ecc19dc32ab76cd24e950228dcf7b1f6d3d9e22f", size = 6828258 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/48/17/325cf6a257d84751a48ae90752b3d8fe0be8f9535b6253add61c49d0d9bc/sphinx-7.1.2-py3-none-any.whl", hash = "sha256:d170a81825b2fcacb6dfd5a0d7f578a053e45d3f2b153fecc948c37344eb4cbe", size = 3169543 }, +] + +[[package]] +name = "sphinx" +version = "7.4.7" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +dependencies = [ + { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "babel", marker = "python_full_version == '3.9.*'" }, + { name = "colorama", marker = "python_full_version == '3.9.*' and sys_platform == 'win32'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "imagesize", marker = "python_full_version == '3.9.*'" }, + { name = "importlib-metadata", marker = "python_full_version == '3.9.*'" }, + { name = "jinja2", marker = "python_full_version == '3.9.*'" }, + { name = "packaging", marker = "python_full_version == '3.9.*'" }, + { name = "pygments", marker = "python_full_version == '3.9.*'" }, + { name = "requests", marker = "python_full_version == '3.9.*'" }, + { name = "snowballstemmer", marker = "python_full_version == '3.9.*'" }, + { name = "sphinxcontrib-applehelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinxcontrib-devhelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinxcontrib-htmlhelp", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.9.*'" }, + { name = "sphinxcontrib-qthelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinxcontrib-serializinghtml", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "tomli", marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/ef/153f6803c5d5f8917dbb7f7fcf6d34a871ede3296fa89c2c703f5f8a6c8e/sphinx-7.4.7-py3-none-any.whl", hash = "sha256:c2419e2135d11f1951cd994d6eb18a1835bd8fdd8429f9ca375dc1f3281bd239", size = 3401624 }, +] + +[[package]] +name = "sphinx" +version = "8.1.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "babel", marker = "python_full_version >= '3.10'" }, + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "imagesize", marker = "python_full_version >= '3.10'" }, + { name = "jinja2", marker = "python_full_version >= '3.10'" }, + { name = "packaging", marker = "python_full_version >= '3.10'" }, + { name = "pygments", marker = "python_full_version >= '3.10'" }, + { name = "requests", marker = "python_full_version >= '3.10'" }, + { name = "snowballstemmer", marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-applehelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-devhelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-htmlhelp", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-qthelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-serializinghtml", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/60/1ddff83a56d33aaf6f10ec8ce84b4c007d9368b21008876fceda7e7381ef/sphinx-8.1.3-py3-none-any.whl", hash = "sha256:09719015511837b76bf6e03e42eb7595ac8c2e41eeb9c29c5b755c6b677992a2", size = 3487125 }, +] + +[[package]] +name = "sphinx-autoapi" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "astroid", version = "3.2.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "astroid", version = "3.3.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "jinja2" }, + { name = "pyyaml" }, + { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "stdlib-list", version = "0.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "stdlib-list", version = "0.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/eb/cc243583bb1d518ca3b10998c203d919a8ed90affd4831f2b61ad09043d2/sphinx_autoapi-3.4.0.tar.gz", hash = "sha256:e6d5371f9411bbb9fca358c00a9e57aef3ac94cbfc5df4bab285946462f69e0c", size = 29292 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/d6/f2acdc2567337fd5f5dc091a4e58d8a0fb14927b9779fc1e5ecee96d9824/sphinx_autoapi-3.4.0-py3-none-any.whl", hash = "sha256:4027fef2875a22c5f2a57107c71641d82f6166bf55beb407a47aaf3ef14e7b92", size = 34095 }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/32/df/45e827f4d7e7fcc84e853bcef1d836effd762d63ccb86f43ede4e98b478c/sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e", size = 24766 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/c1/5e2cafbd03105ce50d8500f9b4e8a6e8d02e22d0475b574c3b3e9451a15f/sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228", size = 120601 }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300 }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "1.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/98/33/dc28393f16385f722c893cb55539c641c9aaec8d1bc1c15b69ce0ac2dbb3/sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4", size = 17398 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/09/5de5ed43a521387f18bdf5f5af31d099605c992fd25372b2b9b825ce48ee/sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", size = 84690 }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530 }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/47/64cff68ea3aa450c373301e5bebfbb9fce0a3e70aca245fcadd4af06cd75/sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff", size = 27967 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/ee/a1f5e39046cbb5f8bc8fba87d1ddf1c6643fbc9194e58d26e606de4b9074/sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903", size = 99833 }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705 }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071 }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "1.0.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/8e/c4846e59f38a5f2b4a0e3b27af38f2fcf904d4bfd82095bf92de0b114ebd/sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", size = 21658 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/14/05f9206cf4e9cfca1afb5fd224c7cd434dcc3a433d6d9e4e0264d29c6cdb/sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6", size = 90609 }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743 }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "1.1.5" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/72/835d6fadb9e5d02304cf39b18f93d227cd93abd3c41ebf58e6853eeb1455/sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952", size = 21019 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/77/5464ec50dd0f1c1037e3c93249b040c8fc8078fdda97530eeb02424b6eea/sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", size = 94021 }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072 }, +] + +[[package]] +name = "stack-data" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521 }, +] + +[[package]] +name = "stdlib-list" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/39/bb/1cdbc326a5ab0026602e0489cbf02357e78140253c4b57cd866d380eb355/stdlib_list-0.10.0.tar.gz", hash = "sha256:6519c50d645513ed287657bfe856d527f277331540691ddeaf77b25459964a14", size = 59447 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/d9/9085375f0d23a4896b307bf14dcc61b49ec8cc67cb33e06cf95bf3af3966/stdlib_list-0.10.0-py3-none-any.whl", hash = "sha256:b3a911bc441d03e0332dd1a9e7d0870ba3bb0a542a74d7524f54fb431256e214", size = 79814 }, +] + +[[package]] +name = "stdlib-list" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/04/6b37a71e92ddca16b190b7df62494ac4779d58ced4787f73584eb32c8f03/stdlib_list-0.11.0.tar.gz", hash = "sha256:b74a7b643a77a12637e907f3f62f0ab9f67300bce4014f6b2d3c8b4c8fd63c66", size = 60335 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/fe/e07300c027a868d32d8ed7a425503401e91a03ff90e7ca525c115c634ffb/stdlib_list-0.11.0-py3-none-any.whl", hash = "sha256:8bf8decfffaaf273d4cfeb5bd852b910a00dec1037dcf163576803622bccf597", size = 83617 }, +] + +[[package]] +name = "toml" +version = "0.10.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588 }, +] + +[[package]] +name = "tomli" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 }, + { url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 }, + { url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 }, + { url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 }, + { url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 }, + { url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 }, + { url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 }, + { url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 }, + { url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 }, + { url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 }, + { url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 }, + { url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 }, + { url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 }, + { url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 }, + { url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 }, + { url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 }, + { url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 }, + { url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 }, + { url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 }, + { url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 }, + { url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 }, + { url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 }, + { url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 }, + { url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 }, + { url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 }, + { url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 }, + { url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 }, + { url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 }, + { url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 }, + { url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 }, + { url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 }, +] + +[[package]] +name = "traitlets" +version = "5.14.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/eb/79/72064e6a701c2183016abbbfedaba506d81e30e232a68c9f0d6f6fcd1574/traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7", size = 161621 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/c0/8f5d070730d7836adc9c9b6408dec68c6ced86b304a9b26a14df072a6e8c/traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f", size = 85359 }, +] + +[[package]] +name = "typing-extensions" +version = "4.12.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/df/db/f35a00659bc03fec321ba8bce9420de607a1d37f8342eee1863174c69557/typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8", size = 85321 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 }, +] + +[[package]] +name = "tzdata" +version = "2024.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/34/943888654477a574a86a98e9896bae89c7aa15078ec29f490fef2f1e5384/tzdata-2024.2.tar.gz", hash = "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc", size = 193282 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/ab/7e5f53c3b9d14972843a647d8d7a853969a58aecc7559cb3267302c94774/tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd", size = 346586 }, +] + +[[package]] +name = "urllib3" +version = "2.2.3" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/63/22ba4ebfe7430b76388e7cd448d5478814d3032121827c12a2cc287e2260/urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9", size = 300677 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", size = 126338 }, +] + +[[package]] +name = "urllib3" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/aa/63/e53da845320b757bf29ef6a9062f5c669fe997973f966045cb019c3f4b66/urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d", size = 307268 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 }, +] + +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166 }, +] + +[[package]] +name = "zipp" +version = "3.20.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/54/bf/5c0000c44ebc80123ecbdddba1f5dcd94a5ada602a9c225d84b5aaa55e86/zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29", size = 24199 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/8b/5ba542fa83c90e09eac972fc9baca7a88e7e7ca4b221a89251954019308b/zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350", size = 9200 }, +] + +[[package]] +name = "zipp" +version = "3.21.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e9468ebd0bcd6505de3b275e06f202c2cb016e3ff56f/zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4", size = 24545 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630 }, +] From dfe4242ce4097a2f923939e443c6686c9d20c0af Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 22 Jan 2025 05:52:32 -0500 Subject: [PATCH 256/413] Update dependencies prior to release (#999) --- Cargo.lock | 247 ++++++++++++++++++++++++++++------------------------- Cargo.toml | 4 +- 2 files changed, 133 insertions(+), 118 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 105cc30c2..5a74a4839 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d" +checksum = "eaf3437355979f1e93ba84ba108c38be5767713051f3c8ffbf07c094e2e61f9f" dependencies = [ "arrow-arith", "arrow-array", @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f" +checksum = "31dce77d2985522288edae7206bffd5fc4996491841dda01a13a58415867e681" dependencies = [ "arrow-array", "arrow-buffer", @@ -216,9 +216,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f" +checksum = "2d45fe6d3faed0435b7313e59a02583b14c6c6339fa7729e94c32a20af319a79" dependencies = [ "ahash", "arrow-buffer", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16" +checksum = "2b02656a35cc103f28084bc80a0159668e0a680d919cef127bd7e0aaccb06ec1" dependencies = [ "bytes", "half", @@ -244,9 +244,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd" +checksum = "c73c6233c5b5d635a56f6010e6eb1ab9e30e94707db21cea03da317f67d84cf3" dependencies = [ "arrow-array", "arrow-buffer", @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97" +checksum = "ec222848d70fea5a32af9c3602b08f5d740d5e2d33fbd76bf6fd88759b5b13a7" dependencies = [ "arrow-array", "arrow-buffer", @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623" +checksum = "b7f2861ffa86f107b8ab577d86cff7c7a490243eabe961ba1e1af4f27542bb79" dependencies = [ "arrow-buffer", "arrow-schema", @@ -296,9 +296,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8" +checksum = "0270dc511f11bb5fa98a25020ad51a99ca5b08d8a8dfbd17503bb9dba0388f0b" dependencies = [ "arrow-array", "arrow-buffer", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85" +checksum = "0eff38eeb8a971ad3a4caf62c5d57f0cff8a48b64a55e3207c4fd696a9234aad" dependencies = [ "arrow-array", "arrow-buffer", @@ -331,9 +331,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" +checksum = "c6f202a879d287099139ff0d121e7f55ae5e0efe634b8cf2106ebc27a8715dee" dependencies = [ "arrow-array", "arrow-buffer", @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff" +checksum = "a8f936954991c360ba762dff23f5dda16300774fafd722353d9683abd97630ae" dependencies = [ "ahash", "arrow-array", @@ -360,18 +360,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678" +checksum = "9579b9d8bce47aa41389fe344f2c6758279983b7c0ebb4013e283e3e91bb450e" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] name = "arrow-select" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" +checksum = "7471ba126d0b0aaa24b50a36bc6c25e4e74869a1fd1a5553357027a0b1c8d1f1" dependencies = [ "ahash", "arrow-array", @@ -383,9 +383,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf" +checksum = "72993b01cb62507b06f1fb49648d7286c8989ecfabdb7b77a750fcb54410731b" dependencies = [ "arrow-array", "arrow-buffer", @@ -444,7 +444,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -455,7 +455,7 @@ checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -528,9 +528,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "blake2" @@ -635,9 +635,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.7" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -671,9 +671,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" +checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f" dependencies = [ "chrono", "chrono-tz-build", @@ -1189,7 +1189,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5de3c8f386ea991696553afe241a326ecbc3c98a12c562867e4be754d3a060c" dependencies = [ "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1406,7 +1406,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1475,6 +1475,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1540,7 +1546,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1662,6 +1668,11 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1916,7 +1927,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -1994,9 +2005,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -2152,9 +2163,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "lz4_flex" @@ -2218,9 +2229,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -2404,9 +2415,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.3.0" +version = "53.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191" +checksum = "8957c0c95a6a1804f3e51a18f69df29be53856a8c5768cc9b6d00fcafcd2917c" dependencies = [ "ahash", "arrow-array", @@ -2579,19 +2590,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.27" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -2622,7 +2633,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.95", + "syn 2.0.96", "tempfile", ] @@ -2636,7 +2647,7 @@ dependencies = [ "itertools", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2726,7 +2737,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2739,7 +2750,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2771,7 +2782,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.10", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -2790,7 +2801,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.10", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time", @@ -2866,7 +2877,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -2875,7 +2886,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -2915,11 +2926,11 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1541daf4e4ed43a0922b7969bdc2170178bcacc5dabf7e39bc508a9fa3953a7a" +checksum = "4f56e622c2378013c6c61e2bd776604c46dc1087b2dc5293275a0c20a44f0771" dependencies = [ - "hashbrown 0.14.5", + "hashbrown 0.15.2", "memchr", ] @@ -3026,7 +3037,7 @@ version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -3035,9 +3046,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "once_cell", "ring", @@ -3139,7 +3150,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3154,7 +3165,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation", "core-foundation-sys", "libc", @@ -3212,7 +3223,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3223,14 +3234,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.136" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "336a0c23cf42a38d9eaa7cd22c7040d04e1228a19a933890805ffd00a16437d2" dependencies = [ "itoa", "memchr", @@ -3247,7 +3258,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3331,7 +3342,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3374,7 +3385,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3418,7 +3429,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3442,7 +3453,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.95", + "syn 2.0.96", "typify", "walkdir", ] @@ -3466,9 +3477,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.95" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -3492,7 +3503,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3526,11 +3537,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.10" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.10", + "thiserror-impl 2.0.11", ] [[package]] @@ -3541,18 +3552,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.10" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3624,7 +3635,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3696,7 +3707,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3762,7 +3773,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -3796,7 +3807,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.95", + "syn 2.0.96", "thiserror 1.0.69", "unicode-ident", ] @@ -3814,7 +3825,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.95", + "syn 2.0.96", "typify-impl", ] @@ -3879,9 +3890,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" dependencies = [ "getrandom", "serde", @@ -3920,34 +3931,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -3958,9 +3970,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3968,22 +3980,25 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -4000,9 +4015,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -4211,7 +4226,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "synstructure", ] @@ -4233,7 +4248,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] @@ -4253,7 +4268,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", "synstructure", ] @@ -4282,7 +4297,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.95", + "syn 2.0.96", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 48219414a..10cffccb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ protoc = [ "datafusion-substrait/protoc" ] substrait = ["dep:datafusion-substrait"] [dependencies] -tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync"] } +tokio = { version = "1.42", features = ["macros", "rt", "rt-multi-thread", "sync"] } pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } pyo3-async-runtimes = { version = "0.22", features = ["tokio-runtime"]} arrow = { version = "53", features = ["pyarrow"] } @@ -43,7 +43,7 @@ datafusion-substrait = { version = "44.0.0", optional = true } datafusion-proto = { version = "44.0.0" } datafusion-ffi = { version = "44.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` -uuid = { version = "1.11", features = ["v4"] } +uuid = { version = "1.12", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } async-trait = "0.1" futures = "0.3" From 78e72c9445db4e78dcda2562e251beea4f1ad470 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 22 Jan 2025 05:53:13 -0500 Subject: [PATCH 257/413] Apply import ordering in ruff check (#1001) --- benchmarks/db-benchmark/groupby-datafusion.py | 12 +-- benchmarks/db-benchmark/join-datafusion.py | 6 +- benchmarks/tpch/tpch.py | 3 +- dev/release/generate-changelog.py | 5 +- examples/export.py | 1 - .../python/tests/_test_table_provider.py | 2 +- examples/import.py | 3 +- examples/python-udaf.py | 5 +- examples/python-udf-comparisons.py | 6 +- examples/python-udf.py | 3 +- examples/python-udwf.py | 7 +- examples/query-pyarrow-data.py | 3 +- examples/sql-parquet-s3.py | 1 + examples/sql-to-pandas.py | 1 - examples/sql-using-python-udaf.py | 2 +- examples/sql-using-python-udf.py | 2 +- examples/tpch/_tests.py | 6 +- examples/tpch/convert_data_to_parquet.py | 3 +- examples/tpch/q01_pricing_summary_report.py | 3 +- examples/tpch/q02_minimum_cost_supplier.py | 3 +- examples/tpch/q03_shipping_priority.py | 3 +- examples/tpch/q04_order_priority_checking.py | 4 +- examples/tpch/q05_local_supplier_volume.py | 5 +- .../tpch/q06_forecasting_revenue_change.py | 4 +- examples/tpch/q07_volume_shipping.py | 4 +- examples/tpch/q08_market_share.py | 4 +- .../tpch/q09_product_type_profit_measure.py | 3 +- examples/tpch/q10_returned_item_reporting.py | 4 +- .../q11_important_stock_identification.py | 3 +- examples/tpch/q12_ship_mode_order_priority.py | 4 +- examples/tpch/q13_customer_distribution.py | 3 +- examples/tpch/q14_promotion_effect.py | 4 +- examples/tpch/q15_top_supplier.py | 4 +- .../tpch/q16_part_supplier_relationship.py | 3 +- examples/tpch/q17_small_quantity_order.py | 3 +- examples/tpch/q18_large_volume_customer.py | 3 +- examples/tpch/q19_discounted_revenue.py | 3 +- examples/tpch/q20_potential_part_promotion.py | 4 +- .../tpch/q21_suppliers_kept_orders_waiting.py | 3 +- examples/tpch/q22_global_sales_opportunity.py | 3 +- pyproject.toml | 4 +- python/datafusion/__init__.py | 30 +++---- python/datafusion/catalog.py | 4 +- python/datafusion/common.py | 3 +- python/datafusion/context.py | 23 ++--- python/datafusion/dataframe.py | 21 +++-- python/datafusion/expr.py | 5 +- python/datafusion/functions.py | 16 ++-- python/datafusion/input/location.py | 2 +- python/datafusion/plan.py | 4 +- python/datafusion/record_batch.py | 3 +- python/datafusion/substrait.py | 8 +- python/datafusion/udf.py | 9 +- python/tests/conftest.py | 2 +- python/tests/test_aggregation.py | 1 - python/tests/test_config.py | 2 +- python/tests/test_context.py | 3 +- python/tests/test_dataframe.py | 5 +- python/tests/test_functions.py | 6 +- python/tests/test_imports.py | 89 +++++++++---------- python/tests/test_indexing.py | 1 - python/tests/test_input.py | 1 + python/tests/test_plans.py | 2 +- python/tests/test_sql.py | 5 +- python/tests/test_store.py | 1 - python/tests/test_substrait.py | 3 +- python/tests/test_udaf.py | 1 - python/tests/test_udf.py | 2 +- python/tests/test_udwf.py | 6 +- 69 files changed, 221 insertions(+), 189 deletions(-) diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 960c8ba9a..04bf7a149 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -15,21 +15,23 @@ # specific language governing permissions and limitations # under the License. -import os import gc +import os import timeit + import datafusion as df +import pyarrow from datafusion import ( - col, - functions as f, RuntimeEnvBuilder, SessionConfig, SessionContext, + col, +) +from datafusion import ( + functions as f, ) -import pyarrow from pyarrow import csv as pacsv - print("# groupby-datafusion.py", flush=True) exec(open("./_helpers/helpers.py").read()) diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py index 811ad8707..b45ebf632 100755 --- a/benchmarks/db-benchmark/join-datafusion.py +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -15,15 +15,15 @@ # specific language governing permissions and limitations # under the License. -import os import gc +import os import timeit + import datafusion as df -from datafusion import functions as f from datafusion import col +from datafusion import functions as f from pyarrow import csv as pacsv - print("# join-datafusion.py", flush=True) exec(open("./_helpers/helpers.py").read()) diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py index daa831b55..fb86b12b6 100644 --- a/benchmarks/tpch/tpch.py +++ b/benchmarks/tpch/tpch.py @@ -16,9 +16,10 @@ # under the License. import argparse -from datafusion import SessionContext import time +from datafusion import SessionContext + def bench(data_path, query_path): with open("results.csv", "w") as results: diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 0f07457d0..2564eea86 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -16,11 +16,12 @@ # limitations under the License. import argparse -import sys -from github import Github import os import re import subprocess +import sys + +from github import Github def print_pulls(repo_name, title, pulls): diff --git a/examples/export.py b/examples/export.py index cc02de52b..c7a387bcb 100644 --- a/examples/export.py +++ b/examples/export.py @@ -17,7 +17,6 @@ import datafusion - # create a context ctx = datafusion.SessionContext() diff --git a/examples/ffi-table-provider/python/tests/_test_table_provider.py b/examples/ffi-table-provider/python/tests/_test_table_provider.py index 56c05e4fa..0db3ec561 100644 --- a/examples/ffi-table-provider/python/tests/_test_table_provider.py +++ b/examples/ffi-table-provider/python/tests/_test_table_provider.py @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. +import pyarrow as pa from datafusion import SessionContext from ffi_table_provider import MyTableProvider -import pyarrow as pa def test_table_loading(): diff --git a/examples/import.py b/examples/import.py index c9d2e8cb6..7b5ab5082 100644 --- a/examples/import.py +++ b/examples/import.py @@ -16,10 +16,9 @@ # under the License. import datafusion -import pyarrow as pa import pandas as pd import polars as pl - +import pyarrow as pa # Create a context ctx = datafusion.SessionContext() diff --git a/examples/python-udaf.py b/examples/python-udaf.py index ed705f5a9..538f69571 100644 --- a/examples/python-udaf.py +++ b/examples/python-udaf.py @@ -15,11 +15,10 @@ # specific language governing permissions and limitations # under the License. +import datafusion import pyarrow import pyarrow.compute -import datafusion -from datafusion import udaf, Accumulator -from datafusion import col +from datafusion import Accumulator, col, udaf class MyAccumulator(Accumulator): diff --git a/examples/python-udf-comparisons.py b/examples/python-udf-comparisons.py index 9a84dd730..c5d5ec8dd 100644 --- a/examples/python-udf-comparisons.py +++ b/examples/python-udf-comparisons.py @@ -15,11 +15,13 @@ # specific language governing permissions and limitations # under the License. -from datafusion import SessionContext, col, lit, udf, functions as F import os +import time + import pyarrow as pa import pyarrow.compute as pc -import time +from datafusion import SessionContext, col, lit, udf +from datafusion import functions as F path = os.path.dirname(os.path.abspath(__file__)) filepath = os.path.join(path, "./tpch/data/lineitem.parquet") diff --git a/examples/python-udf.py b/examples/python-udf.py index 30edd4198..fb2bc253e 100644 --- a/examples/python-udf.py +++ b/examples/python-udf.py @@ -16,7 +16,8 @@ # under the License. import pyarrow -from datafusion import udf, SessionContext, functions as f +from datafusion import SessionContext, udf +from datafusion import functions as f def is_null(array: pyarrow.Array) -> pyarrow.Array: diff --git a/examples/python-udwf.py b/examples/python-udwf.py index 55de2bdc7..32f8fadaa 100644 --- a/examples/python-udwf.py +++ b/examples/python-udwf.py @@ -15,11 +15,12 @@ # specific language governing permissions and limitations # under the License. -import pyarrow as pa import datafusion -from datafusion import udwf, functions as f, col, lit -from datafusion.udf import WindowEvaluator +import pyarrow as pa +from datafusion import col, lit, udwf +from datafusion import functions as f from datafusion.expr import WindowFrame +from datafusion.udf import WindowEvaluator # This example creates five different examples of user defined window functions in order # to demonstrate the variety of ways a user may need to implement. diff --git a/examples/query-pyarrow-data.py b/examples/query-pyarrow-data.py index 83e6884a7..e3456fb5b 100644 --- a/examples/query-pyarrow-data.py +++ b/examples/query-pyarrow-data.py @@ -16,9 +16,8 @@ # under the License. import datafusion -from datafusion import col import pyarrow - +from datafusion import col # create a context ctx = datafusion.SessionContext() diff --git a/examples/sql-parquet-s3.py b/examples/sql-parquet-s3.py index 61f1e0c50..866e2ac68 100644 --- a/examples/sql-parquet-s3.py +++ b/examples/sql-parquet-s3.py @@ -16,6 +16,7 @@ # under the License. import os + import datafusion from datafusion.object_store import AmazonS3 diff --git a/examples/sql-to-pandas.py b/examples/sql-to-pandas.py index 3e99b22de..34f7bde1b 100644 --- a/examples/sql-to-pandas.py +++ b/examples/sql-to-pandas.py @@ -17,7 +17,6 @@ from datafusion import SessionContext - # Create a DataFusion context ctx = SessionContext() diff --git a/examples/sql-using-python-udaf.py b/examples/sql-using-python-udaf.py index 7ccf5d3cb..60ab8d134 100644 --- a/examples/sql-using-python-udaf.py +++ b/examples/sql-using-python-udaf.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from datafusion import udaf, SessionContext, Accumulator import pyarrow as pa +from datafusion import Accumulator, SessionContext, udaf # Define a user-defined aggregation function (UDAF) diff --git a/examples/sql-using-python-udf.py b/examples/sql-using-python-udf.py index d6bbe3ab0..2f0a0b67d 100644 --- a/examples/sql-using-python-udf.py +++ b/examples/sql-using-python-udf.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from datafusion import udf, SessionContext import pyarrow as pa +from datafusion import SessionContext, udf # Define a user-defined function (UDF) diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 3ce9cdfe5..c4d872085 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -15,10 +15,12 @@ # specific language governing permissions and limitations # under the License. -import pytest from importlib import import_module + import pyarrow as pa -from datafusion import DataFrame, col, lit, functions as F +import pytest +from datafusion import DataFrame, col, lit +from datafusion import functions as F from util import get_answer_file diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py index cb0b2f0bd..73097fac5 100644 --- a/examples/tpch/convert_data_to_parquet.py +++ b/examples/tpch/convert_data_to_parquet.py @@ -23,8 +23,9 @@ """ import os -import pyarrow + import datafusion +import pyarrow ctx = datafusion.SessionContext() diff --git a/examples/tpch/q01_pricing_summary_report.py b/examples/tpch/q01_pricing_summary_report.py index cb9485a7a..3f97f00dc 100644 --- a/examples/tpch/q01_pricing_summary_report.py +++ b/examples/tpch/q01_pricing_summary_report.py @@ -30,7 +30,8 @@ """ import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path ctx = SessionContext() diff --git a/examples/tpch/q02_minimum_cost_supplier.py b/examples/tpch/q02_minimum_cost_supplier.py index c4ccf8ad3..7390d0892 100644 --- a/examples/tpch/q02_minimum_cost_supplier.py +++ b/examples/tpch/q02_minimum_cost_supplier.py @@ -30,7 +30,8 @@ """ import datafusion -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path # This is the part we're looking for. Values selected here differ from the spec in order to run diff --git a/examples/tpch/q03_shipping_priority.py b/examples/tpch/q03_shipping_priority.py index 5ebab13c0..fc1231e0a 100644 --- a/examples/tpch/q03_shipping_priority.py +++ b/examples/tpch/q03_shipping_priority.py @@ -27,7 +27,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path SEGMENT_OF_INTEREST = "BUILDING" diff --git a/examples/tpch/q04_order_priority_checking.py b/examples/tpch/q04_order_priority_checking.py index 8bf02cb83..426338aea 100644 --- a/examples/tpch/q04_order_priority_checking.py +++ b/examples/tpch/q04_order_priority_checking.py @@ -27,8 +27,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path # Ideally we could put 3 months into the interval. See note below. diff --git a/examples/tpch/q05_local_supplier_volume.py b/examples/tpch/q05_local_supplier_volume.py index 413a4acb9..fa2b01dea 100644 --- a/examples/tpch/q05_local_supplier_volume.py +++ b/examples/tpch/q05_local_supplier_volume.py @@ -30,11 +30,12 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path - DATE_OF_INTEREST = "1994-01-01" INTERVAL_DAYS = 365 REGION_OF_INTEREST = "ASIA" diff --git a/examples/tpch/q06_forecasting_revenue_change.py b/examples/tpch/q06_forecasting_revenue_change.py index eaf9b0c29..1de5848b1 100644 --- a/examples/tpch/q06_forecasting_revenue_change.py +++ b/examples/tpch/q06_forecasting_revenue_change.py @@ -30,8 +30,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path # Variables from the example query diff --git a/examples/tpch/q07_volume_shipping.py b/examples/tpch/q07_volume_shipping.py index 18c290d9c..a84cf728a 100644 --- a/examples/tpch/q07_volume_shipping.py +++ b/examples/tpch/q07_volume_shipping.py @@ -29,8 +29,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path # Variables of interest to query over diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py index 7138ab65a..d46df30f2 100644 --- a/examples/tpch/q08_market_share.py +++ b/examples/tpch/q08_market_share.py @@ -28,8 +28,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path supplier_nation = lit("BRAZIL") diff --git a/examples/tpch/q09_product_type_profit_measure.py b/examples/tpch/q09_product_type_profit_measure.py index aa47d76c0..e2abbd095 100644 --- a/examples/tpch/q09_product_type_profit_measure.py +++ b/examples/tpch/q09_product_type_profit_measure.py @@ -30,7 +30,8 @@ """ import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path part_color = lit("green") diff --git a/examples/tpch/q10_returned_item_reporting.py b/examples/tpch/q10_returned_item_reporting.py index 94b398c1d..ed822e264 100644 --- a/examples/tpch/q10_returned_item_reporting.py +++ b/examples/tpch/q10_returned_item_reporting.py @@ -30,8 +30,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path DATE_START_OF_QUARTER = "1993-10-01" diff --git a/examples/tpch/q11_important_stock_identification.py b/examples/tpch/q11_important_stock_identification.py index 707265e16..22829ab7c 100644 --- a/examples/tpch/q11_important_stock_identification.py +++ b/examples/tpch/q11_important_stock_identification.py @@ -27,7 +27,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from datafusion import SessionContext, WindowFrame, col, lit +from datafusion import functions as F from util import get_data_path NATION = "GERMANY" diff --git a/examples/tpch/q12_ship_mode_order_priority.py b/examples/tpch/q12_ship_mode_order_priority.py index def2a6c30..f1d894940 100644 --- a/examples/tpch/q12_ship_mode_order_priority.py +++ b/examples/tpch/q12_ship_mode_order_priority.py @@ -30,8 +30,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path SHIP_MODE_1 = "MAIL" diff --git a/examples/tpch/q13_customer_distribution.py b/examples/tpch/q13_customer_distribution.py index 67365a96a..93f082ea3 100644 --- a/examples/tpch/q13_customer_distribution.py +++ b/examples/tpch/q13_customer_distribution.py @@ -28,7 +28,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path WORD_1 = "special" diff --git a/examples/tpch/q14_promotion_effect.py b/examples/tpch/q14_promotion_effect.py index cd26ee2bd..d62f76e3c 100644 --- a/examples/tpch/q14_promotion_effect.py +++ b/examples/tpch/q14_promotion_effect.py @@ -27,8 +27,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path DATE = "1995-09-01" diff --git a/examples/tpch/q15_top_supplier.py b/examples/tpch/q15_top_supplier.py index 0bc316f7a..c321048f2 100644 --- a/examples/tpch/q15_top_supplier.py +++ b/examples/tpch/q15_top_supplier.py @@ -27,8 +27,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from datafusion import SessionContext, WindowFrame, col, lit +from datafusion import functions as F from util import get_data_path DATE = "1996-01-01" diff --git a/examples/tpch/q16_part_supplier_relationship.py b/examples/tpch/q16_part_supplier_relationship.py index a6a0c43eb..65043ffda 100644 --- a/examples/tpch/q16_part_supplier_relationship.py +++ b/examples/tpch/q16_part_supplier_relationship.py @@ -29,7 +29,8 @@ """ import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path BRAND = "Brand#45" diff --git a/examples/tpch/q17_small_quantity_order.py b/examples/tpch/q17_small_quantity_order.py index d7b43d498..6d76fe506 100644 --- a/examples/tpch/q17_small_quantity_order.py +++ b/examples/tpch/q17_small_quantity_order.py @@ -28,7 +28,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from datafusion import SessionContext, WindowFrame, col, lit +from datafusion import functions as F from util import get_data_path BRAND = "Brand#23" diff --git a/examples/tpch/q18_large_volume_customer.py b/examples/tpch/q18_large_volume_customer.py index 165fce033..834d181c9 100644 --- a/examples/tpch/q18_large_volume_customer.py +++ b/examples/tpch/q18_large_volume_customer.py @@ -26,7 +26,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path QUANTITY = 300 diff --git a/examples/tpch/q19_discounted_revenue.py b/examples/tpch/q19_discounted_revenue.py index 4aed0cbae..2b87e1120 100644 --- a/examples/tpch/q19_discounted_revenue.py +++ b/examples/tpch/q19_discounted_revenue.py @@ -27,7 +27,8 @@ """ import pyarrow as pa -from datafusion import SessionContext, col, lit, udf, functions as F +from datafusion import SessionContext, col, lit, udf +from datafusion import functions as F from util import get_data_path items_of_interest = { diff --git a/examples/tpch/q20_potential_part_promotion.py b/examples/tpch/q20_potential_part_promotion.py index d720cdce6..a25188d31 100644 --- a/examples/tpch/q20_potential_part_promotion.py +++ b/examples/tpch/q20_potential_part_promotion.py @@ -28,8 +28,10 @@ """ from datetime import datetime + import pyarrow as pa -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path COLOR_OF_INTEREST = "forest" diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py index 27cf816fa..9bbaad779 100644 --- a/examples/tpch/q21_suppliers_kept_orders_waiting.py +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -26,7 +26,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, col, lit, functions as F +from datafusion import SessionContext, col, lit +from datafusion import functions as F from util import get_data_path NATION_OF_INTEREST = "SAUDI ARABIA" diff --git a/examples/tpch/q22_global_sales_opportunity.py b/examples/tpch/q22_global_sales_opportunity.py index 72dce5289..c4d115b74 100644 --- a/examples/tpch/q22_global_sales_opportunity.py +++ b/examples/tpch/q22_global_sales_opportunity.py @@ -26,7 +26,8 @@ as part of their TPC Benchmark H Specification revision 2.18.0. """ -from datafusion import SessionContext, WindowFrame, col, lit, functions as F +from datafusion import SessionContext, WindowFrame, col, lit +from datafusion import functions as F from util import get_data_path NATION_CODES = [13, 31, 23, 29, 30, 18, 17] diff --git a/pyproject.toml b/pyproject.toml index 6e8acfe71..32bb28d21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ features = ["substrait"] # Enable docstring linting using the google style guide [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "D", "W"] +select = ["E4", "E7", "E9", "F", "D", "W", "I"] [tool.ruff.lint.pydocstyle] convention = "google" @@ -100,4 +100,4 @@ docs = [ "pickleshare>=0.7.5", "sphinx-autoapi>=3.4.0", "setuptools>=75.3.0", -] \ No newline at end of file +] diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 2d8db42c8..85aefcce7 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -26,36 +26,28 @@ except ImportError: import importlib_metadata -from .context import ( - SessionContext, - SessionConfig, - RuntimeEnvBuilder, - SQLOptions, -) - -from .catalog import Catalog, Database, Table +from . import functions, object_store, substrait # The following imports are okay to remain as opaque to the user. from ._internal import Config - -from .record_batch import RecordBatchStream, RecordBatch - -from .udf import ScalarUDF, AggregateUDF, Accumulator, WindowUDF - +from .catalog import Catalog, Database, Table from .common import ( DFSchema, ) - +from .context import ( + RuntimeEnvBuilder, + SessionConfig, + SessionContext, + SQLOptions, +) from .dataframe import DataFrame - from .expr import ( Expr, WindowFrame, ) - -from .plan import LogicalPlan, ExecutionPlan - -from . import functions, object_store, substrait +from .plan import ExecutionPlan, LogicalPlan +from .record_batch import RecordBatch, RecordBatchStream +from .udf import Accumulator, AggregateUDF, ScalarUDF, WindowUDF __version__ = importlib_metadata.version(__name__) diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py index acd28f33d..703037665 100644 --- a/python/datafusion/catalog.py +++ b/python/datafusion/catalog.py @@ -19,10 +19,10 @@ from __future__ import annotations -import datafusion._internal as df_internal - from typing import TYPE_CHECKING +import datafusion._internal as df_internal + if TYPE_CHECKING: import pyarrow diff --git a/python/datafusion/common.py b/python/datafusion/common.py index 7db8333f2..a2298c634 100644 --- a/python/datafusion/common.py +++ b/python/datafusion/common.py @@ -16,9 +16,10 @@ # under the License. """Common data types used throughout the DataFusion project.""" -from ._internal import common as common_internal from enum import Enum +from ._internal import common as common_internal + # TODO these should all have proper wrapper classes DFSchema = common_internal.DFSchema diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 3c284c9f9..864ef1c8b 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -19,26 +19,29 @@ from __future__ import annotations -from ._internal import SessionConfig as SessionConfigInternal -from ._internal import RuntimeEnvBuilder as RuntimeEnvBuilderInternal -from ._internal import SQLOptions as SQLOptionsInternal -from ._internal import SessionContext as SessionContextInternal +from typing import TYPE_CHECKING, Any, Protocol + +from typing_extensions import deprecated from datafusion.catalog import Catalog, Table from datafusion.dataframe import DataFrame from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list from datafusion.record_batch import RecordBatchStream -from datafusion.udf import ScalarUDF, AggregateUDF, WindowUDF +from datafusion.udf import AggregateUDF, ScalarUDF, WindowUDF -from typing import Any, TYPE_CHECKING, Protocol -from typing_extensions import deprecated +from ._internal import RuntimeEnvBuilder as RuntimeEnvBuilderInternal +from ._internal import SessionConfig as SessionConfigInternal +from ._internal import SessionContext as SessionContextInternal +from ._internal import SQLOptions as SQLOptionsInternal if TYPE_CHECKING: - import pyarrow + import pathlib + import pandas import polars - import pathlib - from datafusion.plan import LogicalPlan, ExecutionPlan + import pyarrow + + from datafusion.plan import ExecutionPlan, LogicalPlan class ArrowStreamExportable(Protocol): diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index b0c1abdad..7413a5fa3 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -20,31 +20,36 @@ """ from __future__ import annotations + import warnings from typing import ( + TYPE_CHECKING, Any, Iterable, List, - TYPE_CHECKING, Literal, - overload, Optional, Union, + overload, ) -from datafusion.record_batch import RecordBatchStream + from typing_extensions import deprecated -from datafusion.plan import LogicalPlan, ExecutionPlan + +from datafusion.plan import ExecutionPlan, LogicalPlan +from datafusion.record_batch import RecordBatchStream if TYPE_CHECKING: - import pyarrow as pa - import pandas as pd - import polars as pl import pathlib from typing import Callable, Sequence + import pandas as pd + import polars as pl + import pyarrow as pa + +from enum import Enum + from datafusion._internal import DataFrame as DataFrameInternal from datafusion.expr import Expr, SortExpr, sort_or_default -from enum import Enum # excerpt from deltalake diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 16add16f4..68ddd7c9a 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -22,12 +22,13 @@ from __future__ import annotations -from typing import Any, Optional, Type, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Optional, Type import pyarrow as pa -from datafusion.common import DataTypeMap, NullTreatment, RexType from typing_extensions import deprecated +from datafusion.common import DataTypeMap, NullTreatment, RexType + from ._internal import expr as expr_internal from ._internal import functions as functions_internal diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index c0097c6ab..7c2fa9a8f 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -18,21 +18,21 @@ from __future__ import annotations +from typing import Any, Optional + +import pyarrow as pa + from datafusion._internal import functions as f +from datafusion.common import NullTreatment +from datafusion.context import SessionContext from datafusion.expr import ( CaseBuilder, Expr, - WindowFrame, SortExpr, - sort_list_to_raw_sort_list, + WindowFrame, expr_list_to_raw_expr_list, + sort_list_to_raw_sort_list, ) -from datafusion.context import SessionContext -from datafusion.common import NullTreatment - -from typing import Any, Optional - -import pyarrow as pa __all__ = [ "abs", diff --git a/python/datafusion/input/location.py b/python/datafusion/input/location.py index b274539fc..a8252b53c 100644 --- a/python/datafusion/input/location.py +++ b/python/datafusion/input/location.py @@ -17,8 +17,8 @@ """The default input source for DataFusion.""" -import os import glob +import os from typing import Any from datafusion.common import DataTypeMap, SqlTable diff --git a/python/datafusion/plan.py b/python/datafusion/plan.py index a71965f41..133fc446d 100644 --- a/python/datafusion/plan.py +++ b/python/datafusion/plan.py @@ -19,9 +19,9 @@ from __future__ import annotations -import datafusion._internal as df_internal +from typing import TYPE_CHECKING, Any, List -from typing import List, Any, TYPE_CHECKING +import datafusion._internal as df_internal if TYPE_CHECKING: from datafusion.context import SessionContext diff --git a/python/datafusion/record_batch.py b/python/datafusion/record_batch.py index 75e58998f..772cd9089 100644 --- a/python/datafusion/record_batch.py +++ b/python/datafusion/record_batch.py @@ -27,9 +27,10 @@ if TYPE_CHECKING: import pyarrow - import datafusion._internal as df_internal import typing_extensions + import datafusion._internal as df_internal + class RecordBatch: """This class is essentially a wrapper for :py:class:`pyarrow.RecordBatch`.""" diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index dea47acca..402184d3f 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -23,13 +23,15 @@ from __future__ import annotations -from ._internal import substrait as substrait_internal - +import pathlib from typing import TYPE_CHECKING + from typing_extensions import deprecated -import pathlib + from datafusion.plan import LogicalPlan +from ._internal import substrait as substrait_internal + if TYPE_CHECKING: from datafusion.context import SessionContext diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index d9d994b22..c97f453d0 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -19,14 +19,15 @@ from __future__ import annotations -import datafusion._internal as df_internal -from datafusion.expr import Expr -from typing import Callable, TYPE_CHECKING, TypeVar from abc import ABCMeta, abstractmethod -from typing import List, Optional from enum import Enum +from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar + import pyarrow +import datafusion._internal as df_internal +from datafusion.expr import Expr + if TYPE_CHECKING: _R = TypeVar("_R", bound=pyarrow.DataType) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index 1cc07e500..9548fbfe4 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. +import pyarrow as pa import pytest from datafusion import SessionContext -import pyarrow as pa from pyarrow.csv import write_csv diff --git a/python/tests/test_aggregation.py b/python/tests/test_aggregation.py index 243a8c3c9..5ef46131b 100644 --- a/python/tests/test_aggregation.py +++ b/python/tests/test_aggregation.py @@ -18,7 +18,6 @@ import numpy as np import pyarrow as pa import pytest - from datafusion import SessionContext, column, lit from datafusion import functions as f from datafusion.common import NullTreatment diff --git a/python/tests/test_config.py b/python/tests/test_config.py index 12d9fc3ff..c1d7f97e1 100644 --- a/python/tests/test_config.py +++ b/python/tests/test_config.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from datafusion import Config import pytest +from datafusion import Config @pytest.fixture diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 10e8ad0e9..91046e6b8 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -14,15 +14,14 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import datetime as dt import gzip import os -import datetime as dt import pathlib import pyarrow as pa import pyarrow.dataset as ds import pytest - from datafusion import ( DataFrame, RuntimeEnvBuilder, diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index a1a871e9a..5bc3fb094 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -18,11 +18,8 @@ from typing import Any import pyarrow as pa -from pyarrow.csv import write_csv import pyarrow.parquet as pq import pytest - -from datafusion import functions as f from datafusion import ( DataFrame, SessionContext, @@ -30,7 +27,9 @@ column, literal, ) +from datafusion import functions as f from datafusion.expr import Window +from pyarrow.csv import write_csv @pytest.fixture diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index add170c17..ad6aa7c0a 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -15,15 +15,13 @@ # specific language governing permissions and limitations # under the License. import math +from datetime import datetime import numpy as np import pyarrow as pa import pytest -from datetime import datetime - -from datafusion import SessionContext, column +from datafusion import SessionContext, column, literal, string_literal from datafusion import functions as f -from datafusion import literal, string_literal np.seterr(invalid="ignore") diff --git a/python/tests/test_imports.py b/python/tests/test_imports.py index 6ea77b15f..0c155cbde 100644 --- a/python/tests/test_imports.py +++ b/python/tests/test_imports.py @@ -15,72 +15,69 @@ # specific language governing permissions and limitations # under the License. -import pytest - import datafusion +import pytest from datafusion import ( AggregateUDF, DataFrame, - SessionContext, ScalarUDF, + SessionContext, functions, ) - from datafusion.common import ( DFSchema, ) - from datafusion.expr import ( - Expr, - Column, - Literal, - BinaryExpr, - AggregateFunction, - Projection, - TableScan, - Filter, - Limit, Aggregate, - Sort, - Analyze, - Join, - JoinType, - JoinConstraint, - Union, - Like, - ILike, - SimilarTo, - ScalarVariable, + AggregateFunction, Alias, - Not, - IsNotNull, - IsTrue, - IsFalse, - IsUnknown, - IsNotTrue, - IsNotFalse, - IsNotUnknown, - Negative, - InList, - Exists, - Subquery, - InSubquery, - ScalarSubquery, - GroupingSet, - Placeholder, + Analyze, + Between, + BinaryExpr, Case, Cast, - TryCast, - SubqueryAlias, - Between, - Explain, - Extension, + Column, CreateMemoryTable, CreateView, Distinct, DropTable, - Repartition, + Exists, + Explain, + Expr, + Extension, + Filter, + GroupingSet, + ILike, + InList, + InSubquery, + IsFalse, + IsNotFalse, + IsNotNull, + IsNotTrue, + IsNotUnknown, + IsTrue, + IsUnknown, + Join, + JoinConstraint, + JoinType, + Like, + Limit, + Literal, + Negative, + Not, Partitioning, + Placeholder, + Projection, + Repartition, + ScalarSubquery, + ScalarVariable, + SimilarTo, + Sort, + Subquery, + SubqueryAlias, + TableScan, + TryCast, + Union, ) diff --git a/python/tests/test_indexing.py b/python/tests/test_indexing.py index 8ca3eab19..5b0d08610 100644 --- a/python/tests/test_indexing.py +++ b/python/tests/test_indexing.py @@ -17,7 +17,6 @@ import pyarrow as pa import pytest - from datafusion import SessionContext diff --git a/python/tests/test_input.py b/python/tests/test_input.py index fb53d86e5..806471357 100644 --- a/python/tests/test_input.py +++ b/python/tests/test_input.py @@ -16,6 +16,7 @@ # under the License. import os + from datafusion.input.location import LocationInputPlugin diff --git a/python/tests/test_plans.py b/python/tests/test_plans.py index 0283a4e6a..396acbe97 100644 --- a/python/tests/test_plans.py +++ b/python/tests/test_plans.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from datafusion import SessionContext, LogicalPlan, ExecutionPlan import pytest +from datafusion import ExecutionPlan, LogicalPlan, SessionContext # Note: We must use CSV because memory tables are currently not supported for diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py index a2521dd09..862f745bf 100644 --- a/python/tests/test_sql.py +++ b/python/tests/test_sql.py @@ -19,12 +19,11 @@ import numpy as np import pyarrow as pa -from pyarrow.csv import write_csv import pyarrow.dataset as ds import pytest +from datafusion import col, udf from datafusion.object_store import Http - -from datafusion import udf, col +from pyarrow.csv import write_csv from . import generic as helpers diff --git a/python/tests/test_store.py b/python/tests/test_store.py index f85b28311..53ffc3acf 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -18,7 +18,6 @@ import os import pytest - from datafusion import SessionContext diff --git a/python/tests/test_substrait.py b/python/tests/test_substrait.py index 2071c8f3b..feada7cde 100644 --- a/python/tests/test_substrait.py +++ b/python/tests/test_substrait.py @@ -16,10 +16,9 @@ # under the License. import pyarrow as pa - +import pytest from datafusion import SessionContext from datafusion import substrait as ss -import pytest @pytest.fixture diff --git a/python/tests/test_udaf.py b/python/tests/test_udaf.py index 8f31748e0..0005a3da8 100644 --- a/python/tests/test_udaf.py +++ b/python/tests/test_udaf.py @@ -20,7 +20,6 @@ import pyarrow as pa import pyarrow.compute as pc import pytest - from datafusion import Accumulator, column, udaf diff --git a/python/tests/test_udf.py b/python/tests/test_udf.py index 568a66dbb..3a5dce6d6 100644 --- a/python/tests/test_udf.py +++ b/python/tests/test_udf.py @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -from datafusion import udf, column import pyarrow as pa import pytest +from datafusion import column, udf @pytest.fixture diff --git a/python/tests/test_udwf.py b/python/tests/test_udwf.py index 2099ac9bc..0ffa04179 100644 --- a/python/tests/test_udwf.py +++ b/python/tests/test_udwf.py @@ -17,10 +17,10 @@ import pyarrow as pa import pytest - -from datafusion import SessionContext, column, udwf, lit, functions as f -from datafusion.udf import WindowEvaluator +from datafusion import SessionContext, column, lit, udwf +from datafusion import functions as f from datafusion.expr import WindowFrame +from datafusion.udf import WindowEvaluator class ExponentialSmoothDefault(WindowEvaluator): From 8b513906315a0749b9f5cd6f34bf259ab4dd1add Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 1 Feb 2025 08:29:48 -0500 Subject: [PATCH 258/413] feat: remove DataFusion pyarrow feat (#1000) * Add developer instructions to speed up build processes * Remove pyarrow dep from datafusion. Add in PyScalarValue wrapper and rename DataFusionError to PyDataFusionError to be less confusing * Removed unnecessary cloning of scalar value when going from rust to python. Also removed the rust unit tests copied over from upstream repo that were failing due to #941 in pyo3 * Change return types to PyDataFusionError to simplify code * Update exception handling to fix build errors in recent rust toolchains --- Cargo.lock | 145 +++++++++++------- Cargo.toml | 2 +- .../source/contributor-guide/introduction.rst | 53 +++++++ python/tests/test_indexing.py | 3 +- src/catalog.rs | 8 +- src/common/data_type.rs | 14 ++ src/config.rs | 11 +- src/context.rs | 136 ++++++++-------- src/dataframe.rs | 119 +++++++------- src/dataset_exec.rs | 6 +- src/errors.rs | 42 ++--- src/expr.rs | 38 ++--- src/expr/conditional_expr.rs | 6 +- src/expr/literal.rs | 4 +- src/expr/window.rs | 13 +- src/functions.rs | 55 ++++--- src/lib.rs | 1 + src/physical_plan.rs | 13 +- src/pyarrow_filter_expression.rs | 24 +-- src/pyarrow_util.rs | 61 ++++++++ src/record_batch.rs | 3 +- src/sql/exceptions.rs | 16 +- src/sql/logical.rs | 14 +- src/substrait.rs | 54 ++++--- src/udaf.rs | 21 ++- src/udwf.rs | 4 +- src/utils.rs | 6 +- 27 files changed, 524 insertions(+), 348 deletions(-) create mode 100644 src/pyarrow_util.rs diff --git a/Cargo.lock b/Cargo.lock index 5a74a4839..c6590fd21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,7 +79,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy", @@ -449,9 +449,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.85" +version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", @@ -576,9 +576,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -586,9 +586,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "byteorder" @@ -635,9 +635,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.10" +version = "1.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" dependencies = [ "jobserver", "libc", @@ -692,9 +692,9 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" +checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" dependencies = [ "cc", ] @@ -725,7 +725,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -784,9 +784,9 @@ checksum = "69f3b219d28b6e3b4ac87bc1fc522e0803ab22e055da177bff0068c4150c61a6" [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -817,9 +817,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -961,7 +961,6 @@ dependencies = [ "object_store", "parquet", "paste", - "pyo3", "recursive", "sqlparser", "tokio", @@ -1411,9 +1410,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" +checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" [[package]] name = "either" @@ -1607,10 +1606,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets", +] + [[package]] name = "gimli" version = "0.31.1" @@ -1722,9 +1733,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "humantime" @@ -1734,9 +1745,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -1953,9 +1964,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1975,9 +1986,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "itertools" @@ -2243,7 +2254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2377,9 +2388,9 @@ checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "ordered-float" @@ -2661,9 +2672,9 @@ dependencies = [ [[package]] name = "protobuf-src" -version = "2.1.0+27.1" +version = "2.1.1+27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7edafa3bcc668fa93efafcbdf58d7821bbda0f4b458ac7fae3d57ec0fec8167" +checksum = "6217c3504da19b85a3a4b2e9a5183d635822d83507ba0986624b5c05b83bfc40" dependencies = [ "cmake", ] @@ -2794,7 +2805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring", "rustc-hash", @@ -2857,7 +2868,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -2926,9 +2937,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f56e622c2378013c6c61e2bd776604c46dc1087b2dc5293275a0c20a44f0771" +checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366" dependencies = [ "hashbrown 0.15.2", "memchr", @@ -2997,7 +3008,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -3033,9 +3044,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.43" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ "bitflags 2.8.0", "errno", @@ -3046,9 +3057,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.21" +version = "0.23.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" dependencies = [ "once_cell", "ring", @@ -3081,9 +3092,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time", ] @@ -3107,9 +3118,9 @@ checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "same-file" @@ -3184,9 +3195,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" dependencies = [ "serde", ] @@ -3239,9 +3250,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.136" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "336a0c23cf42a38d9eaa7cd22c7040d04e1228a19a933890805ffd00a16437d2" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -3514,13 +3525,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.15.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", - "getrandom", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3831,9 +3842,9 @@ dependencies = [ [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-segmentation" @@ -3890,11 +3901,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" +checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" dependencies = [ - "getrandom", + "getrandom 0.2.15", "serde", ] @@ -3929,6 +3940,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -4185,6 +4205,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 10cffccb1..003ba36e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,7 +38,7 @@ tokio = { version = "1.42", features = ["macros", "rt", "rt-multi-thread", "sync pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } pyo3-async-runtimes = { version = "0.22", features = ["tokio-runtime"]} arrow = { version = "53", features = ["pyarrow"] } -datafusion = { version = "44.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion = { version = "44.0.0", features = ["avro", "unicode_expressions"] } datafusion-substrait = { version = "44.0.0", optional = true } datafusion-proto = { version = "44.0.0" } datafusion-ffi = { version = "44.0.0" } diff --git a/docs/source/contributor-guide/introduction.rst b/docs/source/contributor-guide/introduction.rst index fb98cfd1d..25f2c21a4 100644 --- a/docs/source/contributor-guide/introduction.rst +++ b/docs/source/contributor-guide/introduction.rst @@ -95,3 +95,56 @@ To update dependencies, run .. code-block:: shell uv sync --dev --no-install-package datafusion + +Improving Build Speed +--------------------- + +The `pyo3 `_ dependency of this project contains a ``build.rs`` file which +can cause it to rebuild frequently. You can prevent this from happening by defining a ``PYO3_CONFIG_FILE`` +environment variable that points to a file with your build configuration. Whenever your build configuration +changes, such as during some major version updates, you will need to regenerate this file. This variable +should point to a fully resolved path on your build machine. + +To generate this file, use the following command: + +.. code-block:: shell + + PYO3_PRINT_CONFIG=1 cargo build + +This will generate some output that looks like the following. You will want to copy these contents intro +a file. If you place this file in your project directory with filename ``.pyo3_build_config`` it will +be ignored by ``git``. + +.. code-block:: + + implementation=CPython + version=3.8 + shared=true + abi3=true + lib_name=python3.12 + lib_dir=/opt/homebrew/opt/python@3.12/Frameworks/Python.framework/Versions/3.12/lib + executable=/Users/myusername/src/datafusion-python/.venv/bin/python + pointer_width=64 + build_flags= + suppress_build_script_link_lines=false + +Add the environment variable to your system. + +.. code-block:: shell + + export PYO3_CONFIG_FILE="/Users//myusername/src/datafusion-python/.pyo3_build_config" + +If you are on a Mac and you use VS Code for your IDE, you will want to add these variables +to your settings. You can find the appropriate rust flags by looking in the +``.cargo/config.toml`` file. + +.. code-block:: + + "rust-analyzer.cargo.extraEnv": { + "RUSTFLAGS": "-C link-arg=-undefined -C link-arg=dynamic_lookup", + "PYO3_CONFIG_FILE": "/Users/myusername/src/datafusion-python/.pyo3_build_config" + }, + "rust-analyzer.runnables.extraEnv": { + "RUSTFLAGS": "-C link-arg=-undefined -C link-arg=dynamic_lookup", + "PYO3_CONFIG_FILE": "/Users/myusername/src/personal/datafusion-python/.pyo3_build_config" + } diff --git a/python/tests/test_indexing.py b/python/tests/test_indexing.py index 5b0d08610..327decd2f 100644 --- a/python/tests/test_indexing.py +++ b/python/tests/test_indexing.py @@ -43,7 +43,8 @@ def test_err(df): with pytest.raises(Exception) as e_info: df["c"] - assert "Schema error: No field named c." in e_info.value.args[0] + for e in ["SchemaError", "FieldNotFound", 'name: "c"']: + assert e in e_info.value.args[0] with pytest.raises(Exception) as e_info: df[1] diff --git a/src/catalog.rs b/src/catalog.rs index 1ce66a4dc..1e189a5aa 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use pyo3::exceptions::PyKeyError; use pyo3::prelude::*; -use crate::errors::DataFusionError; +use crate::errors::{PyDataFusionError, PyDataFusionResult}; use crate::utils::wait_for_future; use datafusion::{ arrow::pyarrow::ToPyArrow, @@ -96,11 +96,13 @@ impl PyDatabase { self.database.table_names().into_iter().collect() } - fn table(&self, name: &str, py: Python) -> PyResult { + fn table(&self, name: &str, py: Python) -> PyDataFusionResult { if let Some(table) = wait_for_future(py, self.database.table(name))? { Ok(PyTable::new(table)) } else { - Err(DataFusionError::Common(format!("Table not found: {name}")).into()) + Err(PyDataFusionError::Common(format!( + "Table not found: {name}" + ))) } } diff --git a/src/common/data_type.rs b/src/common/data_type.rs index 7f9c75bfd..f5f8a6b06 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -23,6 +23,20 @@ use pyo3::{exceptions::PyValueError, prelude::*}; use crate::errors::py_datafusion_err; +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd)] +pub struct PyScalarValue(pub ScalarValue); + +impl From for PyScalarValue { + fn from(value: ScalarValue) -> Self { + Self(value) + } +} +impl From for ScalarValue { + fn from(value: PyScalarValue) -> Self { + value.0 + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] #[pyclass(eq, eq_int, name = "RexType", module = "datafusion.common")] pub enum RexType { diff --git a/src/config.rs b/src/config.rs index 3f2a05580..cc725b9a3 100644 --- a/src/config.rs +++ b/src/config.rs @@ -21,6 +21,8 @@ use pyo3::types::*; use datafusion::common::ScalarValue; use datafusion::config::ConfigOptions; +use crate::errors::PyDataFusionResult; + #[pyclass(name = "Config", module = "datafusion", subclass)] #[derive(Clone)] pub(crate) struct PyConfig { @@ -38,7 +40,7 @@ impl PyConfig { /// Get configurations from environment variables #[staticmethod] - pub fn from_env() -> PyResult { + pub fn from_env() -> PyDataFusionResult { Ok(Self { config: ConfigOptions::from_env()?, }) @@ -56,11 +58,10 @@ impl PyConfig { } /// Set a configuration option - pub fn set(&mut self, key: &str, value: PyObject, py: Python) -> PyResult<()> { + pub fn set(&mut self, key: &str, value: PyObject, py: Python) -> PyDataFusionResult<()> { let scalar_value = py_obj_to_scalar_value(py, value); - self.config - .set(key, scalar_value.to_string().as_str()) - .map_err(|e| e.into()) + self.config.set(key, scalar_value.to_string().as_str())?; + Ok(()) } /// Get all configuration options diff --git a/src/context.rs b/src/context.rs index bab7fd42a..f53b15576 100644 --- a/src/context.rs +++ b/src/context.rs @@ -28,16 +28,17 @@ use object_store::ObjectStore; use url::Url; use uuid::Uuid; -use pyo3::exceptions::{PyKeyError, PyNotImplementedError, PyTypeError, PyValueError}; +use pyo3::exceptions::{PyKeyError, PyValueError}; use pyo3::prelude::*; use crate::catalog::{PyCatalog, PyTable}; use crate::dataframe::PyDataFrame; use crate::dataset::Dataset; -use crate::errors::{py_datafusion_err, DataFusionError}; +use crate::errors::{py_datafusion_err, PyDataFusionResult}; use crate::expr::sort_expr::PySortExpr; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; +use crate::sql::exceptions::py_value_err; use crate::sql::logical::PyLogicalPlan; use crate::store::StorageContexts; use crate::udaf::PyAggregateUDF; @@ -277,7 +278,7 @@ impl PySessionContext { pub fn new( config: Option, runtime: Option, - ) -> PyResult { + ) -> PyDataFusionResult { let config = if let Some(c) = config { c.config } else { @@ -348,7 +349,7 @@ impl PySessionContext { schema: Option>, file_sort_order: Option>>, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let options = ListingOptions::new(Arc::new(ParquetFormat::new())) .with_file_extension(file_extension) .with_table_partition_cols(convert_table_partition_cols(table_partition_cols)?) @@ -365,7 +366,7 @@ impl PySessionContext { None => { let state = self.ctx.state(); let schema = options.infer_schema(&state, &table_path); - wait_for_future(py, schema).map_err(DataFusionError::from)? + wait_for_future(py, schema)? } }; let config = ListingTableConfig::new(table_path) @@ -382,9 +383,9 @@ impl PySessionContext { } /// Returns a PyDataFrame whose plan corresponds to the SQL statement. - pub fn sql(&mut self, query: &str, py: Python) -> PyResult { + pub fn sql(&mut self, query: &str, py: Python) -> PyDataFusionResult { let result = self.ctx.sql(query); - let df = wait_for_future(py, result).map_err(DataFusionError::from)?; + let df = wait_for_future(py, result)?; Ok(PyDataFrame::new(df)) } @@ -394,14 +395,14 @@ impl PySessionContext { query: &str, options: Option, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let options = if let Some(options) = options { options.options } else { SQLOptions::new() }; let result = self.ctx.sql_with_options(query, options); - let df = wait_for_future(py, result).map_err(DataFusionError::from)?; + let df = wait_for_future(py, result)?; Ok(PyDataFrame::new(df)) } @@ -412,14 +413,14 @@ impl PySessionContext { name: Option<&str>, schema: Option>, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let schema = if let Some(schema) = schema { SchemaRef::from(schema.0) } else { partitions.0[0][0].schema() }; - let table = MemTable::try_new(schema, partitions.0).map_err(DataFusionError::from)?; + let table = MemTable::try_new(schema, partitions.0)?; // generate a random (unique) name for this table if none is provided // table name cannot start with numeric digit @@ -433,11 +434,9 @@ impl PySessionContext { } }; - self.ctx - .register_table(&*table_name, Arc::new(table)) - .map_err(DataFusionError::from)?; + self.ctx.register_table(&*table_name, Arc::new(table))?; - let table = wait_for_future(py, self._table(&table_name)).map_err(DataFusionError::from)?; + let table = wait_for_future(py, self._table(&table_name))?; let df = PyDataFrame::new(table); Ok(df) @@ -495,15 +494,14 @@ impl PySessionContext { data: Bound<'_, PyAny>, name: Option<&str>, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let (schema, batches) = if let Ok(stream_reader) = ArrowArrayStreamReader::from_pyarrow_bound(&data) { // Works for any object that implements __arrow_c_stream__ in pycapsule. let schema = stream_reader.schema().as_ref().to_owned(); let batches = stream_reader - .collect::, arrow::error::ArrowError>>() - .map_err(DataFusionError::from)?; + .collect::, arrow::error::ArrowError>>()?; (schema, batches) } else if let Ok(array) = RecordBatch::from_pyarrow_bound(&data) { @@ -512,8 +510,8 @@ impl PySessionContext { (array.schema().as_ref().to_owned(), vec![array]) } else { - return Err(PyTypeError::new_err( - "Expected either a Arrow Array or Arrow Stream in from_arrow().", + return Err(crate::errors::PyDataFusionError::Common( + "Expected either a Arrow Array or Arrow Stream in from_arrow().".to_string(), )); }; @@ -559,17 +557,13 @@ impl PySessionContext { Ok(df) } - pub fn register_table(&mut self, name: &str, table: &PyTable) -> PyResult<()> { - self.ctx - .register_table(name, table.table()) - .map_err(DataFusionError::from)?; + pub fn register_table(&mut self, name: &str, table: &PyTable) -> PyDataFusionResult<()> { + self.ctx.register_table(name, table.table())?; Ok(()) } - pub fn deregister_table(&mut self, name: &str) -> PyResult<()> { - self.ctx - .deregister_table(name) - .map_err(DataFusionError::from)?; + pub fn deregister_table(&mut self, name: &str) -> PyDataFusionResult<()> { + self.ctx.deregister_table(name)?; Ok(()) } @@ -578,10 +572,10 @@ impl PySessionContext { &mut self, name: &str, provider: Bound<'_, PyAny>, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { if provider.hasattr("__datafusion_table_provider__")? { let capsule = provider.getattr("__datafusion_table_provider__")?.call0()?; - let capsule = capsule.downcast::()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; validate_pycapsule(capsule, "datafusion_table_provider")?; let provider = unsafe { capsule.reference::() }; @@ -591,8 +585,9 @@ impl PySessionContext { Ok(()) } else { - Err(PyNotImplementedError::new_err( - "__datafusion_table_provider__ does not exist on Table Provider object.", + Err(crate::errors::PyDataFusionError::Common( + "__datafusion_table_provider__ does not exist on Table Provider object." + .to_string(), )) } } @@ -601,12 +596,10 @@ impl PySessionContext { &mut self, name: &str, partitions: PyArrowType>>, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let schema = partitions.0[0][0].schema(); let table = MemTable::try_new(schema, partitions.0)?; - self.ctx - .register_table(name, Arc::new(table)) - .map_err(DataFusionError::from)?; + self.ctx.register_table(name, Arc::new(table))?; Ok(()) } @@ -628,7 +621,7 @@ impl PySessionContext { schema: Option>, file_sort_order: Option>>, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let mut options = ParquetReadOptions::default() .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) .parquet_pruning(parquet_pruning) @@ -642,7 +635,7 @@ impl PySessionContext { .collect(); let result = self.ctx.register_parquet(name, path, options); - wait_for_future(py, result).map_err(DataFusionError::from)?; + wait_for_future(py, result)?; Ok(()) } @@ -666,12 +659,12 @@ impl PySessionContext { file_extension: &str, file_compression_type: Option, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let delimiter = delimiter.as_bytes(); if delimiter.len() != 1 { - return Err(PyValueError::new_err( + return Err(crate::errors::PyDataFusionError::PythonError(py_value_err( "Delimiter must be a single character", - )); + ))); } let mut options = CsvReadOptions::new() @@ -685,11 +678,11 @@ impl PySessionContext { if path.is_instance_of::() { let paths = path.extract::>()?; let result = self.register_csv_from_multiple_paths(name, paths, options); - wait_for_future(py, result).map_err(DataFusionError::from)?; + wait_for_future(py, result)?; } else { let path = path.extract::()?; let result = self.ctx.register_csv(name, &path, options); - wait_for_future(py, result).map_err(DataFusionError::from)?; + wait_for_future(py, result)?; } Ok(()) @@ -713,7 +706,7 @@ impl PySessionContext { table_partition_cols: Vec<(String, String)>, file_compression_type: Option, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let path = path .to_str() .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; @@ -726,7 +719,7 @@ impl PySessionContext { options.schema = schema.as_ref().map(|x| &x.0); let result = self.ctx.register_json(name, path, options); - wait_for_future(py, result).map_err(DataFusionError::from)?; + wait_for_future(py, result)?; Ok(()) } @@ -745,7 +738,7 @@ impl PySessionContext { file_extension: &str, table_partition_cols: Vec<(String, String)>, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let path = path .to_str() .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; @@ -756,7 +749,7 @@ impl PySessionContext { options.schema = schema.as_ref().map(|x| &x.0); let result = self.ctx.register_avro(name, path, options); - wait_for_future(py, result).map_err(DataFusionError::from)?; + wait_for_future(py, result)?; Ok(()) } @@ -767,12 +760,10 @@ impl PySessionContext { name: &str, dataset: &Bound<'_, PyAny>, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { let table: Arc = Arc::new(Dataset::new(dataset, py)?); - self.ctx - .register_table(name, table) - .map_err(DataFusionError::from)?; + self.ctx.register_table(name, table)?; Ok(()) } @@ -824,11 +815,11 @@ impl PySessionContext { Ok(PyDataFrame::new(x)) } - pub fn table_exist(&self, name: &str) -> PyResult { + pub fn table_exist(&self, name: &str) -> PyDataFusionResult { Ok(self.ctx.table_exist(name)?) } - pub fn empty_table(&self) -> PyResult { + pub fn empty_table(&self) -> PyDataFusionResult { Ok(PyDataFrame::new(self.ctx.read_empty()?)) } @@ -847,7 +838,7 @@ impl PySessionContext { table_partition_cols: Vec<(String, String)>, file_compression_type: Option, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let path = path .to_str() .ok_or_else(|| PyValueError::new_err("Unable to convert path to a string"))?; @@ -859,10 +850,10 @@ impl PySessionContext { let df = if let Some(schema) = schema { options.schema = Some(&schema.0); let result = self.ctx.read_json(path, options); - wait_for_future(py, result).map_err(DataFusionError::from)? + wait_for_future(py, result)? } else { let result = self.ctx.read_json(path, options); - wait_for_future(py, result).map_err(DataFusionError::from)? + wait_for_future(py, result)? }; Ok(PyDataFrame::new(df)) } @@ -888,12 +879,12 @@ impl PySessionContext { table_partition_cols: Vec<(String, String)>, file_compression_type: Option, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let delimiter = delimiter.as_bytes(); if delimiter.len() != 1 { - return Err(PyValueError::new_err( + return Err(crate::errors::PyDataFusionError::PythonError(py_value_err( "Delimiter must be a single character", - )); + ))); }; let mut options = CsvReadOptions::new() @@ -909,12 +900,12 @@ impl PySessionContext { let paths = path.extract::>()?; let paths = paths.iter().map(|p| p as &str).collect::>(); let result = self.ctx.read_csv(paths, options); - let df = PyDataFrame::new(wait_for_future(py, result).map_err(DataFusionError::from)?); + let df = PyDataFrame::new(wait_for_future(py, result)?); Ok(df) } else { let path = path.extract::()?; let result = self.ctx.read_csv(path, options); - let df = PyDataFrame::new(wait_for_future(py, result).map_err(DataFusionError::from)?); + let df = PyDataFrame::new(wait_for_future(py, result)?); Ok(df) } } @@ -938,7 +929,7 @@ impl PySessionContext { schema: Option>, file_sort_order: Option>>, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let mut options = ParquetReadOptions::default() .table_partition_cols(convert_table_partition_cols(table_partition_cols)?) .parquet_pruning(parquet_pruning) @@ -952,7 +943,7 @@ impl PySessionContext { .collect(); let result = self.ctx.read_parquet(path, options); - let df = PyDataFrame::new(wait_for_future(py, result).map_err(DataFusionError::from)?); + let df = PyDataFrame::new(wait_for_future(py, result)?); Ok(df) } @@ -965,26 +956,23 @@ impl PySessionContext { table_partition_cols: Vec<(String, String)>, file_extension: &str, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let mut options = AvroReadOptions::default() .table_partition_cols(convert_table_partition_cols(table_partition_cols)?); options.file_extension = file_extension; let df = if let Some(schema) = schema { options.schema = Some(&schema.0); let read_future = self.ctx.read_avro(path, options); - wait_for_future(py, read_future).map_err(DataFusionError::from)? + wait_for_future(py, read_future)? } else { let read_future = self.ctx.read_avro(path, options); - wait_for_future(py, read_future).map_err(DataFusionError::from)? + wait_for_future(py, read_future)? }; Ok(PyDataFrame::new(df)) } - pub fn read_table(&self, table: &PyTable) -> PyResult { - let df = self - .ctx - .read_table(table.table()) - .map_err(DataFusionError::from)?; + pub fn read_table(&self, table: &PyTable) -> PyDataFusionResult { + let df = self.ctx.read_table(table.table())?; Ok(PyDataFrame::new(df)) } @@ -1011,7 +999,7 @@ impl PySessionContext { plan: PyExecutionPlan, part: usize, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let ctx: TaskContext = TaskContext::from(&self.ctx.state()); // create a Tokio runtime to run the async code let rt = &get_tokio_runtime().0; @@ -1071,13 +1059,13 @@ impl PySessionContext { pub fn convert_table_partition_cols( table_partition_cols: Vec<(String, String)>, -) -> Result, DataFusionError> { +) -> PyDataFusionResult> { table_partition_cols .into_iter() .map(|(name, ty)| match ty.as_str() { "string" => Ok((name, DataType::Utf8)), "int" => Ok((name, DataType::Int32)), - _ => Err(DataFusionError::Common(format!( + _ => Err(crate::errors::PyDataFusionError::Common(format!( "Unsupported data type '{ty}' for partition column. Supported types are 'string' and 'int'" ))), }) diff --git a/src/dataframe.rs b/src/dataframe.rs index b875480a7..6fb08ba25 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -33,20 +33,20 @@ use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::prelude::*; -use pyo3::exceptions::{PyTypeError, PyValueError}; +use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyCapsule, PyTuple, PyTupleMethods}; use tokio::task::JoinHandle; -use crate::errors::py_datafusion_err; +use crate::errors::{py_datafusion_err, PyDataFusionError}; use crate::expr::sort_expr::to_sort_expressions; use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; use crate::utils::{get_tokio_runtime, validate_pycapsule, wait_for_future}; use crate::{ - errors::DataFusionError, + errors::PyDataFusionResult, expr::{sort_expr::PySortExpr, PyExpr}, }; @@ -69,7 +69,7 @@ impl PyDataFrame { #[pymethods] impl PyDataFrame { /// Enable selection for `df[col]`, `df[col1, col2, col3]`, and `df[[col1, col2, col3]]` - fn __getitem__(&self, key: Bound<'_, PyAny>) -> PyResult { + fn __getitem__(&self, key: Bound<'_, PyAny>) -> PyDataFusionResult { if let Ok(key) = key.extract::() { // df[col] self.select_columns(vec![key]) @@ -84,12 +84,12 @@ impl PyDataFrame { // df[[col1, col2, col3]] self.select_columns(keys) } else { - let message = "DataFrame can only be indexed by string index or indices"; - Err(PyTypeError::new_err(message)) + let message = "DataFrame can only be indexed by string index or indices".to_string(); + Err(PyDataFusionError::Common(message)) } } - fn __repr__(&self, py: Python) -> PyResult { + fn __repr__(&self, py: Python) -> PyDataFusionResult { let df = self.df.as_ref().clone().limit(0, Some(10))?; let batches = wait_for_future(py, df.collect())?; let batches_as_string = pretty::pretty_format_batches(&batches); @@ -99,7 +99,7 @@ impl PyDataFrame { } } - fn _repr_html_(&self, py: Python) -> PyResult { + fn _repr_html_(&self, py: Python) -> PyDataFusionResult { let mut html_str = "\n".to_string(); let df = self.df.as_ref().clone().limit(0, Some(10))?; @@ -145,7 +145,7 @@ impl PyDataFrame { } /// Calculate summary statistics for a DataFrame - fn describe(&self, py: Python) -> PyResult { + fn describe(&self, py: Python) -> PyDataFusionResult { let df = self.df.as_ref().clone(); let stat_df = wait_for_future(py, df.describe())?; Ok(Self::new(stat_df)) @@ -157,37 +157,37 @@ impl PyDataFrame { } #[pyo3(signature = (*args))] - fn select_columns(&self, args: Vec) -> PyResult { + fn select_columns(&self, args: Vec) -> PyDataFusionResult { let args = args.iter().map(|s| s.as_ref()).collect::>(); let df = self.df.as_ref().clone().select_columns(&args)?; Ok(Self::new(df)) } #[pyo3(signature = (*args))] - fn select(&self, args: Vec) -> PyResult { + fn select(&self, args: Vec) -> PyDataFusionResult { let expr = args.into_iter().map(|e| e.into()).collect(); let df = self.df.as_ref().clone().select(expr)?; Ok(Self::new(df)) } #[pyo3(signature = (*args))] - fn drop(&self, args: Vec) -> PyResult { + fn drop(&self, args: Vec) -> PyDataFusionResult { let cols = args.iter().map(|s| s.as_ref()).collect::>(); let df = self.df.as_ref().clone().drop_columns(&cols)?; Ok(Self::new(df)) } - fn filter(&self, predicate: PyExpr) -> PyResult { + fn filter(&self, predicate: PyExpr) -> PyDataFusionResult { let df = self.df.as_ref().clone().filter(predicate.into())?; Ok(Self::new(df)) } - fn with_column(&self, name: &str, expr: PyExpr) -> PyResult { + fn with_column(&self, name: &str, expr: PyExpr) -> PyDataFusionResult { let df = self.df.as_ref().clone().with_column(name, expr.into())?; Ok(Self::new(df)) } - fn with_columns(&self, exprs: Vec) -> PyResult { + fn with_columns(&self, exprs: Vec) -> PyDataFusionResult { let mut df = self.df.as_ref().clone(); for expr in exprs { let expr: Expr = expr.into(); @@ -199,7 +199,7 @@ impl PyDataFrame { /// Rename one column by applying a new projection. This is a no-op if the column to be /// renamed does not exist. - fn with_column_renamed(&self, old_name: &str, new_name: &str) -> PyResult { + fn with_column_renamed(&self, old_name: &str, new_name: &str) -> PyDataFusionResult { let df = self .df .as_ref() @@ -208,7 +208,7 @@ impl PyDataFrame { Ok(Self::new(df)) } - fn aggregate(&self, group_by: Vec, aggs: Vec) -> PyResult { + fn aggregate(&self, group_by: Vec, aggs: Vec) -> PyDataFusionResult { let group_by = group_by.into_iter().map(|e| e.into()).collect(); let aggs = aggs.into_iter().map(|e| e.into()).collect(); let df = self.df.as_ref().clone().aggregate(group_by, aggs)?; @@ -216,14 +216,14 @@ impl PyDataFrame { } #[pyo3(signature = (*exprs))] - fn sort(&self, exprs: Vec) -> PyResult { + fn sort(&self, exprs: Vec) -> PyDataFusionResult { let exprs = to_sort_expressions(exprs); let df = self.df.as_ref().clone().sort(exprs)?; Ok(Self::new(df)) } #[pyo3(signature = (count, offset=0))] - fn limit(&self, count: usize, offset: usize) -> PyResult { + fn limit(&self, count: usize, offset: usize) -> PyDataFusionResult { let df = self.df.as_ref().clone().limit(offset, Some(count))?; Ok(Self::new(df)) } @@ -232,14 +232,15 @@ impl PyDataFrame { /// Unless some order is specified in the plan, there is no /// guarantee of the order of the result. fn collect(&self, py: Python) -> PyResult> { - let batches = wait_for_future(py, self.df.as_ref().clone().collect())?; + let batches = wait_for_future(py, self.df.as_ref().clone().collect()) + .map_err(PyDataFusionError::from)?; // cannot use PyResult> return type due to // https://github.com/PyO3/pyo3/issues/1813 batches.into_iter().map(|rb| rb.to_pyarrow(py)).collect() } /// Cache DataFrame. - fn cache(&self, py: Python) -> PyResult { + fn cache(&self, py: Python) -> PyDataFusionResult { let df = wait_for_future(py, self.df.as_ref().clone().cache())?; Ok(Self::new(df)) } @@ -247,7 +248,8 @@ impl PyDataFrame { /// Executes this DataFrame and collects all results into a vector of vector of RecordBatch /// maintaining the input partitioning. fn collect_partitioned(&self, py: Python) -> PyResult>> { - let batches = wait_for_future(py, self.df.as_ref().clone().collect_partitioned())?; + let batches = wait_for_future(py, self.df.as_ref().clone().collect_partitioned()) + .map_err(PyDataFusionError::from)?; batches .into_iter() @@ -257,13 +259,13 @@ impl PyDataFrame { /// Print the result, 20 lines by default #[pyo3(signature = (num=20))] - fn show(&self, py: Python, num: usize) -> PyResult<()> { + fn show(&self, py: Python, num: usize) -> PyDataFusionResult<()> { let df = self.df.as_ref().clone().limit(0, Some(num))?; print_dataframe(py, df) } /// Filter out duplicate rows - fn distinct(&self) -> PyResult { + fn distinct(&self) -> PyDataFusionResult { let df = self.df.as_ref().clone().distinct()?; Ok(Self::new(df)) } @@ -274,7 +276,7 @@ impl PyDataFrame { how: &str, left_on: Vec, right_on: Vec, - ) -> PyResult { + ) -> PyDataFusionResult { let join_type = match how { "inner" => JoinType::Inner, "left" => JoinType::Left, @@ -283,10 +285,9 @@ impl PyDataFrame { "semi" => JoinType::LeftSemi, "anti" => JoinType::LeftAnti, how => { - return Err(DataFusionError::Common(format!( + return Err(PyDataFusionError::Common(format!( "The join type {how} does not exist or is not implemented" - )) - .into()); + ))); } }; @@ -303,7 +304,12 @@ impl PyDataFrame { Ok(Self::new(df)) } - fn join_on(&self, right: PyDataFrame, on_exprs: Vec, how: &str) -> PyResult { + fn join_on( + &self, + right: PyDataFrame, + on_exprs: Vec, + how: &str, + ) -> PyDataFusionResult { let join_type = match how { "inner" => JoinType::Inner, "left" => JoinType::Left, @@ -312,10 +318,9 @@ impl PyDataFrame { "semi" => JoinType::LeftSemi, "anti" => JoinType::LeftAnti, how => { - return Err(DataFusionError::Common(format!( + return Err(PyDataFusionError::Common(format!( "The join type {how} does not exist or is not implemented" - )) - .into()); + ))); } }; let exprs: Vec = on_exprs.into_iter().map(|e| e.into()).collect(); @@ -330,7 +335,7 @@ impl PyDataFrame { /// Print the query plan #[pyo3(signature = (verbose=false, analyze=false))] - fn explain(&self, py: Python, verbose: bool, analyze: bool) -> PyResult<()> { + fn explain(&self, py: Python, verbose: bool, analyze: bool) -> PyDataFusionResult<()> { let df = self.df.as_ref().clone().explain(verbose, analyze)?; print_dataframe(py, df) } @@ -341,18 +346,18 @@ impl PyDataFrame { } /// Get the optimized logical plan for this `DataFrame` - fn optimized_logical_plan(&self) -> PyResult { + fn optimized_logical_plan(&self) -> PyDataFusionResult { Ok(self.df.as_ref().clone().into_optimized_plan()?.into()) } /// Get the execution plan for this `DataFrame` - fn execution_plan(&self, py: Python) -> PyResult { + fn execution_plan(&self, py: Python) -> PyDataFusionResult { let plan = wait_for_future(py, self.df.as_ref().clone().create_physical_plan())?; Ok(plan.into()) } /// Repartition a `DataFrame` based on a logical partitioning scheme. - fn repartition(&self, num: usize) -> PyResult { + fn repartition(&self, num: usize) -> PyDataFusionResult { let new_df = self .df .as_ref() @@ -363,7 +368,7 @@ impl PyDataFrame { /// Repartition a `DataFrame` based on a logical partitioning scheme. #[pyo3(signature = (*args, num))] - fn repartition_by_hash(&self, args: Vec, num: usize) -> PyResult { + fn repartition_by_hash(&self, args: Vec, num: usize) -> PyDataFusionResult { let expr = args.into_iter().map(|py_expr| py_expr.into()).collect(); let new_df = self .df @@ -376,7 +381,7 @@ impl PyDataFrame { /// Calculate the union of two `DataFrame`s, preserving duplicate rows.The /// two `DataFrame`s must have exactly the same schema #[pyo3(signature = (py_df, distinct=false))] - fn union(&self, py_df: PyDataFrame, distinct: bool) -> PyResult { + fn union(&self, py_df: PyDataFrame, distinct: bool) -> PyDataFusionResult { let new_df = if distinct { self.df .as_ref() @@ -391,7 +396,7 @@ impl PyDataFrame { /// Calculate the distinct union of two `DataFrame`s. The /// two `DataFrame`s must have exactly the same schema - fn union_distinct(&self, py_df: PyDataFrame) -> PyResult { + fn union_distinct(&self, py_df: PyDataFrame) -> PyDataFusionResult { let new_df = self .df .as_ref() @@ -401,7 +406,7 @@ impl PyDataFrame { } #[pyo3(signature = (column, preserve_nulls=true))] - fn unnest_column(&self, column: &str, preserve_nulls: bool) -> PyResult { + fn unnest_column(&self, column: &str, preserve_nulls: bool) -> PyDataFusionResult { // TODO: expose RecursionUnnestOptions // REF: https://github.com/apache/datafusion/pull/11577 let unnest_options = UnnestOptions::default().with_preserve_nulls(preserve_nulls); @@ -414,7 +419,11 @@ impl PyDataFrame { } #[pyo3(signature = (columns, preserve_nulls=true))] - fn unnest_columns(&self, columns: Vec, preserve_nulls: bool) -> PyResult { + fn unnest_columns( + &self, + columns: Vec, + preserve_nulls: bool, + ) -> PyDataFusionResult { // TODO: expose RecursionUnnestOptions // REF: https://github.com/apache/datafusion/pull/11577 let unnest_options = UnnestOptions::default().with_preserve_nulls(preserve_nulls); @@ -428,7 +437,7 @@ impl PyDataFrame { } /// Calculate the intersection of two `DataFrame`s. The two `DataFrame`s must have exactly the same schema - fn intersect(&self, py_df: PyDataFrame) -> PyResult { + fn intersect(&self, py_df: PyDataFrame) -> PyDataFusionResult { let new_df = self .df .as_ref() @@ -438,13 +447,13 @@ impl PyDataFrame { } /// Calculate the exception of two `DataFrame`s. The two `DataFrame`s must have exactly the same schema - fn except_all(&self, py_df: PyDataFrame) -> PyResult { + fn except_all(&self, py_df: PyDataFrame) -> PyDataFusionResult { let new_df = self.df.as_ref().clone().except(py_df.df.as_ref().clone())?; Ok(Self::new(new_df)) } /// Write a `DataFrame` to a CSV file. - fn write_csv(&self, path: &str, with_header: bool, py: Python) -> PyResult<()> { + fn write_csv(&self, path: &str, with_header: bool, py: Python) -> PyDataFusionResult<()> { let csv_options = CsvOptions { has_header: Some(with_header), ..Default::default() @@ -472,7 +481,7 @@ impl PyDataFrame { compression: &str, compression_level: Option, py: Python, - ) -> PyResult<()> { + ) -> PyDataFusionResult<()> { fn verify_compression_level(cl: Option) -> Result { cl.ok_or(PyValueError::new_err("compression_level is not defined")) } @@ -496,7 +505,7 @@ impl PyDataFrame { "lz4_raw" => Compression::LZ4_RAW, "uncompressed" => Compression::UNCOMPRESSED, _ => { - return Err(PyValueError::new_err(format!( + return Err(PyDataFusionError::Common(format!( "Unrecognized compression type {compression}" ))); } @@ -522,7 +531,7 @@ impl PyDataFrame { } /// Executes a query and writes the results to a partitioned JSON file. - fn write_json(&self, path: &str, py: Python) -> PyResult<()> { + fn write_json(&self, path: &str, py: Python) -> PyDataFusionResult<()> { wait_for_future( py, self.df @@ -551,7 +560,7 @@ impl PyDataFrame { &'py mut self, py: Python<'py>, requested_schema: Option>, - ) -> PyResult> { + ) -> PyDataFusionResult> { let mut batches = wait_for_future(py, self.df.as_ref().clone().collect())?; let mut schema: Schema = self.df.schema().to_owned().into(); @@ -559,15 +568,14 @@ impl PyDataFrame { validate_pycapsule(&schema_capsule, "arrow_schema")?; let schema_ptr = unsafe { schema_capsule.reference::() }; - let desired_schema = Schema::try_from(schema_ptr).map_err(DataFusionError::from)?; + let desired_schema = Schema::try_from(schema_ptr)?; - schema = project_schema(schema, desired_schema).map_err(DataFusionError::ArrowError)?; + schema = project_schema(schema, desired_schema)?; batches = batches .into_iter() .map(|record_batch| record_batch_into_schema(record_batch, &schema)) - .collect::, ArrowError>>() - .map_err(DataFusionError::ArrowError)?; + .collect::, ArrowError>>()?; } let batches_wrapped = batches.into_iter().map(Ok); @@ -578,9 +586,10 @@ impl PyDataFrame { let ffi_stream = FFI_ArrowArrayStream::new(reader); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); PyCapsule::new_bound(py, ffi_stream, Some(stream_capsule_name)) + .map_err(PyDataFusionError::from) } - fn execute_stream(&self, py: Python) -> PyResult { + fn execute_stream(&self, py: Python) -> PyDataFusionResult { // create a Tokio runtime to run the async code let rt = &get_tokio_runtime().0; let df = self.df.as_ref().clone(); @@ -647,13 +656,13 @@ impl PyDataFrame { } // Executes this DataFrame to get the total number of rows. - fn count(&self, py: Python) -> PyResult { + fn count(&self, py: Python) -> PyDataFusionResult { Ok(wait_for_future(py, self.df.as_ref().clone().count())?) } } /// Print DataFrame -fn print_dataframe(py: Python, df: DataFrame) -> PyResult<()> { +fn print_dataframe(py: Python, df: DataFrame) -> PyDataFusionResult<()> { // Get string representation of record batches let batches = wait_for_future(py, df.collect())?; let batches_as_string = pretty::pretty_format_batches(&batches); diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 9d2559429..ace42115b 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -42,7 +42,7 @@ use datafusion::physical_plan::{ SendableRecordBatchStream, Statistics, }; -use crate::errors::DataFusionError; +use crate::errors::PyDataFusionResult; use crate::pyarrow_filter_expression::PyArrowFilterExpression; struct PyArrowBatchesAdapter { @@ -83,8 +83,8 @@ impl DatasetExec { dataset: &Bound<'_, PyAny>, projection: Option>, filters: &[Expr], - ) -> Result { - let columns: Option, DataFusionError>> = projection.map(|p| { + ) -> PyDataFusionResult { + let columns: Option>> = projection.map(|p| { p.iter() .map(|index| { let name: String = dataset diff --git a/src/errors.rs b/src/errors.rs index d12b6ade1..b02b754a2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -24,10 +24,10 @@ use datafusion::error::DataFusionError as InnerDataFusionError; use prost::EncodeError; use pyo3::{exceptions::PyException, PyErr}; -pub type Result = std::result::Result; +pub type PyDataFusionResult = std::result::Result; #[derive(Debug)] -pub enum DataFusionError { +pub enum PyDataFusionError { ExecutionError(InnerDataFusionError), ArrowError(ArrowError), Common(String), @@ -35,46 +35,46 @@ pub enum DataFusionError { EncodeError(EncodeError), } -impl fmt::Display for DataFusionError { +impl fmt::Display for PyDataFusionError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - DataFusionError::ExecutionError(e) => write!(f, "DataFusion error: {e:?}"), - DataFusionError::ArrowError(e) => write!(f, "Arrow error: {e:?}"), - DataFusionError::PythonError(e) => write!(f, "Python error {e:?}"), - DataFusionError::Common(e) => write!(f, "{e}"), - DataFusionError::EncodeError(e) => write!(f, "Failed to encode substrait plan: {e}"), + PyDataFusionError::ExecutionError(e) => write!(f, "DataFusion error: {e:?}"), + PyDataFusionError::ArrowError(e) => write!(f, "Arrow error: {e:?}"), + PyDataFusionError::PythonError(e) => write!(f, "Python error {e:?}"), + PyDataFusionError::Common(e) => write!(f, "{e}"), + PyDataFusionError::EncodeError(e) => write!(f, "Failed to encode substrait plan: {e}"), } } } -impl From for DataFusionError { - fn from(err: ArrowError) -> DataFusionError { - DataFusionError::ArrowError(err) +impl From for PyDataFusionError { + fn from(err: ArrowError) -> PyDataFusionError { + PyDataFusionError::ArrowError(err) } } -impl From for DataFusionError { - fn from(err: InnerDataFusionError) -> DataFusionError { - DataFusionError::ExecutionError(err) +impl From for PyDataFusionError { + fn from(err: InnerDataFusionError) -> PyDataFusionError { + PyDataFusionError::ExecutionError(err) } } -impl From for DataFusionError { - fn from(err: PyErr) -> DataFusionError { - DataFusionError::PythonError(err) +impl From for PyDataFusionError { + fn from(err: PyErr) -> PyDataFusionError { + PyDataFusionError::PythonError(err) } } -impl From for PyErr { - fn from(err: DataFusionError) -> PyErr { +impl From for PyErr { + fn from(err: PyDataFusionError) -> PyErr { match err { - DataFusionError::PythonError(py_err) => py_err, + PyDataFusionError::PythonError(py_err) => py_err, _ => PyException::new_err(err.to_string()), } } } -impl Error for DataFusionError {} +impl Error for PyDataFusionError {} pub fn py_type_err(e: impl Debug) -> PyErr { PyErr::new::(format!("{e:?}")) diff --git a/src/expr.rs b/src/expr.rs index bca0cd3fa..1e9983d42 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -24,7 +24,6 @@ use std::convert::{From, Into}; use std::sync::Arc; use window::PyWindowFrame; -use arrow::pyarrow::ToPyArrow; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::functions::core::expr_ext::FieldAccessor; @@ -33,15 +32,17 @@ use datafusion::logical_expr::{ expr::{AggregateFunction, InList, InSubquery, ScalarFunction, WindowFunction}, lit, Between, BinaryExpr, Case, Cast, Expr, Like, Operator, TryCast, }; -use datafusion::scalar::ScalarValue; -use crate::common::data_type::{DataTypeMap, NullTreatment, RexType}; -use crate::errors::{py_runtime_err, py_type_err, py_unsupported_variant_err, DataFusionError}; +use crate::common::data_type::{DataTypeMap, NullTreatment, PyScalarValue, RexType}; +use crate::errors::{ + py_runtime_err, py_type_err, py_unsupported_variant_err, PyDataFusionError, PyDataFusionResult, +}; use crate::expr::aggregate_expr::PyAggregateFunction; use crate::expr::binary_expr::PyBinaryExpr; use crate::expr::column::PyColumn; use crate::expr::literal::PyLiteral; use crate::functions::add_builder_fns_to_window; +use crate::pyarrow_util::scalar_to_pyarrow; use crate::sql::logical::PyLogicalPlan; use self::alias::PyAlias; @@ -261,8 +262,8 @@ impl PyExpr { } #[staticmethod] - pub fn literal(value: ScalarValue) -> PyExpr { - lit(value).into() + pub fn literal(value: PyScalarValue) -> PyExpr { + lit(value.0).into() } #[staticmethod] @@ -356,7 +357,7 @@ impl PyExpr { /// Extracts the Expr value into a PyObject that can be shared with Python pub fn python_value(&self, py: Python) -> PyResult { match &self.expr { - Expr::Literal(scalar_value) => Ok(scalar_value.to_pyarrow(py)?), + Expr::Literal(scalar_value) => scalar_to_pyarrow(scalar_value, py), _ => Err(py_type_err(format!( "Non Expr::Literal encountered in types: {:?}", &self.expr @@ -568,7 +569,7 @@ impl PyExpr { window_frame: Option, order_by: Option>, null_treatment: Option, - ) -> PyResult { + ) -> PyDataFusionResult { match &self.expr { Expr::AggregateFunction(agg_fn) => { let window_fn = Expr::WindowFunction(WindowFunction::new( @@ -592,10 +593,9 @@ impl PyExpr { null_treatment, ), _ => Err( - DataFusionError::ExecutionError(datafusion::error::DataFusionError::Plan( + PyDataFusionError::ExecutionError(datafusion::error::DataFusionError::Plan( format!("Using {} with `over` is not allowed. Must use an aggregate or window function.", self.expr.variant_name()), )) - .into(), ), } } @@ -649,34 +649,26 @@ impl PyExprFuncBuilder { .into() } - pub fn build(&self) -> PyResult { - self.builder - .clone() - .build() - .map(|expr| expr.into()) - .map_err(|err| err.into()) + pub fn build(&self) -> PyDataFusionResult { + Ok(self.builder.clone().build().map(|expr| expr.into())?) } } impl PyExpr { - pub fn _column_name(&self, plan: &LogicalPlan) -> Result { + pub fn _column_name(&self, plan: &LogicalPlan) -> PyDataFusionResult { let field = Self::expr_to_field(&self.expr, plan)?; Ok(field.name().to_owned()) } /// Create a [Field] representing an [Expr], given an input [LogicalPlan] to resolve against - pub fn expr_to_field( - expr: &Expr, - input_plan: &LogicalPlan, - ) -> Result, DataFusionError> { + pub fn expr_to_field(expr: &Expr, input_plan: &LogicalPlan) -> PyDataFusionResult> { match expr { Expr::Wildcard { .. } => { // Since * could be any of the valid column names just return the first one Ok(Arc::new(input_plan.schema().field(0).clone())) } _ => { - let fields = - exprlist_to_fields(&[expr.clone()], input_plan).map_err(PyErr::from)?; + let fields = exprlist_to_fields(&[expr.clone()], input_plan)?; Ok(fields[0].1.clone()) } } diff --git a/src/expr/conditional_expr.rs b/src/expr/conditional_expr.rs index a8a885c54..fe3af2e25 100644 --- a/src/expr/conditional_expr.rs +++ b/src/expr/conditional_expr.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::expr::PyExpr; +use crate::{errors::PyDataFusionResult, expr::PyExpr}; use datafusion::logical_expr::conditional_expressions::CaseBuilder; use pyo3::prelude::*; @@ -44,11 +44,11 @@ impl PyCaseBuilder { } } - fn otherwise(&mut self, else_expr: PyExpr) -> PyResult { + fn otherwise(&mut self, else_expr: PyExpr) -> PyDataFusionResult { Ok(self.case_builder.otherwise(else_expr.expr)?.clone().into()) } - fn end(&mut self) -> PyResult { + fn end(&mut self) -> PyDataFusionResult { Ok(self.case_builder.end()?.clone().into()) } } diff --git a/src/expr/literal.rs b/src/expr/literal.rs index 43084ba4b..2cb2079f1 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::errors::DataFusionError; +use crate::errors::PyDataFusionError; use datafusion::common::ScalarValue; use pyo3::prelude::*; @@ -154,5 +154,5 @@ impl PyLiteral { } fn unexpected_literal_value(value: &ScalarValue) -> PyErr { - DataFusionError::Common(format!("getValue() - Unexpected value: {value}")).into() + PyDataFusionError::Common(format!("getValue() - Unexpected value: {value}")).into() } diff --git a/src/expr/window.rs b/src/expr/window.rs index 6486dbb32..4dc6cb9c9 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -21,8 +21,9 @@ use datafusion::logical_expr::{Expr, Window, WindowFrame, WindowFrameBound, Wind use pyo3::prelude::*; use std::fmt::{self, Display, Formatter}; +use crate::common::data_type::PyScalarValue; use crate::common::df_schema::PyDFSchema; -use crate::errors::py_type_err; +use crate::errors::{py_type_err, PyDataFusionResult}; use crate::expr::logical_node::LogicalNode; use crate::expr::sort_expr::{py_sort_expr_list, PySortExpr}; use crate::expr::PyExpr; @@ -171,8 +172,8 @@ impl PyWindowFrame { #[pyo3(signature=(unit, start_bound, end_bound))] pub fn new( unit: &str, - start_bound: Option, - end_bound: Option, + start_bound: Option, + end_bound: Option, ) -> PyResult { let units = unit.to_ascii_lowercase(); let units = match units.as_str() { @@ -187,7 +188,7 @@ impl PyWindowFrame { } }; let start_bound = match start_bound { - Some(start_bound) => WindowFrameBound::Preceding(start_bound), + Some(start_bound) => WindowFrameBound::Preceding(start_bound.0), None => match units { WindowFrameUnits::Range => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), @@ -200,7 +201,7 @@ impl PyWindowFrame { }, }; let end_bound = match end_bound { - Some(end_bound) => WindowFrameBound::Following(end_bound), + Some(end_bound) => WindowFrameBound::Following(end_bound.0), None => match units { WindowFrameUnits::Rows => WindowFrameBound::Following(ScalarValue::UInt64(None)), WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)), @@ -253,7 +254,7 @@ impl PyWindowFrameBound { matches!(self.frame_bound, WindowFrameBound::Following(_)) } /// Returns the offset of the window frame - pub fn get_offset(&self) -> PyResult> { + pub fn get_offset(&self) -> PyDataFusionResult> { match &self.frame_bound { WindowFrameBound::Preceding(val) | WindowFrameBound::Following(val) => match val { x if x.is_null() => Ok(None), diff --git a/src/functions.rs b/src/functions.rs index ae032d702..46c748cf8 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -22,8 +22,10 @@ use datafusion::logical_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; use crate::common::data_type::NullTreatment; +use crate::common::data_type::PyScalarValue; use crate::context::PySessionContext; -use crate::errors::DataFusionError; +use crate::errors::PyDataFusionError; +use crate::errors::PyDataFusionResult; use crate::expr::conditional_expr::PyCaseBuilder; use crate::expr::sort_expr::to_sort_expressions; use crate::expr::sort_expr::PySortExpr; @@ -44,7 +46,7 @@ fn add_builder_fns_to_aggregate( filter: Option, order_by: Option>, null_treatment: Option, -) -> PyResult { +) -> PyDataFusionResult { // Since ExprFuncBuilder::new() is private, we can guarantee initializing // a builder with an `null_treatment` with option None let mut builder = agg_fn.null_treatment(None); @@ -228,7 +230,10 @@ fn when(when: PyExpr, then: PyExpr) -> PyResult { /// 1) If no function has been found, search default aggregate functions. /// /// NOTE: we search the built-ins first because the `UDAF` versions currently do not have the same behavior. -fn find_window_fn(name: &str, ctx: Option) -> PyResult { +fn find_window_fn( + name: &str, + ctx: Option, +) -> PyDataFusionResult { if let Some(ctx) = ctx { // search UDAFs let udaf = ctx @@ -284,7 +289,9 @@ fn find_window_fn(name: &str, ctx: Option) -> PyResult, order_by: Option>, null_treatment: Option - ) -> PyResult { + ) -> PyDataFusionResult { let agg_fn = functions_aggregate::expr_fn::$NAME($($arg.into()),*); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) @@ -362,7 +369,7 @@ macro_rules! aggregate_function_vec_args { filter: Option, order_by: Option>, null_treatment: Option - ) -> PyResult { + ) -> PyDataFusionResult { let agg_fn = functions_aggregate::expr_fn::$NAME(vec![$($arg.into()),*]); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) @@ -642,7 +649,7 @@ pub fn approx_percentile_cont( percentile: f64, num_centroids: Option, // enforces optional arguments at the end, currently filter: Option, -) -> PyResult { +) -> PyDataFusionResult { let args = if let Some(num_centroids) = num_centroids { vec![expression.expr, lit(percentile), lit(num_centroids)] } else { @@ -661,7 +668,7 @@ pub fn approx_percentile_cont_with_weight( weight: PyExpr, percentile: f64, filter: Option, -) -> PyResult { +) -> PyDataFusionResult { let agg_fn = functions_aggregate::expr_fn::approx_percentile_cont_with_weight( expression.expr, weight.expr, @@ -683,7 +690,7 @@ pub fn first_value( filter: Option, order_by: Option>, null_treatment: Option, -) -> PyResult { +) -> PyDataFusionResult { // If we initialize the UDAF with order_by directly, then it gets over-written by the builder let agg_fn = functions_aggregate::expr_fn::first_value(expr.expr, None); @@ -700,7 +707,7 @@ pub fn nth_value( filter: Option, order_by: Option>, null_treatment: Option, -) -> PyResult { +) -> PyDataFusionResult { let agg_fn = datafusion::functions_aggregate::nth_value::nth_value(expr.expr, n, vec![]); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } @@ -715,7 +722,7 @@ pub fn string_agg( filter: Option, order_by: Option>, null_treatment: Option, -) -> PyResult { +) -> PyDataFusionResult { let agg_fn = datafusion::functions_aggregate::string_agg::string_agg(expr.expr, lit(delimiter)); add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) } @@ -726,7 +733,7 @@ pub(crate) fn add_builder_fns_to_window( window_frame: Option, order_by: Option>, null_treatment: Option, -) -> PyResult { +) -> PyDataFusionResult { let null_treatment = null_treatment.map(|n| n.into()); let mut builder = window_fn.null_treatment(null_treatment); @@ -748,7 +755,7 @@ pub(crate) fn add_builder_fns_to_window( builder = builder.window_frame(window_frame.into()); } - builder.build().map(|e| e.into()).map_err(|err| err.into()) + Ok(builder.build().map(|e| e.into())?) } #[pyfunction] @@ -756,10 +763,11 @@ pub(crate) fn add_builder_fns_to_window( pub fn lead( arg: PyExpr, shift_offset: i64, - default_value: Option, + default_value: Option, partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { + let default_value = default_value.map(|v| v.into()); let window_fn = functions_window::expr_fn::lead(arg.expr, Some(shift_offset), default_value); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -770,10 +778,11 @@ pub fn lead( pub fn lag( arg: PyExpr, shift_offset: i64, - default_value: Option, + default_value: Option, partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { + let default_value = default_value.map(|v| v.into()); let window_fn = functions_window::expr_fn::lag(arg.expr, Some(shift_offset), default_value); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -784,7 +793,7 @@ pub fn lag( pub fn row_number( partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { let window_fn = functions_window::expr_fn::row_number(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -795,7 +804,7 @@ pub fn row_number( pub fn rank( partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { let window_fn = functions_window::expr_fn::rank(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -806,7 +815,7 @@ pub fn rank( pub fn dense_rank( partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { let window_fn = functions_window::expr_fn::dense_rank(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -817,7 +826,7 @@ pub fn dense_rank( pub fn percent_rank( partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { let window_fn = functions_window::expr_fn::percent_rank(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -828,7 +837,7 @@ pub fn percent_rank( pub fn cume_dist( partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { let window_fn = functions_window::expr_fn::cume_dist(); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) @@ -840,7 +849,7 @@ pub fn ntile( arg: PyExpr, partition_by: Option>, order_by: Option>, -) -> PyResult { +) -> PyDataFusionResult { let window_fn = functions_window::expr_fn::ntile(arg.into()); add_builder_fns_to_window(window_fn, partition_by, None, order_by, None) diff --git a/src/lib.rs b/src/lib.rs index 1111d5d06..317c3a49a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,7 @@ pub mod expr; mod functions; pub mod physical_plan; mod pyarrow_filter_expression; +pub mod pyarrow_util; mod record_batch; pub mod sql; pub mod store; diff --git a/src/physical_plan.rs b/src/physical_plan.rs index 9ef2f0ebb..295908dc7 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -22,7 +22,7 @@ use std::sync::Arc; use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyBytes}; -use crate::{context::PySessionContext, errors::DataFusionError}; +use crate::{context::PySessionContext, errors::PyDataFusionResult}; #[pyclass(name = "ExecutionPlan", module = "datafusion", subclass)] #[derive(Debug, Clone)] @@ -58,7 +58,7 @@ impl PyExecutionPlan { format!("{}", d.indent(false)) } - pub fn to_proto<'py>(&'py self, py: Python<'py>) -> PyResult> { + pub fn to_proto<'py>(&'py self, py: Python<'py>) -> PyDataFusionResult> { let codec = DefaultPhysicalExtensionCodec {}; let proto = datafusion_proto::protobuf::PhysicalPlanNode::try_from_physical_plan( self.plan.clone(), @@ -70,7 +70,10 @@ impl PyExecutionPlan { } #[staticmethod] - pub fn from_proto(ctx: PySessionContext, proto_msg: Bound<'_, PyBytes>) -> PyResult { + pub fn from_proto( + ctx: PySessionContext, + proto_msg: Bound<'_, PyBytes>, + ) -> PyDataFusionResult { let bytes: &[u8] = proto_msg.extract()?; let proto_plan = datafusion_proto::protobuf::PhysicalPlanNode::decode(bytes).map_err(|e| { @@ -81,9 +84,7 @@ impl PyExecutionPlan { })?; let codec = DefaultPhysicalExtensionCodec {}; - let plan = proto_plan - .try_into_physical_plan(&ctx.ctx, &ctx.ctx.runtime_env(), &codec) - .map_err(DataFusionError::from)?; + let plan = proto_plan.try_into_physical_plan(&ctx.ctx, &ctx.ctx.runtime_env(), &codec)?; Ok(Self::new(plan)) } diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index 0f97ea442..314eebf4f 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -21,11 +21,11 @@ use pyo3::prelude::*; use std::convert::TryFrom; use std::result::Result; -use arrow::pyarrow::ToPyArrow; use datafusion::common::{Column, ScalarValue}; use datafusion::logical_expr::{expr::InList, Between, BinaryExpr, Expr, Operator}; -use crate::errors::DataFusionError; +use crate::errors::{PyDataFusionError, PyDataFusionResult}; +use crate::pyarrow_util::scalar_to_pyarrow; #[derive(Debug)] #[repr(transparent)] @@ -34,7 +34,7 @@ pub(crate) struct PyArrowFilterExpression(PyObject); fn operator_to_py<'py>( operator: &Operator, op: &Bound<'py, PyModule>, -) -> Result, DataFusionError> { +) -> PyDataFusionResult> { let py_op: Bound<'_, PyAny> = match operator { Operator::Eq => op.getattr("eq")?, Operator::NotEq => op.getattr("ne")?, @@ -45,7 +45,7 @@ fn operator_to_py<'py>( Operator::And => op.getattr("and_")?, Operator::Or => op.getattr("or_")?, _ => { - return Err(DataFusionError::Common(format!( + return Err(PyDataFusionError::Common(format!( "Unsupported operator {operator:?}" ))) } @@ -53,8 +53,8 @@ fn operator_to_py<'py>( Ok(py_op) } -fn extract_scalar_list(exprs: &[Expr], py: Python) -> Result, DataFusionError> { - let ret: Result, DataFusionError> = exprs +fn extract_scalar_list(exprs: &[Expr], py: Python) -> PyDataFusionResult> { + let ret = exprs .iter() .map(|expr| match expr { // TODO: should we also leverage `ScalarValue::to_pyarrow` here? @@ -71,11 +71,11 @@ fn extract_scalar_list(exprs: &[Expr], py: Python) -> Result, Data ScalarValue::Float32(Some(f)) => Ok(f.into_py(py)), ScalarValue::Float64(Some(f)) => Ok(f.into_py(py)), ScalarValue::Utf8(Some(s)) => Ok(s.into_py(py)), - _ => Err(DataFusionError::Common(format!( + _ => Err(PyDataFusionError::Common(format!( "PyArrow can't handle ScalarValue: {v:?}" ))), }, - _ => Err(DataFusionError::Common(format!( + _ => Err(PyDataFusionError::Common(format!( "Only a list of Literals are supported got {expr:?}" ))), }) @@ -90,7 +90,7 @@ impl PyArrowFilterExpression { } impl TryFrom<&Expr> for PyArrowFilterExpression { - type Error = DataFusionError; + type Error = PyDataFusionError; // Converts a Datafusion filter Expr into an expression string that can be evaluated by Python // Note that pyarrow.compute.{field,scalar} are put into Python globals() when evaluated @@ -100,9 +100,9 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { Python::with_gil(|py| { let pc = Python::import_bound(py, "pyarrow.compute")?; let op_module = Python::import_bound(py, "operator")?; - let pc_expr: Result, DataFusionError> = match expr { + let pc_expr: PyDataFusionResult> = match expr { Expr::Column(Column { name, .. }) => Ok(pc.getattr("field")?.call1((name,))?), - Expr::Literal(scalar) => Ok(scalar.to_pyarrow(py)?.into_bound(py)), + Expr::Literal(scalar) => Ok(scalar_to_pyarrow(scalar, py)?.into_bound(py)), Expr::BinaryExpr(BinaryExpr { left, op, right }) => { let operator = operator_to_py(op, &op_module)?; let left = PyArrowFilterExpression::try_from(left.as_ref())?.0; @@ -167,7 +167,7 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { Ok(if *negated { invert.call1((ret,))? } else { ret }) } - _ => Err(DataFusionError::Common(format!( + _ => Err(PyDataFusionError::Common(format!( "Unsupported Datafusion expression {expr:?}" ))), }; diff --git a/src/pyarrow_util.rs b/src/pyarrow_util.rs new file mode 100644 index 000000000..2b31467f8 --- /dev/null +++ b/src/pyarrow_util.rs @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Conversions between PyArrow and DataFusion types + +use arrow::array::{Array, ArrayData}; +use arrow::pyarrow::{FromPyArrow, ToPyArrow}; +use datafusion::scalar::ScalarValue; +use pyo3::types::{PyAnyMethods, PyList}; +use pyo3::{Bound, FromPyObject, PyAny, PyObject, PyResult, Python}; + +use crate::common::data_type::PyScalarValue; +use crate::errors::PyDataFusionError; + +impl FromPyArrow for PyScalarValue { + fn from_pyarrow_bound(value: &Bound<'_, PyAny>) -> PyResult { + let py = value.py(); + let typ = value.getattr("type")?; + let val = value.call_method0("as_py")?; + + // construct pyarrow array from the python value and pyarrow type + let factory = py.import_bound("pyarrow")?.getattr("array")?; + let args = PyList::new_bound(py, [val]); + let array = factory.call1((args, typ))?; + + // convert the pyarrow array to rust array using C data interface + let array = arrow::array::make_array(ArrayData::from_pyarrow_bound(&array)?); + let scalar = ScalarValue::try_from_array(&array, 0).map_err(PyDataFusionError::from)?; + + Ok(PyScalarValue(scalar)) + } +} + +impl<'source> FromPyObject<'source> for PyScalarValue { + fn extract_bound(value: &Bound<'source, PyAny>) -> PyResult { + Self::from_pyarrow_bound(value) + } +} + +pub fn scalar_to_pyarrow(scalar: &ScalarValue, py: Python) -> PyResult { + let array = scalar.to_array().map_err(PyDataFusionError::from)?; + // convert to pyarrow array using C data interface + let pyarray = array.to_data().to_pyarrow(py)?; + let pyscalar = pyarray.call_method1(py, "__getitem__", (0,))?; + + Ok(pyscalar) +} diff --git a/src/record_batch.rs b/src/record_batch.rs index eacdb5867..ec61c263f 100644 --- a/src/record_batch.rs +++ b/src/record_batch.rs @@ -17,6 +17,7 @@ use std::sync::Arc; +use crate::errors::PyDataFusionError; use crate::utils::wait_for_future; use datafusion::arrow::pyarrow::ToPyArrow; use datafusion::arrow::record_batch::RecordBatch; @@ -90,7 +91,7 @@ async fn next_stream( let mut stream = stream.lock().await; match stream.next().await { Some(Ok(batch)) => Ok(batch.into()), - Some(Err(e)) => Err(e.into()), + Some(Err(e)) => Err(PyDataFusionError::from(e))?, None => { // Depending on whether the iteration is sync or not, we raise either a // StopIteration or a StopAsyncIteration diff --git a/src/sql/exceptions.rs b/src/sql/exceptions.rs index c458402a0..cfb02274b 100644 --- a/src/sql/exceptions.rs +++ b/src/sql/exceptions.rs @@ -17,13 +17,7 @@ use std::fmt::{Debug, Display}; -use pyo3::{create_exception, PyErr}; - -// Identifies exceptions that occur while attempting to generate a `LogicalPlan` from a SQL string -create_exception!(rust, ParsingException, pyo3::exceptions::PyException); - -// Identifies exceptions that occur during attempts to optimization an existing `LogicalPlan` -create_exception!(rust, OptimizationException, pyo3::exceptions::PyException); +use pyo3::PyErr; pub fn py_type_err(e: impl Debug + Display) -> PyErr { PyErr::new::(format!("{e}")) @@ -33,10 +27,6 @@ pub fn py_runtime_err(e: impl Debug + Display) -> PyErr { PyErr::new::(format!("{e}")) } -pub fn py_parsing_exp(e: impl Debug + Display) -> PyErr { - PyErr::new::(format!("{e}")) -} - -pub fn py_optimization_exp(e: impl Debug + Display) -> PyErr { - PyErr::new::(format!("{e}")) +pub fn py_value_err(e: impl Debug + Display) -> PyErr { + PyErr::new::(format!("{e}")) } diff --git a/src/sql/logical.rs b/src/sql/logical.rs index a541889c7..1be33b75f 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -17,6 +17,7 @@ use std::sync::Arc; +use crate::errors::PyDataFusionResult; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; use crate::expr::distinct::PyDistinct; @@ -34,7 +35,7 @@ use crate::expr::table_scan::PyTableScan; use crate::expr::unnest::PyUnnest; use crate::expr::window::PyWindowExpr; use crate::{context::PySessionContext, errors::py_unsupported_variant_err}; -use datafusion::{error::DataFusionError, logical_expr::LogicalPlan}; +use datafusion::logical_expr::LogicalPlan; use datafusion_proto::logical_plan::{AsLogicalPlan, DefaultLogicalExtensionCodec}; use prost::Message; use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyBytes}; @@ -125,7 +126,7 @@ impl PyLogicalPlan { format!("{}", self.plan.display_graphviz()) } - pub fn to_proto<'py>(&'py self, py: Python<'py>) -> PyResult> { + pub fn to_proto<'py>(&'py self, py: Python<'py>) -> PyDataFusionResult> { let codec = DefaultLogicalExtensionCodec {}; let proto = datafusion_proto::protobuf::LogicalPlanNode::try_from_logical_plan(&self.plan, &codec)?; @@ -135,7 +136,10 @@ impl PyLogicalPlan { } #[staticmethod] - pub fn from_proto(ctx: PySessionContext, proto_msg: Bound<'_, PyBytes>) -> PyResult { + pub fn from_proto( + ctx: PySessionContext, + proto_msg: Bound<'_, PyBytes>, + ) -> PyDataFusionResult { let bytes: &[u8] = proto_msg.extract()?; let proto_plan = datafusion_proto::protobuf::LogicalPlanNode::decode(bytes).map_err(|e| { @@ -146,9 +150,7 @@ impl PyLogicalPlan { })?; let codec = DefaultLogicalExtensionCodec {}; - let plan = proto_plan - .try_into_logical_plan(&ctx.ctx, &codec) - .map_err(DataFusionError::from)?; + let plan = proto_plan.try_into_logical_plan(&ctx.ctx, &codec)?; Ok(Self::new(plan)) } } diff --git a/src/substrait.rs b/src/substrait.rs index 16e8c9507..8dcf3e8a7 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -18,7 +18,7 @@ use pyo3::{prelude::*, types::PyBytes}; use crate::context::PySessionContext; -use crate::errors::{py_datafusion_err, DataFusionError}; +use crate::errors::{py_datafusion_err, PyDataFusionError, PyDataFusionResult}; use crate::sql::logical::PyLogicalPlan; use crate::utils::wait_for_future; @@ -39,7 +39,7 @@ impl PyPlan { let mut proto_bytes = Vec::::new(); self.plan .encode(&mut proto_bytes) - .map_err(DataFusionError::EncodeError)?; + .map_err(PyDataFusionError::EncodeError)?; Ok(PyBytes::new_bound(py, &proto_bytes).unbind().into()) } } @@ -66,41 +66,47 @@ pub struct PySubstraitSerializer; #[pymethods] impl PySubstraitSerializer { #[staticmethod] - pub fn serialize(sql: &str, ctx: PySessionContext, path: &str, py: Python) -> PyResult<()> { - wait_for_future(py, serializer::serialize(sql, &ctx.ctx, path)) - .map_err(DataFusionError::from)?; + pub fn serialize( + sql: &str, + ctx: PySessionContext, + path: &str, + py: Python, + ) -> PyDataFusionResult<()> { + wait_for_future(py, serializer::serialize(sql, &ctx.ctx, path))?; Ok(()) } #[staticmethod] - pub fn serialize_to_plan(sql: &str, ctx: PySessionContext, py: Python) -> PyResult { - match PySubstraitSerializer::serialize_bytes(sql, ctx, py) { - Ok(proto_bytes) => { - let proto_bytes = proto_bytes.bind(py).downcast::().unwrap(); - PySubstraitSerializer::deserialize_bytes(proto_bytes.as_bytes().to_vec(), py) - } - Err(e) => Err(py_datafusion_err(e)), - } + pub fn serialize_to_plan( + sql: &str, + ctx: PySessionContext, + py: Python, + ) -> PyDataFusionResult { + PySubstraitSerializer::serialize_bytes(sql, ctx, py).and_then(|proto_bytes| { + let proto_bytes = proto_bytes.bind(py).downcast::().unwrap(); + PySubstraitSerializer::deserialize_bytes(proto_bytes.as_bytes().to_vec(), py) + }) } #[staticmethod] - pub fn serialize_bytes(sql: &str, ctx: PySessionContext, py: Python) -> PyResult { - let proto_bytes: Vec = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx)) - .map_err(DataFusionError::from)?; + pub fn serialize_bytes( + sql: &str, + ctx: PySessionContext, + py: Python, + ) -> PyDataFusionResult { + let proto_bytes: Vec = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx))?; Ok(PyBytes::new_bound(py, &proto_bytes).unbind().into()) } #[staticmethod] - pub fn deserialize(path: &str, py: Python) -> PyResult { - let plan = - wait_for_future(py, serializer::deserialize(path)).map_err(DataFusionError::from)?; + pub fn deserialize(path: &str, py: Python) -> PyDataFusionResult { + let plan = wait_for_future(py, serializer::deserialize(path))?; Ok(PyPlan { plan: *plan }) } #[staticmethod] - pub fn deserialize_bytes(proto_bytes: Vec, py: Python) -> PyResult { - let plan = wait_for_future(py, serializer::deserialize_bytes(proto_bytes)) - .map_err(DataFusionError::from)?; + pub fn deserialize_bytes(proto_bytes: Vec, py: Python) -> PyDataFusionResult { + let plan = wait_for_future(py, serializer::deserialize_bytes(proto_bytes))?; Ok(PyPlan { plan: *plan }) } } @@ -134,10 +140,10 @@ impl PySubstraitConsumer { ctx: &mut PySessionContext, plan: PyPlan, py: Python, - ) -> PyResult { + ) -> PyDataFusionResult { let session_state = ctx.ctx.state(); let result = consumer::from_substrait_plan(&session_state, &plan.plan); - let logical_plan = wait_for_future(py, result).map_err(DataFusionError::from)?; + let logical_plan = wait_for_future(py, result)?; Ok(PyLogicalPlan::new(logical_plan)) } } diff --git a/src/udaf.rs b/src/udaf.rs index a6aa59ac3..5f21533e0 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -28,6 +28,7 @@ use datafusion::logical_expr::{ create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF, }; +use crate::common::data_type::PyScalarValue; use crate::expr::PyExpr; use crate::utils::parse_volatility; @@ -44,13 +45,25 @@ impl RustAccumulator { impl Accumulator for RustAccumulator { fn state(&mut self) -> Result> { - Python::with_gil(|py| self.accum.bind(py).call_method0("state")?.extract()) - .map_err(|e| DataFusionError::Execution(format!("{e}"))) + Python::with_gil(|py| { + self.accum + .bind(py) + .call_method0("state")? + .extract::>() + }) + .map(|v| v.into_iter().map(|x| x.0).collect()) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) } fn evaluate(&mut self) -> Result { - Python::with_gil(|py| self.accum.bind(py).call_method0("evaluate")?.extract()) - .map_err(|e| DataFusionError::Execution(format!("{e}"))) + Python::with_gil(|py| { + self.accum + .bind(py) + .call_method0("evaluate")? + .extract::() + }) + .map(|v| v.0) + .map_err(|e| DataFusionError::Execution(format!("{e}"))) } fn update_batch(&mut self, values: &[ArrayRef]) -> Result<()> { diff --git a/src/udwf.rs b/src/udwf.rs index 689eb79e3..04a4a1640 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -26,6 +26,7 @@ use datafusion::scalar::ScalarValue; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; +use crate::common::data_type::PyScalarValue; use crate::expr::PyExpr; use crate::utils::parse_volatility; use datafusion::arrow::datatypes::DataType; @@ -133,7 +134,8 @@ impl PartitionEvaluator for RustPartitionEvaluator { self.evaluator .bind(py) .call_method1("evaluate", py_args) - .and_then(|v| v.extract()) + .and_then(|v| v.extract::()) + .map(|v| v.0) .map_err(|e| DataFusionError::Execution(format!("{e}"))) }) } diff --git a/src/utils.rs b/src/utils.rs index 795589752..ed224b364 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::errors::DataFusionError; +use crate::errors::{PyDataFusionError, PyDataFusionResult}; use crate::TokioRuntime; use datafusion::logical_expr::Volatility; use pyo3::exceptions::PyValueError; @@ -47,13 +47,13 @@ where py.allow_threads(|| runtime.block_on(f)) } -pub(crate) fn parse_volatility(value: &str) -> Result { +pub(crate) fn parse_volatility(value: &str) -> PyDataFusionResult { Ok(match value { "immutable" => Volatility::Immutable, "stable" => Volatility::Stable, "volatile" => Volatility::Volatile, value => { - return Err(DataFusionError::Common(format!( + return Err(PyDataFusionError::Common(format!( "Unsupportad volatility type: `{value}`, supported \ values are: immutable, stable and volatile." ))) From d3c4dabe3c24d419911106bdde3dfe1244e1224c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 5 Feb 2025 09:42:03 -0500 Subject: [PATCH 259/413] Fix verify-release-candidate script by removing reference to requirements-310.txt (#1012) * Fix verify-release-candidate script by removing requirements.txt * Update dev/release/verify-release-candidate.sh Co-authored-by: Kevin Liu --------- Co-authored-by: Kevin Liu --- dev/release/verify-release-candidate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 1a9104b55..2bfce0e2d 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -128,7 +128,7 @@ test_source_distribution() { python3 -m venv .venv source .venv/bin/activate python3 -m pip install -U pip - python3 -m pip install -r requirements-310.txt + python3 -m pip install -U maturin maturin develop #TODO: we should really run tests here as well From 93ac6a820353b3ddea014be1eddad8bd004b0fce Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 7 Feb 2025 10:39:51 -0500 Subject: [PATCH 260/413] Prepare release 44.0.0 (#1009) --- Cargo.lock | 2 +- Cargo.toml | 2 +- dev/changelog/44.0.0.md | 58 ++++++++ pyproject.toml | 1 + uv.lock | 301 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 361 insertions(+), 3 deletions(-) create mode 100644 dev/changelog/44.0.0.md diff --git a/Cargo.lock b/Cargo.lock index c6590fd21..50809696b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1327,7 +1327,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "43.0.0" +version = "44.0.0" dependencies = [ "arrow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 003ba36e5..44e6e2244 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "43.0.0" +version = "44.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] diff --git a/dev/changelog/44.0.0.md b/dev/changelog/44.0.0.md new file mode 100644 index 000000000..c5ed4bdb0 --- /dev/null +++ b/dev/changelog/44.0.0.md @@ -0,0 +1,58 @@ + + +# Apache DataFusion Python 44.0.0 Changelog + +This release consists of 12 commits from 5 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: support enable_url_table config [#980](https://github.com/apache/datafusion-python/pull/980) (chenkovsky) +- feat: remove DataFusion pyarrow feat [#1000](https://github.com/apache/datafusion-python/pull/1000) (timsaucer) + +**Fixed bugs:** + +- fix: correct LZ0 to LZO in compression options [#995](https://github.com/apache/datafusion-python/pull/995) (kosiew) + +**Other:** + +- Add arrow cast [#962](https://github.com/apache/datafusion-python/pull/962) (kosiew) +- Fix small issues in pyproject.toml [#976](https://github.com/apache/datafusion-python/pull/976) (kylebarron) +- chore: set validation and type hint for ffi tableprovider [#983](https://github.com/apache/datafusion-python/pull/983) (ion-elgreco) +- Support async iteration of RecordBatchStream [#975](https://github.com/apache/datafusion-python/pull/975) (kylebarron) +- Chore/upgrade datafusion 44 [#973](https://github.com/apache/datafusion-python/pull/973) (timsaucer) +- Default to ZSTD compression when writing Parquet [#981](https://github.com/apache/datafusion-python/pull/981) (kosiew) +- Feat/use uv python management [#994](https://github.com/apache/datafusion-python/pull/994) (timsaucer) +- minor: Update dependencies prior to release [#999](https://github.com/apache/datafusion-python/pull/999) (timsaucer) +- Apply import ordering in ruff check [#1001](https://github.com/apache/datafusion-python/pull/1001) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 5 Tim Saucer + 3 kosiew + 2 Kyle Barron + 1 Chongchen Chen + 1 Ion Koutsouris +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + diff --git a/pyproject.toml b/pyproject.toml index 32bb28d21..f416e02a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,6 +89,7 @@ dev = [ "pytest>=7.4.4", "ruff>=0.9.1", "toml>=0.10.2", + "pygithub==2.5.0", ] docs = [ "sphinx>=7.1.2", diff --git a/uv.lock b/uv.lock index 75d9ed018..587ddc8b7 100644 --- a/uv.lock +++ b/uv.lock @@ -139,6 +139,83 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/32/8f6669fc4798494966bf446c8c4a162e0b5d893dff088afddf76414f70e1/certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56", size = 164927 }, ] +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/07/f44ca684db4e4f08a3fdc6eeb9a0d15dc6883efc7b8c90357fdbf74e186c/cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14", size = 182191 }, + { url = "https://files.pythonhosted.org/packages/08/fd/cc2fedbd887223f9f5d170c96e57cbf655df9831a6546c1727ae13fa977a/cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67", size = 178592 }, + { url = "https://files.pythonhosted.org/packages/de/cc/4635c320081c78d6ffc2cab0a76025b691a91204f4aa317d568ff9280a2d/cffi-1.17.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:edae79245293e15384b51f88b00613ba9f7198016a5948b5dddf4917d4d26382", size = 426024 }, + { url = "https://files.pythonhosted.org/packages/b6/7b/3b2b250f3aab91abe5f8a51ada1b717935fdaec53f790ad4100fe2ec64d1/cffi-1.17.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:45398b671ac6d70e67da8e4224a065cec6a93541bb7aebe1b198a61b58c7b702", size = 448188 }, + { url = "https://files.pythonhosted.org/packages/d3/48/1b9283ebbf0ec065148d8de05d647a986c5f22586b18120020452fff8f5d/cffi-1.17.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ad9413ccdeda48c5afdae7e4fa2192157e991ff761e7ab8fdd8926f40b160cc3", size = 455571 }, + { url = "https://files.pythonhosted.org/packages/40/87/3b8452525437b40f39ca7ff70276679772ee7e8b394934ff60e63b7b090c/cffi-1.17.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5da5719280082ac6bd9aa7becb3938dc9f9cbd57fac7d2871717b1feb0902ab6", size = 436687 }, + { url = "https://files.pythonhosted.org/packages/8d/fb/4da72871d177d63649ac449aec2e8a29efe0274035880c7af59101ca2232/cffi-1.17.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bb1a08b8008b281856e5971307cc386a8e9c5b625ac297e853d36da6efe9c17", size = 446211 }, + { url = "https://files.pythonhosted.org/packages/ab/a0/62f00bcb411332106c02b663b26f3545a9ef136f80d5df746c05878f8c4b/cffi-1.17.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:045d61c734659cc045141be4bae381a41d89b741f795af1dd018bfb532fd0df8", size = 461325 }, + { url = "https://files.pythonhosted.org/packages/36/83/76127035ed2e7e27b0787604d99da630ac3123bfb02d8e80c633f218a11d/cffi-1.17.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6883e737d7d9e4899a8a695e00ec36bd4e5e4f18fabe0aca0efe0a4b44cdb13e", size = 438784 }, + { url = "https://files.pythonhosted.org/packages/21/81/a6cd025db2f08ac88b901b745c163d884641909641f9b826e8cb87645942/cffi-1.17.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6b8b4a92e1c65048ff98cfe1f735ef8f1ceb72e3d5f0c25fdb12087a23da22be", size = 461564 }, + { url = "https://files.pythonhosted.org/packages/f8/fe/4d41c2f200c4a457933dbd98d3cf4e911870877bd94d9656cc0fcb390681/cffi-1.17.1-cp310-cp310-win32.whl", hash = "sha256:c9c3d058ebabb74db66e431095118094d06abf53284d9c81f27300d0e0d8bc7c", size = 171804 }, + { url = "https://files.pythonhosted.org/packages/d1/b6/0b0f5ab93b0df4acc49cae758c81fe4e5ef26c3ae2e10cc69249dfd8b3ab/cffi-1.17.1-cp310-cp310-win_amd64.whl", hash = "sha256:0f048dcf80db46f0098ccac01132761580d28e28bc0f78ae0d58048063317e15", size = 181299 }, + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264 }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651 }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259 }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200 }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235 }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721 }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242 }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999 }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242 }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604 }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727 }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400 }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178 }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840 }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803 }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850 }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729 }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256 }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424 }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568 }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736 }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448 }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976 }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989 }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802 }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792 }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893 }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810 }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200 }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447 }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358 }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469 }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475 }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, + { url = "https://files.pythonhosted.org/packages/48/08/15bf6b43ae9bd06f6b00ad8a91f5a8fe1069d4c9fab550a866755402724e/cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b", size = 182457 }, + { url = "https://files.pythonhosted.org/packages/c2/5b/f1523dd545f92f7df468e5f653ffa4df30ac222f3c884e51e139878f1cb5/cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964", size = 425932 }, + { url = "https://files.pythonhosted.org/packages/53/93/7e547ab4105969cc8c93b38a667b82a835dd2cc78f3a7dad6130cfd41e1d/cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9", size = 448585 }, + { url = "https://files.pythonhosted.org/packages/56/c4/a308f2c332006206bb511de219efeff090e9d63529ba0a77aae72e82248b/cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc", size = 456268 }, + { url = "https://files.pythonhosted.org/packages/ca/5b/b63681518265f2f4060d2b60755c1c77ec89e5e045fc3773b72735ddaad5/cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c", size = 436592 }, + { url = "https://files.pythonhosted.org/packages/bb/19/b51af9f4a4faa4a8ac5a0e5d5c2522dcd9703d07fac69da34a36c4d960d3/cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1", size = 446512 }, + { url = "https://files.pythonhosted.org/packages/e2/63/2bed8323890cb613bbecda807688a31ed11a7fe7afe31f8faaae0206a9a3/cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8", size = 171576 }, + { url = "https://files.pythonhosted.org/packages/2f/70/80c33b044ebc79527447fd4fbc5455d514c3bb840dede4455de97da39b4d/cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1", size = 181229 }, + { url = "https://files.pythonhosted.org/packages/b9/ea/8bb50596b8ffbc49ddd7a1ad305035daa770202a6b782fc164647c2673ad/cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16", size = 182220 }, + { url = "https://files.pythonhosted.org/packages/ae/11/e77c8cd24f58285a82c23af484cf5b124a376b32644e445960d1a4654c3a/cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36", size = 178605 }, + { url = "https://files.pythonhosted.org/packages/ed/65/25a8dc32c53bf5b7b6c2686b42ae2ad58743f7ff644844af7cdb29b49361/cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8", size = 424910 }, + { url = "https://files.pythonhosted.org/packages/42/7a/9d086fab7c66bd7c4d0f27c57a1b6b068ced810afc498cc8c49e0088661c/cffi-1.17.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca74b8dbe6e8e8263c0ffd60277de77dcee6c837a3d0881d8c1ead7268c9e576", size = 447200 }, + { url = "https://files.pythonhosted.org/packages/da/63/1785ced118ce92a993b0ec9e0d0ac8dc3e5dbfbcaa81135be56c69cabbb6/cffi-1.17.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f7f5baafcc48261359e14bcd6d9bff6d4b28d9103847c9e136694cb0501aef87", size = 454565 }, + { url = "https://files.pythonhosted.org/packages/74/06/90b8a44abf3556599cdec107f7290277ae8901a58f75e6fe8f970cd72418/cffi-1.17.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98e3969bcff97cae1b2def8ba499ea3d6f31ddfdb7635374834cf89a1a08ecf0", size = 435635 }, + { url = "https://files.pythonhosted.org/packages/bd/62/a1f468e5708a70b1d86ead5bab5520861d9c7eacce4a885ded9faa7729c3/cffi-1.17.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdf5ce3acdfd1661132f2a9c19cac174758dc2352bfe37d98aa7512c6b7178b3", size = 445218 }, + { url = "https://files.pythonhosted.org/packages/5b/95/b34462f3ccb09c2594aa782d90a90b045de4ff1f70148ee79c69d37a0a5a/cffi-1.17.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9755e4345d1ec879e3849e62222a18c7174d65a6a92d5b346b1863912168b595", size = 460486 }, + { url = "https://files.pythonhosted.org/packages/fc/fc/a1e4bebd8d680febd29cf6c8a40067182b64f00c7d105f8f26b5bc54317b/cffi-1.17.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:f1e22e8c4419538cb197e4dd60acc919d7696e5ef98ee4da4e01d3f8cfa4cc5a", size = 437911 }, + { url = "https://files.pythonhosted.org/packages/e6/c3/21cab7a6154b6a5ea330ae80de386e7665254835b9e98ecc1340b3a7de9a/cffi-1.17.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c03e868a0b3bc35839ba98e74211ed2b05d2119be4e8a0f224fba9384f1fe02e", size = 460632 }, + { url = "https://files.pythonhosted.org/packages/cb/b5/fd9f8b5a84010ca169ee49f4e4ad6f8c05f4e3545b72ee041dbbcb159882/cffi-1.17.1-cp39-cp39-win32.whl", hash = "sha256:e31ae45bc2e29f6b2abd0de1cc3b9d5205aa847cafaecb8af1476a609a2f6eb7", size = 171820 }, + { url = "https://files.pythonhosted.org/packages/8c/52/b08750ce0bce45c143e1b5d7357ee8c55341b52bdef4b0f081af1eb248c2/cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662", size = 181290 }, +] + [[package]] name = "charset-normalizer" version = "3.4.1" @@ -235,9 +312,46 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, ] +[[package]] +name = "cryptography" +version = "44.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/4c/45dfa6829acffa344e3967d6006ee4ae8be57af746ae2eba1c431949b32c/cryptography-44.0.0.tar.gz", hash = "sha256:cd4e834f340b4293430701e772ec543b0fbe6c2dea510a5286fe0acabe153a02", size = 710657 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/55/09/8cc67f9b84730ad330b3b72cf867150744bf07ff113cda21a15a1c6d2c7c/cryptography-44.0.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:84111ad4ff3f6253820e6d3e58be2cc2a00adb29335d4cacb5ab4d4d34f2a123", size = 6541833 }, + { url = "https://files.pythonhosted.org/packages/7e/5b/3759e30a103144e29632e7cb72aec28cedc79e514b2ea8896bb17163c19b/cryptography-44.0.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b15492a11f9e1b62ba9d73c210e2416724633167de94607ec6069ef724fad092", size = 3922710 }, + { url = "https://files.pythonhosted.org/packages/5f/58/3b14bf39f1a0cfd679e753e8647ada56cddbf5acebffe7db90e184c76168/cryptography-44.0.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831c3c4d0774e488fdc83a1923b49b9957d33287de923d58ebd3cec47a0ae43f", size = 4137546 }, + { url = "https://files.pythonhosted.org/packages/98/65/13d9e76ca19b0ba5603d71ac8424b5694415b348e719db277b5edc985ff5/cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb", size = 3915420 }, + { url = "https://files.pythonhosted.org/packages/b1/07/40fe09ce96b91fc9276a9ad272832ead0fddedcba87f1190372af8e3039c/cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b", size = 4154498 }, + { url = "https://files.pythonhosted.org/packages/75/ea/af65619c800ec0a7e4034207aec543acdf248d9bffba0533342d1bd435e1/cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543", size = 3932569 }, + { url = "https://files.pythonhosted.org/packages/c7/af/d1deb0c04d59612e3d5e54203159e284d3e7a6921e565bb0eeb6269bdd8a/cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e", size = 4016721 }, + { url = "https://files.pythonhosted.org/packages/bd/69/7ca326c55698d0688db867795134bdfac87136b80ef373aaa42b225d6dd5/cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e", size = 4240915 }, + { url = "https://files.pythonhosted.org/packages/ef/d4/cae11bf68c0f981e0413906c6dd03ae7fa864347ed5fac40021df1ef467c/cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053", size = 2757925 }, + { url = "https://files.pythonhosted.org/packages/64/b1/50d7739254d2002acae64eed4fc43b24ac0cc44bf0a0d388d1ca06ec5bb1/cryptography-44.0.0-cp37-abi3-win_amd64.whl", hash = "sha256:abc998e0c0eee3c8a1904221d3f67dcfa76422b23620173e28c11d3e626c21bd", size = 3202055 }, + { url = "https://files.pythonhosted.org/packages/11/18/61e52a3d28fc1514a43b0ac291177acd1b4de00e9301aaf7ef867076ff8a/cryptography-44.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:660cb7312a08bc38be15b696462fa7cc7cd85c3ed9c576e81f4dc4d8b2b31591", size = 6542801 }, + { url = "https://files.pythonhosted.org/packages/1a/07/5f165b6c65696ef75601b781a280fc3b33f1e0cd6aa5a92d9fb96c410e97/cryptography-44.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1923cb251c04be85eec9fda837661c67c1049063305d6be5721643c22dd4e2b7", size = 3922613 }, + { url = "https://files.pythonhosted.org/packages/28/34/6b3ac1d80fc174812486561cf25194338151780f27e438526f9c64e16869/cryptography-44.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:404fdc66ee5f83a1388be54300ae978b2efd538018de18556dde92575e05defc", size = 4137925 }, + { url = "https://files.pythonhosted.org/packages/d0/c7/c656eb08fd22255d21bc3129625ed9cd5ee305f33752ef2278711b3fa98b/cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289", size = 3915417 }, + { url = "https://files.pythonhosted.org/packages/ef/82/72403624f197af0db6bac4e58153bc9ac0e6020e57234115db9596eee85d/cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7", size = 4155160 }, + { url = "https://files.pythonhosted.org/packages/a2/cd/2f3c440913d4329ade49b146d74f2e9766422e1732613f57097fea61f344/cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c", size = 3932331 }, + { url = "https://files.pythonhosted.org/packages/7f/df/8be88797f0a1cca6e255189a57bb49237402b1880d6e8721690c5603ac23/cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64", size = 4017372 }, + { url = "https://files.pythonhosted.org/packages/af/36/5ccc376f025a834e72b8e52e18746b927f34e4520487098e283a719c205e/cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285", size = 4239657 }, + { url = "https://files.pythonhosted.org/packages/46/b0/f4f7d0d0bcfbc8dd6296c1449be326d04217c57afb8b2594f017eed95533/cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417", size = 2758672 }, + { url = "https://files.pythonhosted.org/packages/97/9b/443270b9210f13f6ef240eff73fd32e02d381e7103969dc66ce8e89ee901/cryptography-44.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:708ee5f1bafe76d041b53a4f95eb28cdeb8d18da17e597d46d7833ee59b97ede", size = 3202071 }, + { url = "https://files.pythonhosted.org/packages/77/d4/fea74422326388bbac0c37b7489a0fcb1681a698c3b875959430ba550daa/cryptography-44.0.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:37d76e6863da3774cd9db5b409a9ecfd2c71c981c38788d3fcfaf177f447b731", size = 3338857 }, + { url = "https://files.pythonhosted.org/packages/1a/aa/ba8a7467c206cb7b62f09b4168da541b5109838627f582843bbbe0235e8e/cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:f677e1268c4e23420c3acade68fac427fffcb8d19d7df95ed7ad17cdef8404f4", size = 3850615 }, + { url = "https://files.pythonhosted.org/packages/89/fa/b160e10a64cc395d090105be14f399b94e617c879efd401188ce0fea39ee/cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f5e7cb1e5e56ca0933b4873c0220a78b773b24d40d186b6738080b73d3d0a756", size = 4081622 }, + { url = "https://files.pythonhosted.org/packages/47/8f/20ff0656bb0cf7af26ec1d01f780c5cfbaa7666736063378c5f48558b515/cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:8b3e6eae66cf54701ee7d9c83c30ac0a1e3fa17be486033000f2a73a12ab507c", size = 3867546 }, + { url = "https://files.pythonhosted.org/packages/38/d9/28edf32ee2fcdca587146bcde90102a7319b2f2c690edfa627e46d586050/cryptography-44.0.0-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:be4ce505894d15d5c5037167ffb7f0ae90b7be6f2a98f9a5c3442395501c32fa", size = 4090937 }, + { url = "https://files.pythonhosted.org/packages/cc/9d/37e5da7519de7b0b070a3fedd4230fe76d50d2a21403e0f2153d70ac4163/cryptography-44.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:62901fb618f74d7d81bf408c8719e9ec14d863086efe4185afd07c352aee1d2c", size = 3128774 }, +] + [[package]] name = "datafusion" -version = "43.0.0" +version = "44.0.0" source = { editable = "." } dependencies = [ { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, @@ -249,6 +363,7 @@ dependencies = [ dev = [ { name = "maturin" }, { name = "numpy", version = "2.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pygithub" }, { name = "pytest" }, { name = "ruff" }, { name = "toml" }, @@ -282,6 +397,7 @@ requires-dist = [ dev = [ { name = "maturin", specifier = ">=1.8.1" }, { name = "numpy", marker = "python_full_version >= '3.10'", specifier = ">1.24.4" }, + { name = "pygithub", specifier = "==2.5.0" }, { name = "pytest", specifier = ">=7.4.4" }, { name = "ruff", specifier = ">=0.9.1" }, { name = "toml", specifier = ">=0.10.2" }, @@ -307,6 +423,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/50/83c593b07763e1161326b3b8c6686f0f4b0f24d5526546bee538c89837d6/decorator-5.1.1-py3-none-any.whl", hash = "sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186", size = 9073 }, ] +[[package]] +name = "deprecated" +version = "1.2.18" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/97/06afe62762c9a8a86af0cfb7bfdab22a43ad17138b07af5b1a58442690a2/deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d", size = 2928744 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998 }, +] + [[package]] name = "docutils" version = "0.20.1" @@ -1189,6 +1317,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/d8/94161a7ca5c55199484e926165e9e33f318ea1d1b0d7cdbcbc3652b933ec/pyarrow-18.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:a1880dd6772b685e803011a6b43a230c23b566859a6e0c9a276c1e0faf4f4052", size = 25301373 }, ] +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552 }, +] + [[package]] name = "pydata-sphinx-theme" version = "0.8.0" @@ -1206,6 +1343,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/26/0694318d46c7d90ab602ae27b24431e939f1600f9a4c69d1e727ec57289f/pydata_sphinx_theme-0.8.0-py3-none-any.whl", hash = "sha256:fbcbb833a07d3ad8dd997dd40dc94da18d98b41c68123ab0182b58fe92271204", size = 3284997 }, ] +[[package]] +name = "pygithub" +version = "2.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "pyjwt", version = "2.9.0", source = { registry = "https://pypi.org/simple" }, extra = ["crypto"], marker = "python_full_version < '3.9'" }, + { name = "pyjwt", version = "2.10.1", source = { registry = "https://pypi.org/simple" }, extra = ["crypto"], marker = "python_full_version >= '3.9'" }, + { name = "pynacl" }, + { name = "requests" }, + { name = "typing-extensions" }, + { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "urllib3", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/16/ce/aa91d30040d9552c274e7ea8bd10a977600d508d579a4bb262b95eccf961/pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf", size = 3552804 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/05/bfbdbbc5d8aafd8dae9b3b6877edca561fccd8528ef5edc4e7b6d23721b5/PyGithub-2.5.0-py3-none-any.whl", hash = "sha256:b0b635999a658ab8e08720bdd3318893ff20e2275f6446fcf35bf3f44f2c0fd2", size = 375935 }, +] + [[package]] name = "pygments" version = "2.19.1" @@ -1215,6 +1371,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, ] +[[package]] +name = "pyjwt" +version = "2.9.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.9'", +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/68/ce067f09fca4abeca8771fe667d89cc347d1e99da3e093112ac329c6020e/pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c", size = 78825 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/84/0fdf9b18ba31d69877bd39c9cd6052b47f3761e9910c15de788e519f079f/PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850", size = 22344 }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography", marker = "python_full_version < '3.9'" }, +] + +[[package]] +name = "pyjwt" +version = "2.10.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997 }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography", marker = "python_full_version >= '3.9'" }, +] + +[[package]] +name = "pynacl" +version = "1.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/22/27582568be639dfe22ddb3902225f91f2f17ceff88ce80e4db396c8986da/PyNaCl-1.5.0.tar.gz", hash = "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", size = 3392854 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/75/0b8ede18506041c0bf23ac4d8e2971b4161cd6ce630b177d0a08eb0d8857/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", size = 349920 }, + { url = "https://files.pythonhosted.org/packages/59/bb/fddf10acd09637327a97ef89d2a9d621328850a72f1fdc8c08bdf72e385f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", size = 601722 }, + { url = "https://files.pythonhosted.org/packages/5d/70/87a065c37cca41a75f2ce113a5a2c2aa7533be648b184ade58971b5f7ccc/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", size = 680087 }, + { url = "https://files.pythonhosted.org/packages/ee/87/f1bb6a595f14a327e8285b9eb54d41fef76c585a0edef0a45f6fc95de125/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", size = 856678 }, + { url = "https://files.pythonhosted.org/packages/66/28/ca86676b69bf9f90e710571b67450508484388bfce09acf8a46f0b8c785f/PyNaCl-1.5.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", size = 1133660 }, + { url = "https://files.pythonhosted.org/packages/3d/85/c262db650e86812585e2bc59e497a8f59948a005325a11bbbc9ecd3fe26b/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", size = 663824 }, + { url = "https://files.pythonhosted.org/packages/fd/1a/cc308a884bd299b651f1633acb978e8596c71c33ca85e9dc9fa33a5399b9/PyNaCl-1.5.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", size = 1117912 }, + { url = "https://files.pythonhosted.org/packages/25/2d/b7df6ddb0c2a33afdb358f8af6ea3b8c4d1196ca45497dd37a56f0c122be/PyNaCl-1.5.0-cp36-abi3-win32.whl", hash = "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543", size = 204624 }, + { url = "https://files.pythonhosted.org/packages/5e/22/d3db169895faaf3e2eda892f005f433a62db2decbcfbc2f61e6517adfa87/PyNaCl-1.5.0-cp36-abi3-win_amd64.whl", hash = "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", size = 212141 }, +] + [[package]] name = "pytest" version = "8.3.4" @@ -1817,6 +2030,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166 }, ] +[[package]] +name = "wrapt" +version = "1.17.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/fc/e91cc220803d7bc4db93fb02facd8461c37364151b8494762cc88b0fbcef/wrapt-1.17.2.tar.gz", hash = "sha256:41388e9d4d1522446fe79d3213196bd9e3b301a336965b9e27ca2788ebd122f3", size = 55531 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/d1/1daec934997e8b160040c78d7b31789f19b122110a75eca3d4e8da0049e1/wrapt-1.17.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984", size = 53307 }, + { url = "https://files.pythonhosted.org/packages/1b/7b/13369d42651b809389c1a7153baa01d9700430576c81a2f5c5e460df0ed9/wrapt-1.17.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22", size = 38486 }, + { url = "https://files.pythonhosted.org/packages/62/bf/e0105016f907c30b4bd9e377867c48c34dc9c6c0c104556c9c9126bd89ed/wrapt-1.17.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:80dd7db6a7cb57ffbc279c4394246414ec99537ae81ffd702443335a61dbf3a7", size = 38777 }, + { url = "https://files.pythonhosted.org/packages/27/70/0f6e0679845cbf8b165e027d43402a55494779295c4b08414097b258ac87/wrapt-1.17.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a6e821770cf99cc586d33833b2ff32faebdbe886bd6322395606cf55153246c", size = 83314 }, + { url = "https://files.pythonhosted.org/packages/0f/77/0576d841bf84af8579124a93d216f55d6f74374e4445264cb378a6ed33eb/wrapt-1.17.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b60fb58b90c6d63779cb0c0c54eeb38941bae3ecf7a73c764c52c88c2dcb9d72", size = 74947 }, + { url = "https://files.pythonhosted.org/packages/90/ec/00759565518f268ed707dcc40f7eeec38637d46b098a1f5143bff488fe97/wrapt-1.17.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b870b5df5b71d8c3359d21be8f0d6c485fa0ebdb6477dda51a1ea54a9b558061", size = 82778 }, + { url = "https://files.pythonhosted.org/packages/f8/5a/7cffd26b1c607b0b0c8a9ca9d75757ad7620c9c0a9b4a25d3f8a1480fafc/wrapt-1.17.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4011d137b9955791f9084749cba9a367c68d50ab8d11d64c50ba1688c9b457f2", size = 81716 }, + { url = "https://files.pythonhosted.org/packages/7e/09/dccf68fa98e862df7e6a60a61d43d644b7d095a5fc36dbb591bbd4a1c7b2/wrapt-1.17.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1473400e5b2733e58b396a04eb7f35f541e1fb976d0c0724d0223dd607e0f74c", size = 74548 }, + { url = "https://files.pythonhosted.org/packages/b7/8e/067021fa3c8814952c5e228d916963c1115b983e21393289de15128e867e/wrapt-1.17.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3cedbfa9c940fdad3e6e941db7138e26ce8aad38ab5fe9dcfadfed9db7a54e62", size = 81334 }, + { url = "https://files.pythonhosted.org/packages/4b/0d/9d4b5219ae4393f718699ca1c05f5ebc0c40d076f7e65fd48f5f693294fb/wrapt-1.17.2-cp310-cp310-win32.whl", hash = "sha256:582530701bff1dec6779efa00c516496968edd851fba224fbd86e46cc6b73563", size = 36427 }, + { url = "https://files.pythonhosted.org/packages/72/6a/c5a83e8f61aec1e1aeef939807602fb880e5872371e95df2137142f5c58e/wrapt-1.17.2-cp310-cp310-win_amd64.whl", hash = "sha256:58705da316756681ad3c9c73fd15499aa4d8c69f9fd38dc8a35e06c12468582f", size = 38774 }, + { url = "https://files.pythonhosted.org/packages/cd/f7/a2aab2cbc7a665efab072344a8949a71081eed1d2f451f7f7d2b966594a2/wrapt-1.17.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ff04ef6eec3eee8a5efef2401495967a916feaa353643defcc03fc74fe213b58", size = 53308 }, + { url = "https://files.pythonhosted.org/packages/50/ff/149aba8365fdacef52b31a258c4dc1c57c79759c335eff0b3316a2664a64/wrapt-1.17.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4db983e7bca53819efdbd64590ee96c9213894272c776966ca6306b73e4affda", size = 38488 }, + { url = "https://files.pythonhosted.org/packages/65/46/5a917ce85b5c3b490d35c02bf71aedaa9f2f63f2d15d9949cc4ba56e8ba9/wrapt-1.17.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9abc77a4ce4c6f2a3168ff34b1da9b0f311a8f1cfd694ec96b0603dff1c79438", size = 38776 }, + { url = "https://files.pythonhosted.org/packages/ca/74/336c918d2915a4943501c77566db41d1bd6e9f4dbc317f356b9a244dfe83/wrapt-1.17.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b929ac182f5ace000d459c59c2c9c33047e20e935f8e39371fa6e3b85d56f4a", size = 83776 }, + { url = "https://files.pythonhosted.org/packages/09/99/c0c844a5ccde0fe5761d4305485297f91d67cf2a1a824c5f282e661ec7ff/wrapt-1.17.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f09b286faeff3c750a879d336fb6d8713206fc97af3adc14def0cdd349df6000", size = 75420 }, + { url = "https://files.pythonhosted.org/packages/b4/b0/9fc566b0fe08b282c850063591a756057c3247b2362b9286429ec5bf1721/wrapt-1.17.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7ed2d9d039bd41e889f6fb9364554052ca21ce823580f6a07c4ec245c1f5d6", size = 83199 }, + { url = "https://files.pythonhosted.org/packages/9d/4b/71996e62d543b0a0bd95dda485219856def3347e3e9380cc0d6cf10cfb2f/wrapt-1.17.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:129a150f5c445165ff941fc02ee27df65940fcb8a22a61828b1853c98763a64b", size = 82307 }, + { url = "https://files.pythonhosted.org/packages/39/35/0282c0d8789c0dc9bcc738911776c762a701f95cfe113fb8f0b40e45c2b9/wrapt-1.17.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1fb5699e4464afe5c7e65fa51d4f99e0b2eadcc176e4aa33600a3df7801d6662", size = 75025 }, + { url = "https://files.pythonhosted.org/packages/4f/6d/90c9fd2c3c6fee181feecb620d95105370198b6b98a0770cba090441a828/wrapt-1.17.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9a2bce789a5ea90e51a02dfcc39e31b7f1e662bc3317979aa7e5538e3a034f72", size = 81879 }, + { url = "https://files.pythonhosted.org/packages/8f/fa/9fb6e594f2ce03ef03eddbdb5f4f90acb1452221a5351116c7c4708ac865/wrapt-1.17.2-cp311-cp311-win32.whl", hash = "sha256:4afd5814270fdf6380616b321fd31435a462019d834f83c8611a0ce7484c7317", size = 36419 }, + { url = "https://files.pythonhosted.org/packages/47/f8/fb1773491a253cbc123c5d5dc15c86041f746ed30416535f2a8df1f4a392/wrapt-1.17.2-cp311-cp311-win_amd64.whl", hash = "sha256:acc130bc0375999da18e3d19e5a86403667ac0c4042a094fefb7eec8ebac7cf3", size = 38773 }, + { url = "https://files.pythonhosted.org/packages/a1/bd/ab55f849fd1f9a58ed7ea47f5559ff09741b25f00c191231f9f059c83949/wrapt-1.17.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:d5e2439eecc762cd85e7bd37161d4714aa03a33c5ba884e26c81559817ca0925", size = 53799 }, + { url = "https://files.pythonhosted.org/packages/53/18/75ddc64c3f63988f5a1d7e10fb204ffe5762bc663f8023f18ecaf31a332e/wrapt-1.17.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fc7cb4c1c744f8c05cd5f9438a3caa6ab94ce8344e952d7c45a8ed59dd88392", size = 38821 }, + { url = "https://files.pythonhosted.org/packages/48/2a/97928387d6ed1c1ebbfd4efc4133a0633546bec8481a2dd5ec961313a1c7/wrapt-1.17.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8fdbdb757d5390f7c675e558fd3186d590973244fab0c5fe63d373ade3e99d40", size = 38919 }, + { url = "https://files.pythonhosted.org/packages/73/54/3bfe5a1febbbccb7a2f77de47b989c0b85ed3a6a41614b104204a788c20e/wrapt-1.17.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bb1d0dbf99411f3d871deb6faa9aabb9d4e744d67dcaaa05399af89d847a91d", size = 88721 }, + { url = "https://files.pythonhosted.org/packages/25/cb/7262bc1b0300b4b64af50c2720ef958c2c1917525238d661c3e9a2b71b7b/wrapt-1.17.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d18a4865f46b8579d44e4fe1e2bcbc6472ad83d98e22a26c963d46e4c125ef0b", size = 80899 }, + { url = "https://files.pythonhosted.org/packages/2a/5a/04cde32b07a7431d4ed0553a76fdb7a61270e78c5fd5a603e190ac389f14/wrapt-1.17.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc570b5f14a79734437cb7b0500376b6b791153314986074486e0b0fa8d71d98", size = 89222 }, + { url = "https://files.pythonhosted.org/packages/09/28/2e45a4f4771fcfb109e244d5dbe54259e970362a311b67a965555ba65026/wrapt-1.17.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6d9187b01bebc3875bac9b087948a2bccefe464a7d8f627cf6e48b1bbae30f82", size = 86707 }, + { url = "https://files.pythonhosted.org/packages/c6/d2/dcb56bf5f32fcd4bd9aacc77b50a539abdd5b6536872413fd3f428b21bed/wrapt-1.17.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:9e8659775f1adf02eb1e6f109751268e493c73716ca5761f8acb695e52a756ae", size = 79685 }, + { url = "https://files.pythonhosted.org/packages/80/4e/eb8b353e36711347893f502ce91c770b0b0929f8f0bed2670a6856e667a9/wrapt-1.17.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e8b2816ebef96d83657b56306152a93909a83f23994f4b30ad4573b00bd11bb9", size = 87567 }, + { url = "https://files.pythonhosted.org/packages/17/27/4fe749a54e7fae6e7146f1c7d914d28ef599dacd4416566c055564080fe2/wrapt-1.17.2-cp312-cp312-win32.whl", hash = "sha256:468090021f391fe0056ad3e807e3d9034e0fd01adcd3bdfba977b6fdf4213ea9", size = 36672 }, + { url = "https://files.pythonhosted.org/packages/15/06/1dbf478ea45c03e78a6a8c4be4fdc3c3bddea5c8de8a93bc971415e47f0f/wrapt-1.17.2-cp312-cp312-win_amd64.whl", hash = "sha256:ec89ed91f2fa8e3f52ae53cd3cf640d6feff92ba90d62236a81e4e563ac0e991", size = 38865 }, + { url = "https://files.pythonhosted.org/packages/ce/b9/0ffd557a92f3b11d4c5d5e0c5e4ad057bd9eb8586615cdaf901409920b14/wrapt-1.17.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6ed6ffac43aecfe6d86ec5b74b06a5be33d5bb9243d055141e8cabb12aa08125", size = 53800 }, + { url = "https://files.pythonhosted.org/packages/c0/ef/8be90a0b7e73c32e550c73cfb2fa09db62234227ece47b0e80a05073b375/wrapt-1.17.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35621ae4c00e056adb0009f8e86e28eb4a41a4bfa8f9bfa9fca7d343fe94f998", size = 38824 }, + { url = "https://files.pythonhosted.org/packages/36/89/0aae34c10fe524cce30fe5fc433210376bce94cf74d05b0d68344c8ba46e/wrapt-1.17.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a604bf7a053f8362d27eb9fefd2097f82600b856d5abe996d623babd067b1ab5", size = 38920 }, + { url = "https://files.pythonhosted.org/packages/3b/24/11c4510de906d77e0cfb5197f1b1445d4fec42c9a39ea853d482698ac681/wrapt-1.17.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cbabee4f083b6b4cd282f5b817a867cf0b1028c54d445b7ec7cfe6505057cf8", size = 88690 }, + { url = "https://files.pythonhosted.org/packages/71/d7/cfcf842291267bf455b3e266c0c29dcb675b5540ee8b50ba1699abf3af45/wrapt-1.17.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49703ce2ddc220df165bd2962f8e03b84c89fee2d65e1c24a7defff6f988f4d6", size = 80861 }, + { url = "https://files.pythonhosted.org/packages/d5/66/5d973e9f3e7370fd686fb47a9af3319418ed925c27d72ce16b791231576d/wrapt-1.17.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8112e52c5822fc4253f3901b676c55ddf288614dc7011634e2719718eaa187dc", size = 89174 }, + { url = "https://files.pythonhosted.org/packages/a7/d3/8e17bb70f6ae25dabc1aaf990f86824e4fd98ee9cadf197054e068500d27/wrapt-1.17.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9fee687dce376205d9a494e9c121e27183b2a3df18037f89d69bd7b35bcf59e2", size = 86721 }, + { url = "https://files.pythonhosted.org/packages/6f/54/f170dfb278fe1c30d0ff864513cff526d624ab8de3254b20abb9cffedc24/wrapt-1.17.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:18983c537e04d11cf027fbb60a1e8dfd5190e2b60cc27bc0808e653e7b218d1b", size = 79763 }, + { url = "https://files.pythonhosted.org/packages/4a/98/de07243751f1c4a9b15c76019250210dd3486ce098c3d80d5f729cba029c/wrapt-1.17.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:703919b1633412ab54bcf920ab388735832fdcb9f9a00ae49387f0fe67dad504", size = 87585 }, + { url = "https://files.pythonhosted.org/packages/f9/f0/13925f4bd6548013038cdeb11ee2cbd4e37c30f8bfd5db9e5a2a370d6e20/wrapt-1.17.2-cp313-cp313-win32.whl", hash = "sha256:abbb9e76177c35d4e8568e58650aa6926040d6a9f6f03435b7a522bf1c487f9a", size = 36676 }, + { url = "https://files.pythonhosted.org/packages/bf/ae/743f16ef8c2e3628df3ddfd652b7d4c555d12c84b53f3d8218498f4ade9b/wrapt-1.17.2-cp313-cp313-win_amd64.whl", hash = "sha256:69606d7bb691b50a4240ce6b22ebb319c1cfb164e5f6569835058196e0f3a845", size = 38871 }, + { url = "https://files.pythonhosted.org/packages/3d/bc/30f903f891a82d402ffb5fda27ec1d621cc97cb74c16fea0b6141f1d4e87/wrapt-1.17.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:4a721d3c943dae44f8e243b380cb645a709ba5bd35d3ad27bc2ed947e9c68192", size = 56312 }, + { url = "https://files.pythonhosted.org/packages/8a/04/c97273eb491b5f1c918857cd26f314b74fc9b29224521f5b83f872253725/wrapt-1.17.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:766d8bbefcb9e00c3ac3b000d9acc51f1b399513f44d77dfe0eb026ad7c9a19b", size = 40062 }, + { url = "https://files.pythonhosted.org/packages/4e/ca/3b7afa1eae3a9e7fefe499db9b96813f41828b9fdb016ee836c4c379dadb/wrapt-1.17.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e496a8ce2c256da1eb98bd15803a79bee00fc351f5dfb9ea82594a3f058309e0", size = 40155 }, + { url = "https://files.pythonhosted.org/packages/89/be/7c1baed43290775cb9030c774bc53c860db140397047cc49aedaf0a15477/wrapt-1.17.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40d615e4fe22f4ad3528448c193b218e077656ca9ccb22ce2cb20db730f8d306", size = 113471 }, + { url = "https://files.pythonhosted.org/packages/32/98/4ed894cf012b6d6aae5f5cc974006bdeb92f0241775addad3f8cd6ab71c8/wrapt-1.17.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a5aaeff38654462bc4b09023918b7f21790efb807f54c000a39d41d69cf552cb", size = 101208 }, + { url = "https://files.pythonhosted.org/packages/ea/fd/0c30f2301ca94e655e5e057012e83284ce8c545df7661a78d8bfca2fac7a/wrapt-1.17.2-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7d15bbd2bc99e92e39f49a04653062ee6085c0e18b3b7512a4f2fe91f2d681", size = 109339 }, + { url = "https://files.pythonhosted.org/packages/75/56/05d000de894c4cfcb84bcd6b1df6214297b8089a7bd324c21a4765e49b14/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:e3890b508a23299083e065f435a492b5435eba6e304a7114d2f919d400888cc6", size = 110232 }, + { url = "https://files.pythonhosted.org/packages/53/f8/c3f6b2cf9b9277fb0813418e1503e68414cd036b3b099c823379c9575e6d/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:8c8b293cd65ad716d13d8dd3624e42e5a19cc2a2f1acc74b30c2c13f15cb61a6", size = 100476 }, + { url = "https://files.pythonhosted.org/packages/a7/b1/0bb11e29aa5139d90b770ebbfa167267b1fc548d2302c30c8f7572851738/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c82b8785d98cdd9fed4cac84d765d234ed3251bd6afe34cb7ac523cb93e8b4f", size = 106377 }, + { url = "https://files.pythonhosted.org/packages/6a/e1/0122853035b40b3f333bbb25f1939fc1045e21dd518f7f0922b60c156f7c/wrapt-1.17.2-cp313-cp313t-win32.whl", hash = "sha256:13e6afb7fe71fe7485a4550a8844cc9ffbe263c0f1a1eea569bc7091d4898555", size = 37986 }, + { url = "https://files.pythonhosted.org/packages/09/5e/1655cf481e079c1f22d0cabdd4e51733679932718dc23bf2db175f329b76/wrapt-1.17.2-cp313-cp313t-win_amd64.whl", hash = "sha256:eaf675418ed6b3b31c7a989fd007fa7c3be66ce14e5c3b27336383604c9da85c", size = 40750 }, + { url = "https://files.pythonhosted.org/packages/0c/66/95b9e90e6e1274999b183c9c3f984996d870e933ca9560115bd1cd1d6f77/wrapt-1.17.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5c803c401ea1c1c18de70a06a6f79fcc9c5acfc79133e9869e730ad7f8ad8ef9", size = 53234 }, + { url = "https://files.pythonhosted.org/packages/a4/b6/6eced5e2db5924bf6d9223d2bb96b62e00395aae77058e6a9e11bf16b3bd/wrapt-1.17.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f917c1180fdb8623c2b75a99192f4025e412597c50b2ac870f156de8fb101119", size = 38462 }, + { url = "https://files.pythonhosted.org/packages/5d/a4/c8472fe2568978b5532df84273c53ddf713f689d408a4335717ab89547e0/wrapt-1.17.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ecc840861360ba9d176d413a5489b9a0aff6d6303d7e733e2c4623cfa26904a6", size = 38730 }, + { url = "https://files.pythonhosted.org/packages/3c/70/1d259c6b1ad164eb23ff70e3e452dd1950f96e6473f72b7207891d0fd1f0/wrapt-1.17.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb87745b2e6dc56361bfde481d5a378dc314b252a98d7dd19a651a3fa58f24a9", size = 86225 }, + { url = "https://files.pythonhosted.org/packages/a9/68/6b83367e1afb8de91cbea4ef8e85b58acdf62f034f05d78c7b82afaa23d8/wrapt-1.17.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58455b79ec2661c3600e65c0a716955adc2410f7383755d537584b0de41b1d8a", size = 78055 }, + { url = "https://files.pythonhosted.org/packages/0d/21/09573d2443916705c57fdab85d508f592c0a58d57becc53e15755d67fba2/wrapt-1.17.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e42a40a5e164cbfdb7b386c966a588b1047558a990981ace551ed7e12ca9c2", size = 85592 }, + { url = "https://files.pythonhosted.org/packages/45/ce/700e17a852dd5dec894e241c72973ea82363486bcc1fb05d47b4fbd1d683/wrapt-1.17.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:91bd7d1773e64019f9288b7a5101f3ae50d3d8e6b1de7edee9c2ccc1d32f0c0a", size = 83906 }, + { url = "https://files.pythonhosted.org/packages/37/14/bd210faf0a66faeb8529d42b6b45a25d6aa6ce25ddfc19168e4161aed227/wrapt-1.17.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:bb90fb8bda722a1b9d48ac1e6c38f923ea757b3baf8ebd0c82e09c5c1a0e7a04", size = 76763 }, + { url = "https://files.pythonhosted.org/packages/34/0c/85af70d291f44659c422416f0272046109e785bf6db8c081cfeeae5715c5/wrapt-1.17.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:08e7ce672e35efa54c5024936e559469436f8b8096253404faeb54d2a878416f", size = 83573 }, + { url = "https://files.pythonhosted.org/packages/f8/1e/b215068e824878f69ea945804fa26c176f7c2735a3ad5367d78930bd076a/wrapt-1.17.2-cp38-cp38-win32.whl", hash = "sha256:410a92fefd2e0e10d26210e1dfb4a876ddaf8439ef60d6434f21ef8d87efc5b7", size = 36408 }, + { url = "https://files.pythonhosted.org/packages/52/27/3dd9ad5f1097b33c95d05929e409cc86d7c765cb5437b86694dc8f8e9af0/wrapt-1.17.2-cp38-cp38-win_amd64.whl", hash = "sha256:95c658736ec15602da0ed73f312d410117723914a5c91a14ee4cdd72f1d790b3", size = 38737 }, + { url = "https://files.pythonhosted.org/packages/8a/f4/6ed2b8f6f1c832933283974839b88ec7c983fd12905e01e97889dadf7559/wrapt-1.17.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99039fa9e6306880572915728d7f6c24a86ec57b0a83f6b2491e1d8ab0235b9a", size = 53308 }, + { url = "https://files.pythonhosted.org/packages/a2/a9/712a53f8f4f4545768ac532619f6e56d5d0364a87b2212531685e89aeef8/wrapt-1.17.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2696993ee1eebd20b8e4ee4356483c4cb696066ddc24bd70bcbb80fa56ff9061", size = 38489 }, + { url = "https://files.pythonhosted.org/packages/fa/9b/e172c8f28a489a2888df18f953e2f6cb8d33b1a2e78c9dfc52d8bf6a5ead/wrapt-1.17.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:612dff5db80beef9e649c6d803a8d50c409082f1fedc9dbcdfde2983b2025b82", size = 38776 }, + { url = "https://files.pythonhosted.org/packages/cf/cb/7a07b51762dcd59bdbe07aa97f87b3169766cadf240f48d1cbe70a1be9db/wrapt-1.17.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:62c2caa1585c82b3f7a7ab56afef7b3602021d6da34fbc1cf234ff139fed3cd9", size = 83050 }, + { url = "https://files.pythonhosted.org/packages/a5/51/a42757dd41032afd6d8037617aa3bc6803ba971850733b24dfb7d5c627c4/wrapt-1.17.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c958bcfd59bacc2d0249dcfe575e71da54f9dcf4a8bdf89c4cb9a68a1170d73f", size = 74718 }, + { url = "https://files.pythonhosted.org/packages/bf/bb/d552bfe47db02fcfc950fc563073a33500f8108efa5f7b41db2f83a59028/wrapt-1.17.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc78a84e2dfbc27afe4b2bd7c80c8db9bca75cc5b85df52bfe634596a1da846b", size = 82590 }, + { url = "https://files.pythonhosted.org/packages/77/99/77b06b3c3c410dbae411105bf22496facf03a5496bfaca8fbcf9da381889/wrapt-1.17.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ba0f0eb61ef00ea10e00eb53a9129501f52385c44853dbd6c4ad3f403603083f", size = 81462 }, + { url = "https://files.pythonhosted.org/packages/2d/21/cf0bd85ae66f92600829ea1de8e1da778e5e9f6e574ccbe74b66db0d95db/wrapt-1.17.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:1e1fe0e6ab7775fd842bc39e86f6dcfc4507ab0ffe206093e76d61cde37225c8", size = 74309 }, + { url = "https://files.pythonhosted.org/packages/6d/16/112d25e9092398a0dd6fec50ab7ac1b775a0c19b428f049785096067ada9/wrapt-1.17.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c86563182421896d73858e08e1db93afdd2b947a70064b813d515d66549e15f9", size = 81081 }, + { url = "https://files.pythonhosted.org/packages/2b/49/364a615a0cc0872685646c495c7172e4fc7bf1959e3b12a1807a03014e05/wrapt-1.17.2-cp39-cp39-win32.whl", hash = "sha256:f393cda562f79828f38a819f4788641ac7c4085f30f1ce1a68672baa686482bb", size = 36423 }, + { url = "https://files.pythonhosted.org/packages/00/ad/5d2c1b34ba3202cd833d9221833e74d6500ce66730974993a8dc9a94fb8c/wrapt-1.17.2-cp39-cp39-win_amd64.whl", hash = "sha256:36ccae62f64235cf8ddb682073a60519426fdd4725524ae38874adf72b5f2aeb", size = 38772 }, + { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594 }, +] + [[package]] name = "zipp" version = "3.20.2" From d635d56ecdc0cf2667c01cfcc51f26733ec796dc Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 7 Feb 2025 14:36:54 -0500 Subject: [PATCH 261/413] Chore/upgrade datafusion 45 (#1010) * upgrade dep * resolve errors * match new pyo3 version * upgrade dep * back to 43 * use released v45 * remove unnecessary pyarrow feature * Update unit test return type * fix test_relational_expr --------- Co-authored-by: Tim Saucer --- Cargo.lock | 400 +++++++++++--------- Cargo.toml | 16 +- examples/ffi-table-provider/Cargo.lock | 493 ++++++++++++++----------- examples/ffi-table-provider/Cargo.toml | 14 +- examples/ffi-table-provider/src/lib.rs | 2 +- python/tests/test_expr.py | 3 +- python/tests/test_functions.py | 2 +- src/context.rs | 2 +- src/dataframe.rs | 2 +- 9 files changed, 514 insertions(+), 420 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50809696b..f1b1ed50a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf3437355979f1e93ba84ba108c38be5767713051f3c8ffbf07c094e2e61f9f" +checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc" dependencies = [ "arrow-arith", "arrow-array", @@ -201,24 +201,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31dce77d2985522288edae7206bffd5fc4996491841dda01a13a58415867e681" +checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "half", "num", ] [[package]] name = "arrow-array" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d45fe6d3faed0435b7313e59a02583b14c6c6339fa7729e94c32a20af319a79" +checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" dependencies = [ "ahash", "arrow-buffer", @@ -233,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b02656a35cc103f28084bc80a0159668e0a680d919cef127bd7e0aaccb06ec1" +checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89" dependencies = [ "bytes", "half", @@ -244,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c73c6233c5b5d635a56f6010e6eb1ab9e30e94707db21cea03da317f67d84cf3" +checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870" dependencies = [ "arrow-array", "arrow-buffer", @@ -265,28 +264,25 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec222848d70fea5a32af9c3602b08f5d740d5e2d33fbd76bf6fd88759b5b13a7" +checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6" dependencies = [ "arrow-array", - "arrow-buffer", "arrow-cast", - "arrow-data", "arrow-schema", "chrono", "csv", "csv-core", "lazy_static", - "lexical-core", "regex", ] [[package]] name = "arrow-data" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f2861ffa86f107b8ab577d86cff7c7a490243eabe961ba1e1af4f27542bb79" +checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754" dependencies = [ "arrow-buffer", "arrow-schema", @@ -296,13 +292,12 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0270dc511f11bb5fa98a25020ad51a99ca5b08d8a8dfbd17503bb9dba0388f0b" +checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e" dependencies = [ "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-schema", "flatbuffers", @@ -311,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eff38eeb8a971ad3a4caf62c5d57f0cff8a48b64a55e3207c4fd696a9234aad" +checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c" dependencies = [ "arrow-array", "arrow-buffer", @@ -331,26 +326,23 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f202a879d287099139ff0d121e7f55ae5e0efe634b8cf2106ebc27a8715dee" +checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "half", - "num", ] [[package]] name = "arrow-row" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f936954991c360ba762dff23f5dda16300774fafd722353d9683abd97630ae" +checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c" dependencies = [ - "ahash", "arrow-array", "arrow-buffer", "arrow-data", @@ -360,18 +352,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9579b9d8bce47aa41389fe344f2c6758279983b7c0ebb4013e283e3e91bb450e" +checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" dependencies = [ "bitflags 2.8.0", ] [[package]] name = "arrow-select" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7471ba126d0b0aaa24b50a36bc6c25e4e74869a1fd1a5553357027a0b1c8d1f1" +checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" dependencies = [ "ahash", "arrow-array", @@ -383,9 +375,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72993b01cb62507b06f1fb49648d7286c8989ecfabdb7b77a750fcb54410731b" +checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0" dependencies = [ "arrow-array", "arrow-buffer", @@ -444,7 +436,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -455,7 +447,7 @@ checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -598,9 +590,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "bzip2" @@ -635,9 +627,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.11" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" +checksum = "755717a7de9ec452bf7f3f1a3099085deabd7f2962b861dae91ecd7a365903d2" dependencies = [ "jobserver", "libc", @@ -874,9 +866,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014fc8c384ecacedaabb3bc8359c2a6c6e9d8f7bea65be3434eccacfc37f52d9" +checksum = "eae420e7a5b0b7f1c39364cc76cbcd0f5fdc416b2514ae3847c2676bbd60702a" dependencies = [ "apache-avro", "arrow", @@ -888,7 +880,6 @@ dependencies = [ "bytes", "bzip2 0.5.0", "chrono", - "dashmap", "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", @@ -908,7 +899,7 @@ dependencies = [ "flate2", "futures", "glob", - "itertools", + "itertools 0.14.0", "log", "num-traits", "object_store", @@ -928,31 +919,39 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee60d33e210ef96070377ae667ece7caa0e959c8387496773d4a1a72f1a5012e" +checksum = "6f27987bc22b810939e8dfecc55571e9d50355d6ea8ec1c47af8383a76a6d0e1" dependencies = [ - "arrow-schema", + "arrow", "async-trait", + "dashmap", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", "parking_lot", + "sqlparser", ] [[package]] name = "datafusion-common" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b42b7d720fe21ed9cca2ebb635f3f13a12cfab786b41e0fba184fb2e620525b" +checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602" dependencies = [ "ahash", "apache-avro", "arrow", "arrow-array", "arrow-buffer", + "arrow-ipc", "arrow-schema", + "base64 0.22.1", "half", "hashbrown 0.14.5", "indexmap", @@ -969,9 +968,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72fbf14d4079f7ce5306393084fe5057dddfdc2113577e0049310afa12e94281" +checksum = "0d4603c8e8a4baf77660ab7074cc66fc15cc8a18f2ce9dfadb755fc6ee294e48" dependencies = [ "log", "tokio", @@ -979,15 +978,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c278dbd64860ed0bb5240fc1f4cb6aeea437153910aea69bcf7d5a8d6d0454f3" +checksum = "e5bf4bc68623a5cf231eed601ed6eb41f46a37c4d15d11a0bff24cbc8396cd66" [[package]] name = "datafusion-execution" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22cb02af47e756468b3cbfee7a83e3d4f2278d452deb4b033ba933c75169486" +checksum = "88b491c012cdf8e051053426013429a76f74ee3c2db68496c79c323ca1084d27" dependencies = [ "arrow", "dashmap", @@ -1004,9 +1003,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62298eadb1d15b525df1315e61a71519ffc563d41d5c3b2a30fda2d70f77b93c" +checksum = "e5a181408d4fc5dc22f9252781a8f39f2d0e5d1b33ec9bde242844980a2689c1" dependencies = [ "arrow", "chrono", @@ -1025,23 +1024,26 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda7f73c5fc349251cd3dcb05773c5bf55d2505a698ef9d38dfc712161ea2f55" +checksum = "d1129b48e8534d8c03c6543bcdccef0b55c8ac0c1272a15a56c67068b6eb1885" dependencies = [ "arrow", "datafusion-common", - "itertools", + "itertools 0.14.0", + "paste", ] [[package]] name = "datafusion-ffi" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "114e944790756b84c2cc5971eae24f5430980149345601939ac222885d4db5f7" +checksum = "ff47a79d442207c168c6e3e1d970c248589c148e4800e5b285ac1b2cb1a230f8" dependencies = [ "abi_stable", "arrow", + "arrow-array", + "arrow-schema", "async-ffi", "async-trait", "datafusion", @@ -1049,13 +1051,15 @@ dependencies = [ "futures", "log", "prost", + "semver", + "tokio", ] [[package]] name = "datafusion-functions" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd197f3b2975424d3a4898ea46651be855a46721a56727515dbd5c9e2fb597da" +checksum = "6125874e4856dfb09b59886784fcb74cde5cfc5930b3a80a1a728ef7a010df6b" dependencies = [ "arrow", "arrow-buffer", @@ -1071,7 +1075,7 @@ dependencies = [ "datafusion-macros", "hashbrown 0.14.5", "hex", - "itertools", + "itertools 0.14.0", "log", "md-5", "rand", @@ -1083,12 +1087,13 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabbe48fba18f9981b134124381bee9e46f93518b8ad2f9721ee296cef5affb9" +checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6" dependencies = [ "ahash", "arrow", + "arrow-buffer", "arrow-schema", "datafusion-common", "datafusion-doc", @@ -1105,9 +1110,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a3fefed9c8c11268d446d924baca8cabf52fe32f73fdaa20854bac6473590c" +checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c" dependencies = [ "ahash", "arrow", @@ -1118,9 +1123,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6360f27464fab857bec698af39b2ae331dc07c8bf008fb4de387a19cdc6815a5" +checksum = "3ec5ee8cecb0dc370291279673097ddabec03a011f73f30d7f1096457127e03e" dependencies = [ "arrow", "arrow-array", @@ -1128,21 +1133,23 @@ dependencies = [ "arrow-ord", "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-macros", "datafusion-physical-expr-common", - "itertools", + "itertools 0.14.0", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c35c070eb705c12795dab399c3809f4dfbc290678c624d3989490ca9b8449c1" +checksum = "2c403ddd473bbb0952ba880008428b3c7febf0ed3ce1eec35a205db20efb2a36" dependencies = [ "arrow", "async-trait", @@ -1156,9 +1163,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52229bca26b590b140900752226c829f15fc1a99840e1ca3ce1a9534690b82a8" +checksum = "1ab18c2fb835614d06a75f24a9e09136d3a8c12a92d97c95a6af316a1787a9c5" dependencies = [ "datafusion-common", "datafusion-doc", @@ -1173,9 +1180,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "367befc303b64a668a10ae6988a064a9289e1999e71a7f8e526b6e14d6bdd9d6" +checksum = "a77b73bc15e7d1967121fdc7a55d819bfb9d6c03766a6c322247dce9094a53a4" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1183,19 +1190,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5de3c8f386ea991696553afe241a326ecbc3c98a12c562867e4be754d3a060c" +checksum = "09369b8d962291e808977cf94d495fd8b5b38647232d7ef562c27ac0f495b0af" dependencies = [ + "datafusion-expr", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] name = "datafusion-optimizer" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b520413906f755910422b016fb73884ae6e9e1b376de4f9584b6c0e031da75" +checksum = "2403a7e4a84637f3de7d8d4d7a9ccc0cc4be92d89b0161ba3ee5be82f0531c54" dependencies = [ "arrow", "chrono", @@ -1203,7 +1211,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "indexmap", - "itertools", + "itertools 0.14.0", "log", "recursive", "regex", @@ -1212,9 +1220,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acd6ddc378f6ad19af95ccd6790dec8f8e1264bc4c70e99ddc1830c1a1c78ccd" +checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c" dependencies = [ "ahash", "arrow", @@ -1229,48 +1237,54 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", "paste", - "petgraph", + "petgraph 0.7.1", ] [[package]] name = "datafusion-physical-expr-common" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e6c05458eccd74b4c77ed6a1fe63d52434240711de7f6960034794dad1caf5" +checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435" dependencies = [ "ahash", "arrow", + "arrow-buffer", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "itertools", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-optimizer" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dc3a82190f49c37d377f31317e07ab5d7588b837adadba8ac367baad5dc2351" +checksum = "ac5e85c189d5238a5cf181a624e450c4cd4c66ac77ca551d6f3ff9080bac90bb" dependencies = [ "arrow", + "arrow-schema", "datafusion-common", "datafusion-execution", + "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", + "futures", + "itertools 0.14.0", "log", "recursive", + "url", ] [[package]] name = "datafusion-physical-plan" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6608bc9844b4ddb5ed4e687d173e6c88700b1d0482f43894617d18a1fe75da" +checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f" dependencies = [ "ahash", "arrow", @@ -1291,7 +1305,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "pin-project-lite", @@ -1300,9 +1314,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e23b0998195e495bfa7b37cdceb317129a6c40522219f6872d2e0c9ae9f4fcb" +checksum = "2db5d79f0c974041787b899d24dc91bdab2ff112d1942dd71356a4ce3b407e6c" dependencies = [ "arrow", "chrono", @@ -1316,9 +1330,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc59992a29eed2d2c1dd779deac99083b217774ebcf90ee121840607a4d866f" +checksum = "de21bde1603aac0ff32cf478e47081be6e3583c6861fe8f57034da911efe7578" dependencies = [ "arrow", "datafusion-common", @@ -1350,9 +1364,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a884061c79b33d0c8e84a6f4f4be8bdc12c0f53f5af28ddf5d6d95ac0b15fdc" +checksum = "e13caa4daede211ecec53c78b13c503b592794d125f9a3cc3afe992edf9e7f43" dependencies = [ "arrow", "arrow-array", @@ -1369,16 +1383,16 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "44.0.0" +version = "45.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ec36dd38512b1ecc7a3bb92e72046b944611b2f0d709445c1e51b0143bffd4" +checksum = "1634405abd8bd3c64c352f2da2f2aec6d80a815930257e0db0ce4ff5daf00944" dependencies = [ "arrow-buffer", "async-recursion", "async-trait", "chrono", "datafusion", - "itertools", + "itertools 0.14.0", "object_store", "pbjson-types", "prost", @@ -1405,7 +1419,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1448,6 +1462,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flatbuffers" version = "24.12.23" @@ -1545,7 +1565,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1938,7 +1958,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1999,6 +2019,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -2362,7 +2391,7 @@ dependencies = [ "httparse", "humantime", "hyper", - "itertools", + "itertools 0.13.0", "md-5", "parking_lot", "percent-encoding", @@ -2382,9 +2411,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "openssl-probe" @@ -2426,9 +2455,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.4.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8957c0c95a6a1804f3e51a18f69df29be53856a8c5768cc9b6d00fcafcd2917c" +checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" dependencies = [ "ahash", "arrow-array", @@ -2452,6 +2481,7 @@ dependencies = [ "object_store", "paste", "seq-macro", + "simdutf8", "snap", "thrift", "tokio", @@ -2492,7 +2522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" dependencies = [ "heck", - "itertools", + "itertools 0.13.0", "prost", "prost-types", ] @@ -2524,7 +2554,17 @@ version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", + "indexmap", +] + +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset 0.5.7", "indexmap", ] @@ -2606,7 +2646,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2635,16 +2675,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ "heck", - "itertools", + "itertools 0.13.0", "log", "multimap", "once_cell", - "petgraph", + "petgraph 0.6.5", "prettyplease", "prost", "prost-types", "regex", - "syn 2.0.96", + "syn 2.0.98", "tempfile", ] @@ -2655,10 +2695,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2690,9 +2730,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "cfg-if", "indoc", @@ -2708,9 +2748,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2529f0be73ffd2be0cc43c013a640796558aa12d7ca0aab5cc14f375b4733031" +checksum = "977dc837525cfd22919ba6a831413854beb7c99a256c03bf8624ad707e45810e" dependencies = [ "futures", "once_cell", @@ -2721,9 +2761,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -2731,9 +2771,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", @@ -2741,27 +2781,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] name = "pyo3-macros-backend" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2888,7 +2928,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3029,9 +3069,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -3161,7 +3201,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3234,7 +3274,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3245,7 +3285,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3269,7 +3309,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3314,6 +3354,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.1" @@ -3353,7 +3399,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3396,7 +3442,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3440,14 +3486,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] name = "substrait" -version = "0.50.4" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1772d041c37cc7e6477733c76b2acf4ee36bd52b2ae4d9ea0ec9c87d003db32" +checksum = "5db15789cecbfdf6b1fcf2db807e767c92273bdc407ac057c2194b070c597756" dependencies = [ "heck", "pbjson", @@ -3464,7 +3510,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.96", + "syn 2.0.98", "typify", "walkdir", ] @@ -3488,9 +3534,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -3514,7 +3560,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3563,7 +3609,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3574,7 +3620,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3646,7 +3692,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3718,7 +3764,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3784,7 +3830,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3795,9 +3841,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typify" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c644dda9862f0fef3a570d8ddb3c2cfb1d5ac824a1f2ddfa7bc8f071a5ad8a" +checksum = "e03ba3643450cfd95a1aca2e1938fef63c1c1994489337998aff4ad771f21ef8" dependencies = [ "typify-impl", "typify-macro", @@ -3805,9 +3851,9 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59ab345b6c0d8ae9500b9ff334a4c7c0d316c1c628dc55726b95887eb8dbd11" +checksum = "bce48219a2f3154aaa2c56cbf027728b24a3c8fe0a47ed6399781de2b3f3eeaf" dependencies = [ "heck", "log", @@ -3818,16 +3864,16 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.96", - "thiserror 1.0.69", + "syn 2.0.98", + "thiserror 2.0.11", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "785e2cdcef0df8160fdd762ed548a637aaec1e83704fdbc14da0df66013ee8d0" +checksum = "68b5780d745920ed73c5b7447496a9b5c42ed2681a9b70859377aec423ecf02b" dependencies = [ "proc-macro2", "quote", @@ -3836,7 +3882,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.96", + "syn 2.0.98", "typify-impl", ] @@ -3901,11 +3947,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.12.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.3.1", "serde", ] @@ -3971,7 +4017,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-shared", ] @@ -4006,7 +4052,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4255,7 +4301,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -4277,7 +4323,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4297,7 +4343,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -4326,7 +4372,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 44e6e2244..d18e0e8f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,13 +35,13 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.42", features = ["macros", "rt", "rt-multi-thread", "sync"] } -pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } -pyo3-async-runtimes = { version = "0.22", features = ["tokio-runtime"]} -arrow = { version = "53", features = ["pyarrow"] } -datafusion = { version = "44.0.0", features = ["avro", "unicode_expressions"] } -datafusion-substrait = { version = "44.0.0", optional = true } -datafusion-proto = { version = "44.0.0" } -datafusion-ffi = { version = "44.0.0" } +pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py38"] } +pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"]} +arrow = { version = "54", features = ["pyarrow"] } +datafusion = { version = "45.0.0", features = ["avro", "unicode_expressions"] } +datafusion-substrait = { version = "45.0.0", optional = true } +datafusion-proto = { version = "45.0.0" } +datafusion-ffi = { version = "45.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.12", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } @@ -52,7 +52,7 @@ url = "2" [build-dependencies] prost-types = "0.13" # keep in line with `datafusion-substrait` -pyo3-build-config = "0.22" +pyo3-build-config = "0.23" [lib] name = "datafusion_python" diff --git a/examples/ffi-table-provider/Cargo.lock b/examples/ffi-table-provider/Cargo.lock index 3b57cac75..32af85180 100644 --- a/examples/ffi-table-provider/Cargo.lock +++ b/examples/ffi-table-provider/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "abi_stable" @@ -144,9 +144,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" +checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc" dependencies = [ "arrow-arith", "arrow-array", @@ -165,24 +165,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" +checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "half", "num", ] [[package]] name = "arrow-array" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" +checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" dependencies = [ "ahash", "arrow-buffer", @@ -191,15 +190,15 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.1", "num", ] [[package]] name = "arrow-buffer" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" +checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89" dependencies = [ "bytes", "half", @@ -208,9 +207,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" +checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870" dependencies = [ "arrow-array", "arrow-buffer", @@ -229,28 +228,25 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" +checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6" dependencies = [ "arrow-array", - "arrow-buffer", "arrow-cast", - "arrow-data", "arrow-schema", "chrono", "csv", "csv-core", "lazy_static", - "lexical-core", "regex", ] [[package]] name = "arrow-data" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" +checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754" dependencies = [ "arrow-buffer", "arrow-schema", @@ -260,13 +256,12 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" +checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e" dependencies = [ "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-schema", "flatbuffers", @@ -275,9 +270,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" +checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c" dependencies = [ "arrow-array", "arrow-buffer", @@ -295,26 +290,23 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" +checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "arrow-select", - "half", - "num", ] [[package]] name = "arrow-row" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" +checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c" dependencies = [ - "ahash", "arrow-array", "arrow-buffer", "arrow-data", @@ -324,18 +316,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" +checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "arrow-select" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" +checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" dependencies = [ "ahash", "arrow-array", @@ -347,9 +339,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" +checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0" dependencies = [ "arrow-array", "arrow-buffer", @@ -380,10 +372,9 @@ version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" dependencies = [ - "bzip2", + "bzip2 0.4.4", "flate2", "futures-core", - "futures-io", "memchr", "pin-project-lite", "tokio", @@ -448,6 +439,19 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -540,6 +544,16 @@ dependencies = [ "libc", ] +[[package]] +name = "bzip2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafdbf26611df8c14810e268ddceda071c297570a5fb360ceddf617fe417ef58" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2-sys" version = "0.1.11+1.0.8" @@ -751,11 +765,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ - "ahash", "arrow", "arrow-array", "arrow-ipc", @@ -763,9 +775,8 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.0", "chrono", - "dashmap", "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", @@ -774,6 +785,7 @@ dependencies = [ "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", + "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", @@ -784,18 +796,13 @@ dependencies = [ "flate2", "futures", "glob", - "half", - "hashbrown 0.14.5", - "indexmap", - "itertools", + "itertools 0.14.0", "log", - "num_cpus", "object_store", "parking_lot", "parquet", - "paste", - "pin-project-lite", "rand", + "regex", "sqlparser", "tempfile", "tokio", @@ -808,67 +815,74 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ - "arrow-schema", + "arrow", "async-trait", + "dashmap", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", "parking_lot", + "sqlparser", ] [[package]] name = "datafusion-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", + "arrow-ipc", "arrow-schema", - "chrono", + "base64", "half", "hashbrown 0.14.5", "indexmap", - "instant", "libc", - "num_cpus", + "log", "object_store", "parquet", "paste", + "recursive", "sqlparser", "tokio", + "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "log", "tokio", ] +[[package]] +name = "datafusion-doc" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" + [[package]] name = "datafusion-execution" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.5", "log", "object_store", "parking_lot", @@ -879,63 +893,59 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ - "ahash", "arrow", - "arrow-array", - "arrow-buffer", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", "paste", + "recursive", "serde_json", "sqlparser", - "strum", - "strum_macros", ] [[package]] name = "datafusion-expr-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", "datafusion-common", - "itertools", + "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-ffi" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e923c459b53a26d92a8806d1f6a37fdf48bde51507a39eaed6f42a60f2bfd160" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "abi_stable", "arrow", + "arrow-array", + "arrow-schema", "async-ffi", "async-trait", "datafusion", "datafusion-proto", - "doc-comment", "futures", "log", "prost", + "semver", + "tokio", ] [[package]] name = "datafusion-functions" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", "arrow-buffer", @@ -944,11 +954,14 @@ dependencies = [ "blake3", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hashbrown 0.14.5", "hex", - "itertools", + "itertools 0.14.0", "log", "md-5", "rand", @@ -960,44 +973,42 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "ahash", "arrow", + "arrow-buffer", "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", "datafusion-functions-aggregate-common", + "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", "half", - "indexmap", "log", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "ahash", "arrow", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", - "rand", ] [[package]] name = "datafusion-functions-nested" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", "arrow-array", @@ -1005,26 +1016,43 @@ dependencies = [ "arrow-ord", "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", "datafusion-functions", "datafusion-functions-aggregate", + "datafusion-macros", "datafusion-physical-expr-common", - "itertools", + "itertools 0.14.0", "log", "paste", - "rand", +] + +[[package]] +name = "datafusion-functions-table" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", ] [[package]] name = "datafusion-functions-window" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "datafusion-common", + "datafusion-doc", "datafusion-expr", "datafusion-functions-window-common", + "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", "log", @@ -1033,48 +1061,51 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", ] +[[package]] +name = "datafusion-macros" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +dependencies = [ + "datafusion-expr", + "quote", + "syn 2.0.87", +] + [[package]] name = "datafusion-optimizer" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", - "async-trait", "chrono", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", - "paste", + "recursive", + "regex", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", - "arrow-ord", "arrow-schema", - "arrow-string", - "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1083,7 +1114,7 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", "paste", "petgraph", @@ -1091,39 +1122,43 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "ahash", "arrow", + "arrow-buffer", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "rand", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-optimizer" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", "arrow-schema", "datafusion-common", "datafusion-execution", + "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools", + "futures", + "itertools 0.14.0", + "log", + "recursive", + "url", ] [[package]] name = "datafusion-physical-plan" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "ahash", "arrow", @@ -1137,7 +1172,6 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -1145,20 +1179,17 @@ dependencies = [ "half", "hashbrown 0.14.5", "indexmap", - "itertools", + "itertools 0.14.0", "log", - "once_cell", "parking_lot", "pin-project-lite", - "rand", "tokio", ] [[package]] name = "datafusion-proto" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f730f7fc5a20134d4e5ecdf7bbf392002ac58163d58423ea28a702dc077b06e1" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", "chrono", @@ -1172,33 +1203,30 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12c225fe49e4f943e35446b263613ada7a9e9f8d647544e6b07037b9803567df" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", - "chrono", "datafusion-common", - "object_store", "prost", ] [[package]] name = "datafusion-sql" -version = "43.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" +version = "45.0.0" +source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" dependencies = [ "arrow", "arrow-array", "arrow-schema", + "bigdecimal", "datafusion-common", "datafusion-expr", "indexmap", "log", + "recursive", "regex", "sqlparser", - "strum", ] [[package]] @@ -1223,12 +1251,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "either" version = "1.13.0" @@ -1272,15 +1294,15 @@ dependencies = [ [[package]] name = "fixedbitset" -version = "0.4.2" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "24.3.25" +version = "24.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1469,12 +1491,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hex" version = "0.4.3" @@ -1651,9 +1667,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.1", @@ -1665,18 +1681,6 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "integer-encoding" version = "3.0.4" @@ -1692,6 +1696,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -1964,16 +1977,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.36.5" @@ -1994,7 +1997,7 @@ dependencies = [ "chrono", "futures", "humantime", - "itertools", + "itertools 0.13.0", "parking_lot", "percent-encoding", "snafu", @@ -2044,9 +2047,9 @@ dependencies = [ [[package]] name = "parquet" -version = "53.2.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e" +checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" dependencies = [ "ahash", "arrow-array", @@ -2063,13 +2066,14 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.15.1", "lz4_flex", "num", "num-bigint", "object_store", "paste", "seq-macro", + "simdutf8", "snap", "thrift", "tokio", @@ -2101,9 +2105,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.6.5" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", "indexmap", @@ -2206,17 +2210,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", - "itertools", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.87", ] +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + [[package]] name = "pyo3" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" dependencies = [ "cfg-if", "indoc", @@ -2232,9 +2245,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" dependencies = [ "once_cell", "target-lexicon", @@ -2242,9 +2255,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" dependencies = [ "libc", "pyo3-build-config", @@ -2252,9 +2265,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -2264,9 +2277,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.22.6" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" dependencies = [ "heck", "proc-macro2", @@ -2314,6 +2327,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.87", +] + [[package]] name = "redox_syscall" version = "0.5.7" @@ -2418,9 +2451,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.23" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" [[package]] name = "seq-macro" @@ -2477,6 +2510,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "0.3.11" @@ -2527,9 +2566,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "sqlparser" -version = "0.51.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" dependencies = [ "log", "sqlparser_derive", @@ -2537,9 +2576,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", @@ -2552,6 +2591,19 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "stacker" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2563,9 +2615,6 @@ name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros", -] [[package]] name = "strum_macros" @@ -2798,9 +2847,9 @@ checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna", @@ -2906,10 +2955,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] -name = "web-sys" -version = "0.3.72" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/examples/ffi-table-provider/Cargo.toml b/examples/ffi-table-provider/Cargo.toml index 4e6f91f33..0e558fdd0 100644 --- a/examples/ffi-table-provider/Cargo.toml +++ b/examples/ffi-table-provider/Cargo.toml @@ -21,15 +21,15 @@ version = "0.1.0" edition = "2021" [dependencies] -datafusion = { version = "44.0.0" } -datafusion-ffi = { version = "44.0.0" } -pyo3 = { version = "0.22.6", features = ["extension-module", "abi3", "abi3-py38"] } -arrow = { version = "53.2.0" } -arrow-array = { version = "53.2.0" } -arrow-schema = { version = "53.2.0" } +datafusion = { version = "45.0.0" } +datafusion-ffi = { version = "45.0.0" } +pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py38"] } +arrow = { version = "54" } +arrow-array = { version = "54" } +arrow-schema = { version = "54" } [build-dependencies] -pyo3-build-config = "0.22.6" +pyo3-build-config = "0.23" [lib] name = "ffi_table_provider" diff --git a/examples/ffi-table-provider/src/lib.rs b/examples/ffi-table-provider/src/lib.rs index 473244d88..88deeece2 100644 --- a/examples/ffi-table-provider/src/lib.rs +++ b/examples/ffi-table-provider/src/lib.rs @@ -102,7 +102,7 @@ impl MyTableProvider { let provider = self .create_table() .map_err(|e| PyRuntimeError::new_err(e.to_string()))?; - let provider = FFI_TableProvider::new(Arc::new(provider), false); + let provider = FFI_TableProvider::new(Arc::new(provider), false, None); PyCapsule::new_bound(py, provider, Some(name.clone())) } diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 77f88aa44..354c7e180 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -148,8 +148,7 @@ def test_relational_expr(test_ctx): assert df.filter(col("b") == "beta").count() == 1 assert df.filter(col("b") != "beta").count() == 2 - with pytest.raises(Exception): - df.filter(col("a") == "beta").count() + assert df.filter(col("a") == "beta").count() == 0 def test_expr_to_variant(): diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index ad6aa7c0a..796b1f76e 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -732,7 +732,7 @@ def test_array_function_obj_tests(stmt, py_expr): ), ( f.regexp_match(column("a"), literal("(ell|orl)")), - pa.array([["ell"], ["orl"], None]), + pa.array([["ell"], ["orl"], None], type=pa.list_(pa.string_view())), ), ( f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), diff --git a/src/context.rs b/src/context.rs index f53b15576..ebe7db230 100644 --- a/src/context.rs +++ b/src/context.rs @@ -48,7 +48,7 @@ use crate::utils::{get_tokio_runtime, validate_pycapsule, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; -use datafusion::catalog_common::TableReference; +use datafusion::common::TableReference; use datafusion::common::{exec_err, ScalarValue}; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; use datafusion::datasource::file_format::parquet::ParquetFormat; diff --git a/src/dataframe.rs b/src/dataframe.rs index 6fb08ba25..13d7ae838 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -546,7 +546,7 @@ impl PyDataFrame { /// Collect the batches and pass to Arrow Table fn to_arrow_table(&self, py: Python<'_>) -> PyResult { let batches = self.collect(py)?.to_object(py); - let schema: PyObject = self.schema().into_py(py); + let schema: PyObject = self.schema().into_pyobject(py)?.to_object(py); // Instantiate pyarrow Table object and use its from_batches method let table_class = py.import_bound("pyarrow")?.getattr("Table")?; From 40a61c150adee6beb9961302fece81c33639082e Mon Sep 17 00:00:00 2001 From: Chongchen Chen Date: Sun, 16 Feb 2025 02:31:00 +0800 Subject: [PATCH 262/413] add to_timestamp_nanos (#1020) --- python/datafusion/functions.py | 1 + python/tests/test_functions.py | 4 ++++ src/functions.rs | 2 ++ 3 files changed, 7 insertions(+) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 7c2fa9a8f..5c260aade 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -252,6 +252,7 @@ "to_hex", "to_timestamp", "to_timestamp_micros", + "to_timestamp_nanos", "to_timestamp_millis", "to_timestamp_seconds", "to_unixtime", diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 796b1f76e..b1a739b49 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -871,6 +871,7 @@ def test_temporal_functions(df): f.to_timestamp_millis(literal("2023-09-07 05:06:14.523952")), f.to_timestamp_micros(literal("2023-09-07 05:06:14.523952")), f.extract(literal("day"), column("d")), + f.to_timestamp_nanos(literal("2023-09-07 05:06:14.523952")), ) result = df.collect() assert len(result) == 1 @@ -909,6 +910,9 @@ def test_temporal_functions(df): [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") ) assert result.column(10) == pa.array([31, 26, 2], type=pa.int32()) + assert result.column(11) == pa.array( + [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + ) def test_arrow_cast(df): diff --git a/src/functions.rs b/src/functions.rs index 46c748cf8..6a8abb18d 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -553,6 +553,7 @@ expr_fn!( expr_fn!(now); expr_fn_vec!(to_timestamp); expr_fn_vec!(to_timestamp_millis); +expr_fn_vec!(to_timestamp_nanos); expr_fn_vec!(to_timestamp_micros); expr_fn_vec!(to_timestamp_seconds); expr_fn_vec!(to_unixtime); @@ -977,6 +978,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(to_hex))?; m.add_wrapped(wrap_pyfunction!(to_timestamp))?; m.add_wrapped(wrap_pyfunction!(to_timestamp_millis))?; + m.add_wrapped(wrap_pyfunction!(to_timestamp_nanos))?; m.add_wrapped(wrap_pyfunction!(to_timestamp_micros))?; m.add_wrapped(wrap_pyfunction!(to_timestamp_seconds))?; m.add_wrapped(wrap_pyfunction!(to_unixtime))?; From 3584bec8900bcfb33bcae4b85a3c47a46b82c72e Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Wed, 19 Feb 2025 20:50:31 -0500 Subject: [PATCH 263/413] [infra] Fail Clippy on rust build warnings (#1029) * pyo3 update required changes to deprecated interfaces * Substrait feature clippy updates * PyTuple was called twice * add -D warnings option --------- Co-authored-by: Tim Saucer --- .github/workflows/test.yaml | 2 +- .pre-commit-config.yaml | 2 +- src/config.rs | 10 +++--- src/context.rs | 12 +++---- src/dataframe.rs | 17 +++++---- src/dataset.rs | 2 +- src/dataset_exec.rs | 8 ++--- src/errors.rs | 4 +++ src/expr.rs | 61 ++++++++++++++++---------------- src/expr/aggregate.rs | 6 ++-- src/expr/analyze.rs | 6 ++-- src/expr/create_memory_table.rs | 6 ++-- src/expr/create_view.rs | 6 ++-- src/expr/distinct.rs | 6 ++-- src/expr/drop_table.rs | 6 ++-- src/expr/empty_relation.rs | 6 ++-- src/expr/explain.rs | 6 ++-- src/expr/extension.rs | 6 ++-- src/expr/filter.rs | 6 ++-- src/expr/join.rs | 6 ++-- src/expr/limit.rs | 6 ++-- src/expr/literal.rs | 6 ++-- src/expr/logical_node.rs | 4 +-- src/expr/projection.rs | 6 ++-- src/expr/repartition.rs | 6 ++-- src/expr/sort.rs | 6 ++-- src/expr/subquery.rs | 6 ++-- src/expr/subquery_alias.rs | 6 ++-- src/expr/table_scan.rs | 6 ++-- src/expr/union.rs | 6 ++-- src/expr/unnest.rs | 6 ++-- src/expr/window.rs | 6 ++-- src/lib.rs | 10 +++--- src/physical_plan.rs | 2 +- src/pyarrow_filter_expression.rs | 36 ++++++++++--------- src/pyarrow_util.rs | 4 +-- src/sql/logical.rs | 4 +-- src/substrait.rs | 4 +-- src/udaf.rs | 5 +-- src/udf.rs | 5 +-- src/udwf.rs | 44 +++++++++++------------ 41 files changed, 188 insertions(+), 180 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c93d4c06f..c1d9ac838 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -71,7 +71,7 @@ jobs: - name: Run Clippy if: ${{ matrix.python-version == '3.10' && matrix.toolchain == 'stable' }} - run: cargo clippy --all-targets --all-features -- -D clippy::all -A clippy::redundant_closure + run: cargo clippy --all-targets --all-features -- -D clippy::all -D warnings -A clippy::redundant_closure - name: Install dependencies and build uses: astral-sh/setup-uv@v5 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e20fedf5c..b548ff18f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: - id: rust-clippy name: Rust clippy description: Run cargo clippy on files included in the commit. clippy should be installed before-hand. - entry: cargo clippy --all-targets --all-features -- -Dclippy::all -Aclippy::redundant_closure + entry: cargo clippy --all-targets --all-features -- -Dclippy::all -D warnings -Aclippy::redundant_closure pass_filenames: false types: [file, rust] language: system diff --git a/src/config.rs b/src/config.rs index cc725b9a3..667d5c590 100644 --- a/src/config.rs +++ b/src/config.rs @@ -47,14 +47,14 @@ impl PyConfig { } /// Get a configuration option - pub fn get(&mut self, key: &str, py: Python) -> PyResult { + pub fn get<'py>(&mut self, key: &str, py: Python<'py>) -> PyResult> { let options = self.config.to_owned(); for entry in options.entries() { if entry.key == key { - return Ok(entry.value.into_py(py)); + return Ok(entry.value.into_pyobject(py)?); } } - Ok(None::.into_py(py)) + Ok(None::.into_pyobject(py)?) } /// Set a configuration option @@ -66,10 +66,10 @@ impl PyConfig { /// Get all configuration options pub fn get_all(&mut self, py: Python) -> PyResult { - let dict = PyDict::new_bound(py); + let dict = PyDict::new(py); let options = self.config.to_owned(); for entry in options.entries() { - dict.set_item(entry.key, entry.value.clone().into_py(py))?; + dict.set_item(entry.key, entry.value.clone().into_pyobject(py)?)?; } Ok(dict.into()) } diff --git a/src/context.rs b/src/context.rs index ebe7db230..0f962638e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -458,8 +458,8 @@ impl PySessionContext { let py = data.py(); // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[data]); + let table_class = py.import("pyarrow")?.getattr("Table")?; + let args = PyTuple::new(py, &[data])?; let table = table_class.call_method1("from_pylist", args)?; // Convert Arrow Table to datafusion DataFrame @@ -478,8 +478,8 @@ impl PySessionContext { let py = data.py(); // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[data]); + let table_class = py.import("pyarrow")?.getattr("Table")?; + let args = PyTuple::new(py, &[data])?; let table = table_class.call_method1("from_pydict", args)?; // Convert Arrow Table to datafusion DataFrame @@ -533,8 +533,8 @@ impl PySessionContext { let py = data.py(); // Instantiate pyarrow Table object & convert to Arrow Table - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[data]); + let table_class = py.import("pyarrow")?.getattr("Table")?; + let args = PyTuple::new(py, &[data])?; let table = table_class.call_method1("from_pandas", args)?; // Convert Arrow Table to datafusion DataFrame diff --git a/src/dataframe.rs b/src/dataframe.rs index 13d7ae838..ed9578a71 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -545,12 +545,12 @@ impl PyDataFrame { /// Convert to Arrow Table /// Collect the batches and pass to Arrow Table fn to_arrow_table(&self, py: Python<'_>) -> PyResult { - let batches = self.collect(py)?.to_object(py); - let schema: PyObject = self.schema().into_pyobject(py)?.to_object(py); + let batches = self.collect(py)?.into_pyobject(py)?; + let schema = self.schema().into_pyobject(py)?; // Instantiate pyarrow Table object and use its from_batches method - let table_class = py.import_bound("pyarrow")?.getattr("Table")?; - let args = PyTuple::new_bound(py, &[batches, schema]); + let table_class = py.import("pyarrow")?.getattr("Table")?; + let args = PyTuple::new(py, &[batches, schema])?; let table: PyObject = table_class.call_method1("from_batches", args)?.into(); Ok(table) } @@ -585,8 +585,7 @@ impl PyDataFrame { let ffi_stream = FFI_ArrowArrayStream::new(reader); let stream_capsule_name = CString::new("arrow_array_stream").unwrap(); - PyCapsule::new_bound(py, ffi_stream, Some(stream_capsule_name)) - .map_err(PyDataFusionError::from) + PyCapsule::new(py, ffi_stream, Some(stream_capsule_name)).map_err(PyDataFusionError::from) } fn execute_stream(&self, py: Python) -> PyDataFusionResult { @@ -649,8 +648,8 @@ impl PyDataFrame { /// Collect the batches, pass to Arrow Table & then convert to polars DataFrame fn to_polars(&self, py: Python<'_>) -> PyResult { let table = self.to_arrow_table(py)?; - let dataframe = py.import_bound("polars")?.getattr("DataFrame")?; - let args = PyTuple::new_bound(py, &[table]); + let dataframe = py.import("polars")?.getattr("DataFrame")?; + let args = PyTuple::new(py, &[table])?; let result: PyObject = dataframe.call1(args)?.into(); Ok(result) } @@ -673,7 +672,7 @@ fn print_dataframe(py: Python, df: DataFrame) -> PyDataFusionResult<()> { // Import the Python 'builtins' module to access the print function // Note that println! does not print to the Python debug console and is not visible in notebooks for instance - let print = py.import_bound("builtins")?.getattr("print")?; + let print = py.import("builtins")?.getattr("print")?; print.call1((result,))?; Ok(()) } diff --git a/src/dataset.rs b/src/dataset.rs index a8fa21ec5..0baf4da2a 100644 --- a/src/dataset.rs +++ b/src/dataset.rs @@ -48,7 +48,7 @@ impl Dataset { // Creates a Python PyArrow.Dataset pub fn new(dataset: &Bound<'_, PyAny>, py: Python) -> PyResult { // Ensure that we were passed an instance of pyarrow.dataset.Dataset - let ds = PyModule::import_bound(py, "pyarrow.dataset")?; + let ds = PyModule::import(py, "pyarrow.dataset")?; let ds_attr = ds.getattr("Dataset")?; let ds_type = ds_attr.downcast::()?; if dataset.is_instance(ds_type)? { diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index ace42115b..445e4fe74 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -104,7 +104,7 @@ impl DatasetExec { }) .transpose()?; - let kwargs = PyDict::new_bound(py); + let kwargs = PyDict::new(py); kwargs.set_item("columns", columns.clone())?; kwargs.set_item( @@ -121,7 +121,7 @@ impl DatasetExec { .0, ); - let builtins = Python::import_bound(py, "builtins")?; + let builtins = Python::import(py, "builtins")?; let pylist = builtins.getattr("list")?; // Get the fragments or partitions of the dataset @@ -198,7 +198,7 @@ impl ExecutionPlan for DatasetExec { let dataset_schema = dataset .getattr("schema") .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; - let kwargs = PyDict::new_bound(py); + let kwargs = PyDict::new(py); kwargs .set_item("columns", self.columns.clone()) .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; @@ -223,7 +223,7 @@ impl ExecutionPlan for DatasetExec { let record_batches: Bound<'_, PyIterator> = scanner .call_method0("to_batches") .map_err(|err| InnerDataFusionError::External(Box::new(err)))? - .iter() + .try_iter() .map_err(|err| InnerDataFusionError::External(Box::new(err)))?; let record_batches = PyArrowBatchesAdapter { diff --git a/src/errors.rs b/src/errors.rs index b02b754a2..f1d5aeb23 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -91,3 +91,7 @@ pub fn py_datafusion_err(e: impl Debug) -> PyErr { pub fn py_unsupported_variant_err(e: impl Debug) -> PyErr { PyErr::new::(format!("{e:?}")) } + +pub fn to_datafusion_err(e: impl Debug) -> InnerDataFusionError { + InnerDataFusionError::Execution(format!("{e:?}")) +} diff --git a/src/expr.rs b/src/expr.rs index 1e9983d42..e750be6a4 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -19,6 +19,7 @@ use datafusion::logical_expr::utils::exprlist_to_fields; use datafusion::logical_expr::{ ExprFuncBuilder, ExprFunctionExt, LogicalPlan, WindowFunctionDefinition, }; +use pyo3::IntoPyObjectExt; use pyo3::{basic::CompareOp, prelude::*}; use std::convert::{From, Into}; use std::sync::Arc; @@ -126,35 +127,35 @@ pub fn py_expr_list(expr: &[Expr]) -> PyResult> { #[pymethods] impl PyExpr { /// Return the specific expression - fn to_variant(&self, py: Python) -> PyResult { + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { Python::with_gil(|_| { match &self.expr { - Expr::Alias(alias) => Ok(PyAlias::from(alias.clone()).into_py(py)), - Expr::Column(col) => Ok(PyColumn::from(col.clone()).into_py(py)), + Expr::Alias(alias) => Ok(PyAlias::from(alias.clone()).into_bound_py_any(py)?), + Expr::Column(col) => Ok(PyColumn::from(col.clone()).into_bound_py_any(py)?), Expr::ScalarVariable(data_type, variables) => { - Ok(PyScalarVariable::new(data_type, variables).into_py(py)) + Ok(PyScalarVariable::new(data_type, variables).into_bound_py_any(py)?) } - Expr::Like(value) => Ok(PyLike::from(value.clone()).into_py(py)), - Expr::Literal(value) => Ok(PyLiteral::from(value.clone()).into_py(py)), - Expr::BinaryExpr(expr) => Ok(PyBinaryExpr::from(expr.clone()).into_py(py)), - Expr::Not(expr) => Ok(PyNot::new(*expr.clone()).into_py(py)), - Expr::IsNotNull(expr) => Ok(PyIsNotNull::new(*expr.clone()).into_py(py)), - Expr::IsNull(expr) => Ok(PyIsNull::new(*expr.clone()).into_py(py)), - Expr::IsTrue(expr) => Ok(PyIsTrue::new(*expr.clone()).into_py(py)), - Expr::IsFalse(expr) => Ok(PyIsFalse::new(*expr.clone()).into_py(py)), - Expr::IsUnknown(expr) => Ok(PyIsUnknown::new(*expr.clone()).into_py(py)), - Expr::IsNotTrue(expr) => Ok(PyIsNotTrue::new(*expr.clone()).into_py(py)), - Expr::IsNotFalse(expr) => Ok(PyIsNotFalse::new(*expr.clone()).into_py(py)), - Expr::IsNotUnknown(expr) => Ok(PyIsNotUnknown::new(*expr.clone()).into_py(py)), - Expr::Negative(expr) => Ok(PyNegative::new(*expr.clone()).into_py(py)), + Expr::Like(value) => Ok(PyLike::from(value.clone()).into_bound_py_any(py)?), + Expr::Literal(value) => Ok(PyLiteral::from(value.clone()).into_bound_py_any(py)?), + Expr::BinaryExpr(expr) => Ok(PyBinaryExpr::from(expr.clone()).into_bound_py_any(py)?), + Expr::Not(expr) => Ok(PyNot::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsNotNull(expr) => Ok(PyIsNotNull::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsNull(expr) => Ok(PyIsNull::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsTrue(expr) => Ok(PyIsTrue::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsFalse(expr) => Ok(PyIsFalse::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsUnknown(expr) => Ok(PyIsUnknown::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsNotTrue(expr) => Ok(PyIsNotTrue::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsNotFalse(expr) => Ok(PyIsNotFalse::new(*expr.clone()).into_bound_py_any(py)?), + Expr::IsNotUnknown(expr) => Ok(PyIsNotUnknown::new(*expr.clone()).into_bound_py_any(py)?), + Expr::Negative(expr) => Ok(PyNegative::new(*expr.clone()).into_bound_py_any(py)?), Expr::AggregateFunction(expr) => { - Ok(PyAggregateFunction::from(expr.clone()).into_py(py)) + Ok(PyAggregateFunction::from(expr.clone()).into_bound_py_any(py)?) } - Expr::SimilarTo(value) => Ok(PySimilarTo::from(value.clone()).into_py(py)), - Expr::Between(value) => Ok(between::PyBetween::from(value.clone()).into_py(py)), - Expr::Case(value) => Ok(case::PyCase::from(value.clone()).into_py(py)), - Expr::Cast(value) => Ok(cast::PyCast::from(value.clone()).into_py(py)), - Expr::TryCast(value) => Ok(cast::PyTryCast::from(value.clone()).into_py(py)), + Expr::SimilarTo(value) => Ok(PySimilarTo::from(value.clone()).into_bound_py_any(py)?), + Expr::Between(value) => Ok(between::PyBetween::from(value.clone()).into_bound_py_any(py)?), + Expr::Case(value) => Ok(case::PyCase::from(value.clone()).into_bound_py_any(py)?), + Expr::Cast(value) => Ok(cast::PyCast::from(value.clone()).into_bound_py_any(py)?), + Expr::TryCast(value) => Ok(cast::PyTryCast::from(value.clone()).into_bound_py_any(py)?), Expr::ScalarFunction(value) => Err(py_unsupported_variant_err(format!( "Converting Expr::ScalarFunction to a Python object is not implemented: {:?}", value @@ -163,29 +164,29 @@ impl PyExpr { "Converting Expr::WindowFunction to a Python object is not implemented: {:?}", value ))), - Expr::InList(value) => Ok(in_list::PyInList::from(value.clone()).into_py(py)), - Expr::Exists(value) => Ok(exists::PyExists::from(value.clone()).into_py(py)), + Expr::InList(value) => Ok(in_list::PyInList::from(value.clone()).into_bound_py_any(py)?), + Expr::Exists(value) => Ok(exists::PyExists::from(value.clone()).into_bound_py_any(py)?), Expr::InSubquery(value) => { - Ok(in_subquery::PyInSubquery::from(value.clone()).into_py(py)) + Ok(in_subquery::PyInSubquery::from(value.clone()).into_bound_py_any(py)?) } Expr::ScalarSubquery(value) => { - Ok(scalar_subquery::PyScalarSubquery::from(value.clone()).into_py(py)) + Ok(scalar_subquery::PyScalarSubquery::from(value.clone()).into_bound_py_any(py)?) } Expr::Wildcard { qualifier, options } => Err(py_unsupported_variant_err(format!( "Converting Expr::Wildcard to a Python object is not implemented : {:?} {:?}", qualifier, options ))), Expr::GroupingSet(value) => { - Ok(grouping_set::PyGroupingSet::from(value.clone()).into_py(py)) + Ok(grouping_set::PyGroupingSet::from(value.clone()).into_bound_py_any(py)?) } Expr::Placeholder(value) => { - Ok(placeholder::PyPlaceholder::from(value.clone()).into_py(py)) + Ok(placeholder::PyPlaceholder::from(value.clone()).into_bound_py_any(py)?) } Expr::OuterReferenceColumn(data_type, column) => Err(py_unsupported_variant_err(format!( "Converting Expr::OuterReferenceColumn to a Python object is not implemented: {:?} - {:?}", data_type, column ))), - Expr::Unnest(value) => Ok(unnest_expr::PyUnnestExpr::from(value.clone()).into_py(py)), + Expr::Unnest(value) => Ok(unnest_expr::PyUnnestExpr::from(value.clone()).into_bound_py_any(py)?), } }) } diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index 389bfb332..8fc9da5b0 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -19,7 +19,7 @@ use datafusion::common::DataFusionError; use datafusion::logical_expr::expr::{AggregateFunction, Alias}; use datafusion::logical_expr::logical_plan::Aggregate; use datafusion::logical_expr::Expr; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use super::logical_node::LogicalNode; @@ -151,7 +151,7 @@ impl LogicalNode for PyAggregate { vec![PyLogicalPlan::from((*self.aggregate.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/analyze.rs b/src/expr/analyze.rs index 084513971..62f93cd26 100644 --- a/src/expr/analyze.rs +++ b/src/expr/analyze.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::logical_plan::Analyze; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use super::logical_node::LogicalNode; @@ -78,7 +78,7 @@ impl LogicalNode for PyAnalyze { vec![PyLogicalPlan::from((*self.analyze.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/create_memory_table.rs b/src/expr/create_memory_table.rs index 01ebb66b0..8872b2d47 100644 --- a/src/expr/create_memory_table.rs +++ b/src/expr/create_memory_table.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::CreateMemoryTable; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::sql::logical::PyLogicalPlan; @@ -91,7 +91,7 @@ impl LogicalNode for PyCreateMemoryTable { vec![PyLogicalPlan::from((*self.create.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/create_view.rs b/src/expr/create_view.rs index d119f5c21..87bb76876 100644 --- a/src/expr/create_view.rs +++ b/src/expr/create_view.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::{CreateView, DdlStatement, LogicalPlan}; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; @@ -88,8 +88,8 @@ impl LogicalNode for PyCreateView { vec![PyLogicalPlan::from((*self.create.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs index 061ab4824..b62b776f8 100644 --- a/src/expr/distinct.rs +++ b/src/expr/distinct.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::Distinct; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::sql::logical::PyLogicalPlan; @@ -89,7 +89,7 @@ impl LogicalNode for PyDistinct { } } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/drop_table.rs b/src/expr/drop_table.rs index 330156abe..96983c1cf 100644 --- a/src/expr/drop_table.rs +++ b/src/expr/drop_table.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::logical_plan::DropTable; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::sql::logical::PyLogicalPlan; @@ -83,7 +83,7 @@ impl LogicalNode for PyDropTable { vec![] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/empty_relation.rs b/src/expr/empty_relation.rs index ce7163466..a1534ac15 100644 --- a/src/expr/empty_relation.rs +++ b/src/expr/empty_relation.rs @@ -17,7 +17,7 @@ use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; use datafusion::logical_expr::EmptyRelation; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use super::logical_node::LogicalNode; @@ -79,7 +79,7 @@ impl LogicalNode for PyEmptyRelation { vec![] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/explain.rs b/src/expr/explain.rs index 8e7fb8843..fc02fe2b5 100644 --- a/src/expr/explain.rs +++ b/src/expr/explain.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::{logical_plan::Explain, LogicalPlan}; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::{common::df_schema::PyDFSchema, errors::py_type_err, sql::logical::PyLogicalPlan}; @@ -104,7 +104,7 @@ impl LogicalNode for PyExplain { vec![] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/extension.rs b/src/expr/extension.rs index a29802b0b..1e3fbb199 100644 --- a/src/expr/extension.rs +++ b/src/expr/extension.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::Extension; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::sql::logical::PyLogicalPlan; @@ -46,7 +46,7 @@ impl LogicalNode for PyExtension { vec![] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/filter.rs b/src/expr/filter.rs index a6d8aa7ee..9bdb667cd 100644 --- a/src/expr/filter.rs +++ b/src/expr/filter.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::logical_plan::Filter; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -81,7 +81,7 @@ impl LogicalNode for PyFilter { vec![PyLogicalPlan::from((*self.filter.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/join.rs b/src/expr/join.rs index 66e677f8a..76ec532e7 100644 --- a/src/expr/join.rs +++ b/src/expr/join.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::logical_plan::{Join, JoinConstraint, JoinType}; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -193,7 +193,7 @@ impl LogicalNode for PyJoin { ] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/limit.rs b/src/expr/limit.rs index 84ad7d68b..c2a33ff89 100644 --- a/src/expr/limit.rs +++ b/src/expr/limit.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::logical_plan::Limit; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -90,7 +90,7 @@ impl LogicalNode for PyLimit { vec![PyLogicalPlan::from((*self.limit.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/literal.rs b/src/expr/literal.rs index 2cb2079f1..a660ac914 100644 --- a/src/expr/literal.rs +++ b/src/expr/literal.rs @@ -17,7 +17,7 @@ use crate::errors::PyDataFusionError; use datafusion::common::ScalarValue; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; #[pyclass(name = "Literal", module = "datafusion.expr", subclass)] #[derive(Clone)] @@ -144,8 +144,8 @@ impl PyLiteral { } #[allow(clippy::wrong_self_convention)] - fn into_type(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn into_type<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } fn __repr__(&self) -> PyResult { diff --git a/src/expr/logical_node.rs b/src/expr/logical_node.rs index 757e4f94b..5aff70059 100644 --- a/src/expr/logical_node.rs +++ b/src/expr/logical_node.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use pyo3::{PyObject, PyResult, Python}; +use pyo3::{Bound, PyAny, PyResult, Python}; use crate::sql::logical::PyLogicalPlan; @@ -25,5 +25,5 @@ pub trait LogicalNode { /// The input plan to the current logical node instance. fn inputs(&self) -> Vec; - fn to_variant(&self, py: Python) -> PyResult; + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult>; } diff --git a/src/expr/projection.rs b/src/expr/projection.rs index 36534fdb2..dc7e5e3c1 100644 --- a/src/expr/projection.rs +++ b/src/expr/projection.rs @@ -17,7 +17,7 @@ use datafusion::logical_expr::logical_plan::Projection; use datafusion::logical_expr::Expr; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -113,7 +113,7 @@ impl LogicalNode for PyProjection { vec![PyLogicalPlan::from((*self.projection.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/repartition.rs b/src/expr/repartition.rs index 4e680e181..3e782d6af 100644 --- a/src/expr/repartition.rs +++ b/src/expr/repartition.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::{logical_plan::Repartition, Expr, Partitioning}; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::{errors::py_type_err, sql::logical::PyLogicalPlan}; @@ -121,7 +121,7 @@ impl LogicalNode for PyRepartition { vec![PyLogicalPlan::from((*self.repartition.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/sort.rs b/src/expr/sort.rs index a1803ccaf..ed4947591 100644 --- a/src/expr/sort.rs +++ b/src/expr/sort.rs @@ -17,7 +17,7 @@ use datafusion::common::DataFusionError; use datafusion::logical_expr::logical_plan::Sort; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -96,7 +96,7 @@ impl LogicalNode for PySort { vec![PyLogicalPlan::from((*self.sort.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/subquery.rs b/src/expr/subquery.rs index dac8d0a2b..5ebfe6927 100644 --- a/src/expr/subquery.rs +++ b/src/expr/subquery.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::Subquery; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::sql::logical::PyLogicalPlan; @@ -75,7 +75,7 @@ impl LogicalNode for PySubquery { vec![] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/subquery_alias.rs b/src/expr/subquery_alias.rs index a83cff96d..267a4d485 100644 --- a/src/expr/subquery_alias.rs +++ b/src/expr/subquery_alias.rs @@ -18,7 +18,7 @@ use std::fmt::{self, Display, Formatter}; use datafusion::logical_expr::SubqueryAlias; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; @@ -85,7 +85,7 @@ impl LogicalNode for PySubqueryAlias { vec![PyLogicalPlan::from((*self.subquery_alias.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index f61be7fe4..6a0d53f0f 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -17,7 +17,7 @@ use datafusion::common::TableReference; use datafusion::logical_expr::logical_plan::TableScan; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::expr::logical_node::LogicalNode; @@ -146,7 +146,7 @@ impl LogicalNode for PyTableScan { vec![] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/union.rs b/src/expr/union.rs index 62488d9a1..5a08ccc13 100644 --- a/src/expr/union.rs +++ b/src/expr/union.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::logical_plan::Union; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -83,7 +83,7 @@ impl LogicalNode for PyUnion { .collect() } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/unnest.rs b/src/expr/unnest.rs index adc705035..8e70e0990 100644 --- a/src/expr/unnest.rs +++ b/src/expr/unnest.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::logical_expr::logical_plan::Unnest; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::df_schema::PyDFSchema; @@ -79,7 +79,7 @@ impl LogicalNode for PyUnnest { vec![PyLogicalPlan::from((*self.unnest_.input).clone())] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/expr/window.rs b/src/expr/window.rs index 4dc6cb9c9..13deaec25 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -18,7 +18,7 @@ use datafusion::common::{DataFusionError, ScalarValue}; use datafusion::logical_expr::expr::WindowFunction; use datafusion::logical_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; use crate::common::data_type::PyScalarValue; @@ -289,7 +289,7 @@ impl LogicalNode for PyWindowExpr { vec![self.window.input.as_ref().clone().into()] } - fn to_variant(&self, py: Python) -> PyResult { - Ok(self.clone().into_py(py)) + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) } } diff --git a/src/lib.rs b/src/lib.rs index 317c3a49a..ce93ff0c3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -94,21 +94,21 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/ - let common = PyModule::new_bound(py, "common")?; + let common = PyModule::new(py, "common")?; common::init_module(&common)?; m.add_submodule(&common)?; // Register `expr` as a submodule. Matching `datafusion-expr` https://docs.rs/datafusion-expr/latest/datafusion_expr/ - let expr = PyModule::new_bound(py, "expr")?; + let expr = PyModule::new(py, "expr")?; expr::init_module(&expr)?; m.add_submodule(&expr)?; // Register the functions as a submodule - let funcs = PyModule::new_bound(py, "functions")?; + let funcs = PyModule::new(py, "functions")?; functions::init_module(&funcs)?; m.add_submodule(&funcs)?; - let store = PyModule::new_bound(py, "object_store")?; + let store = PyModule::new(py, "object_store")?; store::init_module(&store)?; m.add_submodule(&store)?; @@ -121,7 +121,7 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { #[cfg(feature = "substrait")] fn setup_substrait_module(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { - let substrait = PyModule::new_bound(py, "substrait")?; + let substrait = PyModule::new(py, "substrait")?; substrait::init_module(&substrait)?; m.add_submodule(&substrait)?; Ok(()) diff --git a/src/physical_plan.rs b/src/physical_plan.rs index 295908dc7..f0be45c6a 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -66,7 +66,7 @@ impl PyExecutionPlan { )?; let bytes = proto.encode_to_vec(); - Ok(PyBytes::new_bound(py, &bytes)) + Ok(PyBytes::new(py, &bytes)) } #[staticmethod] diff --git a/src/pyarrow_filter_expression.rs b/src/pyarrow_filter_expression.rs index 314eebf4f..4b4c86597 100644 --- a/src/pyarrow_filter_expression.rs +++ b/src/pyarrow_filter_expression.rs @@ -16,7 +16,7 @@ // under the License. /// Converts a Datafusion logical plan expression (Expr) into a PyArrow compute expression -use pyo3::prelude::*; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::convert::TryFrom; use std::result::Result; @@ -53,24 +53,28 @@ fn operator_to_py<'py>( Ok(py_op) } -fn extract_scalar_list(exprs: &[Expr], py: Python) -> PyDataFusionResult> { +fn extract_scalar_list<'py>( + exprs: &[Expr], + py: Python<'py>, +) -> PyDataFusionResult>> { let ret = exprs .iter() .map(|expr| match expr { // TODO: should we also leverage `ScalarValue::to_pyarrow` here? Expr::Literal(v) => match v { - ScalarValue::Boolean(Some(b)) => Ok(b.into_py(py)), - ScalarValue::Int8(Some(i)) => Ok(i.into_py(py)), - ScalarValue::Int16(Some(i)) => Ok(i.into_py(py)), - ScalarValue::Int32(Some(i)) => Ok(i.into_py(py)), - ScalarValue::Int64(Some(i)) => Ok(i.into_py(py)), - ScalarValue::UInt8(Some(i)) => Ok(i.into_py(py)), - ScalarValue::UInt16(Some(i)) => Ok(i.into_py(py)), - ScalarValue::UInt32(Some(i)) => Ok(i.into_py(py)), - ScalarValue::UInt64(Some(i)) => Ok(i.into_py(py)), - ScalarValue::Float32(Some(f)) => Ok(f.into_py(py)), - ScalarValue::Float64(Some(f)) => Ok(f.into_py(py)), - ScalarValue::Utf8(Some(s)) => Ok(s.into_py(py)), + // The unwraps here are for infallible conversions + ScalarValue::Boolean(Some(b)) => Ok(b.into_bound_py_any(py)?), + ScalarValue::Int8(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::Int16(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::Int32(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::Int64(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::UInt8(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::UInt16(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::UInt32(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::UInt64(Some(i)) => Ok(i.into_bound_py_any(py)?), + ScalarValue::Float32(Some(f)) => Ok(f.into_bound_py_any(py)?), + ScalarValue::Float64(Some(f)) => Ok(f.into_bound_py_any(py)?), + ScalarValue::Utf8(Some(s)) => Ok(s.into_bound_py_any(py)?), _ => Err(PyDataFusionError::Common(format!( "PyArrow can't handle ScalarValue: {v:?}" ))), @@ -98,8 +102,8 @@ impl TryFrom<&Expr> for PyArrowFilterExpression { // https://arrow.apache.org/docs/python/generated/pyarrow.dataset.Expression.html#pyarrow-dataset-expression fn try_from(expr: &Expr) -> Result { Python::with_gil(|py| { - let pc = Python::import_bound(py, "pyarrow.compute")?; - let op_module = Python::import_bound(py, "operator")?; + let pc = Python::import(py, "pyarrow.compute")?; + let op_module = Python::import(py, "operator")?; let pc_expr: PyDataFusionResult> = match expr { Expr::Column(Column { name, .. }) => Ok(pc.getattr("field")?.call1((name,))?), Expr::Literal(scalar) => Ok(scalar_to_pyarrow(scalar, py)?.into_bound(py)), diff --git a/src/pyarrow_util.rs b/src/pyarrow_util.rs index 2b31467f8..cab708458 100644 --- a/src/pyarrow_util.rs +++ b/src/pyarrow_util.rs @@ -33,8 +33,8 @@ impl FromPyArrow for PyScalarValue { let val = value.call_method0("as_py")?; // construct pyarrow array from the python value and pyarrow type - let factory = py.import_bound("pyarrow")?.getattr("array")?; - let args = PyList::new_bound(py, [val]); + let factory = py.import("pyarrow")?.getattr("array")?; + let args = PyList::new(py, [val])?; let array = factory.call1((args, typ))?; // convert the pyarrow array to rust array using C data interface diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 1be33b75f..96561c434 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -64,7 +64,7 @@ impl PyLogicalPlan { #[pymethods] impl PyLogicalPlan { /// Return the specific logical operator - pub fn to_variant(&self, py: Python) -> PyResult { + pub fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { match self.plan.as_ref() { LogicalPlan::Aggregate(plan) => PyAggregate::from(plan.clone()).to_variant(py), LogicalPlan::Analyze(plan) => PyAnalyze::from(plan.clone()).to_variant(py), @@ -132,7 +132,7 @@ impl PyLogicalPlan { datafusion_proto::protobuf::LogicalPlanNode::try_from_logical_plan(&self.plan, &codec)?; let bytes = proto.encode_to_vec(); - Ok(PyBytes::new_bound(py, &bytes)) + Ok(PyBytes::new(py, &bytes)) } #[staticmethod] diff --git a/src/substrait.rs b/src/substrait.rs index 8dcf3e8a7..1fefc0bbd 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -40,7 +40,7 @@ impl PyPlan { self.plan .encode(&mut proto_bytes) .map_err(PyDataFusionError::EncodeError)?; - Ok(PyBytes::new_bound(py, &proto_bytes).unbind().into()) + Ok(PyBytes::new(py, &proto_bytes).into()) } } @@ -95,7 +95,7 @@ impl PySubstraitSerializer { py: Python, ) -> PyDataFusionResult { let proto_bytes: Vec = wait_for_future(py, serializer::serialize_bytes(sql, &ctx.ctx))?; - Ok(PyBytes::new_bound(py, &proto_bytes).unbind().into()) + Ok(PyBytes::new(py, &proto_bytes).into()) } #[staticmethod] diff --git a/src/udaf.rs b/src/udaf.rs index 5f21533e0..34a9cd51d 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -29,6 +29,7 @@ use datafusion::logical_expr::{ }; use crate::common::data_type::PyScalarValue; +use crate::errors::to_datafusion_err; use crate::expr::PyExpr; use crate::utils::parse_volatility; @@ -73,7 +74,7 @@ impl Accumulator for RustAccumulator { .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); - let py_args = PyTuple::new_bound(py, py_args); + let py_args = PyTuple::new(py, py_args).map_err(to_datafusion_err)?; // 2. call function self.accum @@ -119,7 +120,7 @@ impl Accumulator for RustAccumulator { .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()) .collect::>(); - let py_args = PyTuple::new_bound(py, py_args); + let py_args = PyTuple::new(py, py_args).map_err(to_datafusion_err)?; // 2. call function self.accum diff --git a/src/udf.rs b/src/udf.rs index 4570e77a6..574c9d7b5 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -28,6 +28,7 @@ use datafusion::logical_expr::function::ScalarFunctionImplementation; use datafusion::logical_expr::ScalarUDF; use datafusion::logical_expr::{create_udf, ColumnarValue}; +use crate::errors::to_datafusion_err; use crate::expr::PyExpr; use crate::utils::parse_volatility; @@ -46,11 +47,11 @@ fn pyarrow_function_to_rust( .map_err(|e| DataFusionError::Execution(format!("{e:?}"))) }) .collect::, _>>()?; - let py_args = PyTuple::new_bound(py, py_args); + let py_args = PyTuple::new(py, py_args).map_err(to_datafusion_err)?; // 2. call function let value = func - .call_bound(py, py_args, None) + .call(py, py_args, None) .map_err(|e| DataFusionError::Execution(format!("{e:?}")))?; // 3. cast to arrow::array::Array diff --git a/src/udwf.rs b/src/udwf.rs index 04a4a1640..defd9c522 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -27,6 +27,7 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use crate::common::data_type::PyScalarValue; +use crate::errors::to_datafusion_err; use crate::expr::PyExpr; use crate::utils::parse_volatility; use datafusion::arrow::datatypes::DataType; @@ -56,8 +57,8 @@ impl PartitionEvaluator for RustPartitionEvaluator { fn get_range(&self, idx: usize, n_rows: usize) -> Result> { Python::with_gil(|py| { - let py_args = vec![idx.to_object(py), n_rows.to_object(py)]; - let py_args = PyTuple::new_bound(py, py_args); + let py_args = vec![idx.into_pyobject(py)?, n_rows.into_pyobject(py)?]; + let py_args = PyTuple::new(py, py_args)?; self.evaluator .bind(py) @@ -93,17 +94,14 @@ impl PartitionEvaluator for RustPartitionEvaluator { fn evaluate_all(&mut self, values: &[ArrayRef], num_rows: usize) -> Result { println!("evaluate all called with number of values {}", values.len()); Python::with_gil(|py| { - let py_values = PyList::new_bound( + let py_values = PyList::new( py, values .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()), - ); - let py_num_rows = num_rows.to_object(py).into_bound(py); - let py_args = PyTuple::new_bound( - py, - PyTuple::new_bound(py, vec![py_values.as_any(), &py_num_rows]), - ); + )?; + let py_num_rows = num_rows.into_pyobject(py)?; + let py_args = PyTuple::new(py, vec![py_values.as_any(), &py_num_rows])?; self.evaluator .bind(py) @@ -112,32 +110,28 @@ impl PartitionEvaluator for RustPartitionEvaluator { let array_data = ArrayData::from_pyarrow_bound(&v).unwrap(); make_array(array_data) }) - .map_err(|e| DataFusionError::Execution(format!("{e}"))) }) + .map_err(to_datafusion_err) } fn evaluate(&mut self, values: &[ArrayRef], range: &Range) -> Result { Python::with_gil(|py| { - let py_values = PyList::new_bound( + let py_values = PyList::new( py, values .iter() .map(|arg| arg.into_data().to_pyarrow(py).unwrap()), - ); - let range_tuple = - PyTuple::new_bound(py, vec![range.start.to_object(py), range.end.to_object(py)]); - let py_args = PyTuple::new_bound( - py, - PyTuple::new_bound(py, vec![py_values.as_any(), range_tuple.as_any()]), - ); + )?; + let range_tuple = PyTuple::new(py, vec![range.start, range.end])?; + let py_args = PyTuple::new(py, vec![py_values.as_any(), range_tuple.as_any()])?; self.evaluator .bind(py) .call_method1("evaluate", py_args) .and_then(|v| v.extract::()) .map(|v| v.0) - .map_err(|e| DataFusionError::Execution(format!("{e}"))) }) + .map_err(to_datafusion_err) } fn evaluate_all_with_rank( @@ -148,23 +142,27 @@ impl PartitionEvaluator for RustPartitionEvaluator { Python::with_gil(|py| { let ranks = ranks_in_partition .iter() - .map(|r| PyTuple::new_bound(py, vec![r.start, r.end])); + .map(|r| PyTuple::new(py, vec![r.start, r.end])) + .collect::>>()?; // 1. cast args to Pyarrow array - let py_args = vec![num_rows.to_object(py), PyList::new_bound(py, ranks).into()]; + let py_args = vec![ + num_rows.into_pyobject(py)?.into_any(), + PyList::new(py, ranks)?.into_any(), + ]; - let py_args = PyTuple::new_bound(py, py_args); + let py_args = PyTuple::new(py, py_args)?; // 2. call function self.evaluator .bind(py) .call_method1("evaluate_all_with_rank", py_args) - .map_err(|e| DataFusionError::Execution(format!("{e}"))) .map(|v| { let array_data = ArrayData::from_pyarrow_bound(&v).unwrap(); make_array(array_data) }) }) + .map_err(to_datafusion_err) } fn supports_bounded_execution(&self) -> bool { From e6f6e66c1d180246ad933f8bcc0d40faa8426dfa Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 21 Feb 2025 16:03:36 -0500 Subject: [PATCH 264/413] Add user documentation for the FFI approach (#1031) * Initial commit for FFI user documentation * Update readme to point to the online documentation. Fix a small typo. * Small text adjustments for clarity and formatting --- README.md | 11 +- docs/source/contributor-guide/ffi.rst | 212 ++++++++++++++++++++++++++ docs/source/index.rst | 1 + 3 files changed, 220 insertions(+), 4 deletions(-) create mode 100644 docs/source/contributor-guide/ffi.rst diff --git a/README.md b/README.md index 5aaf7f5f3..9c56b62dd 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,8 @@ DataFusion's Python bindings can be used as a foundation for building new data s planning, and logical plan optimizations, and then transpiles the logical plan to Dask operations for execution. - [DataFusion Ballista](https://github.com/apache/datafusion-ballista) is a distributed SQL query engine that extends DataFusion's Python bindings for distributed use cases. - -It is also possible to use these Python bindings directly for DataFrame and SQL operations, but you may find that -[Polars](http://pola.rs/) and [DuckDB](http://www.duckdb.org/) are more suitable for this use case, since they have -more of an end-user focus and are more actively maintained than these Python bindings. +- [DataFusion Ray](https://github.com/apache/datafusion-ray) is another distributed query engine that uses + DataFusion's Python bindings. ## Features @@ -114,6 +112,11 @@ Printing the context will show the current configuration settings. print(ctx) ``` +## Extensions + +For information about how to extend DataFusion Python, please see the extensions page of the +[online documentation](https://datafusion.apache.org/python/). + ## More Examples See [examples](examples/README.md) for more information. diff --git a/docs/source/contributor-guide/ffi.rst b/docs/source/contributor-guide/ffi.rst new file mode 100644 index 000000000..c1f9806b3 --- /dev/null +++ b/docs/source/contributor-guide/ffi.rst @@ -0,0 +1,212 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +Python Extensions +================= + +The DataFusion in Python project is designed to allow users to extend its functionality in a few core +areas. Ideally many users would like to package their extensions as a Python package and easily +integrate that package with this project. This page serves to describe some of the challenges we face +when doing these integrations and the approach our project uses. + +The Primary Issue +----------------- + +Suppose you wish to use DataFusion and you have a custom data source that can produce tables that +can then be queried against, similar to how you can register a :ref:`CSV ` or +:ref:`Parquet ` file. In DataFusion terminology, you likely want to implement a +:ref:`Custom Table Provider `. In an effort to make your data source +as performant as possible and to utilize the features of DataFusion, you may decide to write +your source in Rust and then expose it through `PyO3 `_ as a Python library. + +At first glance, it may appear the best way to do this is to add the ``datafusion-python`` +crate as a dependency, provide a ``PyTable``, and then to register it with the +``SessionContext``. Unfortunately, this will not work. + +When you produce your code as a Python library and it needs to interact with the DataFusion +library, at the lowest level they communicate through an Application Binary Interface (ABI). +The acronym sounds similar to API (Application Programming Interface), but it is distinctly +different. + +The ABI sets the standard for how these libraries can share data and functions between each +other. One of the key differences between Rust and other programming languages is that Rust +does not have a stable ABI. What this means in practice is that if you compile a Rust library +with one version of the ``rustc`` compiler and I compile another library to interface with it +but I use a different version of the compiler, there is no guarantee the interface will be +the same. + +In practice, this means that a Python library built with ``datafusion-python`` as a Rust +dependency will generally **not** be compatible with the DataFusion Python package, even +if they reference the same version of ``datafusion-python``. If you attempt to do this, it may +work on your local computer if you have built both packages with the same optimizations. +This can sometimes lead to a false expectation that the code will work, but it frequently +breaks the moment you try to use your package against the released packages. + +You can find more information about the Rust ABI in their +`online documentation `_. + +The FFI Approach +---------------- + +Rust supports interacting with other programming languages through it's Foreign Function +Interface (FFI). The advantage of using the FFI is that it enables you to write data structures +and functions that have a stable ABI. The allows you to use Rust code with C, Python, and +other languages. In fact, the `PyO3 `_ library uses the FFI to share data +and functions between Python and Rust. + +The approach we are taking in the DataFusion in Python project is to incrementally expose +more portions of the DataFusion project via FFI interfaces. This allows users to write Rust +code that does **not** require the ``datafusion-python`` crate as a dependency, expose their +code in Python via PyO3, and have it interact with the DataFusion Python package. + +Early adopters of this approach include `delta-rs `_ +who has adapted their Table Provider for use in ```datafusion-python``` with only a few lines +of code. Also, the DataFusion Python project uses the existing definitions from +`Apache Arrow CStream Interface `_ +to support importing **and** exporting tables. Any Python package that supports reading +the Arrow C Stream interface can work with DataFusion Python out of the box! You can read +more about working with Arrow sources in the :ref:`Data Sources ` +page. + +To learn more about the Foreign Function Interface in Rust, the +`Rustonomicon `_ is a good resource. + +Inspiration from Arrow +---------------------- + +DataFusion is built upon `Apache Arrow `_. The canonical Python +Arrow implementation, `pyarrow `_ provides +an excellent way to share Arrow data between Python projects without performing any copy +operations on the data. They do this by using a well defined set of interfaces. You can +find the details about their stream interface +`here `_. The +`Rust Arrow Implementation `_ also supports these +``C`` style definitions via the Foreign Function Interface. + +In addition to using these interfaces to transfer Arrow data between libraries, ``pyarrow`` +goes one step further to make sharing the interfaces easier in Python. They do this +by exposing PyCapsules that contain the expected functionality. + +You can learn more about PyCapsules from the official +`Python online documentation `_. PyCapsules +have excellent support in PyO3 already. The +`PyO3 online documentation `_ is a good source +for more details on using PyCapsules in Rust. + +Two lessons we leverage from the Arrow project in DataFusion Python are: + +- We reuse the existing Arrow FFI functionality wherever possible. +- We expose PyCapsules that contain a FFI stable struct. + +Implementation Details +---------------------- + +The bulk of the code necessary to perform our FFI operations is in the upstream +`DataFusion `_ core repository. You can review the code and +documentation in the `datafusion-ffi`_ crate. + +Our FFI implementation is narrowly focused at sharing data and functions with Rust backed +libraries. This allows us to use the `abi_stable crate `_. +This is an excellent crate that allows for easy conversion between Rust native types +and FFI-safe alternatives. For example, if you needed to pass a ``Vec`` via FFI, +you can simply convert it to a ``RVec`` in an intuitive manner. It also supports +features like ``RResult`` and ``ROption`` that do not have an obvious translation to a +C equivalent. + +The `datafusion-ffi`_ crate has been designed to make it easy to convert from DataFusion +traits into their FFI counterparts. For example, if you have defined a custom +`TableProvider `_ +and you want to create a sharable FFI counterpart, you could write: + +.. code-block:: rust + + let my_provider = MyTableProvider::default(); + let ffi_provider = FFI_TableProvider::new(Arc::new(my_provider), false, None); + +If you were interfacing with a library that provided the above ``FFI_TableProvider`` and +you needed to turn it back into an ``TableProvider``, you can turn it into a +``ForeignTableProvider`` with implements the ``TableProvider`` trait. + +.. code-block:: rust + + let foreign_provider: ForeignTableProvider = ffi_provider.into(); + +If you review the code in `datafusion-ffi`_ you will find that each of the traits we share +across the boundary has two portions, one with a ``FFI_`` prefix and one with a ``Foreign`` +prefix. This is used to distinguish which side of the FFI boundary that struct is +designed to be used on. The structures with the ``FFI_`` prefix are to be used on the +**provider** of the structure. In the example we're showing, this means the code that has +written the underlying ``TableProvider`` implementation to access your custom data source. +The structures with the ``Foreign`` prefix are to be used by the receiver. In this case, +it is the ``datafusion-python`` library. + +In order to share these FFI structures, we need to wrap them in some kind of Python object +that can be used to interface from one package to another. As described in the above +section on our inspiration from Arrow, we use ``PyCapsule``. We can create a ``PyCapsule`` +for our provider thusly: + +.. code-block:: rust + + let name = CString::new("datafusion_table_provider")?; + let my_capsule = PyCapsule::new_bound(py, provider, Some(name))?; + +On the receiving side, turn this pycapsule object into the ``FFI_TableProvider``, which +can then be turned into a ``ForeignTableProvider`` the associated code is: + +.. code-block:: rust + + let capsule = capsule.downcast::()?; + let provider = unsafe { capsule.reference::() }; + +By convention the ``datafusion-python`` library expects a Python object that has a +``TableProvider`` PyCapsule to have this capsule accessible by calling a function named +``__datafusion_table_provider__``. You can see a complete working example of how to +share a ``TableProvider`` from one python library to DataFusion Python in the +`repository examples folder `_. + +This section has been written using ``TableProvider`` as an example. It is the first +extension that has been written using this approach and the most thoroughly implemented. +As we continue to expose more of the DataFusion features, we intend to follow this same +design pattern. + +Alternative Approach +-------------------- + +Suppose you needed to expose some other features of DataFusion and you could not wait +for the upstream repository to implement the FFI approach we describe. In this case +you decide to create your dependency on the ``datafusion-python`` crate instead. + +As we discussed, this is not guaranteed to work across different compiler versions and +optimization levels. If you wish to go down this route, there are two approaches we +have identified you can use. + +#. Re-export all of ``datafusion-python`` yourself with your extensions built in. +#. Carefully synchonize your software releases with the ``datafusion-python`` CI build + system so that your libraries use the exact same compiler, features, and + optimization level. + +We currently do not recommend either of these approaches as they are difficult to +maintain over a long period. Additionally, they require a tight version coupling +between libraries. + +Status of Work +-------------- + +At the time of this writing, the FFI features are under active development. To see +the latest status, we recommend reviewing the code in the `datafusion-ffi`_ crate. + +.. _datafusion-ffi: https://crates.io/crates/datafusion-ffi diff --git a/docs/source/index.rst b/docs/source/index.rst index 34eb23b28..558b2d572 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -85,6 +85,7 @@ Example :caption: CONTRIBUTOR GUIDE contributor-guide/introduction + contributor-guide/ffi .. _toc.api: .. toctree:: From 3f3983cc86ffe267cff97480241e8a588ac38fa3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 23 Feb 2025 08:00:52 -0500 Subject: [PATCH 265/413] build(deps): bump arrow from 54.1.0 to 54.2.0 (#1035) Bumps [arrow](https://github.com/apache/arrow-rs) from 54.1.0 to 54.2.0. - [Release notes](https://github.com/apache/arrow-rs/releases) - [Changelog](https://github.com/apache/arrow-rs/blob/main/CHANGELOG-old.md) - [Commits](https://github.com/apache/arrow-rs/compare/54.1.0...54.2.0) --- updated-dependencies: - dependency-name: arrow dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 56 +++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1b1ed50a..d23ed6169 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc" +checksum = "755b6da235ac356a869393c23668c663720b8749dd6f15e52b6c214b4b964cc7" dependencies = [ "arrow-arith", "arrow-array", @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248" +checksum = "64656a1e0b13ca766f8440752e9a93e11014eec7b67909986f83ed0ab1fe37b8" dependencies = [ "arrow-array", "arrow-buffer", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" +checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a" dependencies = [ "ahash", "arrow-buffer", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89" +checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a" dependencies = [ "bytes", "half", @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870" +checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6" +checksum = "90f12542b8164398fc9ec595ff783c4cf6044daa89622c5a7201be920e4c0d4c" dependencies = [ "arrow-array", "arrow-cast", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754" +checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83" dependencies = [ "arrow-buffer", "arrow-schema", @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e" +checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c" +checksum = "9551d9400532f23a370cabbea1dc5a53c49230397d41f96c4c8eedf306199305" dependencies = [ "arrow-array", "arrow-buffer", @@ -326,9 +326,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da" +checksum = "6c07223476f8219d1ace8cd8d85fa18c4ebd8d945013f25ef5c72e85085ca4ee" dependencies = [ "arrow-array", "arrow-buffer", @@ -339,9 +339,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c" +checksum = "91b194b38bfd89feabc23e798238989c6648b2506ad639be42ec8eb1658d82c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -352,18 +352,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" +checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735" dependencies = [ "bitflags 2.8.0", ] [[package]] name = "arrow-select" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" +checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539" dependencies = [ "ahash", "arrow-array", @@ -375,9 +375,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0" +checksum = "d44c8eed43be4ead49128370f7131f054839d3d6003e52aebf64322470b8fbd0" dependencies = [ "arrow-array", "arrow-buffer", From 69ebf70bd821d0ae516d2f61d96058e2252a7a1f Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 24 Feb 2025 21:30:52 +0100 Subject: [PATCH 266/413] Chore: Release datafusion-python 45 (#1024) * Bump version number to prepare for release * Add changelog 45.0.0 * Add deprecated marker from either typing or typing_extensions based on the python version * Limit pyarrow version per issue # 1023 * Bumping the version number to support new release candidate * There was no guarantee that the record batches would be returned in a single partition, so update the unit test to check all partitions. * Revert "Limit pyarrow version per issue # 1023" This reverts commit b48d5872661017ec21ea71f7dbb9569f2f0bf797. * Correct import for python 3.13 and above * Bump minor version due to pypi requirement * Update cargo lock --- Cargo.lock | 113 +++++++++++++-------------------- Cargo.toml | 2 +- dev/changelog/45.0.0.md | 42 ++++++++++++ python/datafusion/context.py | 5 +- python/datafusion/dataframe.py | 5 +- python/datafusion/expr.py | 6 +- python/datafusion/substrait.py | 5 +- python/tests/test_dataframe.py | 21 ++++-- 8 files changed, 118 insertions(+), 81 deletions(-) create mode 100644 dev/changelog/45.0.0.md diff --git a/Cargo.lock b/Cargo.lock index d23ed6169..5c7f2bf3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -606,19 +606,18 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bafdbf26611df8c14810e268ddceda071c297570a5fb360ceddf617fe417ef58" +checksum = "75b89e7c29231c673a61a46e722602bcd138298f6b9e81e71119693534585f5c" dependencies = [ "bzip2-sys", - "libc", ] [[package]] name = "bzip2-sys" -version = "0.1.11+1.0.8" +version = "0.1.12+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "72ebc2f1a417f01e1da30ef264ee86ae31d2dcd2d603ea283d3c244a883ca2a9" dependencies = [ "cc", "libc", @@ -627,9 +626,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.12" +version = "1.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755717a7de9ec452bf7f3f1a3099085deabd7f2962b861dae91ecd7a365903d2" +checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" dependencies = [ "jobserver", "libc", @@ -684,21 +683,20 @@ dependencies = [ [[package]] name = "cmake" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] [[package]] name = "comfy-table" -version = "7.1.3" +version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ - "strum", - "strum_macros", + "unicode-segmentation", "unicode-width", ] @@ -837,9 +835,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -878,7 +876,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2 0.5.0", + "bzip2 0.5.1", "chrono", "datafusion-catalog", "datafusion-common", @@ -1240,7 +1238,7 @@ dependencies = [ "itertools 0.14.0", "log", "paste", - "petgraph 0.7.1", + "petgraph", ] [[package]] @@ -1341,7 +1339,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "44.0.0" +version = "45.2.0" dependencies = [ "arrow", "async-trait", @@ -1436,9 +1434,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -1456,12 +1454,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "fixedbitset" version = "0.5.7" @@ -2269,9 +2261,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" dependencies = [ "adler2", ] @@ -2548,23 +2540,13 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset 0.4.2", - "indexmap", -] - [[package]] name = "petgraph" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ - "fixedbitset 0.5.7", + "fixedbitset", "indexmap", ] @@ -2660,9 +2642,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -2670,16 +2652,16 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" +checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "once_cell", - "petgraph 0.6.5", + "petgraph", "prettyplease", "prost", "prost-types", @@ -2690,12 +2672,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.98", @@ -2703,9 +2685,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] @@ -2721,9 +2703,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" dependencies = [ "cc", ] @@ -2860,9 +2842,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" dependencies = [ "cfg_aliases", "libc", @@ -3042,15 +3024,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24" dependencies = [ "cc", "cfg-if", "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -3097,9 +3078,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.22" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "once_cell", "ring", @@ -3377,9 +3358,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "snafu" @@ -3418,12 +3399,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "sqlparser" version = "0.53.0" @@ -3453,9 +3428,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e" dependencies = [ "cc", "cfg-if", diff --git a/Cargo.toml b/Cargo.toml index d18e0e8f0..5358b1836 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "44.0.0" +version = "45.2.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] diff --git a/dev/changelog/45.0.0.md b/dev/changelog/45.0.0.md new file mode 100644 index 000000000..93659b171 --- /dev/null +++ b/dev/changelog/45.0.0.md @@ -0,0 +1,42 @@ + + +# Apache DataFusion Python 45.0.0 Changelog + +This release consists of 2 commits from 2 contributors. See credits at the end of this changelog for more information. + +**Fixed bugs:** + +- fix: add to_timestamp_nanos [#1020](https://github.com/apache/datafusion-python/pull/1020) (chenkovsky) + +**Other:** + +- Chore/upgrade datafusion 45 [#1010](https://github.com/apache/datafusion-python/pull/1010) (kevinjqliu) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 1 Kevin Liu + 1 Tim Saucer +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 864ef1c8b..21955b6d1 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -21,7 +21,10 @@ from typing import TYPE_CHECKING, Any, Protocol -from typing_extensions import deprecated +try: + from warnings import deprecated # Python 3.13+ +except ImportError: + from typing_extensions import deprecated # Python 3.12 from datafusion.catalog import Catalog, Table from datafusion.dataframe import DataFrame diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 7413a5fa3..23b5d630b 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -33,7 +33,10 @@ overload, ) -from typing_extensions import deprecated +try: + from warnings import deprecated # Python 3.13+ +except ImportError: + from typing_extensions import deprecated # Python 3.12 from datafusion.plan import ExecutionPlan, LogicalPlan from datafusion.record_batch import RecordBatchStream diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 68ddd7c9a..e3d7158eb 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -25,7 +25,11 @@ from typing import TYPE_CHECKING, Any, Optional, Type import pyarrow as pa -from typing_extensions import deprecated + +try: + from warnings import deprecated # Python 3.13+ +except ImportError: + from typing_extensions import deprecated # Python 3.12 from datafusion.common import DataTypeMap, NullTreatment, RexType diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index 402184d3f..06302fe38 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -26,7 +26,10 @@ import pathlib from typing import TYPE_CHECKING -from typing_extensions import deprecated +try: + from warnings import deprecated # Python 3.13+ +except ImportError: + from typing_extensions import deprecated # Python 3.12 from datafusion.plan import LogicalPlan diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 5bc3fb094..c636e896a 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -755,13 +755,20 @@ def test_execution_plan(aggregate_df): assert "CsvExec:" in indent ctx = SessionContext() - stream = ctx.execute(plan, 0) - # get the one and only batch - batch = stream.next() - assert batch is not None - # there should be no more batches - with pytest.raises(StopIteration): - stream.next() + rows_returned = 0 + for idx in range(0, plan.partition_count): + stream = ctx.execute(plan, idx) + try: + batch = stream.next() + assert batch is not None + rows_returned += len(batch.to_pyarrow()[0]) + except StopIteration: + # This is one of the partitions with no values + pass + with pytest.raises(StopIteration): + stream.next() + + assert rows_returned == 5 def test_repartition(df): From a80a788f69cf46ef002b3c537837548cc103748c Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 8 Mar 2025 21:22:36 +0800 Subject: [PATCH 267/413] Enable Dataframe to be converted into views which can be used in register_table (#1016) * add test_view * feat: add into_view method to register DataFrame as a view * add pytableprovider * feat: add as_table method to PyTableProvider and update into_view to return PyTable * refactor: simplify as_table method and update documentation for into_view * test: improve test_register_filtered_dataframe by removing redundant comments and assertions * test: enhance test_register_filtered_dataframe with additional assertions for DataFrame results * ruff formatted * cleanup: remove unused imports from test_view.py * docs: add example for registering a DataFrame as a view in README.md * docs: update docstring for into_view method to clarify usage as ViewTable * chore: add license header to test_view.py * ruff correction * refactor: rename into_view method to _into_view * ruff lint * refactor: simplify into_view method and update Rust binding convention * docs: add views section to user guide with example on registering views * feat: add register_view method to SessionContext for DataFrame registration * docs: update README and user guide to reflect register_view method for DataFrame registration * docs: remove some documentation from PyDataFrame --- README.md | 40 +++++++++++++ .../user-guide/common-operations/index.rst | 1 + .../user-guide/common-operations/views.rst | 58 +++++++++++++++++++ python/datafusion/context.py | 12 ++++ python/datafusion/dataframe.py | 4 ++ python/tests/test_view.py | 49 ++++++++++++++++ src/dataframe.rs | 39 +++++++++++++ 7 files changed, 203 insertions(+) create mode 100644 docs/source/user-guide/common-operations/views.rst create mode 100644 python/tests/test_view.py diff --git a/README.md b/README.md index 9c56b62dd..4f80dbe18 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,46 @@ This produces the following chart: ![Chart](examples/chart.png) +## Registering a DataFrame as a View + +You can use SessionContext's `register_view` method to convert a DataFrame into a view and register it with the context. + +```python +from datafusion import SessionContext, col, literal + +# Create a DataFusion context +ctx = SessionContext() + +# Create sample data +data = {"a": [1, 2, 3, 4, 5], "b": [10, 20, 30, 40, 50]} + +# Create a DataFrame from the dictionary +df = ctx.from_pydict(data, "my_table") + +# Filter the DataFrame (for example, keep rows where a > 2) +df_filtered = df.filter(col("a") > literal(2)) + +# Register the dataframe as a view with the context +ctx.register_view("view1", df_filtered) + +# Now run a SQL query against the registered view +df_view = ctx.sql("SELECT * FROM view1") + +# Collect the results +results = df_view.collect() + +# Convert results to a list of dictionaries for display +result_dicts = [batch.to_pydict() for batch in results] + +print(result_dicts) +``` + +This will output: + +```python +[{'a': [3, 4, 5], 'b': [30, 40, 50]}] +``` + ## Configuration It is possible to configure runtime (memory and disk settings) and configuration settings when creating a context. diff --git a/docs/source/user-guide/common-operations/index.rst b/docs/source/user-guide/common-operations/index.rst index d7c708c21..7abd1f138 100644 --- a/docs/source/user-guide/common-operations/index.rst +++ b/docs/source/user-guide/common-operations/index.rst @@ -23,6 +23,7 @@ The contents of this section are designed to guide a new user through how to use .. toctree:: :maxdepth: 2 + views basic-info select-and-filter expressions diff --git a/docs/source/user-guide/common-operations/views.rst b/docs/source/user-guide/common-operations/views.rst new file mode 100644 index 000000000..df11e3abe --- /dev/null +++ b/docs/source/user-guide/common-operations/views.rst @@ -0,0 +1,58 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +====================== +Registering Views +====================== + +You can use the context's ``register_view`` method to register a DataFrame as a view + +.. code-block:: python + + from datafusion import SessionContext, col, literal + + # Create a DataFusion context + ctx = SessionContext() + + # Create sample data + data = {"a": [1, 2, 3, 4, 5], "b": [10, 20, 30, 40, 50]} + + # Create a DataFrame from the dictionary + df = ctx.from_pydict(data, "my_table") + + # Filter the DataFrame (for example, keep rows where a > 2) + df_filtered = df.filter(col("a") > literal(2)) + + # Register the dataframe as a view with the context + ctx.register_view("view1", df_filtered) + + # Now run a SQL query against the registered view + df_view = ctx.sql("SELECT * FROM view1") + + # Collect the results + results = df_view.collect() + + # Convert results to a list of dictionaries for display + result_dicts = [batch.to_pydict() for batch in results] + + print(result_dicts) + +This will output: + +.. code-block:: python + + [{'a': [3, 4, 5], 'b': [30, 40, 50]}] diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 21955b6d1..befc4dce6 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -707,6 +707,18 @@ def from_polars(self, data: polars.DataFrame, name: str | None = None) -> DataFr """ return DataFrame(self.ctx.from_polars(data, name)) + # https://github.com/apache/datafusion-python/pull/1016#discussion_r1983239116 + # is the discussion on how we arrived at adding register_view + def register_view(self, name: str, df: DataFrame): + """Register a :py:class: `~datafusion.detaframe.DataFrame` as a view. + + Args: + name (str): The name to register the view under. + df (DataFrame): The DataFrame to be converted into a view and registered. + """ + view = df.into_view() + self.ctx.register_table(name, view) + def register_table(self, name: str, table: Table) -> None: """Register a :py:class: `~datafusion.catalog.Table` as a table. diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 23b5d630b..85a179ec9 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -124,6 +124,10 @@ def __init__(self, df: DataFrameInternal) -> None: """ self.df = df + def into_view(self) -> pa.Table: + """Convert DataFrame as a ViewTable which can be used in register_table.""" + return self.df.into_view() + def __getitem__(self, key: str | List[str]) -> DataFrame: """Return a new :py:class`DataFrame` with the specified column or columns. diff --git a/python/tests/test_view.py b/python/tests/test_view.py new file mode 100644 index 000000000..1d92cc0d4 --- /dev/null +++ b/python/tests/test_view.py @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +from datafusion import SessionContext, col, literal + + +def test_register_filtered_dataframe(): + ctx = SessionContext() + + data = {"a": [1, 2, 3, 4, 5], "b": [10, 20, 30, 40, 50]} + + df = ctx.from_pydict(data, "my_table") + + df_filtered = df.filter(col("a") > literal(2)) + + ctx.register_view("view1", df_filtered) + + df_view = ctx.sql("SELECT * FROM view1") + + filtered_results = df_view.collect() + + result_dicts = [batch.to_pydict() for batch in filtered_results] + + expected_results = [{"a": [3, 4, 5], "b": [30, 40, 50]}] + + assert result_dicts == expected_results + + df_results = df.collect() + + df_result_dicts = [batch.to_pydict() for batch in df_results] + + expected_df_results = [{"a": [1, 2, 3, 4, 5], "b": [10, 20, 30, 40, 50]}] + + assert df_result_dicts == expected_df_results diff --git a/src/dataframe.rs b/src/dataframe.rs index ed9578a71..243e2e14f 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -30,6 +30,7 @@ use datafusion::arrow::util::pretty; use datafusion::common::UnnestOptions; use datafusion::config::{CsvOptions, TableParquetOptions}; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; +use datafusion::datasource::TableProvider; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::prelude::*; @@ -39,6 +40,7 @@ use pyo3::pybacked::PyBackedStr; use pyo3::types::{PyCapsule, PyTuple, PyTupleMethods}; use tokio::task::JoinHandle; +use crate::catalog::PyTable; use crate::errors::{py_datafusion_err, PyDataFusionError}; use crate::expr::sort_expr::to_sort_expressions; use crate::physical_plan::PyExecutionPlan; @@ -50,6 +52,25 @@ use crate::{ expr::{sort_expr::PySortExpr, PyExpr}, }; +// https://github.com/apache/datafusion-python/pull/1016#discussion_r1983239116 +// - we have not decided on the table_provider approach yet +// this is an interim implementation +#[pyclass(name = "TableProvider", module = "datafusion")] +pub struct PyTableProvider { + provider: Arc, +} + +impl PyTableProvider { + pub fn new(provider: Arc) -> Self { + Self { provider } + } + + pub fn as_table(&self) -> PyTable { + let table_provider: Arc = self.provider.clone(); + PyTable::new(table_provider) + } +} + /// A PyDataFrame is a representation of a logical plan and an API to compose statements. /// Use it to build a plan and `.collect()` to execute the plan and collect the result. /// The actual execution of a plan runs natively on Rust and Arrow on a multi-threaded environment. @@ -156,6 +177,24 @@ impl PyDataFrame { PyArrowType(self.df.schema().into()) } + /// Convert this DataFrame into a Table that can be used in register_table + /// By convention, into_... methods consume self and return the new object. + /// Disabling the clippy lint, so we can use &self + /// because we're working with Python bindings + /// where objects are shared + /// https://github.com/apache/datafusion-python/pull/1016#discussion_r1983239116 + /// - we have not decided on the table_provider approach yet + #[allow(clippy::wrong_self_convention)] + fn into_view(&self) -> PyDataFusionResult { + // Call the underlying Rust DataFrame::into_view method. + // Note that the Rust method consumes self; here we clone the inner Arc + // so that we don’t invalidate this PyDataFrame. + let table_provider = self.df.as_ref().clone().into_view(); + let table_provider = PyTableProvider::new(table_provider); + + Ok(table_provider.as_table()) + } + #[pyo3(signature = (*args))] fn select_columns(&self, args: Vec) -> PyDataFusionResult { let args = args.iter().map(|s| s.as_ref()).collect::>(); From 9027b4d79fdd7a41dd9c1f25c2ecebc1fabf50f2 Mon Sep 17 00:00:00 2001 From: Chen Chongchen Date: Sat, 8 Mar 2025 21:24:02 +0800 Subject: [PATCH 268/413] fix: type checking (#993) * fix: type checking * update license * format * format * update catalog * revert type annotation * format * format * update --- python/datafusion/catalog.py | 5 +++-- python/datafusion/context.py | 19 ++++++++++++------ python/datafusion/dataframe.py | 3 ++- python/datafusion/expr.py | 8 ++++---- python/datafusion/functions.py | 10 +++++++--- python/datafusion/input/location.py | 10 +++++----- python/datafusion/udf.py | 7 ++++--- python/tests/test_functions.py | 30 +++++++++++++++++++++++++++++ 8 files changed, 68 insertions(+), 24 deletions(-) diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py index 703037665..0560f4704 100644 --- a/python/datafusion/catalog.py +++ b/python/datafusion/catalog.py @@ -66,11 +66,12 @@ def __init__(self, table: df_internal.Table) -> None: """This constructor is not typically called by the end user.""" self.table = table + @property def schema(self) -> pyarrow.Schema: """Returns the schema associated with this table.""" - return self.table.schema() + return self.table.schema @property def kind(self) -> str: """Returns the kind of table.""" - return self.table.kind() + return self.table.kind diff --git a/python/datafusion/context.py b/python/datafusion/context.py index befc4dce6..282b2a477 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -728,7 +728,7 @@ def register_table(self, name: str, table: Table) -> None: name: Name of the resultant table. table: DataFusion table to add to the session context. """ - self.ctx.register_table(name, table) + self.ctx.register_table(name, table.table) def deregister_table(self, name: str) -> None: """Remove a table from the session.""" @@ -767,7 +767,7 @@ def register_parquet( file_extension: str = ".parquet", skip_metadata: bool = True, schema: pyarrow.Schema | None = None, - file_sort_order: list[list[Expr]] | None = None, + file_sort_order: list[list[SortExpr]] | None = None, ) -> None: """Register a Parquet file as a table. @@ -798,7 +798,9 @@ def register_parquet( file_extension, skip_metadata, schema, - file_sort_order, + [sort_list_to_raw_sort_list(exprs) for exprs in file_sort_order] + if file_sort_order is not None + else None, ) def register_csv( @@ -934,7 +936,7 @@ def register_udwf(self, udwf: WindowUDF) -> None: def catalog(self, name: str = "datafusion") -> Catalog: """Retrieve a catalog by name.""" - return self.ctx.catalog(name) + return Catalog(self.ctx.catalog(name)) @deprecated( "Use the catalog provider interface ``SessionContext.Catalog`` to " @@ -1054,7 +1056,7 @@ def read_parquet( file_extension: str = ".parquet", skip_metadata: bool = True, schema: pyarrow.Schema | None = None, - file_sort_order: list[list[Expr]] | None = None, + file_sort_order: list[list[Expr | SortExpr]] | None = None, ) -> DataFrame: """Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. @@ -1078,6 +1080,11 @@ def read_parquet( """ if table_partition_cols is None: table_partition_cols = [] + file_sort_order = ( + [sort_list_to_raw_sort_list(f) for f in file_sort_order] + if file_sort_order is not None + else None + ) return DataFrame( self.ctx.read_parquet( str(path), @@ -1121,7 +1128,7 @@ def read_table(self, table: Table) -> DataFrame: :py:class:`~datafusion.catalog.ListingTable`, create a :py:class:`~datafusion.dataframe.DataFrame`. """ - return DataFrame(self.ctx.read_table(table)) + return DataFrame(self.ctx.read_table(table.table)) def execute(self, plan: ExecutionPlan, partitions: int) -> RecordBatchStream: """Execute the ``plan`` and return the results.""" diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 85a179ec9..de5d8376e 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -52,6 +52,7 @@ from enum import Enum from datafusion._internal import DataFrame as DataFrameInternal +from datafusion._internal import expr as expr_internal from datafusion.expr import Expr, SortExpr, sort_or_default @@ -277,7 +278,7 @@ def with_columns( def _simplify_expression( *exprs: Expr | Iterable[Expr], **named_exprs: Expr - ) -> list[Expr]: + ) -> list[expr_internal.Expr]: expr_list = [] for expr in exprs: if isinstance(expr, Expr): diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index e3d7158eb..3639abec6 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -176,7 +176,7 @@ def sort_or_default(e: Expr | SortExpr) -> expr_internal.SortExpr: """Helper function to return a default Sort if an Expr is provided.""" if isinstance(e, SortExpr): return e.raw_sort - return SortExpr(e.expr, True, True).raw_sort + return SortExpr(e, True, True).raw_sort def sort_list_to_raw_sort_list( @@ -231,7 +231,7 @@ def variant_name(self) -> str: def __richcmp__(self, other: Expr, op: int) -> Expr: """Comparison operator.""" - return Expr(self.expr.__richcmp__(other, op)) + return Expr(self.expr.__richcmp__(other.expr, op)) def __repr__(self) -> str: """Generate a string representation of this expression.""" @@ -417,7 +417,7 @@ def sort(self, ascending: bool = True, nulls_first: bool = True) -> SortExpr: ascending: If true, sort in ascending order. nulls_first: Return null values first. """ - return SortExpr(self.expr, ascending=ascending, nulls_first=nulls_first) + return SortExpr(self, ascending=ascending, nulls_first=nulls_first) def is_null(self) -> Expr: """Returns ``True`` if this expression is null.""" @@ -789,7 +789,7 @@ class SortExpr: def __init__(self, expr: Expr, ascending: bool, nulls_first: bool) -> None: """This constructor should not be called by the end user.""" - self.raw_sort = expr_internal.SortExpr(expr, ascending, nulls_first) + self.raw_sort = expr_internal.SortExpr(expr.expr, ascending, nulls_first) def expr(self) -> Expr: """Return the raw expr backing the SortExpr.""" diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 5c260aade..b449c4868 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -366,7 +366,7 @@ def concat_ws(separator: str, *args: Expr) -> Expr: def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> SortExpr: """Creates a new sort expression.""" - return SortExpr(expr.expr, ascending=ascending, nulls_first=nulls_first) + return SortExpr(expr, ascending=ascending, nulls_first=nulls_first) def alias(expr: Expr, name: str) -> Expr: @@ -942,6 +942,7 @@ def to_timestamp_millis(arg: Expr, *formatters: Expr) -> Expr: See :py:func:`to_timestamp` for a description on how to use formatters. """ + formatters = [f.expr for f in formatters] return Expr(f.to_timestamp_millis(arg.expr, *formatters)) @@ -950,6 +951,7 @@ def to_timestamp_micros(arg: Expr, *formatters: Expr) -> Expr: See :py:func:`to_timestamp` for a description on how to use formatters. """ + formatters = [f.expr for f in formatters] return Expr(f.to_timestamp_micros(arg.expr, *formatters)) @@ -958,6 +960,7 @@ def to_timestamp_nanos(arg: Expr, *formatters: Expr) -> Expr: See :py:func:`to_timestamp` for a description on how to use formatters. """ + formatters = [f.expr for f in formatters] return Expr(f.to_timestamp_nanos(arg.expr, *formatters)) @@ -966,6 +969,7 @@ def to_timestamp_seconds(arg: Expr, *formatters: Expr) -> Expr: See :py:func:`to_timestamp` for a description on how to use formatters. """ + formatters = [f.expr for f in formatters] return Expr(f.to_timestamp_seconds(arg.expr, *formatters)) @@ -1078,9 +1082,9 @@ def range(start: Expr, stop: Expr, step: Expr) -> Expr: return Expr(f.range(start.expr, stop.expr, step.expr)) -def uuid(arg: Expr) -> Expr: +def uuid() -> Expr: """Returns uuid v4 as a string value.""" - return Expr(f.uuid(arg.expr)) + return Expr(f.uuid()) def struct(*args: Expr) -> Expr: diff --git a/python/datafusion/input/location.py b/python/datafusion/input/location.py index a8252b53c..517cd1578 100644 --- a/python/datafusion/input/location.py +++ b/python/datafusion/input/location.py @@ -37,12 +37,12 @@ def is_correct_input(self, input_item: Any, table_name: str, **kwargs): def build_table( self, - input_file: str, + input_item: str, table_name: str, **kwargs, ) -> SqlTable: """Create a table from the input source.""" - _, extension = os.path.splitext(input_file) + _, extension = os.path.splitext(input_item) format = extension.lstrip(".").lower() num_rows = 0 # Total number of rows in the file. Used for statistics columns = [] @@ -50,7 +50,7 @@ def build_table( import pyarrow.parquet as pq # Read the Parquet metadata - metadata = pq.read_metadata(input_file) + metadata = pq.read_metadata(input_item) num_rows = metadata.num_rows # Iterate through the schema and build the SqlTable for col in metadata.schema: @@ -69,7 +69,7 @@ def build_table( # to get that information. However, this should only be occurring # at table creation time and therefore shouldn't # slow down query performance. - with open(input_file, "r") as file: + with open(input_item, "r") as file: reader = csv.reader(file) header_row = next(reader) print(header_row) @@ -84,6 +84,6 @@ def build_table( ) # Input could possibly be multiple files. Create a list if so - input_files = glob.glob(input_file) + input_files = glob.glob(input_item) return SqlTable(table_name, columns, num_rows, input_files) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index c97f453d0..0bba3d723 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -85,7 +85,7 @@ class ScalarUDF: def __init__( self, - name: Optional[str], + name: str, func: Callable[..., _R], input_types: pyarrow.DataType | list[pyarrow.DataType], return_type: _R, @@ -182,7 +182,7 @@ class AggregateUDF: def __init__( self, - name: Optional[str], + name: str, accumulator: Callable[[], Accumulator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, @@ -277,6 +277,7 @@ def sum_bias_10() -> Summarize: ) if name is None: name = accum.__call__().__class__.__qualname__.lower() + assert name is not None if isinstance(input_types, pyarrow.DataType): input_types = [input_types] return AggregateUDF( @@ -462,7 +463,7 @@ class WindowUDF: def __init__( self, - name: Optional[str], + name: str, func: Callable[[], WindowEvaluator], input_types: list[pyarrow.DataType], return_type: pyarrow.DataType, diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index b1a739b49..fca05bb8f 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -871,7 +871,22 @@ def test_temporal_functions(df): f.to_timestamp_millis(literal("2023-09-07 05:06:14.523952")), f.to_timestamp_micros(literal("2023-09-07 05:06:14.523952")), f.extract(literal("day"), column("d")), + f.to_timestamp( + literal("2023-09-07 05:06:14.523952000"), literal("%Y-%m-%d %H:%M:%S.%f") + ), + f.to_timestamp_seconds( + literal("2023-09-07 05:06:14.523952000"), literal("%Y-%m-%d %H:%M:%S.%f") + ), + f.to_timestamp_millis( + literal("2023-09-07 05:06:14.523952000"), literal("%Y-%m-%d %H:%M:%S.%f") + ), + f.to_timestamp_micros( + literal("2023-09-07 05:06:14.523952000"), literal("%Y-%m-%d %H:%M:%S.%f") + ), f.to_timestamp_nanos(literal("2023-09-07 05:06:14.523952")), + f.to_timestamp_nanos( + literal("2023-09-07 05:06:14.523952000"), literal("%Y-%m-%d %H:%M:%S.%f") + ), ) result = df.collect() assert len(result) == 1 @@ -913,6 +928,21 @@ def test_temporal_functions(df): assert result.column(11) == pa.array( [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") ) + assert result.column(12) == pa.array( + [datetime(2023, 9, 7, 5, 6, 14)] * 3, type=pa.timestamp("s") + ) + assert result.column(13) == pa.array( + [datetime(2023, 9, 7, 5, 6, 14, 523000)] * 3, type=pa.timestamp("ms") + ) + assert result.column(14) == pa.array( + [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") + ) + assert result.column(15) == pa.array( + [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + ) + assert result.column(16) == pa.array( + [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + ) def test_arrow_cast(df): From acd70409f73f299a144e7ff4115c6e6035c3ffb5 Mon Sep 17 00:00:00 2001 From: Ion Koutsouris <15728914+ion-elgreco@users.noreply.github.com> Date: Sat, 8 Mar 2025 16:37:10 +0100 Subject: [PATCH 269/413] feat: reads using global ctx (#982) * feat: reads using global ctx * Add text to io methods to describe the context they are using --------- Co-authored-by: Tim Saucer --- python/datafusion/__init__.py | 5 + python/datafusion/io.py | 199 ++++++++++++++++++++++++++ python/tests/test_io.py | 95 ++++++++++++ python/tests/test_wrapper_coverage.py | 2 + src/context.rs | 12 +- src/utils.rs | 8 ++ 6 files changed, 319 insertions(+), 2 deletions(-) create mode 100644 python/datafusion/io.py create mode 100644 python/tests/test_io.py diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 85aefcce7..f11ce54a6 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -45,6 +45,7 @@ Expr, WindowFrame, ) +from .io import read_avro, read_csv, read_json, read_parquet from .plan import ExecutionPlan, LogicalPlan from .record_batch import RecordBatch, RecordBatchStream from .udf import Accumulator, AggregateUDF, ScalarUDF, WindowUDF @@ -81,6 +82,10 @@ "functions", "object_store", "substrait", + "read_parquet", + "read_avro", + "read_csv", + "read_json", ] diff --git a/python/datafusion/io.py b/python/datafusion/io.py new file mode 100644 index 000000000..7f3b77efa --- /dev/null +++ b/python/datafusion/io.py @@ -0,0 +1,199 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""IO read functions using global context.""" + +import pathlib + +import pyarrow + +from datafusion.dataframe import DataFrame +from datafusion.expr import Expr + +from ._internal import SessionContext as SessionContextInternal + + +def read_parquet( + path: str | pathlib.Path, + table_partition_cols: list[tuple[str, str]] | None = None, + parquet_pruning: bool = True, + file_extension: str = ".parquet", + skip_metadata: bool = True, + schema: pyarrow.Schema | None = None, + file_sort_order: list[list[Expr]] | None = None, +) -> DataFrame: + """Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + Args: + path: Path to the Parquet file. + table_partition_cols: Partition columns. + parquet_pruning: Whether the parquet reader should use the predicate + to prune row groups. + file_extension: File extension; only files with this extension are + selected for data input. + skip_metadata: Whether the parquet reader should skip any metadata + that may be in the file schema. This can help avoid schema + conflicts due to metadata. + schema: An optional schema representing the parquet files. If None, + the parquet reader will try to infer it based on data in the + file. + file_sort_order: Sort order for the file. + + Returns: + DataFrame representation of the read Parquet files + """ + if table_partition_cols is None: + table_partition_cols = [] + return DataFrame( + SessionContextInternal._global_ctx().read_parquet( + str(path), + table_partition_cols, + parquet_pruning, + file_extension, + skip_metadata, + schema, + file_sort_order, + ) + ) + + +def read_json( + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + schema_infer_max_records: int = 1000, + file_extension: str = ".json", + table_partition_cols: list[tuple[str, str]] | None = None, + file_compression_type: str | None = None, +) -> DataFrame: + """Read a line-delimited JSON data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + Args: + path: Path to the JSON file. + schema: The data source schema. + schema_infer_max_records: Maximum number of rows to read from JSON + files for schema inference if needed. + file_extension: File extension; only files with this extension are + selected for data input. + table_partition_cols: Partition columns. + file_compression_type: File compression type. + + Returns: + DataFrame representation of the read JSON files. + """ + if table_partition_cols is None: + table_partition_cols = [] + return DataFrame( + SessionContextInternal._global_ctx().read_json( + str(path), + schema, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, + ) + ) + + +def read_csv( + path: str | pathlib.Path | list[str] | list[pathlib.Path], + schema: pyarrow.Schema | None = None, + has_header: bool = True, + delimiter: str = ",", + schema_infer_max_records: int = 1000, + file_extension: str = ".csv", + table_partition_cols: list[tuple[str, str]] | None = None, + file_compression_type: str | None = None, +) -> DataFrame: + """Read a CSV data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + Args: + path: Path to the CSV file + schema: An optional schema representing the CSV files. If None, the + CSV reader will try to infer it based on data in file. + has_header: Whether the CSV file have a header. If schema inference + is run on a file with no headers, default column names are + created. + delimiter: An optional column delimiter. + schema_infer_max_records: Maximum number of rows to read from CSV + files for schema inference if needed. + file_extension: File extension; only files with this extension are + selected for data input. + table_partition_cols: Partition columns. + file_compression_type: File compression type. + + Returns: + DataFrame representation of the read CSV files + """ + if table_partition_cols is None: + table_partition_cols = [] + + path = [str(p) for p in path] if isinstance(path, list) else str(path) + + return DataFrame( + SessionContextInternal._global_ctx().read_csv( + path, + schema, + has_header, + delimiter, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, + ) + ) + + +def read_avro( + path: str | pathlib.Path, + schema: pyarrow.Schema | None = None, + file_partition_cols: list[tuple[str, str]] | None = None, + file_extension: str = ".avro", +) -> DataFrame: + """Create a :py:class:`DataFrame` for reading Avro data source. + + This function will use the global context. Any functions or tables registered + with another context may not be accessible when used with a DataFrame created + using this function. + + Args: + path: Path to the Avro file. + schema: The data source schema. + file_partition_cols: Partition columns. + file_extension: File extension to select. + + Returns: + DataFrame representation of the read Avro file + """ + if file_partition_cols is None: + file_partition_cols = [] + return DataFrame( + SessionContextInternal._global_ctx().read_avro( + str(path), schema, file_partition_cols, file_extension + ) + ) diff --git a/python/tests/test_io.py b/python/tests/test_io.py new file mode 100644 index 000000000..21ad188ee --- /dev/null +++ b/python/tests/test_io.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os +import pathlib + +import pyarrow as pa +from datafusion import column +from datafusion.io import read_avro, read_csv, read_json, read_parquet + + +def test_read_json_global_ctx(ctx): + path = os.path.dirname(os.path.abspath(__file__)) + + # Default + test_data_path = os.path.join(path, "data_test_context", "data.json") + df = read_json(test_data_path) + result = df.collect() + + assert result[0].column(0) == pa.array(["a", "b", "c"]) + assert result[0].column(1) == pa.array([1, 2, 3]) + + # Schema + schema = pa.schema( + [ + pa.field("A", pa.string(), nullable=True), + ] + ) + df = read_json(test_data_path, schema=schema) + result = df.collect() + + assert result[0].column(0) == pa.array(["a", "b", "c"]) + assert result[0].schema == schema + + # File extension + test_data_path = os.path.join(path, "data_test_context", "data.json") + df = read_json(test_data_path, file_extension=".json") + result = df.collect() + + assert result[0].column(0) == pa.array(["a", "b", "c"]) + assert result[0].column(1) == pa.array([1, 2, 3]) + + +def test_read_parquet_global(): + parquet_df = read_parquet(path="parquet/data/alltypes_plain.parquet") + parquet_df.show() + assert parquet_df is not None + + path = pathlib.Path.cwd() / "parquet/data/alltypes_plain.parquet" + parquet_df = read_parquet(path=path) + assert parquet_df is not None + + +def test_read_csv(): + csv_df = read_csv(path="testing/data/csv/aggregate_test_100.csv") + csv_df.select(column("c1")).show() + + +def test_read_csv_list(): + csv_df = read_csv(path=["testing/data/csv/aggregate_test_100.csv"]) + expected = csv_df.count() * 2 + + double_csv_df = read_csv( + path=[ + "testing/data/csv/aggregate_test_100.csv", + "testing/data/csv/aggregate_test_100.csv", + ] + ) + actual = double_csv_df.count() + + double_csv_df.select(column("c1")).show() + assert actual == expected + + +def test_read_avro(): + avro_df = read_avro(path="testing/data/avro/alltypes_plain.avro") + avro_df.show() + assert avro_df is not None + + path = pathlib.Path.cwd() / "testing/data/avro/alltypes_plain.avro" + avro_df = read_avro(path=path) + assert avro_df is not None diff --git a/python/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py index 86f2d57f2..ac064ba95 100644 --- a/python/tests/test_wrapper_coverage.py +++ b/python/tests/test_wrapper_coverage.py @@ -34,6 +34,8 @@ def missing_exports(internal_obj, wrapped_obj) -> None: return for attr in dir(internal_obj): + if attr in ["_global_ctx"]: + continue assert attr in dir(wrapped_obj) internal_attr = getattr(internal_obj, attr) diff --git a/src/context.rs b/src/context.rs index 0f962638e..9ba87eb8a 100644 --- a/src/context.rs +++ b/src/context.rs @@ -44,7 +44,7 @@ use crate::store::StorageContexts; use crate::udaf::PyAggregateUDF; use crate::udf::PyScalarUDF; use crate::udwf::PyWindowUDF; -use crate::utils::{get_tokio_runtime, validate_pycapsule, wait_for_future}; +use crate::utils::{get_global_ctx, get_tokio_runtime, validate_pycapsule, wait_for_future}; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; @@ -69,7 +69,7 @@ use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; use datafusion_ffi::table_provider::{FFI_TableProvider, ForeignTableProvider}; -use pyo3::types::{PyCapsule, PyDict, PyList, PyTuple}; +use pyo3::types::{PyCapsule, PyDict, PyList, PyTuple, PyType}; use tokio::task::JoinHandle; /// Configuration options for a SessionContext @@ -306,6 +306,14 @@ impl PySessionContext { }) } + #[classmethod] + #[pyo3(signature = ())] + fn _global_ctx(_cls: &Bound<'_, PyType>) -> PyResult { + Ok(Self { + ctx: get_global_ctx().clone(), + }) + } + /// Register an object store with the given name #[pyo3(signature = (scheme, store, host=None))] pub fn register_object_store( diff --git a/src/utils.rs b/src/utils.rs index ed224b364..999aad755 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -17,6 +17,7 @@ use crate::errors::{PyDataFusionError, PyDataFusionResult}; use crate::TokioRuntime; +use datafusion::execution::context::SessionContext; use datafusion::logical_expr::Volatility; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; @@ -37,6 +38,13 @@ pub(crate) fn get_tokio_runtime() -> &'static TokioRuntime { RUNTIME.get_or_init(|| TokioRuntime(tokio::runtime::Runtime::new().unwrap())) } +/// Utility to get the Global Datafussion CTX +#[inline] +pub(crate) fn get_global_ctx() -> &'static SessionContext { + static CTX: OnceLock = OnceLock::new(); + CTX.get_or_init(|| SessionContext::new()) +} + /// Utility to collect rust futures with GIL released pub fn wait_for_future(py: Python, f: F) -> F::Output where From 973d7ec4a8196a78bc4fb32db4f24e523997ba4c Mon Sep 17 00:00:00 2001 From: Crystal Zhou <45134936+CrystalZhou0529@users.noreply.github.com> Date: Sat, 8 Mar 2025 16:23:54 -0500 Subject: [PATCH 270/413] feat: Implementation of udf and udaf decorator (#1040) * Implementation of udf and udaf decorator * Rename decorators back to udf and udaf, update documentations * Minor typo fixes * Fixing linting errors * ruff formatting --------- Co-authored-by: Tim Saucer --- python/datafusion/udf.py | 257 +++++++++++++++++++++++++++----------- python/tests/test_udaf.py | 42 +++++++ python/tests/test_udf.py | 42 ++++++- 3 files changed, 265 insertions(+), 76 deletions(-) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index 0bba3d723..af7bcf2ed 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -19,6 +19,7 @@ from __future__ import annotations +import functools from abc import ABCMeta, abstractmethod from enum import Enum from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar @@ -110,43 +111,102 @@ def __call__(self, *args: Expr) -> Expr: args_raw = [arg.expr for arg in args] return Expr(self._udf.__call__(*args_raw)) - @staticmethod - def udf( - func: Callable[..., _R], - input_types: list[pyarrow.DataType], - return_type: _R, - volatility: Volatility | str, - name: Optional[str] = None, - ) -> ScalarUDF: - """Create a new User-Defined Function. + class udf: + """Create a new User-Defined Function (UDF). + + This class can be used both as a **function** and as a **decorator**. + + Usage: + - **As a function**: Call `udf(func, input_types, return_type, volatility, + name)`. + - **As a decorator**: Use `@udf(input_types, return_type, volatility, + name)`. In this case, do **not** pass `func` explicitly. Args: - func: A callable python function. - input_types: The data types of the arguments to ``func``. This list - must be of the same length as the number of arguments. - return_type: The data type of the return value from the python - function. - volatility: See ``Volatility`` for allowed values. - name: A descriptive name for the function. + func (Callable, optional): **Only needed when calling as a function.** + Skip this argument when using `udf` as a decorator. + input_types (list[pyarrow.DataType]): The data types of the arguments + to `func`. This list must be of the same length as the number of + arguments. + return_type (_R): The data type of the return value from the function. + volatility (Volatility | str): See `Volatility` for allowed values. + name (Optional[str]): A descriptive name for the function. Returns: - A user-defined aggregate function, which can be used in either data - aggregation or window function calls. + A user-defined function that can be used in SQL expressions, + data aggregation, or window function calls. + + Example: + **Using `udf` as a function:** + ``` + def double_func(x): + return x * 2 + double_udf = udf(double_func, [pyarrow.int32()], pyarrow.int32(), + "volatile", "double_it") + ``` + + **Using `udf` as a decorator:** + ``` + @udf([pyarrow.int32()], pyarrow.int32(), "volatile", "double_it") + def double_udf(x): + return x * 2 + ``` """ - if not callable(func): - raise TypeError("`func` argument must be callable") - if name is None: - if hasattr(func, "__qualname__"): - name = func.__qualname__.lower() + + def __new__(cls, *args, **kwargs): + """Create a new UDF. + + Trigger UDF function or decorator depending on if the first args is callable + """ + if args and callable(args[0]): + # Case 1: Used as a function, require the first parameter to be callable + return cls._function(*args, **kwargs) else: - name = func.__class__.__name__.lower() - return ScalarUDF( - name=name, - func=func, - input_types=input_types, - return_type=return_type, - volatility=volatility, - ) + # Case 2: Used as a decorator with parameters + return cls._decorator(*args, **kwargs) + + @staticmethod + def _function( + func: Callable[..., _R], + input_types: list[pyarrow.DataType], + return_type: _R, + volatility: Volatility | str, + name: Optional[str] = None, + ) -> ScalarUDF: + if not callable(func): + raise TypeError("`func` argument must be callable") + if name is None: + if hasattr(func, "__qualname__"): + name = func.__qualname__.lower() + else: + name = func.__class__.__name__.lower() + return ScalarUDF( + name=name, + func=func, + input_types=input_types, + return_type=return_type, + volatility=volatility, + ) + + @staticmethod + def _decorator( + input_types: list[pyarrow.DataType], + return_type: _R, + volatility: Volatility | str, + name: Optional[str] = None, + ): + def decorator(func): + udf_caller = ScalarUDF.udf( + func, input_types, return_type, volatility, name + ) + + @functools.wraps(func) + def wrapper(*args, **kwargs): + return udf_caller(*args, **kwargs) + + return wrapper + + return decorator class Accumulator(metaclass=ABCMeta): @@ -212,25 +272,27 @@ def __call__(self, *args: Expr) -> Expr: args_raw = [arg.expr for arg in args] return Expr(self._udaf.__call__(*args_raw)) - @staticmethod - def udaf( - accum: Callable[[], Accumulator], - input_types: pyarrow.DataType | list[pyarrow.DataType], - return_type: pyarrow.DataType, - state_type: list[pyarrow.DataType], - volatility: Volatility | str, - name: Optional[str] = None, - ) -> AggregateUDF: - """Create a new User-Defined Aggregate Function. + class udaf: + """Create a new User-Defined Aggregate Function (UDAF). - If your :py:class:`Accumulator` can be instantiated with no arguments, you - can simply pass it's type as ``accum``. If you need to pass additional arguments - to it's constructor, you can define a lambda or a factory method. During runtime - the :py:class:`Accumulator` will be constructed for every instance in - which this UDAF is used. The following examples are all valid. + This class allows you to define an **aggregate function** that can be used in + data aggregation or window function calls. - .. code-block:: python + Usage: + - **As a function**: Call `udaf(accum, input_types, return_type, state_type, + volatility, name)`. + - **As a decorator**: Use `@udaf(input_types, return_type, state_type, + volatility, name)`. + When using `udaf` as a decorator, **do not pass `accum` explicitly**. + **Function example:** + + If your `:py:class:Accumulator` can be instantiated with no arguments, you + can simply pass it's type as `accum`. If you need to pass additional + arguments to it's constructor, you can define a lambda or a factory method. + During runtime the `:py:class:Accumulator` will be constructed for every + instance in which this UDAF is used. The following examples are all valid. + ``` import pyarrow as pa import pyarrow.compute as pc @@ -253,12 +315,24 @@ def evaluate(self) -> pa.Scalar: def sum_bias_10() -> Summarize: return Summarize(10.0) - udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], "immutable") - udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], "immutable") - udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), [pa.float64()], "immutable") + udaf1 = udaf(Summarize, pa.float64(), pa.float64(), [pa.float64()], + "immutable") + udaf2 = udaf(sum_bias_10, pa.float64(), pa.float64(), [pa.float64()], + "immutable") + udaf3 = udaf(lambda: Summarize(20.0), pa.float64(), pa.float64(), + [pa.float64()], "immutable") + ``` + + **Decorator example:** + ``` + @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable") + def udf4() -> Summarize: + return Summarize(10.0) + ``` Args: - accum: The accumulator python function. + accum: The accumulator python function. **Only needed when calling as a + function. Skip this argument when using `udaf` as a decorator.** input_types: The data types of the arguments to ``accum``. return_type: The data type of the return value. state_type: The data types of the intermediate accumulation. @@ -268,26 +342,69 @@ def sum_bias_10() -> Summarize: Returns: A user-defined aggregate function, which can be used in either data aggregation or window function calls. - """ # noqa W505 - if not callable(accum): - raise TypeError("`func` must be callable.") - if not isinstance(accum.__call__(), Accumulator): - raise TypeError( - "Accumulator must implement the abstract base class Accumulator" + """ + + def __new__(cls, *args, **kwargs): + """Create a new UDAF. + + Trigger UDAF function or decorator depending on if the first args is + callable + """ + if args and callable(args[0]): + # Case 1: Used as a function, require the first parameter to be callable + return cls._function(*args, **kwargs) + else: + # Case 2: Used as a decorator with parameters + return cls._decorator(*args, **kwargs) + + @staticmethod + def _function( + accum: Callable[[], Accumulator], + input_types: pyarrow.DataType | list[pyarrow.DataType], + return_type: pyarrow.DataType, + state_type: list[pyarrow.DataType], + volatility: Volatility | str, + name: Optional[str] = None, + ) -> AggregateUDF: + if not callable(accum): + raise TypeError("`func` must be callable.") + if not isinstance(accum.__call__(), Accumulator): + raise TypeError( + "Accumulator must implement the abstract base class Accumulator" + ) + if name is None: + name = accum.__call__().__class__.__qualname__.lower() + if isinstance(input_types, pyarrow.DataType): + input_types = [input_types] + return AggregateUDF( + name=name, + accumulator=accum, + input_types=input_types, + return_type=return_type, + state_type=state_type, + volatility=volatility, ) - if name is None: - name = accum.__call__().__class__.__qualname__.lower() - assert name is not None - if isinstance(input_types, pyarrow.DataType): - input_types = [input_types] - return AggregateUDF( - name=name, - accumulator=accum, - input_types=input_types, - return_type=return_type, - state_type=state_type, - volatility=volatility, - ) + + @staticmethod + def _decorator( + input_types: pyarrow.DataType | list[pyarrow.DataType], + return_type: pyarrow.DataType, + state_type: list[pyarrow.DataType], + volatility: Volatility | str, + name: Optional[str] = None, + ): + def decorator(accum: Callable[[], Accumulator]): + udaf_caller = AggregateUDF.udaf( + accum, input_types, return_type, state_type, volatility, name + ) + + @functools.wraps(accum) + def wrapper(*args, **kwargs): + return udaf_caller(*args, **kwargs) + + return wrapper + + return decorator class WindowEvaluator(metaclass=ABCMeta): diff --git a/python/tests/test_udaf.py b/python/tests/test_udaf.py index 0005a3da8..e69c77d3c 100644 --- a/python/tests/test_udaf.py +++ b/python/tests/test_udaf.py @@ -117,6 +117,26 @@ def test_udaf_aggregate(df): assert result.column(0) == pa.array([1.0 + 2.0 + 3.0]) +def test_udaf_decorator_aggregate(df): + @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable") + def summarize(): + return Summarize() + + df1 = df.aggregate([], [summarize(column("a"))]) + + # execute and collect the first (and only) batch + result = df1.collect()[0] + + assert result.column(0) == pa.array([1.0 + 2.0 + 3.0]) + + df2 = df.aggregate([], [summarize(column("a"))]) + + # Run a second time to ensure the state is properly reset + result = df2.collect()[0] + + assert result.column(0) == pa.array([1.0 + 2.0 + 3.0]) + + def test_udaf_aggregate_with_arguments(df): bias = 10.0 @@ -143,6 +163,28 @@ def test_udaf_aggregate_with_arguments(df): assert result.column(0) == pa.array([bias + 1.0 + 2.0 + 3.0]) +def test_udaf_decorator_aggregate_with_arguments(df): + bias = 10.0 + + @udaf(pa.float64(), pa.float64(), [pa.float64()], "immutable") + def summarize(): + return Summarize(bias) + + df1 = df.aggregate([], [summarize(column("a"))]) + + # execute and collect the first (and only) batch + result = df1.collect()[0] + + assert result.column(0) == pa.array([bias + 1.0 + 2.0 + 3.0]) + + df2 = df.aggregate([], [summarize(column("a"))]) + + # Run a second time to ensure the state is properly reset + result = df2.collect()[0] + + assert result.column(0) == pa.array([bias + 1.0 + 2.0 + 3.0]) + + def test_group_by(df): summarize = udaf( Summarize, diff --git a/python/tests/test_udf.py b/python/tests/test_udf.py index 3a5dce6d6..a6c047552 100644 --- a/python/tests/test_udf.py +++ b/python/tests/test_udf.py @@ -24,7 +24,7 @@ def df(ctx): # create a RecordBatch and a new DataFrame from it batch = pa.RecordBatch.from_arrays( - [pa.array([1, 2, 3]), pa.array([4, 4, 6])], + [pa.array([1, 2, 3]), pa.array([4, 4, None])], names=["a", "b"], ) return ctx.create_dataframe([[batch]], name="test_table") @@ -39,10 +39,20 @@ def test_udf(df): volatility="immutable", ) - df = df.select(is_null(column("a"))) + df = df.select(is_null(column("b"))) result = df.collect()[0].column(0) - assert result == pa.array([False, False, False]) + assert result == pa.array([False, False, True]) + + +def test_udf_decorator(df): + @udf([pa.int64()], pa.bool_(), "immutable") + def is_null(x: pa.Array) -> pa.Array: + return x.is_null() + + df = df.select(is_null(column("b"))) + result = df.collect()[0].column(0) + assert result == pa.array([False, False, True]) def test_register_udf(ctx, df) -> None: @@ -56,10 +66,10 @@ def test_register_udf(ctx, df) -> None: ctx.register_udf(is_null) - df_result = ctx.sql("select is_null(a) from test_table") + df_result = ctx.sql("select is_null(b) from test_table") result = df_result.collect()[0].column(0) - assert result == pa.array([False, False, False]) + assert result == pa.array([False, False, True]) class OverThresholdUDF: @@ -70,7 +80,7 @@ def __call__(self, values: pa.Array) -> pa.Array: return pa.array(v.as_py() >= self.threshold for v in values) -def test_udf_with_parameters(df) -> None: +def test_udf_with_parameters_function(df) -> None: udf_no_param = udf( OverThresholdUDF(), pa.int64(), @@ -94,3 +104,23 @@ def test_udf_with_parameters(df) -> None: result = df2.collect()[0].column(0) assert result == pa.array([False, True, True]) + + +def test_udf_with_parameters_decorator(df) -> None: + @udf([pa.int64()], pa.bool_(), "immutable") + def udf_no_param(values: pa.Array) -> pa.Array: + return OverThresholdUDF()(values) + + df1 = df.select(udf_no_param(column("a"))) + result = df1.collect()[0].column(0) + + assert result == pa.array([True, True, True]) + + @udf([pa.int64()], pa.bool_(), "immutable") + def udf_with_param(values: pa.Array) -> pa.Array: + return OverThresholdUDF(2)(values) + + df2 = df.select(udf_with_param(column("a"))) + result = df2.collect()[0].column(0) + + assert result == pa.array([False, True, True]) From d72f5605b3d523585d04857505793920f96242ba Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 10 Mar 2025 06:56:12 -0400 Subject: [PATCH 271/413] Enable FA ruff lint (#1052) --- examples/python-udwf.py | 2 ++ pyproject.toml | 2 +- python/datafusion/io.py | 2 ++ python/tests/test_udaf.py | 2 ++ python/tests/test_udwf.py | 2 ++ 5 files changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/python-udwf.py b/examples/python-udwf.py index 32f8fadaa..7d39dc1b8 100644 --- a/examples/python-udwf.py +++ b/examples/python-udwf.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + import datafusion import pyarrow as pa from datafusion import col, lit, udwf diff --git a/pyproject.toml b/pyproject.toml index f416e02a5..d16a18aa6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ features = ["substrait"] # Enable docstring linting using the google style guide [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "D", "W", "I"] +select = ["E4", "E7", "E9", "F", "FA", "D", "W", "I"] [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/python/datafusion/io.py b/python/datafusion/io.py index 7f3b77efa..3b6264948 100644 --- a/python/datafusion/io.py +++ b/python/datafusion/io.py @@ -17,6 +17,8 @@ """IO read functions using global context.""" +from __future__ import annotations + import pathlib import pyarrow diff --git a/python/tests/test_udaf.py b/python/tests/test_udaf.py index e69c77d3c..97cf81f3c 100644 --- a/python/tests/test_udaf.py +++ b/python/tests/test_udaf.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + from typing import List import pyarrow as pa diff --git a/python/tests/test_udwf.py b/python/tests/test_udwf.py index 0ffa04179..2fea34aa3 100644 --- a/python/tests/test_udwf.py +++ b/python/tests/test_udwf.py @@ -15,6 +15,8 @@ # specific language governing permissions and limitations # under the License. +from __future__ import annotations + import pyarrow as pa import pytest from datafusion import SessionContext, column, lit, udwf From 0002372ccdb780e011631c797ec9613174cf0a94 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Mon, 10 Mar 2025 14:22:42 -0400 Subject: [PATCH 272/413] Enable take comments to assign issues to users (#1058) --- .github/workflows/take.yml | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 .github/workflows/take.yml diff --git a/.github/workflows/take.yml b/.github/workflows/take.yml new file mode 100644 index 000000000..86dc190ad --- /dev/null +++ b/.github/workflows/take.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Assign the issue via a `take` comment +on: + issue_comment: + types: created + +permissions: + issues: write + +jobs: + issue_assign: + runs-on: ubuntu-latest + if: (!github.event.issue.pull_request) && github.event.comment.body == 'take' + concurrency: + group: ${{ github.actor }}-issue-assign + steps: + - run: | + CODE=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -LI https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees/${{ github.event.comment.user.login }} -o /dev/null -w '%{http_code}\n' -s) + if [ "$CODE" -eq "204" ] + then + echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees + else + echo "Cannot assign issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}" + fi \ No newline at end of file From 9d634de6df2f8b76bd303ab1f5972f01deb2210d Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Mon, 10 Mar 2025 14:24:40 -0400 Subject: [PATCH 273/413] Update python min version to 3.9 (#1043) * 3.8 -> 3.9 * upgrade pyo3 abi3-py38 -> abi3-py39 --- Cargo.toml | 2 +- .../source/contributor-guide/introduction.rst | 2 +- examples/ffi-table-provider/Cargo.lock | 75 +- examples/ffi-table-provider/Cargo.toml | 2 +- examples/ffi-table-provider/pyproject.toml | 2 +- pyproject.toml | 3 +- uv.lock | 707 ++---------------- 7 files changed, 121 insertions(+), 672 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5358b1836..50967a219 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,7 @@ substrait = ["dep:datafusion-substrait"] [dependencies] tokio = { version = "1.42", features = ["macros", "rt", "rt-multi-thread", "sync"] } -pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py38"] } +pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"]} arrow = { version = "54", features = ["pyarrow"] } datafusion = { version = "45.0.0", features = ["avro", "unicode_expressions"] } diff --git a/docs/source/contributor-guide/introduction.rst b/docs/source/contributor-guide/introduction.rst index 25f2c21a4..2fba64111 100644 --- a/docs/source/contributor-guide/introduction.rst +++ b/docs/source/contributor-guide/introduction.rst @@ -118,7 +118,7 @@ be ignored by ``git``. .. code-block:: implementation=CPython - version=3.8 + version=3.9 shared=true abi3=true lib_name=python3.12 diff --git a/examples/ffi-table-provider/Cargo.lock b/examples/ffi-table-provider/Cargo.lock index 32af85180..8d0edd515 100644 --- a/examples/ffi-table-provider/Cargo.lock +++ b/examples/ffi-table-provider/Cargo.lock @@ -766,7 +766,8 @@ dependencies = [ [[package]] name = "datafusion" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eae420e7a5b0b7f1c39364cc76cbcd0f5fdc416b2514ae3847c2676bbd60702a" dependencies = [ "arrow", "arrow-array", @@ -816,7 +817,8 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f27987bc22b810939e8dfecc55571e9d50355d6ea8ec1c47af8383a76a6d0e1" dependencies = [ "arrow", "async-trait", @@ -836,7 +838,8 @@ dependencies = [ [[package]] name = "datafusion-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602" dependencies = [ "ahash", "arrow", @@ -862,7 +865,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4603c8e8a4baf77660ab7074cc66fc15cc8a18f2ce9dfadb755fc6ee294e48" dependencies = [ "log", "tokio", @@ -871,12 +875,14 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5bf4bc68623a5cf231eed601ed6eb41f46a37c4d15d11a0bff24cbc8396cd66" [[package]] name = "datafusion-execution" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b491c012cdf8e051053426013429a76f74ee3c2db68496c79c323ca1084d27" dependencies = [ "arrow", "dashmap", @@ -894,7 +900,8 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a181408d4fc5dc22f9252781a8f39f2d0e5d1b33ec9bde242844980a2689c1" dependencies = [ "arrow", "chrono", @@ -914,7 +921,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1129b48e8534d8c03c6543bcdccef0b55c8ac0c1272a15a56c67068b6eb1885" dependencies = [ "arrow", "datafusion-common", @@ -925,7 +933,8 @@ dependencies = [ [[package]] name = "datafusion-ffi" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff47a79d442207c168c6e3e1d970c248589c148e4800e5b285ac1b2cb1a230f8" dependencies = [ "abi_stable", "arrow", @@ -945,7 +954,8 @@ dependencies = [ [[package]] name = "datafusion-functions" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6125874e4856dfb09b59886784fcb74cde5cfc5930b3a80a1a728ef7a010df6b" dependencies = [ "arrow", "arrow-buffer", @@ -974,7 +984,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6" dependencies = [ "ahash", "arrow", @@ -996,7 +1007,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c" dependencies = [ "ahash", "arrow", @@ -1008,7 +1020,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec5ee8cecb0dc370291279673097ddabec03a011f73f30d7f1096457127e03e" dependencies = [ "arrow", "arrow-array", @@ -1031,7 +1044,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c403ddd473bbb0952ba880008428b3c7febf0ed3ce1eec35a205db20efb2a36" dependencies = [ "arrow", "async-trait", @@ -1046,7 +1060,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab18c2fb835614d06a75f24a9e09136d3a8c12a92d97c95a6af316a1787a9c5" dependencies = [ "datafusion-common", "datafusion-doc", @@ -1062,7 +1077,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77b73bc15e7d1967121fdc7a55d819bfb9d6c03766a6c322247dce9094a53a4" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1071,7 +1087,8 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09369b8d962291e808977cf94d495fd8b5b38647232d7ef562c27ac0f495b0af" dependencies = [ "datafusion-expr", "quote", @@ -1081,7 +1098,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2403a7e4a84637f3de7d8d4d7a9ccc0cc4be92d89b0161ba3ee5be82f0531c54" dependencies = [ "arrow", "chrono", @@ -1099,7 +1117,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c" dependencies = [ "ahash", "arrow", @@ -1123,7 +1142,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435" dependencies = [ "ahash", "arrow", @@ -1137,7 +1157,8 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac5e85c189d5238a5cf181a624e450c4cd4c66ac77ca551d6f3ff9080bac90bb" dependencies = [ "arrow", "arrow-schema", @@ -1158,7 +1179,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f" dependencies = [ "ahash", "arrow", @@ -1189,7 +1211,8 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2db5d79f0c974041787b899d24dc91bdab2ff112d1942dd71356a4ce3b407e6c" dependencies = [ "arrow", "chrono", @@ -1204,7 +1227,8 @@ dependencies = [ [[package]] name = "datafusion-proto-common" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de21bde1603aac0ff32cf478e47081be6e3583c6861fe8f57034da911efe7578" dependencies = [ "arrow", "datafusion-common", @@ -1214,7 +1238,8 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "45.0.0" -source = "git+https://github.com/apache/datafusion.git?rev=1a29bd3#1a29bd3b62f1759c557aca9eed937ac38f5a5602" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13caa4daede211ecec53c78b13c503b592794d125f9a3cc3afe992edf9e7f43" dependencies = [ "arrow", "arrow-array", diff --git a/examples/ffi-table-provider/Cargo.toml b/examples/ffi-table-provider/Cargo.toml index 0e558fdd0..f4e4fda79 100644 --- a/examples/ffi-table-provider/Cargo.toml +++ b/examples/ffi-table-provider/Cargo.toml @@ -23,7 +23,7 @@ edition = "2021" [dependencies] datafusion = { version = "45.0.0" } datafusion-ffi = { version = "45.0.0" } -pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py38"] } +pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } arrow = { version = "54" } arrow-array = { version = "54" } arrow-schema = { version = "54" } diff --git a/examples/ffi-table-provider/pyproject.toml b/examples/ffi-table-provider/pyproject.toml index 116efae9c..9cd25b423 100644 --- a/examples/ffi-table-provider/pyproject.toml +++ b/examples/ffi-table-provider/pyproject.toml @@ -21,7 +21,7 @@ build-backend = "maturin" [project] name = "ffi_table_provider" -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", diff --git a/pyproject.toml b/pyproject.toml index d16a18aa6..1c2733677 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ name = "datafusion" description = "Build and run queries against data" readme = "README.md" license = { file = "LICENSE.txt" } -requires-python = ">=3.8" +requires-python = ">=3.9" keywords = ["datafusion", "dataframe", "rust", "query-engine"] classifiers = [ "Development Status :: 2 - Pre-Alpha", @@ -35,7 +35,6 @@ classifiers = [ "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/uv.lock b/uv.lock index 587ddc8b7..619b92856 100644 --- a/uv.lock +++ b/uv.lock @@ -1,23 +1,10 @@ version = 1 -requires-python = ">=3.8" +requires-python = ">=3.9" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", - "python_full_version < '3.9'", -] - -[[package]] -name = "alabaster" -version = "0.7.13" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/94/71/a8ee96d1fd95ca04a0d2e2d9c4081dac4c2d2b12f7ddb899c8cb9bfd1532/alabaster-0.7.13.tar.gz", hash = "sha256:a27a4a084d5e690e16e01e03ad2b2e552c61a65469419b907243193de1a84ae2", size = 11454 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/88/c7083fc61120ab661c5d0b82cb77079fc1429d3f913a456c1c82cf4658f7/alabaster-0.7.13-py3-none-any.whl", hash = "sha256:1ee19aca801bbabb5ba3f5f258e4422dfa86f82f3e9cefb0859b283cdd7f62a3", size = 13857 }, + "python_full_version < '3.10'", ] [[package]] @@ -25,7 +12,7 @@ name = "alabaster" version = "0.7.16" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", + "python_full_version < '3.10'", ] sdist = { url = "https://files.pythonhosted.org/packages/c9/3e/13dd8e5ed9094e734ac430b5d0eb4f2bb001708a8b7856cbf8e084e001ba/alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65", size = 23776 } wheels = [ @@ -46,42 +33,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929 }, ] -[[package]] -name = "appnope" -version = "0.1.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/35/5d/752690df9ef5b76e169e68d6a129fa6d08a7100ca7f754c89495db3c6019/appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee", size = 4170 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/29/5ecc3a15d5a33e31b26c11426c45c501e439cb865d0bff96315d86443b78/appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c", size = 4321 }, -] - -[[package]] -name = "astroid" -version = "3.2.4" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.9'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/53/1067e1113ecaf58312357f2cd93063674924119d80d173adc3f6f2387aa2/astroid-3.2.4.tar.gz", hash = "sha256:0e14202810b30da1b735827f78f5157be2bbd4a7a59b7707ca0bfc2fb4c0063a", size = 397576 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/80/96/b32bbbb46170a1c8b8b1f28c794202e25cfe743565e9d3469b8eb1e0cc05/astroid-3.2.4-py3-none-any.whl", hash = "sha256:413658a61eeca6202a59231abb473f932038fbcbf1666587f66d482083413a25", size = 276348 }, -] - [[package]] name = "astroid" version = "3.3.8" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] dependencies = [ - { name = "typing-extensions", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/80/c5/5c83c48bbf547f3dd8b587529db7cf5a265a3368b33e85e76af8ff6061d3/astroid-3.3.8.tar.gz", hash = "sha256:a88c7994f914a4ea8572fac479459f4955eeccc877be3f2d959a33273b0cf40b", size = 398196 } wheels = [ @@ -101,23 +58,11 @@ wheels = [ name = "babel" version = "2.16.0" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pytz", marker = "python_full_version < '3.9'" }, -] sdist = { url = "https://files.pythonhosted.org/packages/2a/74/f1bc80f23eeba13393b7222b11d95ca3af2c1e28edca18af487137eefed9/babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316", size = 9348104 } wheels = [ { url = "https://files.pythonhosted.org/packages/ed/20/bc79bc575ba2e2a7f70e8a1155618bb1301eaa5132a8271373a6903f73f8/babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b", size = 9587599 }, ] -[[package]] -name = "backcall" -version = "0.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a2/40/764a663805d84deee23043e1426a9175567db89c8b3287b5c2ad9f71aa93/backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e", size = 18041 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255", size = 11157 }, -] - [[package]] name = "beautifulsoup4" version = "4.12.3" @@ -194,14 +139,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469 }, { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475 }, { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009 }, - { url = "https://files.pythonhosted.org/packages/48/08/15bf6b43ae9bd06f6b00ad8a91f5a8fe1069d4c9fab550a866755402724e/cffi-1.17.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:636062ea65bd0195bc012fea9321aca499c0504409f413dc88af450b57ffd03b", size = 182457 }, - { url = "https://files.pythonhosted.org/packages/c2/5b/f1523dd545f92f7df468e5f653ffa4df30ac222f3c884e51e139878f1cb5/cffi-1.17.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7eac2ef9b63c79431bc4b25f1cd649d7f061a28808cbc6c47b534bd789ef964", size = 425932 }, - { url = "https://files.pythonhosted.org/packages/53/93/7e547ab4105969cc8c93b38a667b82a835dd2cc78f3a7dad6130cfd41e1d/cffi-1.17.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e221cf152cff04059d011ee126477f0d9588303eb57e88923578ace7baad17f9", size = 448585 }, - { url = "https://files.pythonhosted.org/packages/56/c4/a308f2c332006206bb511de219efeff090e9d63529ba0a77aae72e82248b/cffi-1.17.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:31000ec67d4221a71bd3f67df918b1f88f676f1c3b535a7eb473255fdc0b83fc", size = 456268 }, - { url = "https://files.pythonhosted.org/packages/ca/5b/b63681518265f2f4060d2b60755c1c77ec89e5e045fc3773b72735ddaad5/cffi-1.17.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f17be4345073b0a7b8ea599688f692ac3ef23ce28e5df79c04de519dbc4912c", size = 436592 }, - { url = "https://files.pythonhosted.org/packages/bb/19/b51af9f4a4faa4a8ac5a0e5d5c2522dcd9703d07fac69da34a36c4d960d3/cffi-1.17.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2b1fac190ae3ebfe37b979cc1ce69c81f4e4fe5746bb401dca63a9062cdaf1", size = 446512 }, - { url = "https://files.pythonhosted.org/packages/e2/63/2bed8323890cb613bbecda807688a31ed11a7fe7afe31f8faaae0206a9a3/cffi-1.17.1-cp38-cp38-win32.whl", hash = "sha256:7596d6620d3fa590f677e9ee430df2958d2d6d6de2feeae5b20e82c00b76fbf8", size = 171576 }, - { url = "https://files.pythonhosted.org/packages/2f/70/80c33b044ebc79527447fd4fbc5455d514c3bb840dede4455de97da39b4d/cffi-1.17.1-cp38-cp38-win_amd64.whl", hash = "sha256:78122be759c3f8a014ce010908ae03364d00a1f81ab5c7f4a7a5120607ea56e1", size = 181229 }, { url = "https://files.pythonhosted.org/packages/b9/ea/8bb50596b8ffbc49ddd7a1ad305035daa770202a6b782fc164647c2673ad/cffi-1.17.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b2ab587605f4ba0bf81dc0cb08a41bd1c0a5906bd59243d56bad7668a6fc6c16", size = 182220 }, { url = "https://files.pythonhosted.org/packages/ae/11/e77c8cd24f58285a82c23af484cf5b124a376b32644e445960d1a4654c3a/cffi-1.17.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:28b16024becceed8c6dfbc75629e27788d8a3f9030691a1dbf9821a128b22c36", size = 178605 }, { url = "https://files.pythonhosted.org/packages/ed/65/25a8dc32c53bf5b7b6c2686b42ae2ad58743f7ff644844af7cdb29b49361/cffi-1.17.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1d599671f396c4723d016dbddb72fe8e0397082b0a77a4fab8028923bec050e8", size = 424910 }, @@ -274,19 +211,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e4/93/946a86ce20790e11312c87c75ba68d5f6ad2208cfb52b2d6a2c32840d922/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd", size = 145732 }, { url = "https://files.pythonhosted.org/packages/cd/e5/131d2fb1b0dddafc37be4f3a2fa79aa4c037368be9423061dccadfd90091/charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407", size = 95391 }, { url = "https://files.pythonhosted.org/packages/27/f2/4f9a69cc7712b9b5ad8fdb87039fd89abba997ad5cbe690d1835d40405b0/charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971", size = 102702 }, - { url = "https://files.pythonhosted.org/packages/10/bd/6517ea94f2672e801011d50b5d06be2a0deaf566aea27bcdcd47e5195357/charset_normalizer-3.4.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:ecddf25bee22fe4fe3737a399d0d177d72bc22be6913acfab364b40bce1ba83c", size = 195653 }, - { url = "https://files.pythonhosted.org/packages/e5/0d/815a2ba3f283b4eeaa5ece57acade365c5b4135f65a807a083c818716582/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c60ca7339acd497a55b0ea5d506b2a2612afb2826560416f6894e8b5770d4a9", size = 140701 }, - { url = "https://files.pythonhosted.org/packages/aa/17/c94be7ee0d142687e047fe1de72060f6d6837f40eedc26e87e6e124a3fc6/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b7b2d86dd06bfc2ade3312a83a5c364c7ec2e3498f8734282c6c3d4b07b346b8", size = 150495 }, - { url = "https://files.pythonhosted.org/packages/f7/33/557ac796c47165fc141e4fb71d7b0310f67e05cb420756f3a82e0a0068e0/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd78cfcda14a1ef52584dbb008f7ac81c1328c0f58184bf9a84c49c605002da6", size = 142946 }, - { url = "https://files.pythonhosted.org/packages/1e/0d/38ef4ae41e9248d63fc4998d933cae22473b1b2ac4122cf908d0f5eb32aa/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e27f48bcd0957c6d4cb9d6fa6b61d192d0b13d5ef563e5f2ae35feafc0d179c", size = 144737 }, - { url = "https://files.pythonhosted.org/packages/43/01/754cdb29dd0560f58290aaaa284d43eea343ad0512e6ad3b8b5c11f08592/charset_normalizer-3.4.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:01ad647cdd609225c5350561d084b42ddf732f4eeefe6e678765636791e78b9a", size = 147471 }, - { url = "https://files.pythonhosted.org/packages/ba/cd/861883ba5160c7a9bd242c30b2c71074cda2aefcc0addc91118e0d4e0765/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:619a609aa74ae43d90ed2e89bdd784765de0a25ca761b93e196d938b8fd1dbbd", size = 140801 }, - { url = "https://files.pythonhosted.org/packages/6f/7f/0c0dad447819e90b93f8ed238cc8f11b91353c23c19e70fa80483a155bed/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:89149166622f4db9b4b6a449256291dc87a99ee53151c74cbd82a53c8c2f6ccd", size = 149312 }, - { url = "https://files.pythonhosted.org/packages/8e/09/9f8abcc6fff60fb727268b63c376c8c79cc37b833c2dfe1f535dfb59523b/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:7709f51f5f7c853f0fb938bcd3bc59cdfdc5203635ffd18bf354f6967ea0f824", size = 152347 }, - { url = "https://files.pythonhosted.org/packages/be/e5/3f363dad2e24378f88ccf63ecc39e817c29f32e308ef21a7a6d9c1201165/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:345b0426edd4e18138d6528aed636de7a9ed169b4aaf9d61a8c19e39d26838ca", size = 149888 }, - { url = "https://files.pythonhosted.org/packages/e4/10/a78c0e91f487b4ad0ef7480ac765e15b774f83de2597f1b6ef0eaf7a2f99/charset_normalizer-3.4.1-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0907f11d019260cdc3f94fbdb23ff9125f6b5d1039b76003b5b0ac9d6a6c9d5b", size = 145169 }, - { url = "https://files.pythonhosted.org/packages/d3/81/396e7d7f5d7420da8273c91175d2e9a3f569288e3611d521685e4b9ac9cc/charset_normalizer-3.4.1-cp38-cp38-win32.whl", hash = "sha256:ea0d8d539afa5eb2728aa1932a988a9a7af94f18582ffae4bc10b3fbdad0626e", size = 95094 }, - { url = "https://files.pythonhosted.org/packages/40/bb/20affbbd9ea29c71ea123769dc568a6d42052ff5089c5fe23e21e21084a6/charset_normalizer-3.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:329ce159e82018d646c7ac45b01a430369d526569ec08516081727a20e9e4af4", size = 102139 }, { url = "https://files.pythonhosted.org/packages/7f/c0/b913f8f02836ed9ab32ea643c6fe4d3325c3d8627cf6e78098671cafff86/charset_normalizer-3.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b97e690a2118911e39b4042088092771b4ae3fc3aa86518f84b8cf6888dbdb41", size = 197867 }, { url = "https://files.pythonhosted.org/packages/0f/6c/2bee440303d705b6fb1e2ec789543edec83d32d258299b16eed28aad48e0/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:78baa6d91634dfb69ec52a463534bc0df05dbd546209b79a3880a34487f4b84f", size = 141385 }, { url = "https://files.pythonhosted.org/packages/3d/04/cb42585f07f6f9fd3219ffb6f37d5a39b4fd2db2355b23683060029c35f7/charset_normalizer-3.4.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1a2bc9f351a75ef49d664206d51f8e5ede9da246602dc2d2726837620ea034b2", size = 151367 }, @@ -351,11 +275,9 @@ wheels = [ [[package]] name = "datafusion" -version = "44.0.0" source = { editable = "." } dependencies = [ - { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "pyarrow", version = "18.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "pyarrow" }, { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] @@ -369,20 +291,16 @@ dev = [ { name = "toml" }, ] docs = [ - { name = "ipython", version = "8.12.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "ipython", version = "8.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "ipython", version = "8.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "ipython", version = "8.31.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "jinja2" }, { name = "myst-parser", version = "3.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "myst-parser", version = "4.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pandas", version = "2.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "pandas", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "pandas" }, { name = "pickleshare" }, { name = "pydata-sphinx-theme" }, - { name = "setuptools", version = "75.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "setuptools", version = "75.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "setuptools" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "sphinx-autoapi" }, ] @@ -435,28 +353,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6e/c6/ac0b6c1e2d138f1002bcf799d330bd6d85084fece321e662a14223794041/Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec", size = 9998 }, ] -[[package]] -name = "docutils" -version = "0.20.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/1f/53/a5da4f2c5739cf66290fac1431ee52aff6851c7c8ffd8264f13affd7bcdd/docutils-0.20.1.tar.gz", hash = "sha256:f08a4e276c3a1583a86dce3e34aba3fe04d02bba2dd51ed16106244e8a923e3b", size = 2058365 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/26/87/f238c0670b94533ac0353a4e2a1a771a0cc73277b88bff23d3ae35a256c1/docutils-0.20.1-py3-none-any.whl", hash = "sha256:96f387a2c5562db4476f09f13bbab2192e764cac08ebbf3a34a95d9b1e4a59d6", size = 572666 }, -] - [[package]] name = "docutils" version = "0.21.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/ae/ed/aefcc8cd0ba62a0560c3c18c33925362d46c6075480bfa4df87b28e169a9/docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f", size = 2204444 } wheels = [ { url = "https://files.pythonhosted.org/packages/8f/d7/9322c609343d929e75e7e5e6255e614fcc67572cfd083959cdef3b7aad79/docutils-0.21.2-py3-none-any.whl", hash = "sha256:dafca5b9e384f0e419294eb4d2ff9fa826435bf15f15b7bd45723e8ad76811b2", size = 587408 }, @@ -503,8 +403,7 @@ name = "importlib-metadata" version = "8.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp", version = "3.20.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "zipp", version = "3.21.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "zipp", marker = "python_full_version < '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304 } wheels = [ @@ -520,52 +419,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892 }, ] -[[package]] -name = "ipython" -version = "8.12.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -dependencies = [ - { name = "appnope", marker = "python_full_version < '3.9' and sys_platform == 'darwin'" }, - { name = "backcall", marker = "python_full_version < '3.9'" }, - { name = "colorama", marker = "python_full_version < '3.9' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version < '3.9'" }, - { name = "jedi", marker = "python_full_version < '3.9'" }, - { name = "matplotlib-inline", marker = "python_full_version < '3.9'" }, - { name = "pexpect", marker = "python_full_version < '3.9' and sys_platform != 'win32'" }, - { name = "pickleshare", marker = "python_full_version < '3.9'" }, - { name = "prompt-toolkit", marker = "python_full_version < '3.9'" }, - { name = "pygments", marker = "python_full_version < '3.9'" }, - { name = "stack-data", marker = "python_full_version < '3.9'" }, - { name = "traitlets", marker = "python_full_version < '3.9'" }, - { name = "typing-extensions", marker = "python_full_version < '3.9'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9e/6a/44ef299b1762f5a73841e87fae8a73a8cc8aee538d6dc8c77a5afe1fd2ce/ipython-8.12.3.tar.gz", hash = "sha256:3910c4b54543c2ad73d06579aa771041b7d5707b033bd488669b4cf544e3b363", size = 5470171 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/8d/97/8fe103906cd81bc42d3b0175b5534a9f67dccae47d6451131cf8d0d70bb2/ipython-8.12.3-py3-none-any.whl", hash = "sha256:b0340d46a933d27c657b211a329d0be23793c36595acf9e6ef4164bc01a1804c", size = 798307 }, -] - [[package]] name = "ipython" version = "8.18.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", + "python_full_version < '3.10'", ] dependencies = [ - { name = "colorama", marker = "python_full_version == '3.9.*' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version == '3.9.*'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.9.*'" }, - { name = "jedi", marker = "python_full_version == '3.9.*'" }, - { name = "matplotlib-inline", marker = "python_full_version == '3.9.*'" }, - { name = "pexpect", marker = "python_full_version == '3.9.*' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version == '3.9.*'" }, - { name = "pygments", marker = "python_full_version == '3.9.*'" }, - { name = "stack-data", marker = "python_full_version == '3.9.*'" }, - { name = "traitlets", marker = "python_full_version == '3.9.*'" }, - { name = "typing-extensions", marker = "python_full_version == '3.9.*'" }, + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "decorator", marker = "python_full_version < '3.10'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, + { name = "jedi", marker = "python_full_version < '3.10'" }, + { name = "matplotlib-inline", marker = "python_full_version < '3.10'" }, + { name = "pexpect", marker = "python_full_version < '3.10' and sys_platform != 'win32'" }, + { name = "prompt-toolkit", marker = "python_full_version < '3.10'" }, + { name = "pygments", marker = "python_full_version < '3.10'" }, + { name = "stack-data", marker = "python_full_version < '3.10'" }, + { name = "traitlets", marker = "python_full_version < '3.10'" }, + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/b9/3ba6c45a6df813c09a48bac313c22ff83efa26cbb55011218d925a46e2ad/ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27", size = 5486330 } wheels = [ @@ -616,8 +488,7 @@ name = "jinja2" version = "3.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe", version = "2.1.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "markupsafe", version = "3.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "markupsafe" }, ] sdist = { url = "https://files.pythonhosted.org/packages/af/92/b3130cbbf5591acf9ade8708c365f3238046ac7cb8ccba6e81abccb0ccff/jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb", size = 244674 } wheels = [ @@ -636,77 +507,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528 }, ] -[[package]] -name = "markupsafe" -version = "2.1.5" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/87/5b/aae44c6655f3801e81aa3eef09dbbf012431987ba564d7231722f68df02d/MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", size = 19384 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e4/54/ad5eb37bf9d51800010a74e4665425831a9db4e7c4e0fde4352e391e808e/MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", size = 18206 }, - { url = "https://files.pythonhosted.org/packages/6a/4a/a4d49415e600bacae038c67f9fecc1d5433b9d3c71a4de6f33537b89654c/MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", size = 14079 }, - { url = "https://files.pythonhosted.org/packages/0a/7b/85681ae3c33c385b10ac0f8dd025c30af83c78cec1c37a6aa3b55e67f5ec/MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", size = 26620 }, - { url = "https://files.pythonhosted.org/packages/7c/52/2b1b570f6b8b803cef5ac28fdf78c0da318916c7d2fe9402a84d591b394c/MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", size = 25818 }, - { url = "https://files.pythonhosted.org/packages/29/fe/a36ba8c7ca55621620b2d7c585313efd10729e63ef81e4e61f52330da781/MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", size = 25493 }, - { url = "https://files.pythonhosted.org/packages/60/ae/9c60231cdfda003434e8bd27282b1f4e197ad5a710c14bee8bea8a9ca4f0/MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", size = 30630 }, - { url = "https://files.pythonhosted.org/packages/65/dc/1510be4d179869f5dafe071aecb3f1f41b45d37c02329dfba01ff59e5ac5/MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", size = 29745 }, - { url = "https://files.pythonhosted.org/packages/30/39/8d845dd7d0b0613d86e0ef89549bfb5f61ed781f59af45fc96496e897f3a/MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", size = 30021 }, - { url = "https://files.pythonhosted.org/packages/c7/5c/356a6f62e4f3c5fbf2602b4771376af22a3b16efa74eb8716fb4e328e01e/MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", size = 16659 }, - { url = "https://files.pythonhosted.org/packages/69/48/acbf292615c65f0604a0c6fc402ce6d8c991276e16c80c46a8f758fbd30c/MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", size = 17213 }, - { url = "https://files.pythonhosted.org/packages/11/e7/291e55127bb2ae67c64d66cef01432b5933859dfb7d6949daa721b89d0b3/MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", size = 18219 }, - { url = "https://files.pythonhosted.org/packages/6b/cb/aed7a284c00dfa7c0682d14df85ad4955a350a21d2e3b06d8240497359bf/MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", size = 14098 }, - { url = "https://files.pythonhosted.org/packages/1c/cf/35fe557e53709e93feb65575c93927942087e9b97213eabc3fe9d5b25a55/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", size = 29014 }, - { url = "https://files.pythonhosted.org/packages/97/18/c30da5e7a0e7f4603abfc6780574131221d9148f323752c2755d48abad30/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", size = 28220 }, - { url = "https://files.pythonhosted.org/packages/0c/40/2e73e7d532d030b1e41180807a80d564eda53babaf04d65e15c1cf897e40/MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", size = 27756 }, - { url = "https://files.pythonhosted.org/packages/18/46/5dca760547e8c59c5311b332f70605d24c99d1303dd9a6e1fc3ed0d73561/MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", size = 33988 }, - { url = "https://files.pythonhosted.org/packages/6d/c5/27febe918ac36397919cd4a67d5579cbbfa8da027fa1238af6285bb368ea/MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", size = 32718 }, - { url = "https://files.pythonhosted.org/packages/f8/81/56e567126a2c2bc2684d6391332e357589a96a76cb9f8e5052d85cb0ead8/MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", size = 33317 }, - { url = "https://files.pythonhosted.org/packages/00/0b/23f4b2470accb53285c613a3ab9ec19dc944eaf53592cb6d9e2af8aa24cc/MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", size = 16670 }, - { url = "https://files.pythonhosted.org/packages/b7/a2/c78a06a9ec6d04b3445a949615c4c7ed86a0b2eb68e44e7541b9d57067cc/MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", size = 17224 }, - { url = "https://files.pythonhosted.org/packages/53/bd/583bf3e4c8d6a321938c13f49d44024dbe5ed63e0a7ba127e454a66da974/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", size = 18215 }, - { url = "https://files.pythonhosted.org/packages/48/d6/e7cd795fc710292c3af3a06d80868ce4b02bfbbf370b7cee11d282815a2a/MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", size = 14069 }, - { url = "https://files.pythonhosted.org/packages/51/b5/5d8ec796e2a08fc814a2c7d2584b55f889a55cf17dd1a90f2beb70744e5c/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", size = 29452 }, - { url = "https://files.pythonhosted.org/packages/0a/0d/2454f072fae3b5a137c119abf15465d1771319dfe9e4acbb31722a0fff91/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", size = 28462 }, - { url = "https://files.pythonhosted.org/packages/2d/75/fd6cb2e68780f72d47e6671840ca517bda5ef663d30ada7616b0462ad1e3/MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", size = 27869 }, - { url = "https://files.pythonhosted.org/packages/b0/81/147c477391c2750e8fc7705829f7351cf1cd3be64406edcf900dc633feb2/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", size = 33906 }, - { url = "https://files.pythonhosted.org/packages/8b/ff/9a52b71839d7a256b563e85d11050e307121000dcebc97df120176b3ad93/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", size = 32296 }, - { url = "https://files.pythonhosted.org/packages/88/07/2dc76aa51b481eb96a4c3198894f38b480490e834479611a4053fbf08623/MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", size = 33038 }, - { url = "https://files.pythonhosted.org/packages/96/0c/620c1fb3661858c0e37eb3cbffd8c6f732a67cd97296f725789679801b31/MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", size = 16572 }, - { url = "https://files.pythonhosted.org/packages/3f/14/c3554d512d5f9100a95e737502f4a2323a1959f6d0d01e0d0997b35f7b10/MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", size = 17127 }, - { url = "https://files.pythonhosted.org/packages/f8/ff/2c942a82c35a49df5de3a630ce0a8456ac2969691b230e530ac12314364c/MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", size = 18192 }, - { url = "https://files.pythonhosted.org/packages/4f/14/6f294b9c4f969d0c801a4615e221c1e084722ea6114ab2114189c5b8cbe0/MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", size = 14072 }, - { url = "https://files.pythonhosted.org/packages/81/d4/fd74714ed30a1dedd0b82427c02fa4deec64f173831ec716da11c51a50aa/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", size = 26928 }, - { url = "https://files.pythonhosted.org/packages/c7/bd/50319665ce81bb10e90d1cf76f9e1aa269ea6f7fa30ab4521f14d122a3df/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", size = 26106 }, - { url = "https://files.pythonhosted.org/packages/4c/6f/f2b0f675635b05f6afd5ea03c094557bdb8622fa8e673387444fe8d8e787/MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68", size = 25781 }, - { url = "https://files.pythonhosted.org/packages/51/e0/393467cf899b34a9d3678e78961c2c8cdf49fb902a959ba54ece01273fb1/MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", size = 30518 }, - { url = "https://files.pythonhosted.org/packages/f6/02/5437e2ad33047290dafced9df741d9efc3e716b75583bbd73a9984f1b6f7/MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", size = 29669 }, - { url = "https://files.pythonhosted.org/packages/0e/7d/968284145ffd9d726183ed6237c77938c021abacde4e073020f920e060b2/MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", size = 29933 }, - { url = "https://files.pythonhosted.org/packages/bf/f3/ecb00fc8ab02b7beae8699f34db9357ae49d9f21d4d3de6f305f34fa949e/MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", size = 16656 }, - { url = "https://files.pythonhosted.org/packages/92/21/357205f03514a49b293e214ac39de01fadd0970a6e05e4bf1ddd0ffd0881/MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", size = 17206 }, - { url = "https://files.pythonhosted.org/packages/0f/31/780bb297db036ba7b7bbede5e1d7f1e14d704ad4beb3ce53fb495d22bc62/MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", size = 18193 }, - { url = "https://files.pythonhosted.org/packages/6c/77/d77701bbef72892affe060cdacb7a2ed7fd68dae3b477a8642f15ad3b132/MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", size = 14073 }, - { url = "https://files.pythonhosted.org/packages/d9/a7/1e558b4f78454c8a3a0199292d96159eb4d091f983bc35ef258314fe7269/MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", size = 26486 }, - { url = "https://files.pythonhosted.org/packages/5f/5a/360da85076688755ea0cceb92472923086993e86b5613bbae9fbc14136b0/MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", size = 25685 }, - { url = "https://files.pythonhosted.org/packages/6a/18/ae5a258e3401f9b8312f92b028c54d7026a97ec3ab20bfaddbdfa7d8cce8/MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", size = 25338 }, - { url = "https://files.pythonhosted.org/packages/0b/cc/48206bd61c5b9d0129f4d75243b156929b04c94c09041321456fd06a876d/MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", size = 30439 }, - { url = "https://files.pythonhosted.org/packages/d1/06/a41c112ab9ffdeeb5f77bc3e331fdadf97fa65e52e44ba31880f4e7f983c/MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", size = 29531 }, - { url = "https://files.pythonhosted.org/packages/02/8c/ab9a463301a50dab04d5472e998acbd4080597abc048166ded5c7aa768c8/MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", size = 29823 }, - { url = "https://files.pythonhosted.org/packages/bc/29/9bc18da763496b055d8e98ce476c8e718dcfd78157e17f555ce6dd7d0895/MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", size = 16658 }, - { url = "https://files.pythonhosted.org/packages/f6/f8/4da07de16f10551ca1f640c92b5f316f9394088b183c6a57183df6de5ae4/MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", size = 17211 }, -] - [[package]] name = "markupsafe" version = "3.0.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } wheels = [ { url = "https://files.pythonhosted.org/packages/04/90/d08277ce111dd22f77149fd1a5d4653eeb3b3eaacbdfcbae5afb2600eebd/MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8", size = 14357 }, @@ -832,18 +636,15 @@ name = "myst-parser" version = "3.0.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", - "python_full_version < '3.9'", + "python_full_version < '3.10'", ] dependencies = [ - { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "docutils", marker = "python_full_version < '3.10'" }, { name = "jinja2", marker = "python_full_version < '3.10'" }, { name = "markdown-it-py", marker = "python_full_version < '3.10'" }, { name = "mdit-py-plugins", marker = "python_full_version < '3.10'" }, { name = "pyyaml", marker = "python_full_version < '3.10'" }, - { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/64/e2f13dac02f599980798c01156393b781aec983b52a6e4057ee58f07c43a/myst_parser-3.0.1.tar.gz", hash = "sha256:88f0cb406cb363b077d176b51c476f62d60604d68a8dcdf4832e080441301a87", size = 92392 } wheels = [ @@ -860,7 +661,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "docutils", marker = "python_full_version >= '3.10'" }, { name = "jinja2", marker = "python_full_version >= '3.10'" }, { name = "markdown-it-py", marker = "python_full_version >= '3.10'" }, { name = "mdit-py-plugins", marker = "python_full_version >= '3.10'" }, @@ -872,50 +673,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ca/b4/b036f8fdb667587bb37df29dc6644681dd78b7a2a6321a34684b79412b28/myst_parser-4.0.0-py3-none-any.whl", hash = "sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d", size = 84563 }, ] -[[package]] -name = "numpy" -version = "1.24.4" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/a4/9b/027bec52c633f6556dba6b722d9a0befb40498b9ceddd29cbe67a45a127c/numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463", size = 10911229 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6b/80/6cdfb3e275d95155a34659163b83c09e3a3ff9f1456880bec6cc63d71083/numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64", size = 19789140 }, - { url = "https://files.pythonhosted.org/packages/64/5f/3f01d753e2175cfade1013eea08db99ba1ee4bdb147ebcf3623b75d12aa7/numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1", size = 13854297 }, - { url = "https://files.pythonhosted.org/packages/5a/b3/2f9c21d799fa07053ffa151faccdceeb69beec5a010576b8991f614021f7/numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4", size = 13995611 }, - { url = "https://files.pythonhosted.org/packages/10/be/ae5bf4737cb79ba437879915791f6f26d92583c738d7d960ad94e5c36adf/numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6", size = 17282357 }, - { url = "https://files.pythonhosted.org/packages/c0/64/908c1087be6285f40e4b3e79454552a701664a079321cff519d8c7051d06/numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc", size = 12429222 }, - { url = "https://files.pythonhosted.org/packages/22/55/3d5a7c1142e0d9329ad27cece17933b0e2ab4e54ddc5c1861fbfeb3f7693/numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e", size = 14841514 }, - { url = "https://files.pythonhosted.org/packages/a9/cc/5ed2280a27e5dab12994c884f1f4d8c3bd4d885d02ae9e52a9d213a6a5e2/numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810", size = 19775508 }, - { url = "https://files.pythonhosted.org/packages/c0/bc/77635c657a3668cf652806210b8662e1aff84b818a55ba88257abf6637a8/numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254", size = 13840033 }, - { url = "https://files.pythonhosted.org/packages/a7/4c/96cdaa34f54c05e97c1c50f39f98d608f96f0677a6589e64e53104e22904/numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7", size = 13991951 }, - { url = "https://files.pythonhosted.org/packages/22/97/dfb1a31bb46686f09e68ea6ac5c63fdee0d22d7b23b8f3f7ea07712869ef/numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5", size = 17278923 }, - { url = "https://files.pythonhosted.org/packages/35/e2/76a11e54139654a324d107da1d98f99e7aa2a7ef97cfd7c631fba7dbde71/numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d", size = 12422446 }, - { url = "https://files.pythonhosted.org/packages/d8/ec/ebef2f7d7c28503f958f0f8b992e7ce606fb74f9e891199329d5f5f87404/numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694", size = 14834466 }, - { url = "https://files.pythonhosted.org/packages/11/10/943cfb579f1a02909ff96464c69893b1d25be3731b5d3652c2e0cf1281ea/numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61", size = 19780722 }, - { url = "https://files.pythonhosted.org/packages/a7/ae/f53b7b265fdc701e663fbb322a8e9d4b14d9cb7b2385f45ddfabfc4327e4/numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f", size = 13843102 }, - { url = "https://files.pythonhosted.org/packages/25/6f/2586a50ad72e8dbb1d8381f837008a0321a3516dfd7cb57fc8cf7e4bb06b/numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e", size = 14039616 }, - { url = "https://files.pythonhosted.org/packages/98/5d/5738903efe0ecb73e51eb44feafba32bdba2081263d40c5043568ff60faf/numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc", size = 17316263 }, - { url = "https://files.pythonhosted.org/packages/d1/57/8d328f0b91c733aa9aa7ee540dbc49b58796c862b4fbcb1146c701e888da/numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2", size = 12455660 }, - { url = "https://files.pythonhosted.org/packages/69/65/0d47953afa0ad569d12de5f65d964321c208492064c38fe3b0b9744f8d44/numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706", size = 14868112 }, - { url = "https://files.pythonhosted.org/packages/9a/cd/d5b0402b801c8a8b56b04c1e85c6165efab298d2f0ab741c2406516ede3a/numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400", size = 19816549 }, - { url = "https://files.pythonhosted.org/packages/14/27/638aaa446f39113a3ed38b37a66243e21b38110d021bfcb940c383e120f2/numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f", size = 13879950 }, - { url = "https://files.pythonhosted.org/packages/8f/27/91894916e50627476cff1a4e4363ab6179d01077d71b9afed41d9e1f18bf/numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9", size = 14030228 }, - { url = "https://files.pythonhosted.org/packages/7a/7c/d7b2a0417af6428440c0ad7cb9799073e507b1a465f827d058b826236964/numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d", size = 17311170 }, - { url = "https://files.pythonhosted.org/packages/18/9d/e02ace5d7dfccee796c37b995c63322674daf88ae2f4a4724c5dd0afcc91/numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835", size = 12454918 }, - { url = "https://files.pythonhosted.org/packages/63/38/6cc19d6b8bfa1d1a459daf2b3fe325453153ca7019976274b6f33d8b5663/numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8", size = 14867441 }, - { url = "https://files.pythonhosted.org/packages/a4/fd/8dff40e25e937c94257455c237b9b6bf5a30d42dd1cc11555533be099492/numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef", size = 19156590 }, - { url = "https://files.pythonhosted.org/packages/42/e7/4bf953c6e05df90c6d351af69966384fed8e988d0e8c54dad7103b59f3ba/numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a", size = 16705744 }, - { url = "https://files.pythonhosted.org/packages/fc/dd/9106005eb477d022b60b3817ed5937a43dad8fd1f20b0610ea8a32fcb407/numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2", size = 14734290 }, -] - [[package]] name = "numpy" version = "2.0.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", + "python_full_version < '3.10'", ] sdist = { url = "https://files.pythonhosted.org/packages/a9/75/10dd1f8116a8b796cb2c737b674e02d02e80454bda953fa7e65d8c12b016/numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78", size = 18902015 } wheels = [ @@ -1041,63 +804,16 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/ef/eb23f262cca3c0c4eb7ab1933c3b1f03d021f2c48f54763065b6f0e321be/packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759", size = 65451 }, ] -[[package]] -name = "pandas" -version = "2.0.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -dependencies = [ - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "python-dateutil", marker = "python_full_version < '3.9'" }, - { name = "pytz", marker = "python_full_version < '3.9'" }, - { name = "tzdata", marker = "python_full_version < '3.9'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/a7/824332581e258b5aa4f3763ecb2a797e5f9a54269044ba2e50ac19936b32/pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c", size = 5284455 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3c/b2/0d4a5729ce1ce11630c4fc5d5522a33b967b3ca146c210f58efde7c40e99/pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8", size = 11760908 }, - { url = "https://files.pythonhosted.org/packages/4a/f6/f620ca62365d83e663a255a41b08d2fc2eaf304e0b8b21bb6d62a7390fe3/pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f", size = 10823486 }, - { url = "https://files.pythonhosted.org/packages/c2/59/cb4234bc9b968c57e81861b306b10cd8170272c57b098b724d3de5eda124/pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183", size = 11571897 }, - { url = "https://files.pythonhosted.org/packages/e3/59/35a2892bf09ded9c1bf3804461efe772836a5261ef5dfb4e264ce813ff99/pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0", size = 12306421 }, - { url = "https://files.pythonhosted.org/packages/94/71/3a0c25433c54bb29b48e3155b959ac78f4c4f2f06f94d8318aac612cb80f/pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210", size = 9540792 }, - { url = "https://files.pythonhosted.org/packages/ed/30/b97456e7063edac0e5a405128065f0cd2033adfe3716fb2256c186bd41d0/pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e", size = 10664333 }, - { url = "https://files.pythonhosted.org/packages/b3/92/a5e5133421b49e901a12e02a6a7ef3a0130e10d13db8cb657fdd0cba3b90/pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8", size = 11645672 }, - { url = "https://files.pythonhosted.org/packages/8f/bb/aea1fbeed5b474cb8634364718abe9030d7cc7a30bf51f40bd494bbc89a2/pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26", size = 10693229 }, - { url = "https://files.pythonhosted.org/packages/d6/90/e7d387f1a416b14e59290baa7a454a90d719baebbf77433ff1bdcc727800/pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d", size = 11581591 }, - { url = "https://files.pythonhosted.org/packages/d0/28/88b81881c056376254618fad622a5e94b5126db8c61157ea1910cd1c040a/pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df", size = 12219370 }, - { url = "https://files.pythonhosted.org/packages/e4/a5/212b9039e25bf8ebb97e417a96660e3dc925dacd3f8653d531b8f7fd9be4/pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd", size = 9482935 }, - { url = "https://files.pythonhosted.org/packages/9e/71/756a1be6bee0209d8c0d8c5e3b9fc72c00373f384a4017095ec404aec3ad/pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b", size = 10607692 }, - { url = "https://files.pythonhosted.org/packages/78/a8/07dd10f90ca915ed914853cd57f79bfc22e1ef4384ab56cb4336d2fc1f2a/pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061", size = 11653303 }, - { url = "https://files.pythonhosted.org/packages/53/c3/f8e87361f7fdf42012def602bfa2a593423c729f5cb7c97aed7f51be66ac/pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5", size = 10710932 }, - { url = "https://files.pythonhosted.org/packages/a7/87/828d50c81ce0f434163bf70b925a0eec6076808e0bca312a79322b141f66/pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089", size = 11684018 }, - { url = "https://files.pythonhosted.org/packages/f8/7f/5b047effafbdd34e52c9e2d7e44f729a0655efafb22198c45cf692cdc157/pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0", size = 12353723 }, - { url = "https://files.pythonhosted.org/packages/ea/ae/26a2eda7fa581347d69e51f93892493b2074ef3352ac71033c9f32c52389/pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02", size = 9646403 }, - { url = "https://files.pythonhosted.org/packages/c3/6c/ea362eef61f05553aaf1a24b3e96b2d0603f5dc71a3bd35688a24ed88843/pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78", size = 10777638 }, - { url = "https://files.pythonhosted.org/packages/f8/c7/cfef920b7b457dff6928e824896cb82367650ea127d048ee0b820026db4f/pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b", size = 11834160 }, - { url = "https://files.pythonhosted.org/packages/6c/1c/689c9d99bc4e5d366a5fd871f0bcdee98a6581e240f96b78d2d08f103774/pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e", size = 10862752 }, - { url = "https://files.pythonhosted.org/packages/cc/b8/4d082f41c27c95bf90485d1447b647cc7e5680fea75e315669dc6e4cb398/pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b", size = 11715852 }, - { url = "https://files.pythonhosted.org/packages/9e/0d/91a9fd2c202f2b1d97a38ab591890f86480ecbb596cbc56d035f6f23fdcc/pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641", size = 12398496 }, - { url = "https://files.pythonhosted.org/packages/26/7d/d8aa0a2c4f3f5f8ea59fb946c8eafe8f508090ca73e2b08a9af853c1103e/pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682", size = 9630766 }, - { url = "https://files.pythonhosted.org/packages/9a/f2/0ad053856debbe90c83de1b4f05915f85fd2146f20faf9daa3b320d36df3/pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc", size = 10755902 }, -] - [[package]] name = "pandas" version = "2.2.3" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "numpy", version = "2.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.9'" }, - { name = "pytz", marker = "python_full_version >= '3.9'" }, - { name = "tzdata", marker = "python_full_version >= '3.9'" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/d6/9f8431bacc2e19dca897724cd097b1bb224a6ad5433784a44b587c7c13af/pandas-2.2.3.tar.gz", hash = "sha256:4f18ba62b61d7e192368b84517265a99b4d7ee8912f8708660fb4a366cc82667", size = 4399213 } wheels = [ @@ -1213,65 +929,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/37/efad0257dc6e593a18957422533ff0f87ede7c9c6ea010a2177d738fb82f/pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0", size = 11842 }, ] -[[package]] -name = "pyarrow" -version = "17.0.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -dependencies = [ - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/27/4e/ea6d43f324169f8aec0e57569443a38bab4b398d09769ca64f7b4d467de3/pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28", size = 1112479 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/5d/78d4b040bc5ff2fc6c3d03e80fca396b742f6c125b8af06bcf7427f931bc/pyarrow-17.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a5c8b238d47e48812ee577ee20c9a2779e6a5904f1708ae240f53ecbee7c9f07", size = 28994846 }, - { url = "https://files.pythonhosted.org/packages/3b/73/8ed168db7642e91180330e4ea9f3ff8bab404678f00d32d7df0871a4933b/pyarrow-17.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:db023dc4c6cae1015de9e198d41250688383c3f9af8f565370ab2b4cb5f62655", size = 27165908 }, - { url = "https://files.pythonhosted.org/packages/81/36/e78c24be99242063f6d0590ef68c857ea07bdea470242c361e9a15bd57a4/pyarrow-17.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da1e060b3876faa11cee287839f9cc7cdc00649f475714b8680a05fd9071d545", size = 39264209 }, - { url = "https://files.pythonhosted.org/packages/18/4c/3db637d7578f683b0a8fb8999b436bdbedd6e3517bd4f90c70853cf3ad20/pyarrow-17.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c06d4624c0ad6674364bb46ef38c3132768139ddec1c56582dbac54f2663e2", size = 39862883 }, - { url = "https://files.pythonhosted.org/packages/81/3c/0580626896c842614a523e66b351181ed5bb14e5dfc263cd68cea2c46d90/pyarrow-17.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:fa3c246cc58cb5a4a5cb407a18f193354ea47dd0648194e6265bd24177982fe8", size = 38723009 }, - { url = "https://files.pythonhosted.org/packages/ee/fb/c1b47f0ada36d856a352da261a44d7344d8f22e2f7db3945f8c3b81be5dd/pyarrow-17.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:f7ae2de664e0b158d1607699a16a488de3d008ba99b3a7aa5de1cbc13574d047", size = 39855626 }, - { url = "https://files.pythonhosted.org/packages/19/09/b0a02908180a25d57312ab5919069c39fddf30602568980419f4b02393f6/pyarrow-17.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5984f416552eea15fd9cee03da53542bf4cddaef5afecefb9aa8d1010c335087", size = 25147242 }, - { url = "https://files.pythonhosted.org/packages/f9/46/ce89f87c2936f5bb9d879473b9663ce7a4b1f4359acc2f0eb39865eaa1af/pyarrow-17.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:1c8856e2ef09eb87ecf937104aacfa0708f22dfeb039c363ec99735190ffb977", size = 29028748 }, - { url = "https://files.pythonhosted.org/packages/8d/8e/ce2e9b2146de422f6638333c01903140e9ada244a2a477918a368306c64c/pyarrow-17.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e19f569567efcbbd42084e87f948778eb371d308e137a0f97afe19bb860ccb3", size = 27190965 }, - { url = "https://files.pythonhosted.org/packages/3b/c8/5675719570eb1acd809481c6d64e2136ffb340bc387f4ca62dce79516cea/pyarrow-17.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b244dc8e08a23b3e352899a006a26ae7b4d0da7bb636872fa8f5884e70acf15", size = 39269081 }, - { url = "https://files.pythonhosted.org/packages/5e/78/3931194f16ab681ebb87ad252e7b8d2c8b23dad49706cadc865dff4a1dd3/pyarrow-17.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b72e87fe3e1db343995562f7fff8aee354b55ee83d13afba65400c178ab2597", size = 39864921 }, - { url = "https://files.pythonhosted.org/packages/d8/81/69b6606093363f55a2a574c018901c40952d4e902e670656d18213c71ad7/pyarrow-17.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:dc5c31c37409dfbc5d014047817cb4ccd8c1ea25d19576acf1a001fe07f5b420", size = 38740798 }, - { url = "https://files.pythonhosted.org/packages/4c/21/9ca93b84b92ef927814cb7ba37f0774a484c849d58f0b692b16af8eebcfb/pyarrow-17.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e3343cb1e88bc2ea605986d4b94948716edc7a8d14afd4e2c097232f729758b4", size = 39871877 }, - { url = "https://files.pythonhosted.org/packages/30/d1/63a7c248432c71c7d3ee803e706590a0b81ce1a8d2b2ae49677774b813bb/pyarrow-17.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:a27532c38f3de9eb3e90ecab63dfda948a8ca859a66e3a47f5f42d1e403c4d03", size = 25151089 }, - { url = "https://files.pythonhosted.org/packages/d4/62/ce6ac1275a432b4a27c55fe96c58147f111d8ba1ad800a112d31859fae2f/pyarrow-17.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9b8a823cea605221e61f34859dcc03207e52e409ccf6354634143e23af7c8d22", size = 29019418 }, - { url = "https://files.pythonhosted.org/packages/8e/0a/dbd0c134e7a0c30bea439675cc120012337202e5fac7163ba839aa3691d2/pyarrow-17.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f1e70de6cb5790a50b01d2b686d54aaf73da01266850b05e3af2a1bc89e16053", size = 27152197 }, - { url = "https://files.pythonhosted.org/packages/cb/05/3f4a16498349db79090767620d6dc23c1ec0c658a668d61d76b87706c65d/pyarrow-17.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0071ce35788c6f9077ff9ecba4858108eebe2ea5a3f7cf2cf55ebc1dbc6ee24a", size = 39263026 }, - { url = "https://files.pythonhosted.org/packages/c2/0c/ea2107236740be8fa0e0d4a293a095c9f43546a2465bb7df34eee9126b09/pyarrow-17.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:757074882f844411fcca735e39aae74248a1531367a7c80799b4266390ae51cc", size = 39880798 }, - { url = "https://files.pythonhosted.org/packages/f6/b0/b9164a8bc495083c10c281cc65064553ec87b7537d6f742a89d5953a2a3e/pyarrow-17.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ba11c4f16976e89146781a83833df7f82077cdab7dc6232c897789343f7891a", size = 38715172 }, - { url = "https://files.pythonhosted.org/packages/f1/c4/9625418a1413005e486c006e56675334929fad864347c5ae7c1b2e7fe639/pyarrow-17.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:b0c6ac301093b42d34410b187bba560b17c0330f64907bfa4f7f7f2444b0cf9b", size = 39874508 }, - { url = "https://files.pythonhosted.org/packages/ae/49/baafe2a964f663413be3bd1cf5c45ed98c5e42e804e2328e18f4570027c1/pyarrow-17.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:392bc9feabc647338e6c89267635e111d71edad5fcffba204425a7c8d13610d7", size = 25099235 }, - { url = "https://files.pythonhosted.org/packages/8d/bd/8f52c1d7b430260f80a349cffa2df351750a737b5336313d56dcadeb9ae1/pyarrow-17.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:af5ff82a04b2171415f1410cff7ebb79861afc5dae50be73ce06d6e870615204", size = 28999345 }, - { url = "https://files.pythonhosted.org/packages/64/d9/51e35550f2f18b8815a2ab25948f735434db32000c0e91eba3a32634782a/pyarrow-17.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:edca18eaca89cd6382dfbcff3dd2d87633433043650c07375d095cd3517561d8", size = 27168441 }, - { url = "https://files.pythonhosted.org/packages/18/d8/7161d87d07ea51be70c49f615004c1446d5723622a18b2681f7e4b71bf6e/pyarrow-17.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c7916bff914ac5d4a8fe25b7a25e432ff921e72f6f2b7547d1e325c1ad9d155", size = 39363163 }, - { url = "https://files.pythonhosted.org/packages/3f/08/bc497130789833de09e345e3ce4647e3ce86517c4f70f2144f0367ca378b/pyarrow-17.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f553ca691b9e94b202ff741bdd40f6ccb70cdd5fbf65c187af132f1317de6145", size = 39965253 }, - { url = "https://files.pythonhosted.org/packages/d3/2e/493dd7db889402b4c7871ca7dfdd20f2c5deedbff802d3eb8576359930f9/pyarrow-17.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0cdb0e627c86c373205a2f94a510ac4376fdc523f8bb36beab2e7f204416163c", size = 38805378 }, - { url = "https://files.pythonhosted.org/packages/e6/c1/4c6bcdf7a820034aa91a8b4d25fef38809be79b42ca7aaa16d4680b0bbac/pyarrow-17.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:d7d192305d9d8bc9082d10f361fc70a73590a4c65cf31c3e6926cd72b76bc35c", size = 39958364 }, - { url = "https://files.pythonhosted.org/packages/d1/db/42ac644453cfdfc60fe002b46d647fe7a6dfad753ef7b28e99b4c936ad5d/pyarrow-17.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:02dae06ce212d8b3244dd3e7d12d9c4d3046945a5933d28026598e9dbbda1fca", size = 25229211 }, - { url = "https://files.pythonhosted.org/packages/43/e0/a898096d35be240aa61fb2d54db58b86d664b10e1e51256f9300f47565e8/pyarrow-17.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:13d7a460b412f31e4c0efa1148e1d29bdf18ad1411eb6757d38f8fbdcc8645fb", size = 29007881 }, - { url = "https://files.pythonhosted.org/packages/59/22/f7d14907ed0697b5dd488d393129f2738629fa5bcba863e00931b7975946/pyarrow-17.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9b564a51fbccfab5a04a80453e5ac6c9954a9c5ef2890d1bcf63741909c3f8df", size = 27178117 }, - { url = "https://files.pythonhosted.org/packages/bf/ee/661211feac0ed48467b1d5c57298c91403809ec3ab78b1d175e1d6ad03cf/pyarrow-17.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32503827abbc5aadedfa235f5ece8c4f8f8b0a3cf01066bc8d29de7539532687", size = 39273896 }, - { url = "https://files.pythonhosted.org/packages/af/61/bcd9b58e38ead6ad42b9ed00da33a3f862bc1d445e3d3164799c25550ac2/pyarrow-17.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a155acc7f154b9ffcc85497509bcd0d43efb80d6f733b0dc3bb14e281f131c8b", size = 39875438 }, - { url = "https://files.pythonhosted.org/packages/75/63/29d1bfcc57af73cde3fc3baccab2f37548de512dbe0ab294b033cd203516/pyarrow-17.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:dec8d129254d0188a49f8a1fc99e0560dc1b85f60af729f47de4046015f9b0a5", size = 38735092 }, - { url = "https://files.pythonhosted.org/packages/39/f4/90258b4de753df7cc61cefb0312f8abcf226672e96cc64996e66afce817a/pyarrow-17.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:a48ddf5c3c6a6c505904545c25a4ae13646ae1f8ba703c4df4a1bfe4f4006bda", size = 39867610 }, - { url = "https://files.pythonhosted.org/packages/e7/f6/b75d4816c32f1618ed31a005ee635dd1d91d8164495d94f2ea092f594661/pyarrow-17.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:42bf93249a083aca230ba7e2786c5f673507fa97bbd9725a1e2754715151a204", size = 25148611 }, -] - [[package]] name = "pyarrow" version = "18.1.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/7f/7b/640785a9062bb00314caa8a387abce547d2a420cf09bd6c715fe659ccffb/pyarrow-18.1.0.tar.gz", hash = "sha256:9386d3ca9c145b5539a1cfc75df07757dff870168c959b473a0bccbc3abc8c73", size = 1118671 } wheels = [ { url = "https://files.pythonhosted.org/packages/1a/bb/8d4a1573f66e0684f190dd2b55fd0b97a7214de8882d58a3867e777bf640/pyarrow-18.1.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:e21488d5cfd3d8b500b3238a6c4b075efabc18f0f6d80b29239737ebd69caa6c", size = 29531620 }, @@ -1332,10 +993,8 @@ version = "0.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "beautifulsoup4" }, - { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "docutils" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/d6/3921de802cf1ee771f0e76c9068b52498aeb8eeec6b830ff931c81c7ecf3/pydata_sphinx_theme-0.8.0.tar.gz", hash = "sha256:9f72015d9c572ea92e3007ab221a8325767c426783b6b9941813e65fa988dc90", size = 1123746 } @@ -1349,13 +1008,11 @@ version = "2.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "deprecated" }, - { name = "pyjwt", version = "2.9.0", source = { registry = "https://pypi.org/simple" }, extra = ["crypto"], marker = "python_full_version < '3.9'" }, - { name = "pyjwt", version = "2.10.1", source = { registry = "https://pypi.org/simple" }, extra = ["crypto"], marker = "python_full_version >= '3.9'" }, + { name = "pyjwt", extra = ["crypto"] }, { name = "pynacl" }, { name = "requests" }, { name = "typing-extensions" }, - { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "urllib3", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/16/ce/aa91d30040d9552c274e7ea8bd10a977600d508d579a4bb262b95eccf961/pygithub-2.5.0.tar.gz", hash = "sha256:e1613ac508a9be710920d26eb18b1905ebd9926aa49398e88151c1b526aad3cf", size = 3552804 } wheels = [ @@ -1371,33 +1028,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, ] -[[package]] -name = "pyjwt" -version = "2.9.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/fb/68/ce067f09fca4abeca8771fe667d89cc347d1e99da3e093112ac329c6020e/pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c", size = 78825 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/79/84/0fdf9b18ba31d69877bd39c9cd6052b47f3761e9910c15de788e519f079f/PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850", size = 22344 }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography", marker = "python_full_version < '3.9'" }, -] - [[package]] name = "pyjwt" version = "2.10.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/e7/46/bd74733ff231675599650d3e47f361794b22ef3e3770998dda30d3b63726/pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953", size = 87785 } wheels = [ { url = "https://files.pythonhosted.org/packages/61/ad/689f02752eeec26aed679477e80e632ef1b682313be70793d798c1d5fc8f/PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb", size = 22997 }, @@ -1405,7 +1039,7 @@ wheels = [ [package.optional-dependencies] crypto = [ - { name = "cryptography", marker = "python_full_version >= '3.9'" }, + { name = "cryptography" }, ] [[package]] @@ -1508,13 +1142,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/0f/25911a9f080464c59fab9027482f822b86bf0608957a5fcc6eaac85aa515/PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652", size = 751597 }, { url = "https://files.pythonhosted.org/packages/14/0d/e2c3b43bbce3cf6bd97c840b46088a3031085179e596d4929729d8d68270/PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183", size = 140527 }, { url = "https://files.pythonhosted.org/packages/fa/de/02b54f42487e3d3c6efb3f89428677074ca7bf43aae402517bc7cca949f3/PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563", size = 156446 }, - { url = "https://files.pythonhosted.org/packages/74/d9/323a59d506f12f498c2097488d80d16f4cf965cee1791eab58b56b19f47a/PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a", size = 183218 }, - { url = "https://files.pythonhosted.org/packages/74/cc/20c34d00f04d785f2028737e2e2a8254e1425102e730fee1d6396f832577/PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5", size = 728067 }, - { url = "https://files.pythonhosted.org/packages/20/52/551c69ca1501d21c0de51ddafa8c23a0191ef296ff098e98358f69080577/PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d", size = 757812 }, - { url = "https://files.pythonhosted.org/packages/fd/7f/2c3697bba5d4aa5cc2afe81826d73dfae5f049458e44732c7a0938baa673/PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083", size = 746531 }, - { url = "https://files.pythonhosted.org/packages/8c/ab/6226d3df99900e580091bb44258fde77a8433511a86883bd4681ea19a858/PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706", size = 800820 }, - { url = "https://files.pythonhosted.org/packages/a0/99/a9eb0f3e710c06c5d922026f6736e920d431812ace24aae38228d0d64b04/PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a", size = 145514 }, - { url = "https://files.pythonhosted.org/packages/75/8a/ee831ad5fafa4431099aa4e078d4c8efd43cd5e48fbc774641d233b683a9/PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff", size = 162702 }, { url = "https://files.pythonhosted.org/packages/65/d8/b7a1db13636d7fb7d4ff431593c510c8b8fca920ade06ca8ef20015493c5/PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d", size = 184777 }, { url = "https://files.pythonhosted.org/packages/0a/02/6ec546cd45143fdf9840b2c6be8d875116a64076218b61d68e12548e5839/PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f", size = 172318 }, { url = "https://files.pythonhosted.org/packages/0e/9a/8cc68be846c972bda34f6c2a93abb644fb2476f4dcc924d52175786932c9/PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290", size = 720891 }, @@ -1534,8 +1161,7 @@ dependencies = [ { name = "certifi" }, { name = "charset-normalizer" }, { name = "idna" }, - { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "urllib3", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "urllib3" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218 } wheels = [ @@ -1567,28 +1193,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b2/94/0498cdb7316ed67a1928300dd87d659c933479f44dec51b4f62bfd1f8028/ruff-0.9.1-py3-none-win_arm64.whl", hash = "sha256:1cd76c7f9c679e6e8f2af8f778367dca82b95009bc7b1a85a47f1521ae524fa7", size = 9145708 }, ] -[[package]] -name = "setuptools" -version = "75.3.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/22/a438e0caa4576f8c383fa4d35f1cc01655a46c75be358960d815bfbb12bd/setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686", size = 1351577 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/12/282ee9bce8b58130cb762fbc9beabd531549952cac11fc56add11dcb7ea0/setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd", size = 1251070 }, -] - [[package]] name = "setuptools" version = "75.8.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/92/ec/089608b791d210aec4e7f97488e67ab0d33add3efccb83a056cbafe3a2a6/setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", size = 1343222 } wheels = [ { url = "https://files.pythonhosted.org/packages/69/8a/b9dc7678803429e4a3bc9ba462fa3dd9066824d3c607490235c6a796be5a/setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3", size = 1228782 }, @@ -1621,63 +1229,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/c2/fe97d779f3ef3b15f05c94a2f1e3d21732574ed441687474db9d342a7315/soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9", size = 36186 }, ] -[[package]] -name = "sphinx" -version = "7.1.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -dependencies = [ - { name = "alabaster", version = "0.7.13", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "babel", marker = "python_full_version < '3.9'" }, - { name = "colorama", marker = "python_full_version < '3.9' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.20.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "imagesize", marker = "python_full_version < '3.9'" }, - { name = "importlib-metadata", marker = "python_full_version < '3.9'" }, - { name = "jinja2", marker = "python_full_version < '3.9'" }, - { name = "packaging", marker = "python_full_version < '3.9'" }, - { name = "pygments", marker = "python_full_version < '3.9'" }, - { name = "requests", marker = "python_full_version < '3.9'" }, - { name = "snowballstemmer", marker = "python_full_version < '3.9'" }, - { name = "sphinxcontrib-applehelp", version = "1.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinxcontrib-devhelp", version = "1.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinxcontrib-htmlhelp", version = "2.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.9'" }, - { name = "sphinxcontrib-qthelp", version = "1.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinxcontrib-serializinghtml", version = "1.1.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dc/01/688bdf9282241dca09fe6e3a1110eda399fa9b10d0672db609e37c2e7a39/sphinx-7.1.2.tar.gz", hash = "sha256:780f4d32f1d7d1126576e0e5ecc19dc32ab76cd24e950228dcf7b1f6d3d9e22f", size = 6828258 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/17/325cf6a257d84751a48ae90752b3d8fe0be8f9535b6253add61c49d0d9bc/sphinx-7.1.2-py3-none-any.whl", hash = "sha256:d170a81825b2fcacb6dfd5a0d7f578a053e45d3f2b153fecc948c37344eb4cbe", size = 3169543 }, -] - [[package]] name = "sphinx" version = "7.4.7" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", + "python_full_version < '3.10'", ] dependencies = [ - { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "babel", marker = "python_full_version == '3.9.*'" }, - { name = "colorama", marker = "python_full_version == '3.9.*' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "imagesize", marker = "python_full_version == '3.9.*'" }, - { name = "importlib-metadata", marker = "python_full_version == '3.9.*'" }, - { name = "jinja2", marker = "python_full_version == '3.9.*'" }, - { name = "packaging", marker = "python_full_version == '3.9.*'" }, - { name = "pygments", marker = "python_full_version == '3.9.*'" }, - { name = "requests", marker = "python_full_version == '3.9.*'" }, - { name = "snowballstemmer", marker = "python_full_version == '3.9.*'" }, - { name = "sphinxcontrib-applehelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "sphinxcontrib-devhelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "sphinxcontrib-htmlhelp", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "sphinxcontrib-jsmath", marker = "python_full_version == '3.9.*'" }, - { name = "sphinxcontrib-qthelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "sphinxcontrib-serializinghtml", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "tomli", marker = "python_full_version == '3.9.*'" }, + { name = "alabaster", version = "0.7.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "babel", marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "docutils", marker = "python_full_version < '3.10'" }, + { name = "imagesize", marker = "python_full_version < '3.10'" }, + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, + { name = "jinja2", marker = "python_full_version < '3.10'" }, + { name = "packaging", marker = "python_full_version < '3.10'" }, + { name = "pygments", marker = "python_full_version < '3.10'" }, + { name = "requests", marker = "python_full_version < '3.10'" }, + { name = "snowballstemmer", marker = "python_full_version < '3.10'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version < '3.10'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version < '3.10'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version < '3.10'" }, + { name = "sphinxcontrib-jsmath", marker = "python_full_version < '3.10'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version < '3.10'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/be/50e50cb4f2eff47df05673d361095cafd95521d2a22521b920c67a372dcb/sphinx-7.4.7.tar.gz", hash = "sha256:242f92a7ea7e6c5b406fdc2615413890ba9f699114a9c09192d7dfead2ee9cfe", size = 8067911 } wheels = [ @@ -1697,19 +1274,19 @@ dependencies = [ { name = "alabaster", version = "1.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "babel", marker = "python_full_version >= '3.10'" }, { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "docutils", version = "0.21.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "docutils", marker = "python_full_version >= '3.10'" }, { name = "imagesize", marker = "python_full_version >= '3.10'" }, { name = "jinja2", marker = "python_full_version >= '3.10'" }, { name = "packaging", marker = "python_full_version >= '3.10'" }, { name = "pygments", marker = "python_full_version >= '3.10'" }, { name = "requests", marker = "python_full_version >= '3.10'" }, { name = "snowballstemmer", marker = "python_full_version >= '3.10'" }, - { name = "sphinxcontrib-applehelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "sphinxcontrib-devhelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "sphinxcontrib-htmlhelp", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-applehelp", marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-devhelp", marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-htmlhelp", marker = "python_full_version >= '3.10'" }, { name = "sphinxcontrib-jsmath", marker = "python_full_version >= '3.10'" }, - { name = "sphinxcontrib-qthelp", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "sphinxcontrib-serializinghtml", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-qthelp", marker = "python_full_version >= '3.10'" }, + { name = "sphinxcontrib-serializinghtml", marker = "python_full_version >= '3.10'" }, { name = "tomli", marker = "python_full_version == '3.10.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/be0b61178fe2cdcb67e2a92fc9ebb488e3c51c4f74a36a7824c0adf23425/sphinx-8.1.3.tar.gz", hash = "sha256:43c1911eecb0d3e161ad78611bc905d1ad0e523e4ddc202a58a821773dc4c927", size = 8184611 } @@ -1722,97 +1299,40 @@ name = "sphinx-autoapi" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "astroid", version = "3.2.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "astroid", version = "3.3.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "astroid" }, { name = "jinja2" }, { name = "pyyaml" }, - { name = "sphinx", version = "7.1.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "stdlib-list", version = "0.10.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "stdlib-list", version = "0.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "stdlib-list", marker = "python_full_version < '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4a/eb/cc243583bb1d518ca3b10998c203d919a8ed90affd4831f2b61ad09043d2/sphinx_autoapi-3.4.0.tar.gz", hash = "sha256:e6d5371f9411bbb9fca358c00a9e57aef3ac94cbfc5df4bab285946462f69e0c", size = 29292 } wheels = [ { url = "https://files.pythonhosted.org/packages/de/d6/f2acdc2567337fd5f5dc091a4e58d8a0fb14927b9779fc1e5ecee96d9824/sphinx_autoapi-3.4.0-py3-none-any.whl", hash = "sha256:4027fef2875a22c5f2a57107c71641d82f6166bf55beb407a47aaf3ef14e7b92", size = 34095 }, ] -[[package]] -name = "sphinxcontrib-applehelp" -version = "1.0.4" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/32/df/45e827f4d7e7fcc84e853bcef1d836effd762d63ccb86f43ede4e98b478c/sphinxcontrib-applehelp-1.0.4.tar.gz", hash = "sha256:828f867945bbe39817c210a1abfd1bc4895c8b73fcaade56d45357a348a07d7e", size = 24766 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/c1/5e2cafbd03105ce50d8500f9b4e8a6e8d02e22d0475b574c3b3e9451a15f/sphinxcontrib_applehelp-1.0.4-py3-none-any.whl", hash = "sha256:29d341f67fb0f6f586b23ad80e072c8e6ad0b48417db2bde114a4c9746feb228", size = 120601 }, -] - [[package]] name = "sphinxcontrib-applehelp" version = "2.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053 } wheels = [ { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300 }, ] -[[package]] -name = "sphinxcontrib-devhelp" -version = "1.0.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/98/33/dc28393f16385f722c893cb55539c641c9aaec8d1bc1c15b69ce0ac2dbb3/sphinxcontrib-devhelp-1.0.2.tar.gz", hash = "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4", size = 17398 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c5/09/5de5ed43a521387f18bdf5f5af31d099605c992fd25372b2b9b825ce48ee/sphinxcontrib_devhelp-1.0.2-py2.py3-none-any.whl", hash = "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", size = 84690 }, -] - [[package]] name = "sphinxcontrib-devhelp" version = "2.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967 } wheels = [ { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530 }, ] -[[package]] -name = "sphinxcontrib-htmlhelp" -version = "2.0.1" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/47/64cff68ea3aa450c373301e5bebfbb9fce0a3e70aca245fcadd4af06cd75/sphinxcontrib-htmlhelp-2.0.1.tar.gz", hash = "sha256:0cbdd302815330058422b98a113195c9249825d681e18f11e8b1f78a2f11efff", size = 27967 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6e/ee/a1f5e39046cbb5f8bc8fba87d1ddf1c6643fbc9194e58d26e606de4b9074/sphinxcontrib_htmlhelp-2.0.1-py3-none-any.whl", hash = "sha256:c38cb46dccf316c79de6e5515e1770414b797162b23cd3d06e67020e1d2a6903", size = 99833 }, -] - [[package]] name = "sphinxcontrib-htmlhelp" version = "2.1.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617 } wheels = [ { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705 }, @@ -1827,55 +1347,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071 }, ] -[[package]] -name = "sphinxcontrib-qthelp" -version = "1.0.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/b1/8e/c4846e59f38a5f2b4a0e3b27af38f2fcf904d4bfd82095bf92de0b114ebd/sphinxcontrib-qthelp-1.0.3.tar.gz", hash = "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", size = 21658 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/14/05f9206cf4e9cfca1afb5fd224c7cd434dcc3a433d6d9e4e0264d29c6cdb/sphinxcontrib_qthelp-1.0.3-py2.py3-none-any.whl", hash = "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6", size = 90609 }, -] - [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165 } wheels = [ { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743 }, ] -[[package]] -name = "sphinxcontrib-serializinghtml" -version = "1.1.5" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/72/835d6fadb9e5d02304cf39b18f93d227cd93abd3c41ebf58e6853eeb1455/sphinxcontrib-serializinghtml-1.1.5.tar.gz", hash = "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952", size = 21019 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c6/77/5464ec50dd0f1c1037e3c93249b040c8fc8078fdda97530eeb02424b6eea/sphinxcontrib_serializinghtml-1.1.5-py2.py3-none-any.whl", hash = "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", size = 94021 }, -] - [[package]] name = "sphinxcontrib-serializinghtml" version = "2.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080 } wheels = [ { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072 }, @@ -1895,25 +1379,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521 }, ] -[[package]] -name = "stdlib-list" -version = "0.10.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/39/bb/1cdbc326a5ab0026602e0489cbf02357e78140253c4b57cd866d380eb355/stdlib_list-0.10.0.tar.gz", hash = "sha256:6519c50d645513ed287657bfe856d527f277331540691ddeaf77b25459964a14", size = 59447 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/13/d9/9085375f0d23a4896b307bf14dcc61b49ec8cc67cb33e06cf95bf3af3966/stdlib_list-0.10.0-py3-none-any.whl", hash = "sha256:b3a911bc441d03e0332dd1a9e7d0870ba3bb0a542a74d7524f54fb431256e214", size = 79814 }, -] - [[package]] name = "stdlib-list" version = "0.11.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/5d/04/6b37a71e92ddca16b190b7df62494ac4779d58ced4787f73584eb32c8f03/stdlib_list-0.11.0.tar.gz", hash = "sha256:b74a7b643a77a12637e907f3f62f0ab9f67300bce4014f6b2d3c8b4c8fd63c66", size = 60335 } wheels = [ { url = "https://files.pythonhosted.org/packages/16/fe/e07300c027a868d32d8ed7a425503401e91a03ff90e7ca525c115c634ffb/stdlib_list-0.11.0-py3-none-any.whl", hash = "sha256:8bf8decfffaaf273d4cfeb5bd852b910a00dec1037dcf163576803622bccf597", size = 83617 }, @@ -1994,28 +1463,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/ab/7e5f53c3b9d14972843a647d8d7a853969a58aecc7559cb3267302c94774/tzdata-2024.2-py2.py3-none-any.whl", hash = "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd", size = 346586 }, ] -[[package]] -name = "urllib3" -version = "2.2.3" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/63/22ba4ebfe7430b76388e7cd448d5478814d3032121827c12a2cc287e2260/urllib3-2.2.3.tar.gz", hash = "sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9", size = 300677 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/d9/5f4c13cecde62396b0d3fe530a50ccea91e7dfc1ccf0e09c228841bb5ba8/urllib3-2.2.3-py3-none-any.whl", hash = "sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac", size = 126338 }, -] - [[package]] name = "urllib3" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/aa/63/e53da845320b757bf29ef6a9062f5c669fe997973f966045cb019c3f4b66/urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d", size = 307268 } wheels = [ { url = "https://files.pythonhosted.org/packages/c8/19/4ec628951a74043532ca2cf5d97b7b14863931476d117c471e8e2b1eb39f/urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df", size = 128369 }, @@ -2091,17 +1542,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a7/b1/0bb11e29aa5139d90b770ebbfa167267b1fc548d2302c30c8f7572851738/wrapt-1.17.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c82b8785d98cdd9fed4cac84d765d234ed3251bd6afe34cb7ac523cb93e8b4f", size = 106377 }, { url = "https://files.pythonhosted.org/packages/6a/e1/0122853035b40b3f333bbb25f1939fc1045e21dd518f7f0922b60c156f7c/wrapt-1.17.2-cp313-cp313t-win32.whl", hash = "sha256:13e6afb7fe71fe7485a4550a8844cc9ffbe263c0f1a1eea569bc7091d4898555", size = 37986 }, { url = "https://files.pythonhosted.org/packages/09/5e/1655cf481e079c1f22d0cabdd4e51733679932718dc23bf2db175f329b76/wrapt-1.17.2-cp313-cp313t-win_amd64.whl", hash = "sha256:eaf675418ed6b3b31c7a989fd007fa7c3be66ce14e5c3b27336383604c9da85c", size = 40750 }, - { url = "https://files.pythonhosted.org/packages/0c/66/95b9e90e6e1274999b183c9c3f984996d870e933ca9560115bd1cd1d6f77/wrapt-1.17.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5c803c401ea1c1c18de70a06a6f79fcc9c5acfc79133e9869e730ad7f8ad8ef9", size = 53234 }, - { url = "https://files.pythonhosted.org/packages/a4/b6/6eced5e2db5924bf6d9223d2bb96b62e00395aae77058e6a9e11bf16b3bd/wrapt-1.17.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:f917c1180fdb8623c2b75a99192f4025e412597c50b2ac870f156de8fb101119", size = 38462 }, - { url = "https://files.pythonhosted.org/packages/5d/a4/c8472fe2568978b5532df84273c53ddf713f689d408a4335717ab89547e0/wrapt-1.17.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ecc840861360ba9d176d413a5489b9a0aff6d6303d7e733e2c4623cfa26904a6", size = 38730 }, - { url = "https://files.pythonhosted.org/packages/3c/70/1d259c6b1ad164eb23ff70e3e452dd1950f96e6473f72b7207891d0fd1f0/wrapt-1.17.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb87745b2e6dc56361bfde481d5a378dc314b252a98d7dd19a651a3fa58f24a9", size = 86225 }, - { url = "https://files.pythonhosted.org/packages/a9/68/6b83367e1afb8de91cbea4ef8e85b58acdf62f034f05d78c7b82afaa23d8/wrapt-1.17.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58455b79ec2661c3600e65c0a716955adc2410f7383755d537584b0de41b1d8a", size = 78055 }, - { url = "https://files.pythonhosted.org/packages/0d/21/09573d2443916705c57fdab85d508f592c0a58d57becc53e15755d67fba2/wrapt-1.17.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b4e42a40a5e164cbfdb7b386c966a588b1047558a990981ace551ed7e12ca9c2", size = 85592 }, - { url = "https://files.pythonhosted.org/packages/45/ce/700e17a852dd5dec894e241c72973ea82363486bcc1fb05d47b4fbd1d683/wrapt-1.17.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:91bd7d1773e64019f9288b7a5101f3ae50d3d8e6b1de7edee9c2ccc1d32f0c0a", size = 83906 }, - { url = "https://files.pythonhosted.org/packages/37/14/bd210faf0a66faeb8529d42b6b45a25d6aa6ce25ddfc19168e4161aed227/wrapt-1.17.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:bb90fb8bda722a1b9d48ac1e6c38f923ea757b3baf8ebd0c82e09c5c1a0e7a04", size = 76763 }, - { url = "https://files.pythonhosted.org/packages/34/0c/85af70d291f44659c422416f0272046109e785bf6db8c081cfeeae5715c5/wrapt-1.17.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:08e7ce672e35efa54c5024936e559469436f8b8096253404faeb54d2a878416f", size = 83573 }, - { url = "https://files.pythonhosted.org/packages/f8/1e/b215068e824878f69ea945804fa26c176f7c2735a3ad5367d78930bd076a/wrapt-1.17.2-cp38-cp38-win32.whl", hash = "sha256:410a92fefd2e0e10d26210e1dfb4a876ddaf8439ef60d6434f21ef8d87efc5b7", size = 36408 }, - { url = "https://files.pythonhosted.org/packages/52/27/3dd9ad5f1097b33c95d05929e409cc86d7c765cb5437b86694dc8f8e9af0/wrapt-1.17.2-cp38-cp38-win_amd64.whl", hash = "sha256:95c658736ec15602da0ed73f312d410117723914a5c91a14ee4cdd72f1d790b3", size = 38737 }, { url = "https://files.pythonhosted.org/packages/8a/f4/6ed2b8f6f1c832933283974839b88ec7c983fd12905e01e97889dadf7559/wrapt-1.17.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99039fa9e6306880572915728d7f6c24a86ec57b0a83f6b2491e1d8ab0235b9a", size = 53308 }, { url = "https://files.pythonhosted.org/packages/a2/a9/712a53f8f4f4545768ac532619f6e56d5d0364a87b2212531685e89aeef8/wrapt-1.17.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2696993ee1eebd20b8e4ee4356483c4cb696066ddc24bd70bcbb80fa56ff9061", size = 38489 }, { url = "https://files.pythonhosted.org/packages/fa/9b/e172c8f28a489a2888df18f953e2f6cb8d33b1a2e78c9dfc52d8bf6a5ead/wrapt-1.17.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:612dff5db80beef9e649c6d803a8d50c409082f1fedc9dbcdfde2983b2025b82", size = 38776 }, @@ -2116,25 +1556,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/82/f56956041adef78f849db6b289b282e72b55ab8045a75abad81898c28d19/wrapt-1.17.2-py3-none-any.whl", hash = "sha256:b18f2d1533a71f069c7f82d524a52599053d4c7166e9dd374ae2136b7f40f7c8", size = 23594 }, ] -[[package]] -name = "zipp" -version = "3.20.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.9'", -] -sdist = { url = "https://files.pythonhosted.org/packages/54/bf/5c0000c44ebc80123ecbdddba1f5dcd94a5ada602a9c225d84b5aaa55e86/zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29", size = 24199 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/8b/5ba542fa83c90e09eac972fc9baca7a88e7e7ca4b221a89251954019308b/zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350", size = 9200 }, -] - [[package]] name = "zipp" version = "3.21.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version == '3.9.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/3f/50/bad581df71744867e9468ebd0bcd6505de3b275e06f202c2cb016e3ff56f/zipp-3.21.0.tar.gz", hash = "sha256:2c9958f6430a2040341a52eb608ed6dd93ef4392e02ffe219417c1b28b5dd1f4", size = 24545 } wheels = [ { url = "https://files.pythonhosted.org/packages/b7/1a/7e4798e9339adc931158c9d69ecc34f5e6791489d469f5e50ec15e35f458/zipp-3.21.0-py3-none-any.whl", hash = "sha256:ac1bbe05fd2991f160ebce24ffbac5f6d11d83dc90891255885223d42b3cd931", size = 9630 }, From b194a8772e58ccefc697e11671113127a8038716 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 12 Mar 2025 14:25:32 -0400 Subject: [PATCH 274/413] feat/improve ruff test coverage (#1055) * Run python tests on all currently supported python versions * Update ruff checks to select all * Ruff auto fix * Applying ruff suggestions * noqa rules updates per ruff checks * Working through more ruff suggestions * Working through more ruff suggestions * update timestamps on tests * More ruff updates * More ruff updates * Instead of importing udf static functions as variables, import * More ruff formatting suggestions * more ruff formatting suggestions * More ruff formatting * More ruff formatting * Cut off lint errors for this PR * Working through more ruff checks and disabling a bunch for now * Address CI difference from local ruff * UDWF isn't a proper abstract base class right now since users can opt in to all methods * Update pre-commit to match the version of ruff used in CI * To enable testing in python 3.9 we need numpy. Also going to the current minimal supported version * Update min requried version of python to 3.9 in pyproject.toml. The other changes will come in #1043 that is soon to be merged. * Suppress UP035 * ruff format --- .github/workflows/test.yaml | 2 + .pre-commit-config.yaml | 2 +- benchmarks/tpch/tpch.py | 14 +- dev/release/check-rat-report.py | 2 +- dev/release/generate-changelog.py | 10 +- docs/source/conf.py | 2 +- examples/python-udwf.py | 2 +- examples/tpch/_tests.py | 15 +- pyproject.toml | 76 +++++- python/datafusion/__init__.py | 50 ++-- python/datafusion/common.py | 14 +- python/datafusion/context.py | 4 +- python/datafusion/dataframe.py | 15 +- python/datafusion/expr.py | 94 +++---- python/datafusion/functions.py | 46 ++-- python/datafusion/input/__init__.py | 2 +- python/datafusion/input/base.py | 6 +- python/datafusion/input/location.py | 40 +-- python/datafusion/io.py | 20 +- python/datafusion/object_store.py | 2 +- python/datafusion/plan.py | 8 +- python/datafusion/record_batch.py | 8 +- python/datafusion/substrait.py | 21 +- python/datafusion/udf.py | 236 +++++++++-------- python/tests/generic.py | 19 +- python/tests/test_aggregation.py | 16 +- python/tests/test_catalog.py | 9 +- python/tests/test_context.py | 53 ++-- python/tests/test_dataframe.py | 38 ++- python/tests/test_expr.py | 11 +- python/tests/test_functions.py | 358 ++++++++++++++------------ python/tests/test_imports.py | 7 +- python/tests/test_input.py | 12 +- python/tests/test_io.py | 13 +- python/tests/test_sql.py | 35 +-- python/tests/test_store.py | 13 +- python/tests/test_substrait.py | 2 +- python/tests/test_udaf.py | 10 +- python/tests/test_udwf.py | 2 +- python/tests/test_wrapper_coverage.py | 7 +- 40 files changed, 697 insertions(+), 599 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c1d9ac838..da3582766 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,9 +33,11 @@ jobs: fail-fast: false matrix: python-version: + - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" toolchain: - "stable" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b548ff18f..abcfcf321 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: actionlint-docker - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.3.0 + rev: v0.9.10 hooks: # Run the linter. - id: ruff diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py index fb86b12b6..bfb9ac398 100644 --- a/benchmarks/tpch/tpch.py +++ b/benchmarks/tpch/tpch.py @@ -59,13 +59,13 @@ def bench(data_path, query_path): end = time.time() time_millis = (end - start) * 1000 total_time_millis += time_millis - print("setup,{}".format(round(time_millis, 1))) - results.write("setup,{}\n".format(round(time_millis, 1))) + print(f"setup,{round(time_millis, 1)}") + results.write(f"setup,{round(time_millis, 1)}\n") results.flush() # run queries for query in range(1, 23): - with open("{}/q{}.sql".format(query_path, query)) as f: + with open(f"{query_path}/q{query}.sql") as f: text = f.read() tmp = text.split(";") queries = [] @@ -83,14 +83,14 @@ def bench(data_path, query_path): end = time.time() time_millis = (end - start) * 1000 total_time_millis += time_millis - print("q{},{}".format(query, round(time_millis, 1))) - results.write("q{},{}\n".format(query, round(time_millis, 1))) + print(f"q{query},{round(time_millis, 1)}") + results.write(f"q{query},{round(time_millis, 1)}\n") results.flush() except Exception as e: print("query", query, "failed", e) - print("total,{}".format(round(total_time_millis, 1))) - results.write("total,{}\n".format(round(total_time_millis, 1))) + print(f"total,{round(total_time_millis, 1)}") + results.write(f"total,{round(total_time_millis, 1)}\n") if __name__ == "__main__": diff --git a/dev/release/check-rat-report.py b/dev/release/check-rat-report.py index d3dd7c5dd..0c9f4c326 100644 --- a/dev/release/check-rat-report.py +++ b/dev/release/check-rat-report.py @@ -29,7 +29,7 @@ exclude_globs_filename = sys.argv[1] xml_filename = sys.argv[2] -globs = [line.strip() for line in open(exclude_globs_filename, "r")] +globs = [line.strip() for line in open(exclude_globs_filename)] tree = ET.parse(xml_filename) root = tree.getroot() diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 2564eea86..e30e2def2 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -26,15 +26,11 @@ def print_pulls(repo_name, title, pulls): if len(pulls) > 0: - print("**{}:**".format(title)) + print(f"**{title}:**") print() for pull, commit in pulls: - url = "https://github.com/{}/pull/{}".format(repo_name, pull.number) - print( - "- {} [#{}]({}) ({})".format( - pull.title, pull.number, url, commit.author.login - ) - ) + url = f"https://github.com/{repo_name}/pull/{pull.number}" + print(f"- {pull.title} [#{pull.number}]({url}) ({commit.author.login})") print() diff --git a/docs/source/conf.py b/docs/source/conf.py index 2e5a41339..c82a189e0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -73,7 +73,7 @@ autoapi_python_class_content = "both" -def autoapi_skip_member_fn(app, what, name, obj, skip, options): +def autoapi_skip_member_fn(app, what, name, obj, skip, options): # noqa: ARG001 skip_contents = [ # Re-exports ("class", "datafusion.DataFrame"), diff --git a/examples/python-udwf.py b/examples/python-udwf.py index 7d39dc1b8..98d118bf2 100644 --- a/examples/python-udwf.py +++ b/examples/python-udwf.py @@ -59,7 +59,7 @@ def __init__(self, alpha: float) -> None: def supports_bounded_execution(self) -> bool: return True - def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: + def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: # noqa: ARG002 # Override the default range of current row since uses_window_frame is False # So for the purpose of this test we just smooth from the previous row to # current. diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index c4d872085..2be4dfabd 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -27,28 +27,25 @@ def df_selection(col_name, col_type): if col_type == pa.float64() or isinstance(col_type, pa.Decimal128Type): return F.round(col(col_name), lit(2)).alias(col_name) - elif col_type == pa.string() or col_type == pa.string_view(): + if col_type == pa.string() or col_type == pa.string_view(): return F.trim(col(col_name)).alias(col_name) - else: - return col(col_name) + return col(col_name) def load_schema(col_name, col_type): if col_type == pa.int64() or col_type == pa.int32(): return col_name, pa.string() - elif isinstance(col_type, pa.Decimal128Type): + if isinstance(col_type, pa.Decimal128Type): return col_name, pa.float64() - else: - return col_name, col_type + return col_name, col_type def expected_selection(col_name, col_type): if col_type == pa.int64() or col_type == pa.int32(): return F.trim(col(col_name)).cast(col_type).alias(col_name) - elif col_type == pa.string() or col_type == pa.string_view(): + if col_type == pa.string() or col_type == pa.string_view(): return F.trim(col(col_name)).alias(col_name) - else: - return col(col_name) + return col(col_name) def selections_and_schema(original_schema): diff --git a/pyproject.toml b/pyproject.toml index 1c2733677..060e3b80a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,57 @@ features = ["substrait"] # Enable docstring linting using the google style guide [tool.ruff.lint] -select = ["E4", "E7", "E9", "F", "FA", "D", "W", "I"] +select = ["ALL" ] +ignore = [ + "A001", # Allow using words like min as variable names + "A002", # Allow using words like filter as variable names + "ANN401", # Allow Any for wrapper classes + "COM812", # Recommended to ignore these rules when using with ruff-format + "FIX002", # Allow TODO lines - consider removing at some point + "FBT001", # Allow boolean positional args + "FBT002", # Allow boolean positional args + "ISC001", # Recommended to ignore these rules when using with ruff-format + "SLF001", # Allow accessing private members + "TD002", + "TD003", # Allow TODO lines + "UP007", # Disallowing Union is pedantic + # TODO: Enable all of the following, but this PR is getting too large already + "PT001", + "ANN204", + "B008", + "EM101", + "PLR0913", + "PLR1714", + "ANN201", + "C400", + "TRY003", + "B904", + "UP006", + "RUF012", + "FBT003", + "C416", + "SIM102", + "PGH003", + "PLR2004", + "PERF401", + "PD901", + "EM102", + "ERA001", + "SIM108", + "ICN001", + "ANN001", + "ANN202", + "PTH", + "N812", + "INP001", + "DTZ007", + "PLW2901", + "RET503", + "RUF015", + "A005", + "TC001", + "UP035", +] [tool.ruff.lint.pydocstyle] convention = "google" @@ -75,16 +125,30 @@ max-doc-length = 88 # Disable docstring checking for these directories [tool.ruff.lint.per-file-ignores] -"python/tests/*" = ["D"] -"examples/*" = ["D", "W505"] -"dev/*" = ["D"] -"benchmarks/*" = ["D", "F"] +"python/tests/*" = [ + "ANN", + "ARG", + "BLE001", + "D", + "S101", + "SLF", + "PD", + "PLR2004", + "PT011", + "RUF015", + "S608", + "PLR0913", + "PT004", +] +"examples/*" = ["D", "W505", "E501", "T201", "S101"] +"dev/*" = ["D", "E", "T", "S", "PLR", "C", "SIM", "UP", "EXE", "N817"] +"benchmarks/*" = ["D", "F", "T", "BLE", "FURB", "PLR", "E", "TD", "TRY", "S", "SIM", "EXE", "UP"] "docs/*" = ["D"] [dependency-groups] dev = [ "maturin>=1.8.1", - "numpy>1.24.4 ; python_full_version >= '3.10'", + "numpy>1.25.0", "pytest>=7.4.4", "ruff>=0.9.1", "toml>=0.10.2", diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index f11ce54a6..286e5dc31 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -48,44 +48,47 @@ from .io import read_avro, read_csv, read_json, read_parquet from .plan import ExecutionPlan, LogicalPlan from .record_batch import RecordBatch, RecordBatchStream -from .udf import Accumulator, AggregateUDF, ScalarUDF, WindowUDF +from .udf import Accumulator, AggregateUDF, ScalarUDF, WindowUDF, udaf, udf, udwf __version__ = importlib_metadata.version(__name__) __all__ = [ "Accumulator", + "AggregateUDF", + "Catalog", "Config", - "DataFrame", - "SessionContext", - "SessionConfig", - "SQLOptions", - "RuntimeEnvBuilder", - "Expr", - "ScalarUDF", - "WindowFrame", - "column", - "col", - "literal", - "lit", "DFSchema", - "Catalog", + "DataFrame", "Database", - "Table", - "AggregateUDF", - "WindowUDF", - "LogicalPlan", "ExecutionPlan", + "Expr", + "LogicalPlan", "RecordBatch", "RecordBatchStream", + "RuntimeEnvBuilder", + "SQLOptions", + "ScalarUDF", + "SessionConfig", + "SessionContext", + "Table", + "WindowFrame", + "WindowUDF", + "col", + "column", "common", "expr", "functions", + "lit", + "literal", "object_store", - "substrait", - "read_parquet", "read_avro", "read_csv", "read_json", + "read_parquet", + "substrait", + "udaf", + "udf", + "udwf", ] @@ -120,10 +123,3 @@ def str_lit(value): def lit(value): """Create a literal expression.""" return Expr.literal(value) - - -udf = ScalarUDF.udf - -udaf = AggregateUDF.udaf - -udwf = WindowUDF.udwf diff --git a/python/datafusion/common.py b/python/datafusion/common.py index a2298c634..e762a993b 100644 --- a/python/datafusion/common.py +++ b/python/datafusion/common.py @@ -20,7 +20,7 @@ from ._internal import common as common_internal -# TODO these should all have proper wrapper classes +# TODO: these should all have proper wrapper classes DFSchema = common_internal.DFSchema DataType = common_internal.DataType @@ -38,15 +38,15 @@ "DFSchema", "DataType", "DataTypeMap", - "RexType", - "PythonType", - "SqlType", "NullTreatment", - "SqlTable", + "PythonType", + "RexType", + "SqlFunction", "SqlSchema", - "SqlView", "SqlStatistics", - "SqlFunction", + "SqlTable", + "SqlType", + "SqlView", ] diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 282b2a477..0ab1a908a 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -393,8 +393,6 @@ def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeEnvBuilder: class RuntimeConfig(RuntimeEnvBuilder): """See `RuntimeEnvBuilder`.""" - pass - class SQLOptions: """Options to be used when performing SQL queries.""" @@ -498,7 +496,7 @@ def __init__( self.ctx = SessionContextInternal(config, runtime) - def enable_url_table(self) -> "SessionContext": + def enable_url_table(self) -> SessionContext: """Control if local files can be queried as tables. Returns: diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index de5d8376e..d1c71c2bb 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -29,6 +29,7 @@ List, Literal, Optional, + Type, Union, overload, ) @@ -49,10 +50,11 @@ import polars as pl import pyarrow as pa + from datafusion._internal import DataFrame as DataFrameInternal + from datafusion._internal import expr as expr_internal + from enum import Enum -from datafusion._internal import DataFrame as DataFrameInternal -from datafusion._internal import expr as expr_internal from datafusion.expr import Expr, SortExpr, sort_or_default @@ -73,7 +75,7 @@ class Compression(Enum): LZ4_RAW = "lz4_raw" @classmethod - def from_str(cls, value: str) -> "Compression": + def from_str(cls: Type[Compression], value: str) -> Compression: """Convert a string to a Compression enum value. Args: @@ -88,8 +90,9 @@ def from_str(cls, value: str) -> "Compression": try: return cls(value.lower()) except ValueError: + valid_values = str([item.value for item in Compression]) raise ValueError( - f"{value} is not a valid Compression. Valid values are: {[item.value for item in Compression]}" + f"{value} is not a valid Compression. Valid values are: {valid_values}" ) def get_default_level(self) -> Optional[int]: @@ -104,9 +107,9 @@ def get_default_level(self) -> Optional[int]: # https://github.com/apache/datafusion-python/pull/981#discussion_r1904789223 if self == Compression.GZIP: return 6 - elif self == Compression.BROTLI: + if self == Compression.BROTLI: return 1 - elif self == Compression.ZSTD: + if self == Compression.ZSTD: return 4 return None diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 3639abec6..702f75aed 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -101,63 +101,63 @@ WindowExpr = expr_internal.WindowExpr __all__ = [ - "Expr", - "Column", - "Literal", - "BinaryExpr", - "Literal", + "Aggregate", "AggregateFunction", - "Not", - "IsNotNull", - "IsNull", - "IsTrue", - "IsFalse", - "IsUnknown", - "IsNotTrue", - "IsNotFalse", - "IsNotUnknown", - "Negative", - "Like", - "ILike", - "SimilarTo", - "ScalarVariable", "Alias", - "InList", - "Exists", - "Subquery", - "InSubquery", - "ScalarSubquery", - "Placeholder", - "GroupingSet", + "Analyze", + "Between", + "BinaryExpr", "Case", "CaseBuilder", "Cast", - "TryCast", - "Between", + "Column", + "CreateMemoryTable", + "CreateView", + "Distinct", + "DropTable", + "EmptyRelation", + "Exists", "Explain", + "Expr", + "Extension", + "Filter", + "GroupingSet", + "ILike", + "InList", + "InSubquery", + "IsFalse", + "IsNotFalse", + "IsNotNull", + "IsNotTrue", + "IsNotUnknown", + "IsNull", + "IsTrue", + "IsUnknown", + "Join", + "JoinConstraint", + "JoinType", + "Like", "Limit", - "Aggregate", + "Literal", + "Literal", + "Negative", + "Not", + "Partitioning", + "Placeholder", + "Projection", + "Repartition", + "ScalarSubquery", + "ScalarVariable", + "SimilarTo", "Sort", "SortExpr", - "Analyze", - "EmptyRelation", - "Join", - "JoinType", - "JoinConstraint", + "Subquery", + "SubqueryAlias", + "TableScan", + "TryCast", "Union", "Unnest", "UnnestExpr", - "Extension", - "Filter", - "Projection", - "TableScan", - "CreateMemoryTable", - "CreateView", - "Distinct", - "SubqueryAlias", - "DropTable", - "Partitioning", - "Repartition", "Window", "WindowExpr", "WindowFrame", @@ -311,7 +311,7 @@ def __getitem__(self, key: str | int) -> Expr: ) return Expr(self.expr.__getitem__(key)) - def __eq__(self, rhs: Any) -> Expr: + def __eq__(self, rhs: object) -> Expr: """Equal to. Accepts either an expression or any valid PyArrow scalar literal value. @@ -320,7 +320,7 @@ def __eq__(self, rhs: Any) -> Expr: rhs = Expr.literal(rhs) return Expr(self.expr.__eq__(rhs.expr)) - def __ne__(self, rhs: Any) -> Expr: + def __ne__(self, rhs: object) -> Expr: """Not equal to. Accepts either an expression or any valid PyArrow scalar literal value. diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index b449c4868..0cc7434cf 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -18,13 +18,12 @@ from __future__ import annotations -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional import pyarrow as pa from datafusion._internal import functions as f from datafusion.common import NullTreatment -from datafusion.context import SessionContext from datafusion.expr import ( CaseBuilder, Expr, @@ -34,6 +33,9 @@ sort_list_to_raw_sort_list, ) +if TYPE_CHECKING: + from datafusion.context import SessionContext + __all__ = [ "abs", "acos", @@ -81,8 +83,8 @@ "array_sort", "array_to_string", "array_union", - "arrow_typeof", "arrow_cast", + "arrow_typeof", "ascii", "asin", "asinh", @@ -97,6 +99,7 @@ "bool_and", "bool_or", "btrim", + "cardinality", "case", "cbrt", "ceil", @@ -116,6 +119,7 @@ "covar", "covar_pop", "covar_samp", + "cume_dist", "current_date", "current_time", "date_bin", @@ -125,17 +129,17 @@ "datetrunc", "decode", "degrees", + "dense_rank", "digest", "empty", "encode", "ends_with", - "extract", "exp", + "extract", "factorial", "find_in_set", "first_value", "flatten", - "cardinality", "floor", "from_unixtime", "gcd", @@ -143,8 +147,10 @@ "initcap", "isnan", "iszero", + "lag", "last_value", "lcm", + "lead", "left", "length", "levenshtein", @@ -166,10 +172,10 @@ "list_prepend", "list_push_back", "list_push_front", - "list_repeat", "list_remove", "list_remove_all", "list_remove_n", + "list_repeat", "list_replace", "list_replace_all", "list_replace_n", @@ -180,14 +186,14 @@ "list_union", "ln", "log", - "log10", "log2", + "log10", "lower", "lpad", "ltrim", "make_array", - "make_list", "make_date", + "make_list", "max", "md5", "mean", @@ -195,19 +201,22 @@ "min", "named_struct", "nanvl", - "nvl", "now", "nth_value", + "ntile", "nullif", + "nvl", "octet_length", "order_by", "overlay", + "percent_rank", "pi", "pow", "power", "radians", "random", "range", + "rank", "regexp_like", "regexp_match", "regexp_replace", @@ -225,6 +234,7 @@ "reverse", "right", "round", + "row_number", "rpad", "rtrim", "sha224", @@ -252,8 +262,8 @@ "to_hex", "to_timestamp", "to_timestamp_micros", - "to_timestamp_nanos", "to_timestamp_millis", + "to_timestamp_nanos", "to_timestamp_seconds", "to_unixtime", "translate", @@ -268,14 +278,6 @@ "when", # Window Functions "window", - "lead", - "lag", - "row_number", - "rank", - "dense_rank", - "percent_rank", - "cume_dist", - "ntile", ] @@ -292,14 +294,14 @@ def nullif(expr1: Expr, expr2: Expr) -> Expr: return Expr(f.nullif(expr1.expr, expr2.expr)) -def encode(input: Expr, encoding: Expr) -> Expr: +def encode(expr: Expr, encoding: Expr) -> Expr: """Encode the ``input``, using the ``encoding``. encoding can be base64 or hex.""" - return Expr(f.encode(input.expr, encoding.expr)) + return Expr(f.encode(expr.expr, encoding.expr)) -def decode(input: Expr, encoding: Expr) -> Expr: +def decode(expr: Expr, encoding: Expr) -> Expr: """Decode the ``input``, using the ``encoding``. encoding can be base64 or hex.""" - return Expr(f.decode(input.expr, encoding.expr)) + return Expr(f.decode(expr.expr, encoding.expr)) def array_to_string(expr: Expr, delimiter: Expr) -> Expr: diff --git a/python/datafusion/input/__init__.py b/python/datafusion/input/__init__.py index f85ce21f0..f0c1f42b4 100644 --- a/python/datafusion/input/__init__.py +++ b/python/datafusion/input/__init__.py @@ -23,5 +23,5 @@ from .location import LocationInputPlugin __all__ = [ - LocationInputPlugin, + "LocationInputPlugin", ] diff --git a/python/datafusion/input/base.py b/python/datafusion/input/base.py index 4eba19784..f67dde2a1 100644 --- a/python/datafusion/input/base.py +++ b/python/datafusion/input/base.py @@ -38,11 +38,9 @@ class BaseInputSource(ABC): """ @abstractmethod - def is_correct_input(self, input_item: Any, table_name: str, **kwargs) -> bool: + def is_correct_input(self, input_item: Any, table_name: str, **kwargs: Any) -> bool: """Returns `True` if the input is valid.""" - pass @abstractmethod - def build_table(self, input_item: Any, table_name: str, **kwarg) -> SqlTable: + def build_table(self, input_item: Any, table_name: str, **kwarg: Any) -> SqlTable: # type: ignore[invalid-type-form] """Create a table from the input source.""" - pass diff --git a/python/datafusion/input/location.py b/python/datafusion/input/location.py index 517cd1578..08d98d115 100644 --- a/python/datafusion/input/location.py +++ b/python/datafusion/input/location.py @@ -18,7 +18,7 @@ """The default input source for DataFusion.""" import glob -import os +from pathlib import Path from typing import Any from datafusion.common import DataTypeMap, SqlTable @@ -31,7 +31,7 @@ class LocationInputPlugin(BaseInputSource): This can be read in from a file (on disk, remote etc.). """ - def is_correct_input(self, input_item: Any, table_name: str, **kwargs): + def is_correct_input(self, input_item: Any, table_name: str, **kwargs: Any) -> bool: # noqa: ARG002 """Returns `True` if the input is valid.""" return isinstance(input_item, str) @@ -39,27 +39,28 @@ def build_table( self, input_item: str, table_name: str, - **kwargs, - ) -> SqlTable: + **kwargs: Any, # noqa: ARG002 + ) -> SqlTable: # type: ignore[invalid-type-form] """Create a table from the input source.""" - _, extension = os.path.splitext(input_item) - format = extension.lstrip(".").lower() + extension = Path(input_item).suffix + file_format = extension.lstrip(".").lower() num_rows = 0 # Total number of rows in the file. Used for statistics columns = [] - if format == "parquet": + if file_format == "parquet": import pyarrow.parquet as pq # Read the Parquet metadata metadata = pq.read_metadata(input_item) num_rows = metadata.num_rows # Iterate through the schema and build the SqlTable - for col in metadata.schema: - columns.append( - ( - col.name, - DataTypeMap.from_parquet_type_str(col.physical_type), - ) + columns = [ + ( + col.name, + DataTypeMap.from_parquet_type_str(col.physical_type), ) + for col in metadata.schema + ] + elif format == "csv": import csv @@ -69,19 +70,18 @@ def build_table( # to get that information. However, this should only be occurring # at table creation time and therefore shouldn't # slow down query performance. - with open(input_item, "r") as file: + with Path(input_item).open() as file: reader = csv.reader(file) - header_row = next(reader) - print(header_row) + _header_row = next(reader) for _ in reader: num_rows += 1 # TODO: Need to actually consume this row into reasonable columns - raise RuntimeError("TODO: Currently unable to support CSV input files.") + msg = "TODO: Currently unable to support CSV input files." + raise RuntimeError(msg) else: - raise RuntimeError( - f"Input of format: `{format}` is currently not supported.\ + msg = f"Input of format: `{format}` is currently not supported.\ Only Parquet and CSV." - ) + raise RuntimeError(msg) # Input could possibly be multiple files. Create a list if so input_files = glob.glob(input_item) diff --git a/python/datafusion/io.py b/python/datafusion/io.py index 3b6264948..3e39703e3 100644 --- a/python/datafusion/io.py +++ b/python/datafusion/io.py @@ -19,15 +19,19 @@ from __future__ import annotations -import pathlib - -import pyarrow +from typing import TYPE_CHECKING from datafusion.dataframe import DataFrame -from datafusion.expr import Expr from ._internal import SessionContext as SessionContextInternal +if TYPE_CHECKING: + import pathlib + + import pyarrow as pa + + from datafusion.expr import Expr + def read_parquet( path: str | pathlib.Path, @@ -35,7 +39,7 @@ def read_parquet( parquet_pruning: bool = True, file_extension: str = ".parquet", skip_metadata: bool = True, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_sort_order: list[list[Expr]] | None = None, ) -> DataFrame: """Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. @@ -79,7 +83,7 @@ def read_parquet( def read_json( path: str | pathlib.Path, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = ".json", table_partition_cols: list[tuple[str, str]] | None = None, @@ -120,7 +124,7 @@ def read_json( def read_csv( path: str | pathlib.Path | list[str] | list[pathlib.Path], - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, has_header: bool = True, delimiter: str = ",", schema_infer_max_records: int = 1000, @@ -173,7 +177,7 @@ def read_csv( def read_avro( path: str | pathlib.Path, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = ".avro", ) -> DataFrame: diff --git a/python/datafusion/object_store.py b/python/datafusion/object_store.py index 7cc17506f..6298526f5 100644 --- a/python/datafusion/object_store.py +++ b/python/datafusion/object_store.py @@ -24,4 +24,4 @@ MicrosoftAzure = object_store.MicrosoftAzure Http = object_store.Http -__all__ = ["AmazonS3", "GoogleCloud", "LocalFileSystem", "MicrosoftAzure", "Http"] +__all__ = ["AmazonS3", "GoogleCloud", "Http", "LocalFileSystem", "MicrosoftAzure"] diff --git a/python/datafusion/plan.py b/python/datafusion/plan.py index 133fc446d..0b7bebcb3 100644 --- a/python/datafusion/plan.py +++ b/python/datafusion/plan.py @@ -19,7 +19,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, List +from typing import TYPE_CHECKING, Any import datafusion._internal as df_internal @@ -27,8 +27,8 @@ from datafusion.context import SessionContext __all__ = [ - "LogicalPlan", "ExecutionPlan", + "LogicalPlan", ] @@ -54,7 +54,7 @@ def to_variant(self) -> Any: """Convert the logical plan into its specific variant.""" return self._raw_plan.to_variant() - def inputs(self) -> List[LogicalPlan]: + def inputs(self) -> list[LogicalPlan]: """Returns the list of inputs to the logical plan.""" return [LogicalPlan(p) for p in self._raw_plan.inputs()] @@ -106,7 +106,7 @@ def __init__(self, plan: df_internal.ExecutionPlan) -> None: """This constructor should not be called by the end user.""" self._raw_plan = plan - def children(self) -> List[ExecutionPlan]: + def children(self) -> list[ExecutionPlan]: """Get a list of children `ExecutionPlan` that act as inputs to this plan. The returned list will be empty for leaf nodes such as scans, will contain a diff --git a/python/datafusion/record_batch.py b/python/datafusion/record_batch.py index 772cd9089..556eaa786 100644 --- a/python/datafusion/record_batch.py +++ b/python/datafusion/record_batch.py @@ -26,14 +26,14 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - import pyarrow + import pyarrow as pa import typing_extensions import datafusion._internal as df_internal class RecordBatch: - """This class is essentially a wrapper for :py:class:`pyarrow.RecordBatch`.""" + """This class is essentially a wrapper for :py:class:`pa.RecordBatch`.""" def __init__(self, record_batch: df_internal.RecordBatch) -> None: """This constructor is generally not called by the end user. @@ -42,8 +42,8 @@ def __init__(self, record_batch: df_internal.RecordBatch) -> None: """ self.record_batch = record_batch - def to_pyarrow(self) -> pyarrow.RecordBatch: - """Convert to :py:class:`pyarrow.RecordBatch`.""" + def to_pyarrow(self) -> pa.RecordBatch: + """Convert to :py:class:`pa.RecordBatch`.""" return self.record_batch.to_pyarrow() diff --git a/python/datafusion/substrait.py b/python/datafusion/substrait.py index 06302fe38..f10adfb0c 100644 --- a/python/datafusion/substrait.py +++ b/python/datafusion/substrait.py @@ -23,7 +23,6 @@ from __future__ import annotations -import pathlib from typing import TYPE_CHECKING try: @@ -36,11 +35,13 @@ from ._internal import substrait as substrait_internal if TYPE_CHECKING: + import pathlib + from datafusion.context import SessionContext __all__ = [ - "Plan", "Consumer", + "Plan", "Producer", "Serde", ] @@ -68,11 +69,9 @@ def encode(self) -> bytes: @deprecated("Use `Plan` instead.") -class plan(Plan): +class plan(Plan): # noqa: N801 """See `Plan`.""" - pass - class Serde: """Provides the ``Substrait`` serialization and deserialization.""" @@ -140,11 +139,9 @@ def deserialize_bytes(proto_bytes: bytes) -> Plan: @deprecated("Use `Serde` instead.") -class serde(Serde): +class serde(Serde): # noqa: N801 """See `Serde` instead.""" - pass - class Producer: """Generates substrait plans from a logical plan.""" @@ -168,11 +165,9 @@ def to_substrait_plan(logical_plan: LogicalPlan, ctx: SessionContext) -> Plan: @deprecated("Use `Producer` instead.") -class producer(Producer): +class producer(Producer): # noqa: N801 """Use `Producer` instead.""" - pass - class Consumer: """Generates a logical plan from a substrait plan.""" @@ -194,7 +189,5 @@ def from_substrait_plan(ctx: SessionContext, plan: Plan) -> LogicalPlan: @deprecated("Use `Consumer` instead.") -class consumer(Consumer): +class consumer(Consumer): # noqa: N801 """Use `Consumer` instead.""" - - pass diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index af7bcf2ed..603b7063d 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -22,15 +22,15 @@ import functools from abc import ABCMeta, abstractmethod from enum import Enum -from typing import TYPE_CHECKING, Callable, List, Optional, TypeVar +from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, overload -import pyarrow +import pyarrow as pa import datafusion._internal as df_internal from datafusion.expr import Expr if TYPE_CHECKING: - _R = TypeVar("_R", bound=pyarrow.DataType) + _R = TypeVar("_R", bound=pa.DataType) class Volatility(Enum): @@ -72,7 +72,7 @@ class Volatility(Enum): for each output row, resulting in a unique random value for each row. """ - def __str__(self): + def __str__(self) -> str: """Returns the string equivalent.""" return self.name.lower() @@ -88,7 +88,7 @@ def __init__( self, name: str, func: Callable[..., _R], - input_types: pyarrow.DataType | list[pyarrow.DataType], + input_types: pa.DataType | list[pa.DataType], return_type: _R, volatility: Volatility | str, ) -> None: @@ -96,7 +96,7 @@ def __init__( See helper method :py:func:`udf` for argument details. """ - if isinstance(input_types, pyarrow.DataType): + if isinstance(input_types, pa.DataType): input_types = [input_types] self._udf = df_internal.ScalarUDF( name, func, input_types, return_type, str(volatility) @@ -111,7 +111,27 @@ def __call__(self, *args: Expr) -> Expr: args_raw = [arg.expr for arg in args] return Expr(self._udf.__call__(*args_raw)) - class udf: + @overload + @staticmethod + def udf( + input_types: list[pa.DataType], + return_type: _R, + volatility: Volatility | str, + name: Optional[str] = None, + ) -> Callable[..., ScalarUDF]: ... + + @overload + @staticmethod + def udf( + func: Callable[..., _R], + input_types: list[pa.DataType], + return_type: _R, + volatility: Volatility | str, + name: Optional[str] = None, + ) -> ScalarUDF: ... + + @staticmethod + def udf(*args: Any, **kwargs: Any): # noqa: D417 """Create a new User-Defined Function (UDF). This class can be used both as a **function** and as a **decorator**. @@ -125,7 +145,7 @@ class udf: Args: func (Callable, optional): **Only needed when calling as a function.** Skip this argument when using `udf` as a decorator. - input_types (list[pyarrow.DataType]): The data types of the arguments + input_types (list[pa.DataType]): The data types of the arguments to `func`. This list must be of the same length as the number of arguments. return_type (_R): The data type of the return value from the function. @@ -141,40 +161,28 @@ class udf: ``` def double_func(x): return x * 2 - double_udf = udf(double_func, [pyarrow.int32()], pyarrow.int32(), + double_udf = udf(double_func, [pa.int32()], pa.int32(), "volatile", "double_it") ``` **Using `udf` as a decorator:** ``` - @udf([pyarrow.int32()], pyarrow.int32(), "volatile", "double_it") + @udf([pa.int32()], pa.int32(), "volatile", "double_it") def double_udf(x): return x * 2 ``` """ - def __new__(cls, *args, **kwargs): - """Create a new UDF. - - Trigger UDF function or decorator depending on if the first args is callable - """ - if args and callable(args[0]): - # Case 1: Used as a function, require the first parameter to be callable - return cls._function(*args, **kwargs) - else: - # Case 2: Used as a decorator with parameters - return cls._decorator(*args, **kwargs) - - @staticmethod def _function( func: Callable[..., _R], - input_types: list[pyarrow.DataType], + input_types: list[pa.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None, ) -> ScalarUDF: if not callable(func): - raise TypeError("`func` argument must be callable") + msg = "`func` argument must be callable" + raise TypeError(msg) if name is None: if hasattr(func, "__qualname__"): name = func.__qualname__.lower() @@ -188,49 +196,50 @@ def _function( volatility=volatility, ) - @staticmethod def _decorator( - input_types: list[pyarrow.DataType], + input_types: list[pa.DataType], return_type: _R, volatility: Volatility | str, name: Optional[str] = None, - ): - def decorator(func): + ) -> Callable: + def decorator(func: Callable): udf_caller = ScalarUDF.udf( func, input_types, return_type, volatility, name ) @functools.wraps(func) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any): return udf_caller(*args, **kwargs) return wrapper return decorator + if args and callable(args[0]): + # Case 1: Used as a function, require the first parameter to be callable + return _function(*args, **kwargs) + # Case 2: Used as a decorator with parameters + return _decorator(*args, **kwargs) + class Accumulator(metaclass=ABCMeta): """Defines how an :py:class:`AggregateUDF` accumulates values.""" @abstractmethod - def state(self) -> List[pyarrow.Scalar]: + def state(self) -> list[pa.Scalar]: """Return the current state.""" - pass @abstractmethod - def update(self, *values: pyarrow.Array) -> None: + def update(self, *values: pa.Array) -> None: """Evaluate an array of values and update state.""" - pass @abstractmethod - def merge(self, states: List[pyarrow.Array]) -> None: + def merge(self, states: list[pa.Array]) -> None: """Merge a set of states.""" - pass @abstractmethod - def evaluate(self) -> pyarrow.Scalar: + def evaluate(self) -> pa.Scalar: """Return the resultant value.""" - pass class AggregateUDF: @@ -244,9 +253,9 @@ def __init__( self, name: str, accumulator: Callable[[], Accumulator], - input_types: list[pyarrow.DataType], - return_type: pyarrow.DataType, - state_type: list[pyarrow.DataType], + input_types: list[pa.DataType], + return_type: pa.DataType, + state_type: list[pa.DataType], volatility: Volatility | str, ) -> None: """Instantiate a user-defined aggregate function (UDAF). @@ -272,7 +281,29 @@ def __call__(self, *args: Expr) -> Expr: args_raw = [arg.expr for arg in args] return Expr(self._udaf.__call__(*args_raw)) - class udaf: + @overload + @staticmethod + def udaf( + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + state_type: list[pa.DataType], + volatility: Volatility | str, + name: Optional[str] = None, + ) -> Callable[..., AggregateUDF]: ... + + @overload + @staticmethod + def udaf( + accum: Callable[[], Accumulator], + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + state_type: list[pa.DataType], + volatility: Volatility | str, + name: Optional[str] = None, + ) -> AggregateUDF: ... + + @staticmethod + def udaf(*args: Any, **kwargs: Any): # noqa: D417 """Create a new User-Defined Aggregate Function (UDAF). This class allows you to define an **aggregate function** that can be used in @@ -300,13 +331,13 @@ class Summarize(Accumulator): def __init__(self, bias: float = 0.0): self._sum = pa.scalar(bias) - def state(self) -> List[pa.Scalar]: + def state(self) -> list[pa.Scalar]: return [self._sum] def update(self, values: pa.Array) -> None: self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py()) - def merge(self, states: List[pa.Array]) -> None: + def merge(self, states: list[pa.Array]) -> None: self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py()) def evaluate(self) -> pa.Scalar: @@ -344,37 +375,23 @@ def udf4() -> Summarize: aggregation or window function calls. """ - def __new__(cls, *args, **kwargs): - """Create a new UDAF. - - Trigger UDAF function or decorator depending on if the first args is - callable - """ - if args and callable(args[0]): - # Case 1: Used as a function, require the first parameter to be callable - return cls._function(*args, **kwargs) - else: - # Case 2: Used as a decorator with parameters - return cls._decorator(*args, **kwargs) - - @staticmethod def _function( accum: Callable[[], Accumulator], - input_types: pyarrow.DataType | list[pyarrow.DataType], - return_type: pyarrow.DataType, - state_type: list[pyarrow.DataType], + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + state_type: list[pa.DataType], volatility: Volatility | str, name: Optional[str] = None, ) -> AggregateUDF: if not callable(accum): - raise TypeError("`func` must be callable.") - if not isinstance(accum.__call__(), Accumulator): - raise TypeError( - "Accumulator must implement the abstract base class Accumulator" - ) + msg = "`func` must be callable." + raise TypeError(msg) + if not isinstance(accum(), Accumulator): + msg = "Accumulator must implement the abstract base class Accumulator" + raise TypeError(msg) if name is None: - name = accum.__call__().__class__.__qualname__.lower() - if isinstance(input_types, pyarrow.DataType): + name = accum().__class__.__qualname__.lower() + if isinstance(input_types, pa.DataType): input_types = [input_types] return AggregateUDF( name=name, @@ -385,29 +402,34 @@ def _function( volatility=volatility, ) - @staticmethod def _decorator( - input_types: pyarrow.DataType | list[pyarrow.DataType], - return_type: pyarrow.DataType, - state_type: list[pyarrow.DataType], + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + state_type: list[pa.DataType], volatility: Volatility | str, name: Optional[str] = None, - ): - def decorator(accum: Callable[[], Accumulator]): + ) -> Callable[..., Callable[..., Expr]]: + def decorator(accum: Callable[[], Accumulator]) -> Callable[..., Expr]: udaf_caller = AggregateUDF.udaf( accum, input_types, return_type, state_type, volatility, name ) @functools.wraps(accum) - def wrapper(*args, **kwargs): + def wrapper(*args: Any, **kwargs: Any) -> Expr: return udaf_caller(*args, **kwargs) return wrapper return decorator + if args and callable(args[0]): + # Case 1: Used as a function, require the first parameter to be callable + return _function(*args, **kwargs) + # Case 2: Used as a decorator with parameters + return _decorator(*args, **kwargs) + -class WindowEvaluator(metaclass=ABCMeta): +class WindowEvaluator: """Evaluator class for user-defined window functions (UDWF). It is up to the user to decide which evaluate function is appropriate. @@ -423,7 +445,7 @@ class WindowEvaluator(metaclass=ABCMeta): +------------------------+--------------------------------+------------------+---------------------------+ | True | True/False | True/False | ``evaluate`` | +------------------------+--------------------------------+------------------+---------------------------+ - """ # noqa: W505 + """ # noqa: W505, E501 def memoize(self) -> None: """Perform a memoize operation to improve performance. @@ -436,9 +458,8 @@ def memoize(self) -> None: `memoize` is called after each input batch is processed, and such functions can save whatever they need """ - pass - def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: + def get_range(self, idx: int, num_rows: int) -> tuple[int, int]: # noqa: ARG002 """Return the range for the window fuction. If `uses_window_frame` flag is `false`. This method is used to @@ -460,14 +481,17 @@ def is_causal(self) -> bool: """Get whether evaluator needs future data for its result.""" return False - def evaluate_all(self, values: list[pyarrow.Array], num_rows: int) -> pyarrow.Array: + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: """Evaluate a window function on an entire input partition. This function is called once per input *partition* for window functions that *do not use* values from the window frame, such as - :py:func:`~datafusion.functions.row_number`, :py:func:`~datafusion.functions.rank`, - :py:func:`~datafusion.functions.dense_rank`, :py:func:`~datafusion.functions.percent_rank`, - :py:func:`~datafusion.functions.cume_dist`, :py:func:`~datafusion.functions.lead`, + :py:func:`~datafusion.functions.row_number`, + :py:func:`~datafusion.functions.rank`, + :py:func:`~datafusion.functions.dense_rank`, + :py:func:`~datafusion.functions.percent_rank`, + :py:func:`~datafusion.functions.cume_dist`, + :py:func:`~datafusion.functions.lead`, and :py:func:`~datafusion.functions.lag`. It produces the result of all rows in a single pass. It @@ -499,12 +523,11 @@ def evaluate_all(self, values: list[pyarrow.Array], num_rows: int) -> pyarrow.Ar .. code-block:: text avg(x) OVER (PARTITION BY y ORDER BY z ROWS BETWEEN 2 PRECEDING AND 3 FOLLOWING) - """ # noqa: W505 - pass + """ # noqa: W505, E501 def evaluate( - self, values: list[pyarrow.Array], eval_range: tuple[int, int] - ) -> pyarrow.Scalar: + self, values: list[pa.Array], eval_range: tuple[int, int] + ) -> pa.Scalar: """Evaluate window function on a range of rows in an input partition. This is the simplest and most general function to implement @@ -519,11 +542,10 @@ def evaluate( and evaluation results of ORDER BY expressions. If function has a single argument, `values[1..]` will contain ORDER BY expression results. """ - pass def evaluate_all_with_rank( self, num_rows: int, ranks_in_partition: list[tuple[int, int]] - ) -> pyarrow.Array: + ) -> pa.Array: """Called for window functions that only need the rank of a row. Evaluate the partition evaluator against the partition using @@ -552,7 +574,6 @@ def evaluate_all_with_rank( The user must implement this method if ``include_rank`` returns True. """ - pass def supports_bounded_execution(self) -> bool: """Can the window function be incrementally computed using bounded memory?""" @@ -567,10 +588,6 @@ def include_rank(self) -> bool: return False -if TYPE_CHECKING: - _W = TypeVar("_W", bound=WindowEvaluator) - - class WindowUDF: """Class for performing window user-defined functions (UDF). @@ -582,8 +599,8 @@ def __init__( self, name: str, func: Callable[[], WindowEvaluator], - input_types: list[pyarrow.DataType], - return_type: pyarrow.DataType, + input_types: list[pa.DataType], + return_type: pa.DataType, volatility: Volatility | str, ) -> None: """Instantiate a user-defined window function (UDWF). @@ -607,8 +624,8 @@ def __call__(self, *args: Expr) -> Expr: @staticmethod def udwf( func: Callable[[], WindowEvaluator], - input_types: pyarrow.DataType | list[pyarrow.DataType], - return_type: pyarrow.DataType, + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, volatility: Volatility | str, name: Optional[str] = None, ) -> WindowUDF: @@ -648,16 +665,16 @@ def bias_10() -> BiasedNumbers: Returns: A user-defined window function. - """ # noqa W505 + """ # noqa: W505, E501 if not callable(func): - raise TypeError("`func` must be callable.") - if not isinstance(func.__call__(), WindowEvaluator): - raise TypeError( - "`func` must implement the abstract base class WindowEvaluator" - ) + msg = "`func` must be callable." + raise TypeError(msg) + if not isinstance(func(), WindowEvaluator): + msg = "`func` must implement the abstract base class WindowEvaluator" + raise TypeError(msg) if name is None: - name = func.__call__().__class__.__qualname__.lower() - if isinstance(input_types, pyarrow.DataType): + name = func().__class__.__qualname__.lower() + if isinstance(input_types, pa.DataType): input_types = [input_types] return WindowUDF( name=name, @@ -666,3 +683,10 @@ def bias_10() -> BiasedNumbers: return_type=return_type, volatility=volatility, ) + + +# Convenience exports so we can import instead of treating as +# variables at the package root +udf = ScalarUDF.udf +udaf = AggregateUDF.udaf +udwf = WindowUDF.udwf diff --git a/python/tests/generic.py b/python/tests/generic.py index 0177e2df0..1b98fdf9e 100644 --- a/python/tests/generic.py +++ b/python/tests/generic.py @@ -16,6 +16,7 @@ # under the License. import datetime +from datetime import timezone import numpy as np import pyarrow as pa @@ -26,29 +27,29 @@ def data(): - np.random.seed(1) + rng = np.random.default_rng(1) data = np.concatenate( [ - np.random.normal(0, 0.01, size=50), - np.random.normal(50, 0.01, size=50), + rng.normal(0, 0.01, size=50), + rng.normal(50, 0.01, size=50), ] ) return pa.array(data) def data_with_nans(): - np.random.seed(0) - data = np.random.normal(0, 0.01, size=50) - mask = np.random.randint(0, 2, size=50) + rng = np.random.default_rng(0) + data = rng.normal(0, 0.01, size=50) + mask = rng.normal(0, 2, size=50) data[mask == 0] = np.nan return data def data_datetime(f): data = [ - datetime.datetime.now(), - datetime.datetime.now() - datetime.timedelta(days=1), - datetime.datetime.now() + datetime.timedelta(days=1), + datetime.datetime.now(tz=timezone.utc), + datetime.datetime.now(tz=timezone.utc) - datetime.timedelta(days=1), + datetime.datetime.now(tz=timezone.utc) + datetime.timedelta(days=1), ] return pa.array(data, type=pa.timestamp(f), mask=np.array([False, True, False])) diff --git a/python/tests/test_aggregation.py b/python/tests/test_aggregation.py index 5ef46131b..61b1c7d80 100644 --- a/python/tests/test_aggregation.py +++ b/python/tests/test_aggregation.py @@ -66,7 +66,7 @@ def df_aggregate_100(): @pytest.mark.parametrize( - "agg_expr, calc_expected", + ("agg_expr", "calc_expected"), [ (f.avg(column("a")), lambda a, b, c, d: np.array(np.average(a))), ( @@ -114,7 +114,7 @@ def test_aggregation_stats(df, agg_expr, calc_expected): @pytest.mark.parametrize( - "agg_expr, expected, array_sort", + ("agg_expr", "expected", "array_sort"), [ (f.approx_distinct(column("b")), pa.array([2], type=pa.uint64()), False), ( @@ -182,12 +182,11 @@ def test_aggregation(df, agg_expr, expected, array_sort): agg_df.show() result = agg_df.collect()[0] - print(result) assert result.column(0) == expected @pytest.mark.parametrize( - "name,expr,expected", + ("name", "expr", "expected"), [ ( "approx_percentile_cont", @@ -299,7 +298,9 @@ def test_aggregate_100(df_aggregate_100, name, expr, expected): ] -@pytest.mark.parametrize("name,expr,result", data_test_bitwise_and_boolean_functions) +@pytest.mark.parametrize( + ("name", "expr", "result"), data_test_bitwise_and_boolean_functions +) def test_bit_and_bool_fns(df, name, expr, result): df = df.aggregate([], [expr.alias(name)]) @@ -311,7 +312,7 @@ def test_bit_and_bool_fns(df, name, expr, result): @pytest.mark.parametrize( - "name,expr,result", + ("name", "expr", "result"), [ ("first_value", f.first_value(column("a")), [0, 4]), ( @@ -361,7 +362,6 @@ def test_bit_and_bool_fns(df, name, expr, result): ), [8, 9], ), - ("first_value", f.first_value(column("a")), [0, 4]), ( "nth_value_ordered", f.nth_value(column("a"), 2, order_by=[column("a").sort(ascending=False)]), @@ -401,7 +401,7 @@ def test_first_last_value(df_partitioned, name, expr, result) -> None: @pytest.mark.parametrize( - "name,expr,result", + ("name", "expr", "result"), [ ("string_agg", f.string_agg(column("a"), ","), "one,two,three,two"), ("string_agg", f.string_agg(column("b"), ""), "03124"), diff --git a/python/tests/test_catalog.py b/python/tests/test_catalog.py index 214f6b165..23b328458 100644 --- a/python/tests/test_catalog.py +++ b/python/tests/test_catalog.py @@ -19,6 +19,9 @@ import pytest +# Note we take in `database` as a variable even though we don't use +# it because that will cause the fixture to set up the context with +# the tables we need. def test_basic(ctx, database): with pytest.raises(KeyError): ctx.catalog("non-existent") @@ -26,10 +29,10 @@ def test_basic(ctx, database): default = ctx.catalog() assert default.names() == ["public"] - for database in [default.database("public"), default.database()]: - assert database.names() == {"csv1", "csv", "csv2"} + for db in [default.database("public"), default.database()]: + assert db.names() == {"csv1", "csv", "csv2"} - table = database.table("csv") + table = db.table("csv") assert table.kind == "physical" assert table.schema == pa.schema( [ diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 91046e6b8..7a0a7aa08 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -16,7 +16,6 @@ # under the License. import datetime as dt import gzip -import os import pathlib import pyarrow as pa @@ -45,7 +44,7 @@ def test_create_context_runtime_config_only(): SessionContext(runtime=RuntimeEnvBuilder()) -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_runtime_configs(tmp_path, path_to_str): path1 = tmp_path / "dir1" path2 = tmp_path / "dir2" @@ -62,7 +61,7 @@ def test_runtime_configs(tmp_path, path_to_str): assert db is not None -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_temporary_files(tmp_path, path_to_str): path = str(tmp_path) if path_to_str else tmp_path @@ -79,14 +78,14 @@ def test_create_context_with_all_valid_args(): runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() - .with_create_default_catalog_and_schema(True) + .with_create_default_catalog_and_schema(enabled=True) .with_default_catalog_and_schema("foo", "bar") .with_target_partitions(1) - .with_information_schema(True) - .with_repartition_joins(False) - .with_repartition_aggregations(False) - .with_repartition_windows(False) - .with_parquet_pruning(False) + .with_information_schema(enabled=True) + .with_repartition_joins(enabled=False) + .with_repartition_aggregations(enabled=False) + .with_repartition_windows(enabled=False) + .with_parquet_pruning(enabled=False) ) ctx = SessionContext(config, runtime) @@ -167,7 +166,7 @@ def test_from_arrow_table(ctx): def record_batch_generator(num_batches: int): schema = pa.schema([("a", pa.int64()), ("b", pa.int64())]) - for i in range(num_batches): + for _i in range(num_batches): yield pa.RecordBatch.from_arrays( [pa.array([1, 2, 3]), pa.array([4, 5, 6])], schema=schema ) @@ -492,10 +491,10 @@ def test_table_not_found(ctx): def test_read_json(ctx): - path = os.path.dirname(os.path.abspath(__file__)) + path = pathlib.Path(__file__).parent.resolve() # Default - test_data_path = os.path.join(path, "data_test_context", "data.json") + test_data_path = path / "data_test_context" / "data.json" df = ctx.read_json(test_data_path) result = df.collect() @@ -515,7 +514,7 @@ def test_read_json(ctx): assert result[0].schema == schema # File extension - test_data_path = os.path.join(path, "data_test_context", "data.json") + test_data_path = path / "data_test_context" / "data.json" df = ctx.read_json(test_data_path, file_extension=".json") result = df.collect() @@ -524,15 +523,17 @@ def test_read_json(ctx): def test_read_json_compressed(ctx, tmp_path): - path = os.path.dirname(os.path.abspath(__file__)) - test_data_path = os.path.join(path, "data_test_context", "data.json") + path = pathlib.Path(__file__).parent.resolve() + test_data_path = path / "data_test_context" / "data.json" # File compression type gzip_path = tmp_path / "data.json.gz" - with open(test_data_path, "rb") as csv_file: - with gzip.open(gzip_path, "wb") as gzipped_file: - gzipped_file.writelines(csv_file) + with ( + pathlib.Path.open(test_data_path, "rb") as csv_file, + gzip.open(gzip_path, "wb") as gzipped_file, + ): + gzipped_file.writelines(csv_file) df = ctx.read_json(gzip_path, file_extension=".gz", file_compression_type="gz") result = df.collect() @@ -563,14 +564,16 @@ def test_read_csv_list(ctx): def test_read_csv_compressed(ctx, tmp_path): - test_data_path = "testing/data/csv/aggregate_test_100.csv" + test_data_path = pathlib.Path("testing/data/csv/aggregate_test_100.csv") # File compression type gzip_path = tmp_path / "aggregate_test_100.csv.gz" - with open(test_data_path, "rb") as csv_file: - with gzip.open(gzip_path, "wb") as gzipped_file: - gzipped_file.writelines(csv_file) + with ( + pathlib.Path.open(test_data_path, "rb") as csv_file, + gzip.open(gzip_path, "wb") as gzipped_file, + ): + gzipped_file.writelines(csv_file) csv_df = ctx.read_csv(gzip_path, file_extension=".gz", file_compression_type="gz") csv_df.select(column("c1")).show() @@ -603,7 +606,7 @@ def test_create_sql_options(): def test_sql_with_options_no_ddl(ctx): sql = "CREATE TABLE IF NOT EXISTS valuetable AS VALUES(1,'HELLO'),(12,'DATAFUSION')" ctx.sql(sql) - options = SQLOptions().with_allow_ddl(False) + options = SQLOptions().with_allow_ddl(allow=False) with pytest.raises(Exception, match="DDL"): ctx.sql_with_options(sql, options=options) @@ -618,7 +621,7 @@ def test_sql_with_options_no_dml(ctx): ctx.register_dataset(table_name, dataset) sql = f'INSERT INTO "{table_name}" VALUES (1, 2), (2, 3);' ctx.sql(sql) - options = SQLOptions().with_allow_dml(False) + options = SQLOptions().with_allow_dml(allow=False) with pytest.raises(Exception, match="DML"): ctx.sql_with_options(sql, options=options) @@ -626,6 +629,6 @@ def test_sql_with_options_no_dml(ctx): def test_sql_with_options_no_statements(ctx): sql = "SET time zone = 1;" ctx.sql(sql) - options = SQLOptions().with_allow_statements(False) + options = SQLOptions().with_allow_statements(allow=False) with pytest.raises(Exception, match="SetVariable"): ctx.sql_with_options(sql, options=options) diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index c636e896a..d084f12dd 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -339,7 +339,7 @@ def test_join(): # Verify we don't make a breaking change to pre-43.0.0 # where users would pass join_keys as a positional argument - df2 = df.join(df1, (["a"], ["a"]), how="inner") # type: ignore + df2 = df.join(df1, (["a"], ["a"]), how="inner") df2.show() df2 = df2.sort(column("l.a")) table = pa.Table.from_batches(df2.collect()) @@ -375,17 +375,17 @@ def test_join_invalid_params(): with pytest.raises( ValueError, match=r"`left_on` or `right_on` should not provided with `on`" ): - df2 = df.join(df1, on="a", how="inner", right_on="test") # type: ignore + df2 = df.join(df1, on="a", how="inner", right_on="test") with pytest.raises( ValueError, match=r"`left_on` and `right_on` should both be provided." ): - df2 = df.join(df1, left_on="a", how="inner") # type: ignore + df2 = df.join(df1, left_on="a", how="inner") with pytest.raises( ValueError, match=r"either `on` or `left_on` and `right_on` should be provided." ): - df2 = df.join(df1, how="inner") # type: ignore + df2 = df.join(df1, how="inner") def test_join_on(): @@ -567,7 +567,7 @@ def test_distinct(): ] -@pytest.mark.parametrize("name,expr,result", data_test_window_functions) +@pytest.mark.parametrize(("name", "expr", "result"), data_test_window_functions) def test_window_functions(partitioned_df, name, expr, result): df = partitioned_df.select( column("a"), column("b"), column("c"), f.alias(expr, name) @@ -731,7 +731,7 @@ def test_execution_plan(aggregate_df): plan = aggregate_df.execution_plan() expected = ( - "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[sum(test.c2)]\n" # noqa: E501 + "AggregateExec: mode=FinalPartitioned, gby=[c1@0 as c1], aggr=[sum(test.c2)]\n" ) assert expected == plan.display() @@ -756,7 +756,7 @@ def test_execution_plan(aggregate_df): ctx = SessionContext() rows_returned = 0 - for idx in range(0, plan.partition_count): + for idx in range(plan.partition_count): stream = ctx.execute(plan, idx) try: batch = stream.next() @@ -885,7 +885,7 @@ def test_union_distinct(ctx): ) df_c = ctx.create_dataframe([[batch]]).sort(column("a")) - df_a_u_b = df_a.union(df_b, True).sort(column("a")) + df_a_u_b = df_a.union(df_b, distinct=True).sort(column("a")) assert df_c.collect() == df_a_u_b.collect() assert df_c.collect() == df_a_u_b.collect() @@ -954,8 +954,6 @@ def test_to_arrow_table(df): def test_execute_stream(df): stream = df.execute_stream() - for s in stream: - print(type(s)) assert all(batch is not None for batch in stream) assert not list(stream) # after one iteration the generator must be exhausted @@ -969,7 +967,7 @@ def test_execute_stream_to_arrow_table(df, schema): (batch.to_pyarrow() for batch in stream), schema=df.schema() ) else: - pyarrow_table = pa.Table.from_batches((batch.to_pyarrow() for batch in stream)) + pyarrow_table = pa.Table.from_batches(batch.to_pyarrow() for batch in stream) assert isinstance(pyarrow_table, pa.Table) assert pyarrow_table.shape == (3, 3) @@ -1033,7 +1031,7 @@ def test_describe(df): } -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_write_csv(ctx, df, tmp_path, path_to_str): path = str(tmp_path) if path_to_str else tmp_path @@ -1046,7 +1044,7 @@ def test_write_csv(ctx, df, tmp_path, path_to_str): assert result == expected -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_write_json(ctx, df, tmp_path, path_to_str): path = str(tmp_path) if path_to_str else tmp_path @@ -1059,7 +1057,7 @@ def test_write_json(ctx, df, tmp_path, path_to_str): assert result == expected -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_write_parquet(df, tmp_path, path_to_str): path = str(tmp_path) if path_to_str else tmp_path @@ -1071,7 +1069,7 @@ def test_write_parquet(df, tmp_path, path_to_str): @pytest.mark.parametrize( - "compression, compression_level", + ("compression", "compression_level"), [("gzip", 6), ("brotli", 7), ("zstd", 15)], ) def test_write_compressed_parquet(df, tmp_path, compression, compression_level): @@ -1082,7 +1080,7 @@ def test_write_compressed_parquet(df, tmp_path, compression, compression_level): ) # test that the actual compression scheme is the one written - for root, dirs, files in os.walk(path): + for _root, _dirs, files in os.walk(path): for file in files: if file.endswith(".parquet"): metadata = pq.ParquetFile(tmp_path / file).metadata.to_dict() @@ -1097,7 +1095,7 @@ def test_write_compressed_parquet(df, tmp_path, compression, compression_level): @pytest.mark.parametrize( - "compression, compression_level", + ("compression", "compression_level"), [("gzip", 12), ("brotli", 15), ("zstd", 23), ("wrong", 12)], ) def test_write_compressed_parquet_wrong_compression_level( @@ -1152,7 +1150,7 @@ def test_dataframe_export(df) -> None: table = pa.table(df, schema=desired_schema) assert table.num_columns == 1 assert table.num_rows == 3 - for i in range(0, 3): + for i in range(3): assert table[0][i].as_py() is None # Expect an error when we cannot convert schema @@ -1186,8 +1184,8 @@ def add_with_parameter(df_internal, value: Any) -> DataFrame: result = df.to_pydict() assert result["a"] == [1, 2, 3] - assert result["string_col"] == ["string data" for _i in range(0, 3)] - assert result["new_col"] == [3 for _i in range(0, 3)] + assert result["string_col"] == ["string data" for _i in range(3)] + assert result["new_col"] == [3 for _i in range(3)] def test_dataframe_repr_html(df) -> None: diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 354c7e180..926e69845 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -85,18 +85,14 @@ def test_limit(test_ctx): plan = plan.to_variant() assert isinstance(plan, Limit) - # TODO: Upstream now has expressions for skip and fetch - # REF: https://github.com/apache/datafusion/pull/12836 - # assert plan.skip() == 0 + assert "Skip: None" in str(plan) df = test_ctx.sql("select c1 from test LIMIT 10 OFFSET 5") plan = df.logical_plan() plan = plan.to_variant() assert isinstance(plan, Limit) - # TODO: Upstream now has expressions for skip and fetch - # REF: https://github.com/apache/datafusion/pull/12836 - # assert plan.skip() == 5 + assert "Skip: Some(Literal(Int64(5)))" in str(plan) def test_aggregate_query(test_ctx): @@ -165,6 +161,7 @@ def traverse_logical_plan(plan): res = traverse_logical_plan(input_plan) if res is not None: return res + return None ctx = SessionContext() data = {"id": [1, 2, 3], "name": ["Alice", "Bob", "Charlie"]} @@ -176,7 +173,7 @@ def traverse_logical_plan(plan): assert variant.expr().to_variant().qualified_name() == "table1.name" assert ( str(variant.list()) - == '[Expr(Utf8("dfa")), Expr(Utf8("ad")), Expr(Utf8("dfre")), Expr(Utf8("vsa"))]' + == '[Expr(Utf8("dfa")), Expr(Utf8("ad")), Expr(Utf8("dfre")), Expr(Utf8("vsa"))]' # noqa: E501 ) assert not variant.negated() diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index fca05bb8f..ed88a16e3 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. import math -from datetime import datetime +from datetime import datetime, timezone import numpy as np import pyarrow as pa @@ -25,6 +25,8 @@ np.seterr(invalid="ignore") +DEFAULT_TZ = timezone.utc + @pytest.fixture def df(): @@ -37,9 +39,9 @@ def df(): pa.array(["hello ", " world ", " !"], type=pa.string_view()), pa.array( [ - datetime(2022, 12, 31), - datetime(2027, 6, 26), - datetime(2020, 7, 2), + datetime(2022, 12, 31, tzinfo=DEFAULT_TZ), + datetime(2027, 6, 26, tzinfo=DEFAULT_TZ), + datetime(2020, 7, 2, tzinfo=DEFAULT_TZ), ] ), pa.array([False, True, True]), @@ -221,12 +223,12 @@ def py_indexof(arr, v): def py_arr_remove(arr, v, n=None): new_arr = arr[:] found = 0 - while found != n: - try: + try: + while found != n: new_arr.remove(v) found += 1 - except ValueError: - break + except ValueError: + pass return new_arr @@ -234,13 +236,13 @@ def py_arr_remove(arr, v, n=None): def py_arr_replace(arr, from_, to, n=None): new_arr = arr[:] found = 0 - while found != n: - try: + try: + while found != n: idx = new_arr.index(from_) new_arr[idx] = to found += 1 - except ValueError: - break + except ValueError: + pass return new_arr @@ -268,266 +270,266 @@ def py_flatten(arr): @pytest.mark.parametrize( ("stmt", "py_expr"), [ - [ + ( lambda col: f.array_append(col, literal(99.0)), lambda data: [np.append(arr, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.array_push_back(col, literal(99.0)), lambda data: [np.append(arr, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_append(col, literal(99.0)), lambda data: [np.append(arr, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_push_back(col, literal(99.0)), lambda data: [np.append(arr, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.array_concat(col, col), lambda data: [np.concatenate([arr, arr]) for arr in data], - ], - [ + ), + ( lambda col: f.array_cat(col, col), lambda data: [np.concatenate([arr, arr]) for arr in data], - ], - [ + ), + ( lambda col: f.list_cat(col, col), lambda data: [np.concatenate([arr, arr]) for arr in data], - ], - [ + ), + ( lambda col: f.list_concat(col, col), lambda data: [np.concatenate([arr, arr]) for arr in data], - ], - [ + ), + ( lambda col: f.array_dims(col), lambda data: [[len(r)] for r in data], - ], - [ + ), + ( lambda col: f.array_distinct(col), lambda data: [list(set(r)) for r in data], - ], - [ + ), + ( lambda col: f.list_distinct(col), lambda data: [list(set(r)) for r in data], - ], - [ + ), + ( lambda col: f.list_dims(col), lambda data: [[len(r)] for r in data], - ], - [ + ), + ( lambda col: f.array_element(col, literal(1)), lambda data: [r[0] for r in data], - ], - [ + ), + ( lambda col: f.array_empty(col), lambda data: [len(r) == 0 for r in data], - ], - [ + ), + ( lambda col: f.empty(col), lambda data: [len(r) == 0 for r in data], - ], - [ + ), + ( lambda col: f.array_extract(col, literal(1)), lambda data: [r[0] for r in data], - ], - [ + ), + ( lambda col: f.list_element(col, literal(1)), lambda data: [r[0] for r in data], - ], - [ + ), + ( lambda col: f.list_extract(col, literal(1)), lambda data: [r[0] for r in data], - ], - [ + ), + ( lambda col: f.array_length(col), lambda data: [len(r) for r in data], - ], - [ + ), + ( lambda col: f.list_length(col), lambda data: [len(r) for r in data], - ], - [ + ), + ( lambda col: f.array_has(col, literal(1.0)), lambda data: [1.0 in r for r in data], - ], - [ + ), + ( lambda col: f.array_has_all( col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) ), lambda data: [np.all([v in r for v in [1.0, 3.0, 5.0]]) for r in data], - ], - [ + ), + ( lambda col: f.array_has_any( col, f.make_array(*[literal(v) for v in [1.0, 3.0, 5.0]]) ), lambda data: [np.any([v in r for v in [1.0, 3.0, 5.0]]) for r in data], - ], - [ + ), + ( lambda col: f.array_position(col, literal(1.0)), lambda data: [py_indexof(r, 1.0) for r in data], - ], - [ + ), + ( lambda col: f.array_indexof(col, literal(1.0)), lambda data: [py_indexof(r, 1.0) for r in data], - ], - [ + ), + ( lambda col: f.list_position(col, literal(1.0)), lambda data: [py_indexof(r, 1.0) for r in data], - ], - [ + ), + ( lambda col: f.list_indexof(col, literal(1.0)), lambda data: [py_indexof(r, 1.0) for r in data], - ], - [ + ), + ( lambda col: f.array_positions(col, literal(1.0)), lambda data: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], - ], - [ + ), + ( lambda col: f.list_positions(col, literal(1.0)), lambda data: [[i + 1 for i, _v in enumerate(r) if _v == 1.0] for r in data], - ], - [ + ), + ( lambda col: f.array_ndims(col), lambda data: [np.array(r).ndim for r in data], - ], - [ + ), + ( lambda col: f.list_ndims(col), lambda data: [np.array(r).ndim for r in data], - ], - [ + ), + ( lambda col: f.array_prepend(literal(99.0), col), lambda data: [np.insert(arr, 0, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.array_push_front(literal(99.0), col), lambda data: [np.insert(arr, 0, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_prepend(literal(99.0), col), lambda data: [np.insert(arr, 0, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_push_front(literal(99.0), col), lambda data: [np.insert(arr, 0, 99.0) for arr in data], - ], - [ + ), + ( lambda col: f.array_pop_back(col), lambda data: [arr[:-1] for arr in data], - ], - [ + ), + ( lambda col: f.array_pop_front(col), lambda data: [arr[1:] for arr in data], - ], - [ + ), + ( lambda col: f.array_remove(col, literal(3.0)), lambda data: [py_arr_remove(arr, 3.0, 1) for arr in data], - ], - [ + ), + ( lambda col: f.list_remove(col, literal(3.0)), lambda data: [py_arr_remove(arr, 3.0, 1) for arr in data], - ], - [ + ), + ( lambda col: f.array_remove_n(col, literal(3.0), literal(2)), lambda data: [py_arr_remove(arr, 3.0, 2) for arr in data], - ], - [ + ), + ( lambda col: f.list_remove_n(col, literal(3.0), literal(2)), lambda data: [py_arr_remove(arr, 3.0, 2) for arr in data], - ], - [ + ), + ( lambda col: f.array_remove_all(col, literal(3.0)), lambda data: [py_arr_remove(arr, 3.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_remove_all(col, literal(3.0)), lambda data: [py_arr_remove(arr, 3.0) for arr in data], - ], - [ + ), + ( lambda col: f.array_repeat(col, literal(2)), lambda data: [[arr] * 2 for arr in data], - ], - [ + ), + ( lambda col: f.list_repeat(col, literal(2)), lambda data: [[arr] * 2 for arr in data], - ], - [ + ), + ( lambda col: f.array_replace(col, literal(3.0), literal(4.0)), lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], - ], - [ + ), + ( lambda col: f.list_replace(col, literal(3.0), literal(4.0)), lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], - ], - [ + ), + ( lambda col: f.array_replace_n(col, literal(3.0), literal(4.0), literal(1)), lambda data: [py_arr_replace(arr, 3.0, 4.0, 1) for arr in data], - ], - [ + ), + ( lambda col: f.list_replace_n(col, literal(3.0), literal(4.0), literal(2)), lambda data: [py_arr_replace(arr, 3.0, 4.0, 2) for arr in data], - ], - [ + ), + ( lambda col: f.array_replace_all(col, literal(3.0), literal(4.0)), lambda data: [py_arr_replace(arr, 3.0, 4.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_replace_all(col, literal(3.0), literal(4.0)), lambda data: [py_arr_replace(arr, 3.0, 4.0) for arr in data], - ], - [ + ), + ( lambda col: f.array_sort(col, descending=True, null_first=True), lambda data: [np.sort(arr)[::-1] for arr in data], - ], - [ + ), + ( lambda col: f.list_sort(col, descending=False, null_first=False), lambda data: [np.sort(arr) for arr in data], - ], - [ + ), + ( lambda col: f.array_slice(col, literal(2), literal(4)), lambda data: [arr[1:4] for arr in data], - ], + ), pytest.param( lambda col: f.list_slice(col, literal(-1), literal(2)), lambda data: [arr[-1:2] for arr in data], ), - [ + ( lambda col: f.array_intersect(col, literal([3.0, 4.0])), lambda data: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], - ], - [ + ), + ( lambda col: f.list_intersect(col, literal([3.0, 4.0])), lambda data: [np.intersect1d(arr, [3.0, 4.0]) for arr in data], - ], - [ + ), + ( lambda col: f.array_union(col, literal([12.0, 999.0])), lambda data: [np.union1d(arr, [12.0, 999.0]) for arr in data], - ], - [ + ), + ( lambda col: f.list_union(col, literal([12.0, 999.0])), lambda data: [np.union1d(arr, [12.0, 999.0]) for arr in data], - ], - [ + ), + ( lambda col: f.array_except(col, literal([3.0])), lambda data: [np.setdiff1d(arr, [3.0]) for arr in data], - ], - [ + ), + ( lambda col: f.list_except(col, literal([3.0])), lambda data: [np.setdiff1d(arr, [3.0]) for arr in data], - ], - [ + ), + ( lambda col: f.array_resize(col, literal(10), literal(0.0)), lambda data: [py_arr_resize(arr, 10, 0.0) for arr in data], - ], - [ + ), + ( lambda col: f.list_resize(col, literal(10), literal(0.0)), lambda data: [py_arr_resize(arr, 10, 0.0) for arr in data], - ], - [ + ), + ( lambda col: f.range(literal(1), literal(5), literal(2)), lambda data: [np.arange(1, 5, 2)], - ], + ), ], ) def test_array_functions(stmt, py_expr): @@ -611,22 +613,22 @@ def test_make_array_functions(make_func): @pytest.mark.parametrize( ("stmt", "py_expr"), [ - [ + ( f.array_to_string(column("arr"), literal(",")), lambda data: [",".join([str(int(v)) for v in r]) for r in data], - ], - [ + ), + ( f.array_join(column("arr"), literal(",")), lambda data: [",".join([str(int(v)) for v in r]) for r in data], - ], - [ + ), + ( f.list_to_string(column("arr"), literal(",")), lambda data: [",".join([str(int(v)) for v in r]) for r in data], - ], - [ + ), + ( f.list_join(column("arr"), literal(",")), lambda data: [",".join([str(int(v)) for v in r]) for r in data], - ], + ), ], ) def test_array_function_obj_tests(stmt, py_expr): @@ -640,7 +642,7 @@ def test_array_function_obj_tests(stmt, py_expr): @pytest.mark.parametrize( - "function, expected_result", + ("function", "expected_result"), [ ( f.ascii(column("a")), @@ -894,54 +896,72 @@ def test_temporal_functions(df): assert result.column(0) == pa.array([12, 6, 7], type=pa.int32()) assert result.column(1) == pa.array([2022, 2027, 2020], type=pa.int32()) assert result.column(2) == pa.array( - [datetime(2022, 12, 1), datetime(2027, 6, 1), datetime(2020, 7, 1)], - type=pa.timestamp("us"), + [ + datetime(2022, 12, 1, tzinfo=DEFAULT_TZ), + datetime(2027, 6, 1, tzinfo=DEFAULT_TZ), + datetime(2020, 7, 1, tzinfo=DEFAULT_TZ), + ], + type=pa.timestamp("ns", tz=DEFAULT_TZ), ) assert result.column(3) == pa.array( - [datetime(2022, 12, 31), datetime(2027, 6, 26), datetime(2020, 7, 2)], - type=pa.timestamp("us"), + [ + datetime(2022, 12, 31, tzinfo=DEFAULT_TZ), + datetime(2027, 6, 26, tzinfo=DEFAULT_TZ), + datetime(2020, 7, 2, tzinfo=DEFAULT_TZ), + ], + type=pa.timestamp("ns", tz=DEFAULT_TZ), ) assert result.column(4) == pa.array( [ - datetime(2022, 12, 30, 23, 47, 30), - datetime(2027, 6, 25, 23, 47, 30), - datetime(2020, 7, 1, 23, 47, 30), + datetime(2022, 12, 30, 23, 47, 30, tzinfo=DEFAULT_TZ), + datetime(2027, 6, 25, 23, 47, 30, tzinfo=DEFAULT_TZ), + datetime(2020, 7, 1, 23, 47, 30, tzinfo=DEFAULT_TZ), ], - type=pa.timestamp("ns"), + type=pa.timestamp("ns", tz=DEFAULT_TZ), ) assert result.column(5) == pa.array( - [datetime(2023, 1, 10, 20, 52, 54)] * 3, type=pa.timestamp("s") + [datetime(2023, 1, 10, 20, 52, 54, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("s"), ) assert result.column(6) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + [datetime(2023, 9, 7, 5, 6, 14, 523952, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("ns"), ) assert result.column(7) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14)] * 3, type=pa.timestamp("s") + [datetime(2023, 9, 7, 5, 6, 14, tzinfo=DEFAULT_TZ)] * 3, type=pa.timestamp("s") ) assert result.column(8) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523000)] * 3, type=pa.timestamp("ms") + [datetime(2023, 9, 7, 5, 6, 14, 523000, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("ms"), ) assert result.column(9) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") + [datetime(2023, 9, 7, 5, 6, 14, 523952, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("us"), ) assert result.column(10) == pa.array([31, 26, 2], type=pa.int32()) assert result.column(11) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + [datetime(2023, 9, 7, 5, 6, 14, 523952, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("ns"), ) assert result.column(12) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14)] * 3, type=pa.timestamp("s") + [datetime(2023, 9, 7, 5, 6, 14, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("s"), ) assert result.column(13) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523000)] * 3, type=pa.timestamp("ms") + [datetime(2023, 9, 7, 5, 6, 14, 523000, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("ms"), ) assert result.column(14) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") + [datetime(2023, 9, 7, 5, 6, 14, 523952, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("us"), ) assert result.column(15) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + [datetime(2023, 9, 7, 5, 6, 14, 523952, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("ns"), ) assert result.column(16) == pa.array( - [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("ns") + [datetime(2023, 9, 7, 5, 6, 14, 523952, tzinfo=DEFAULT_TZ)] * 3, + type=pa.timestamp("ns"), ) @@ -1057,7 +1077,7 @@ def test_regr_funcs_sql_2(): @pytest.mark.parametrize( - "func, expected", + ("func", "expected"), [ pytest.param(f.regr_slope(column("c2"), column("c1")), [4.6], id="regr_slope"), pytest.param( @@ -1160,7 +1180,7 @@ def test_binary_string_functions(df): @pytest.mark.parametrize( - "python_datatype, name, expected", + ("python_datatype", "name", "expected"), [ pytest.param(bool, "e", pa.bool_(), id="bool"), pytest.param(int, "b", pa.int64(), id="int"), @@ -1179,7 +1199,7 @@ def test_cast(df, python_datatype, name: str, expected): @pytest.mark.parametrize( - "negated, low, high, expected", + ("negated", "low", "high", "expected"), [ pytest.param(False, 3, 5, {"filtered": [4, 5]}), pytest.param(False, 4, 5, {"filtered": [4, 5]}), diff --git a/python/tests/test_imports.py b/python/tests/test_imports.py index 0c155cbde..9ef7ed89a 100644 --- a/python/tests/test_imports.py +++ b/python/tests/test_imports.py @@ -169,14 +169,15 @@ def test_class_module_is_datafusion(): def test_import_from_functions_submodule(): - from datafusion.functions import abs, sin # noqa + from datafusion.functions import abs as df_abs + from datafusion.functions import sin - assert functions.abs is abs + assert functions.abs is df_abs assert functions.sin is sin msg = "cannot import name 'foobar' from 'datafusion.functions'" with pytest.raises(ImportError, match=msg): - from datafusion.functions import foobar # noqa + from datafusion.functions import foobar # noqa: F401 def test_classes_are_inheritable(): diff --git a/python/tests/test_input.py b/python/tests/test_input.py index 806471357..4663f6148 100644 --- a/python/tests/test_input.py +++ b/python/tests/test_input.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import os +import pathlib from datafusion.input.location import LocationInputPlugin @@ -23,10 +23,10 @@ def test_location_input(): location_input = LocationInputPlugin() - cwd = os.getcwd() - input_file = cwd + "/testing/data/parquet/generated_simple_numerics/blogs.parquet" + cwd = pathlib.Path.cwd() + input_file = cwd / "testing/data/parquet/generated_simple_numerics/blogs.parquet" table_name = "blog" - tbl = location_input.build_table(input_file, table_name) - assert "blog" == tbl.name - assert 3 == len(tbl.columns) + tbl = location_input.build_table(str(input_file), table_name) + assert tbl.name == "blog" + assert len(tbl.columns) == 3 assert "blogs.parquet" in tbl.filepaths[0] diff --git a/python/tests/test_io.py b/python/tests/test_io.py index 21ad188ee..7ca509689 100644 --- a/python/tests/test_io.py +++ b/python/tests/test_io.py @@ -14,8 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import os -import pathlib +from pathlib import Path import pyarrow as pa from datafusion import column @@ -23,10 +22,10 @@ def test_read_json_global_ctx(ctx): - path = os.path.dirname(os.path.abspath(__file__)) + path = Path(__file__).parent.resolve() # Default - test_data_path = os.path.join(path, "data_test_context", "data.json") + test_data_path = Path(path) / "data_test_context" / "data.json" df = read_json(test_data_path) result = df.collect() @@ -46,7 +45,7 @@ def test_read_json_global_ctx(ctx): assert result[0].schema == schema # File extension - test_data_path = os.path.join(path, "data_test_context", "data.json") + test_data_path = Path(path) / "data_test_context" / "data.json" df = read_json(test_data_path, file_extension=".json") result = df.collect() @@ -59,7 +58,7 @@ def test_read_parquet_global(): parquet_df.show() assert parquet_df is not None - path = pathlib.Path.cwd() / "parquet/data/alltypes_plain.parquet" + path = Path.cwd() / "parquet/data/alltypes_plain.parquet" parquet_df = read_parquet(path=path) assert parquet_df is not None @@ -90,6 +89,6 @@ def test_read_avro(): avro_df.show() assert avro_df is not None - path = pathlib.Path.cwd() / "testing/data/avro/alltypes_plain.avro" + path = Path.cwd() / "testing/data/avro/alltypes_plain.avro" avro_df = read_avro(path=path) assert avro_df is not None diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py index 862f745bf..b6348e3a0 100644 --- a/python/tests/test_sql.py +++ b/python/tests/test_sql.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. import gzip -import os +from pathlib import Path import numpy as np import pyarrow as pa @@ -47,9 +47,8 @@ def test_register_csv(ctx, tmp_path): ) write_csv(table, path) - with open(path, "rb") as csv_file: - with gzip.open(gzip_path, "wb") as gzipped_file: - gzipped_file.writelines(csv_file) + with Path.open(path, "rb") as csv_file, gzip.open(gzip_path, "wb") as gzipped_file: + gzipped_file.writelines(csv_file) ctx.register_csv("csv", path) ctx.register_csv("csv1", str(path)) @@ -158,7 +157,7 @@ def test_register_parquet(ctx, tmp_path): assert result.to_pydict() == {"cnt": [100]} -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_register_parquet_partitioned(ctx, tmp_path, path_to_str): dir_root = tmp_path / "dataset_parquet_partitioned" dir_root.mkdir(exist_ok=False) @@ -194,7 +193,7 @@ def test_register_parquet_partitioned(ctx, tmp_path, path_to_str): assert dict(zip(rd["grp"], rd["cnt"])) == {"a": 3, "b": 1} -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_register_dataset(ctx, tmp_path, path_to_str): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) path = str(path) if path_to_str else path @@ -209,13 +208,15 @@ def test_register_dataset(ctx, tmp_path, path_to_str): def test_register_json(ctx, tmp_path): - path = os.path.dirname(os.path.abspath(__file__)) - test_data_path = os.path.join(path, "data_test_context", "data.json") + path = Path(__file__).parent.resolve() + test_data_path = Path(path) / "data_test_context" / "data.json" gzip_path = tmp_path / "data.json.gz" - with open(test_data_path, "rb") as json_file: - with gzip.open(gzip_path, "wb") as gzipped_file: - gzipped_file.writelines(json_file) + with ( + Path.open(test_data_path, "rb") as json_file, + gzip.open(gzip_path, "wb") as gzipped_file, + ): + gzipped_file.writelines(json_file) ctx.register_json("json", test_data_path) ctx.register_json("json1", str(test_data_path)) @@ -470,16 +471,18 @@ def test_simple_select(ctx, tmp_path, arr): # In DF 43.0.0 we now default to having BinaryView and StringView # so the array that is saved to the parquet is slightly different # than the array read. Convert to values for comparison. - if isinstance(result, pa.BinaryViewArray) or isinstance(result, pa.StringViewArray): + if isinstance(result, (pa.BinaryViewArray, pa.StringViewArray)): arr = arr.tolist() result = result.tolist() np.testing.assert_equal(result, arr) -@pytest.mark.parametrize("file_sort_order", (None, [[col("int").sort(True, True)]])) -@pytest.mark.parametrize("pass_schema", (True, False)) -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize( + "file_sort_order", [None, [[col("int").sort(ascending=True, nulls_first=True)]]] +) +@pytest.mark.parametrize("pass_schema", [True, False]) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_register_listing_table( ctx, tmp_path, pass_schema, file_sort_order, path_to_str ): @@ -528,7 +531,7 @@ def test_register_listing_table( assert dict(zip(rd["grp"], rd["count"])) == {"a": 5, "b": 2} result = ctx.sql( - "SELECT grp, COUNT(*) AS count FROM my_table WHERE date_id=20201005 GROUP BY grp" + "SELECT grp, COUNT(*) AS count FROM my_table WHERE date_id=20201005 GROUP BY grp" # noqa: E501 ).collect() result = pa.Table.from_batches(result) diff --git a/python/tests/test_store.py b/python/tests/test_store.py index 53ffc3acf..ac9af98f3 100644 --- a/python/tests/test_store.py +++ b/python/tests/test_store.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -import os +from pathlib import Path import pytest from datafusion import SessionContext @@ -23,17 +23,16 @@ @pytest.fixture def ctx(): - ctx = SessionContext() - return ctx + return SessionContext() def test_read_parquet(ctx): ctx.register_parquet( "test", - f"file://{os.getcwd()}/parquet/data/alltypes_plain.parquet", - [], - True, - ".parquet", + f"file://{Path.cwd()}/parquet/data/alltypes_plain.parquet", + table_partition_cols=[], + parquet_pruning=True, + file_extension=".parquet", ) df = ctx.sql("SELECT * FROM test") assert isinstance(df.collect(), list) diff --git a/python/tests/test_substrait.py b/python/tests/test_substrait.py index feada7cde..f367a447d 100644 --- a/python/tests/test_substrait.py +++ b/python/tests/test_substrait.py @@ -50,7 +50,7 @@ def test_substrait_serialization(ctx): substrait_plan = ss.Producer.to_substrait_plan(df.logical_plan(), ctx) -@pytest.mark.parametrize("path_to_str", (True, False)) +@pytest.mark.parametrize("path_to_str", [True, False]) def test_substrait_file_serialization(ctx, tmp_path, path_to_str): batch = pa.RecordBatch.from_arrays( [pa.array([1, 2, 3]), pa.array([4, 5, 6])], diff --git a/python/tests/test_udaf.py b/python/tests/test_udaf.py index 97cf81f3c..453ff6f4f 100644 --- a/python/tests/test_udaf.py +++ b/python/tests/test_udaf.py @@ -17,8 +17,6 @@ from __future__ import annotations -from typing import List - import pyarrow as pa import pyarrow.compute as pc import pytest @@ -31,7 +29,7 @@ class Summarize(Accumulator): def __init__(self, initial_value: float = 0.0): self._sum = pa.scalar(initial_value) - def state(self) -> List[pa.Scalar]: + def state(self) -> list[pa.Scalar]: return [self._sum] def update(self, values: pa.Array) -> None: @@ -39,7 +37,7 @@ def update(self, values: pa.Array) -> None: # This breaks on `None` self._sum = pa.scalar(self._sum.as_py() + pc.sum(values).as_py()) - def merge(self, states: List[pa.Array]) -> None: + def merge(self, states: list[pa.Array]) -> None: # Not nice since pyarrow scalars can't be summed yet. # This breaks on `None` self._sum = pa.scalar(self._sum.as_py() + pc.sum(states[0]).as_py()) @@ -56,7 +54,7 @@ class MissingMethods(Accumulator): def __init__(self): self._sum = pa.scalar(0) - def state(self) -> List[pa.Scalar]: + def state(self) -> list[pa.Scalar]: return [self._sum] @@ -86,7 +84,7 @@ def test_errors(df): "evaluate, merge, update)" ) with pytest.raises(Exception, match=msg): - accum = udaf( # noqa F841 + accum = udaf( # noqa: F841 MissingMethods, pa.int64(), pa.int64(), diff --git a/python/tests/test_udwf.py b/python/tests/test_udwf.py index 2fea34aa3..3d6dcf9d8 100644 --- a/python/tests/test_udwf.py +++ b/python/tests/test_udwf.py @@ -298,7 +298,7 @@ def test_udwf_errors(df): ] -@pytest.mark.parametrize("name,expr,expected", data_test_udwf_functions) +@pytest.mark.parametrize(("name", "expr", "expected"), data_test_udwf_functions) def test_udwf_functions(df, name, expr, expected): df = df.select("a", "b", f.round(expr, lit(3)).alias(name)) diff --git a/python/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py index ac064ba95..d7f6f6e35 100644 --- a/python/tests/test_wrapper_coverage.py +++ b/python/tests/test_wrapper_coverage.py @@ -19,6 +19,7 @@ import datafusion.functions import datafusion.object_store import datafusion.substrait +import pytest # EnumType introduced in 3.11. 3.10 and prior it was called EnumMeta. try: @@ -41,10 +42,8 @@ def missing_exports(internal_obj, wrapped_obj) -> None: internal_attr = getattr(internal_obj, attr) wrapped_attr = getattr(wrapped_obj, attr) - if internal_attr is not None: - if wrapped_attr is None: - print("Missing attribute: ", attr) - assert False + if internal_attr is not None and wrapped_attr is None: + pytest.fail(f"Missing attribute: {attr}") if attr in ["__self__", "__class__"]: continue From 3dcf7c7e5c0af0eb3c5e3bdf9c6e33fd4541b070 Mon Sep 17 00:00:00 2001 From: jsai28 <54253219+jsai28@users.noreply.github.com> Date: Thu, 13 Mar 2025 04:09:03 -0600 Subject: [PATCH 275/413] feat/making global context accessible for users (#1060) * Rename _global_ctx to global_ctx * Add global context to python wrapper code * Update context.py * singleton for global context * formatting * remove udf from import * remove _global_instance * formatting * formatting * unnecessary test * fix test_io.py * ran ruff * ran ruff format --- python/datafusion/context.py | 12 +++++++ python/datafusion/io.py | 63 ++++++++++++++++-------------------- python/tests/test_context.py | 18 +++++++++++ src/context.rs | 2 +- 4 files changed, 58 insertions(+), 37 deletions(-) diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 0ab1a908a..58ad9a943 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -496,6 +496,18 @@ def __init__( self.ctx = SessionContextInternal(config, runtime) + @classmethod + def global_ctx(cls) -> SessionContext: + """Retrieve the global context as a `SessionContext` wrapper. + + Returns: + A `SessionContext` object that wraps the global `SessionContextInternal`. + """ + internal_ctx = SessionContextInternal.global_ctx() + wrapper = cls() + wrapper.ctx = internal_ctx + return wrapper + def enable_url_table(self) -> SessionContext: """Control if local files can be queried as tables. diff --git a/python/datafusion/io.py b/python/datafusion/io.py index 3e39703e3..ef5ebf96f 100644 --- a/python/datafusion/io.py +++ b/python/datafusion/io.py @@ -21,10 +21,9 @@ from typing import TYPE_CHECKING +from datafusion.context import SessionContext from datafusion.dataframe import DataFrame -from ._internal import SessionContext as SessionContextInternal - if TYPE_CHECKING: import pathlib @@ -68,16 +67,14 @@ def read_parquet( """ if table_partition_cols is None: table_partition_cols = [] - return DataFrame( - SessionContextInternal._global_ctx().read_parquet( - str(path), - table_partition_cols, - parquet_pruning, - file_extension, - skip_metadata, - schema, - file_sort_order, - ) + return SessionContext.global_ctx().read_parquet( + str(path), + table_partition_cols, + parquet_pruning, + file_extension, + skip_metadata, + schema, + file_sort_order, ) @@ -110,15 +107,13 @@ def read_json( """ if table_partition_cols is None: table_partition_cols = [] - return DataFrame( - SessionContextInternal._global_ctx().read_json( - str(path), - schema, - schema_infer_max_records, - file_extension, - table_partition_cols, - file_compression_type, - ) + return SessionContext.global_ctx().read_json( + str(path), + schema, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, ) @@ -161,17 +156,15 @@ def read_csv( path = [str(p) for p in path] if isinstance(path, list) else str(path) - return DataFrame( - SessionContextInternal._global_ctx().read_csv( - path, - schema, - has_header, - delimiter, - schema_infer_max_records, - file_extension, - table_partition_cols, - file_compression_type, - ) + return SessionContext.global_ctx().read_csv( + path, + schema, + has_header, + delimiter, + schema_infer_max_records, + file_extension, + table_partition_cols, + file_compression_type, ) @@ -198,8 +191,6 @@ def read_avro( """ if file_partition_cols is None: file_partition_cols = [] - return DataFrame( - SessionContextInternal._global_ctx().read_avro( - str(path), schema, file_partition_cols, file_extension - ) + return SessionContext.global_ctx().read_avro( + str(path), schema, file_partition_cols, file_extension ) diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 7a0a7aa08..4a15ac9cf 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -632,3 +632,21 @@ def test_sql_with_options_no_statements(ctx): options = SQLOptions().with_allow_statements(allow=False) with pytest.raises(Exception, match="SetVariable"): ctx.sql_with_options(sql, options=options) + + +@pytest.fixture +def batch(): + return pa.RecordBatch.from_arrays( + [pa.array([4, 5, 6])], + names=["a"], + ) + + +def test_create_dataframe_with_global_ctx(batch): + ctx = SessionContext.global_ctx() + + df = ctx.create_dataframe([[batch]]) + + result = df.collect()[0].column(0) + + assert result == pa.array([4, 5, 6]) diff --git a/src/context.rs b/src/context.rs index 9ba87eb8a..0db0f4d7e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -308,7 +308,7 @@ impl PySessionContext { #[classmethod] #[pyo3(signature = ())] - fn _global_ctx(_cls: &Bound<'_, PyType>) -> PyResult { + fn global_ctx(_cls: &Bound<'_, PyType>) -> PyResult { Ok(Self { ctx: get_global_ctx().clone(), }) From 55141bad7c2270c14742e962d8bab1d4f1be27f5 Mon Sep 17 00:00:00 2001 From: Spaarsh <67336892+Spaarsh@users.noreply.github.com> Date: Fri, 14 Mar 2025 18:26:31 +0530 Subject: [PATCH 276/413] Renaming Internal Structs (#1059) * Renamed Expr to RawExpr * Fixed CI test for exported classes to include RawExpr as well * Fixed CI test for exported classes to check if Expr class covers RawExpr * Generalized Raw* class checking * fixes * fixes * fixed the CI test to not look for Raw classes in the datafusion module * Add additional text to unit test describing operation and ensure wrapped Raw classes are checked * New ruff rule on main * Resolve ruff errors --------- Co-authored-by: Tim Saucer --- python/datafusion/expr.py | 8 ++-- python/tests/test_wrapper_coverage.py | 55 +++++++++++++++++++-------- src/expr.rs | 2 +- 3 files changed, 45 insertions(+), 20 deletions(-) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 702f75aed..77b6c272d 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -193,7 +193,7 @@ class Expr: :ref:`Expressions` in the online documentation for more information. """ - def __init__(self, expr: expr_internal.Expr) -> None: + def __init__(self, expr: expr_internal.RawExpr) -> None: """This constructor should not be called by the end user.""" self.expr = expr @@ -383,7 +383,7 @@ def literal(value: Any) -> Expr: value = pa.scalar(value, type=pa.string_view()) if not isinstance(value, pa.Scalar): value = pa.scalar(value) - return Expr(expr_internal.Expr.literal(value)) + return Expr(expr_internal.RawExpr.literal(value)) @staticmethod def string_literal(value: str) -> Expr: @@ -398,13 +398,13 @@ def string_literal(value: str) -> Expr: """ if isinstance(value, str): value = pa.scalar(value, type=pa.string()) - return Expr(expr_internal.Expr.literal(value)) + return Expr(expr_internal.RawExpr.literal(value)) return Expr.literal(value) @staticmethod def column(value: str) -> Expr: """Creates a new expression representing a column.""" - return Expr(expr_internal.Expr.column(value)) + return Expr(expr_internal.RawExpr.column(value)) def alias(self, name: str) -> Expr: """Assign a name to the expression.""" diff --git a/python/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py index d7f6f6e35..a2de2d32b 100644 --- a/python/tests/test_wrapper_coverage.py +++ b/python/tests/test_wrapper_coverage.py @@ -28,37 +28,62 @@ from enum import EnumMeta as EnumType -def missing_exports(internal_obj, wrapped_obj) -> None: - # Special case enums - just make sure they exist since dir() - # and other functions get overridden. +def missing_exports(internal_obj, wrapped_obj) -> None: # noqa: C901 + """ + Identify if any of the rust exposted structs or functions do not have wrappers. + + Special handling for: + - Raw* classes: Internal implementation details that shouldn't be exposed + - _global_ctx: Internal implementation detail + - __self__, __class__: Python special attributes + """ + # Special case enums - EnumType overrides a some of the internal functions, + # so check all of the values exist and move on if isinstance(wrapped_obj, EnumType): + expected_values = [v for v in dir(internal_obj) if not v.startswith("__")] + for value in expected_values: + assert value in dir(wrapped_obj) return - for attr in dir(internal_obj): - if attr in ["_global_ctx"]: - continue - assert attr in dir(wrapped_obj) + for internal_attr_name in dir(internal_obj): + wrapped_attr_name = internal_attr_name.removeprefix("Raw") + assert wrapped_attr_name in dir(wrapped_obj) - internal_attr = getattr(internal_obj, attr) - wrapped_attr = getattr(wrapped_obj, attr) + internal_attr = getattr(internal_obj, internal_attr_name) + wrapped_attr = getattr(wrapped_obj, wrapped_attr_name) - if internal_attr is not None and wrapped_attr is None: - pytest.fail(f"Missing attribute: {attr}") + # There are some auto generated attributes that can be None, such as + # __kwdefaults__ and __doc__. As long as these are None on the internal + # object, it's okay to skip them. However if they do exist on the internal + # object they must also exist on the wrapped object. + if internal_attr is not None: + if wrapped_attr is None: + pytest.fail(f"Missing attribute: {internal_attr_name}") - if attr in ["__self__", "__class__"]: + if internal_attr_name in ["__self__", "__class__"]: continue + if isinstance(internal_attr, list): assert isinstance(wrapped_attr, list) + + # We have cases like __all__ that are a list and we want to be certain that + # every value in the list in the internal object is also in the wrapper list for val in internal_attr: - assert val in wrapped_attr + if isinstance(val, str) and val.startswith("Raw"): + assert val[3:] in wrapped_attr + else: + assert val in wrapped_attr elif hasattr(internal_attr, "__dict__"): + # Check all submodules recursively missing_exports(internal_attr, wrapped_attr) def test_datafusion_missing_exports() -> None: """Check for any missing python exports. - This test verifies that every exposed class, attribute, and function in - the internal (pyo3) module is also exposed in our python wrappers. + This test verifies that every exposed class, attribute, + and function in the internal (pyo3) module - datafusion._internal + is also exposed in our python wrappers - datafusion - + i.e., the ones exposed to the public. """ missing_exports(datafusion._internal, datafusion) diff --git a/src/expr.rs b/src/expr.rs index e750be6a4..d3c528eb4 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -101,7 +101,7 @@ pub mod window; use sort_expr::{to_sort_expressions, PySortExpr}; /// A PyExpr that can be used on a DataFrame -#[pyclass(name = "Expr", module = "datafusion.expr", subclass)] +#[pyclass(name = "RawExpr", module = "datafusion.expr", subclass)] #[derive(Debug, Clone)] pub struct PyExpr { pub expr: Expr, From 4f457030f171a26d0c4cce4d55cf541519956fcc Mon Sep 17 00:00:00 2001 From: jsai28 <54253219+jsai28@users.noreply.github.com> Date: Sat, 15 Mar 2025 04:57:38 -0600 Subject: [PATCH 277/413] added pytest asyncio tests (#1063) --- pyproject.toml | 1 + python/tests/test_dataframe.py | 54 ++++++++++++++++++++++++++++++++++ uv.lock | 17 ++++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 060e3b80a..a4ed18c4c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -150,6 +150,7 @@ dev = [ "maturin>=1.8.1", "numpy>1.25.0", "pytest>=7.4.4", + "pytest-asyncio>=0.23.3", "ruff>=0.9.1", "toml>=0.10.2", "pygithub==2.5.0", diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index d084f12dd..384b17878 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -771,6 +771,16 @@ def test_execution_plan(aggregate_df): assert rows_returned == 5 +@pytest.mark.asyncio +async def test_async_iteration_of_df(aggregate_df): + rows_returned = 0 + async for batch in aggregate_df.execute_stream(): + assert batch is not None + rows_returned += len(batch.to_pyarrow()[0]) + + assert rows_returned == 5 + + def test_repartition(df): df.repartition(2) @@ -958,6 +968,18 @@ def test_execute_stream(df): assert not list(stream) # after one iteration the generator must be exhausted +@pytest.mark.asyncio +async def test_execute_stream_async(df): + stream = df.execute_stream() + batches = [batch async for batch in stream] + + assert all(batch is not None for batch in batches) + + # After consuming all batches, the stream should be exhausted + remaining_batches = [batch async for batch in stream] + assert not remaining_batches + + @pytest.mark.parametrize("schema", [True, False]) def test_execute_stream_to_arrow_table(df, schema): stream = df.execute_stream() @@ -974,6 +996,25 @@ def test_execute_stream_to_arrow_table(df, schema): assert set(pyarrow_table.column_names) == {"a", "b", "c"} +@pytest.mark.asyncio +@pytest.mark.parametrize("schema", [True, False]) +async def test_execute_stream_to_arrow_table_async(df, schema): + stream = df.execute_stream() + + if schema: + pyarrow_table = pa.Table.from_batches( + [batch.to_pyarrow() async for batch in stream], schema=df.schema() + ) + else: + pyarrow_table = pa.Table.from_batches( + [batch.to_pyarrow() async for batch in stream] + ) + + assert isinstance(pyarrow_table, pa.Table) + assert pyarrow_table.shape == (3, 3) + assert set(pyarrow_table.column_names) == {"a", "b", "c"} + + def test_execute_stream_partitioned(df): streams = df.execute_stream_partitioned() assert all(batch is not None for stream in streams for batch in stream) @@ -982,6 +1023,19 @@ def test_execute_stream_partitioned(df): ) # after one iteration all generators must be exhausted +@pytest.mark.asyncio +async def test_execute_stream_partitioned_async(df): + streams = df.execute_stream_partitioned() + + for stream in streams: + batches = [batch async for batch in stream] + assert all(batch is not None for batch in batches) + + # Ensure the stream is exhausted after iteration + remaining_batches = [batch async for batch in stream] + assert not remaining_batches + + def test_empty_to_arrow_table(df): # Convert empty datafusion dataframe to pyarrow Table pyarrow_table = df.limit(0).to_arrow_table() diff --git a/uv.lock b/uv.lock index 619b92856..7e4bc4c6b 100644 --- a/uv.lock +++ b/uv.lock @@ -284,9 +284,11 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "maturin" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "numpy", version = "2.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pygithub" }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "ruff" }, { name = "toml" }, ] @@ -314,9 +316,10 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "maturin", specifier = ">=1.8.1" }, - { name = "numpy", marker = "python_full_version >= '3.10'", specifier = ">1.24.4" }, + { name = "numpy", specifier = ">1.25.0" }, { name = "pygithub", specifier = "==2.5.0" }, { name = "pytest", specifier = ">=7.4.4" }, + { name = "pytest-asyncio", specifier = ">=0.23.3" }, { name = "ruff", specifier = ">=0.9.1" }, { name = "toml", specifier = ">=0.10.2" }, ] @@ -1079,6 +1082,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/11/92/76a1c94d3afee238333bc0a42b82935dd8f9cf8ce9e336ff87ee14d9e1cf/pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6", size = 343083 }, ] +[[package]] +name = "pytest-asyncio" +version = "0.25.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/a8/ecbc8ede70921dd2f544ab1cadd3ff3bf842af27f87bbdea774c7baa1d38/pytest_asyncio-0.25.3.tar.gz", hash = "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a", size = 54239 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/17/3493c5624e48fd97156ebaec380dcaafee9506d7e2c46218ceebbb57d7de/pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3", size = 19467 }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From 2f52688d76e84794343c17ffaf3002534ecfd716 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 15 Mar 2025 19:00:50 +0800 Subject: [PATCH 278/413] Add decorator for udwf (#1061) * feat: Introduce create_udwf method for User-Defined Window Functions - Added `create_udwf` static method to `WindowUDF` class, allowing users to create User-Defined Window Functions (UDWF) as both a function and a decorator. - Updated type hinting for `_R` using `TypeAlias` for better clarity. - Enhanced documentation with usage examples for both function and decorator styles, improving usability and understanding. * refactor: Simplify UDWF test suite and introduce SimpleWindowCount evaluator - Removed multiple exponential smoothing classes to streamline the code. - Introduced SimpleWindowCount class for basic row counting functionality. - Updated test cases to validate the new SimpleWindowCount evaluator. - Refactored fixture and test functions for clarity and consistency. - Enhanced error handling in UDWF creation tests. * fix: Update type alias import to use typing_extensions for compatibility * Add udwf tests for multiple input types and decorator syntax * replace old def udwf * refactor: Simplify df fixture by passing ctx as an argument * refactor: Rename DataFrame fixtures and update test functions - Renamed `df` fixture to `complex_window_df` for clarity. - Renamed `simple_df` fixture to `count_window_df` to better reflect its purpose. - Updated test functions to use the new fixture names, enhancing readability and maintainability. * refactor: Update udwf calls in WindowUDF to use BiasedNumbers directly - Changed udwf1 to use BiasedNumbers instead of bias_10. - Added udwf2 to call udwf with bias_10. - Introduced udwf3 to demonstrate a lambda function returning BiasedNumbers(20). * feat: Add overloads for udwf function to support multiple input types and decorator syntax * refactor: Simplify udwf method signature by removing redundant type hints * refactor: Remove state_type from udwf method signature and update return type handling - Eliminated the state_type parameter from the udwf method to simplify the function signature. - Updated return type handling in the _function and _decorator methods to use a generic type _R for better type flexibility. - Enhanced the decorator to wrap the original function, allowing for improved argument handling and expression return. * refactor: Update volatility parameter type in udwf method signature to support Volatility enum * Fix ruff errors * fix C901 for def udwf * refactor: Update udwf method signature and simplify input handling - Changed the type hint for the return type in the _create_window_udf_decorator method to use pa.DataType directly instead of a TypeVar. - Simplified the handling of input types by removing redundant checks and directly using the input types list. - Removed unnecessary comments and cleaned up the code for better readability. - Updated the test for udwf to use parameterized tests for better coverage and maintainability. * refactor: Rename input_type to input_types in udwf method signature for clarity * refactor: Enhance typing in udf.py by introducing Protocol for WindowEvaluator and improving import organization * Revert "refactor: Enhance typing in udf.py by introducing Protocol for WindowEvaluator and improving import organization" This reverts commit 16dbe5f3fd88f42d0a304384b162009bd9e49a35. --- python/datafusion/udf.py | 123 +++++++++++++++++++++------ python/tests/test_udwf.py | 170 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 264 insertions(+), 29 deletions(-) diff --git a/python/datafusion/udf.py b/python/datafusion/udf.py index 603b7063d..e93a34ca5 100644 --- a/python/datafusion/udf.py +++ b/python/datafusion/udf.py @@ -621,6 +621,16 @@ def __call__(self, *args: Expr) -> Expr: args_raw = [arg.expr for arg in args] return Expr(self._udwf.__call__(*args_raw)) + @overload + @staticmethod + def udwf( + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + volatility: Volatility | str, + name: Optional[str] = None, + ) -> Callable[..., WindowUDF]: ... + + @overload @staticmethod def udwf( func: Callable[[], WindowEvaluator], @@ -628,24 +638,31 @@ def udwf( return_type: pa.DataType, volatility: Volatility | str, name: Optional[str] = None, - ) -> WindowUDF: - """Create a new User-Defined Window Function. + ) -> WindowUDF: ... - If your :py:class:`WindowEvaluator` can be instantiated with no arguments, you - can simply pass it's type as ``func``. If you need to pass additional arguments - to it's constructor, you can define a lambda or a factory method. During runtime - the :py:class:`WindowEvaluator` will be constructed for every instance in - which this UDWF is used. The following examples are all valid. + @staticmethod + def udwf(*args: Any, **kwargs: Any): # noqa: D417 + """Create a new User-Defined Window Function (UDWF). - .. code-block:: python + This class can be used both as a **function** and as a **decorator**. + + Usage: + - **As a function**: Call `udwf(func, input_types, return_type, volatility, + name)`. + - **As a decorator**: Use `@udwf(input_types, return_type, volatility, + name)`. When using `udwf` as a decorator, **do not pass `func` + explicitly**. + **Function example:** + ``` import pyarrow as pa class BiasedNumbers(WindowEvaluator): def __init__(self, start: int = 0) -> None: self.start = start - def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + def evaluate_all(self, values: list[pa.Array], + num_rows: int) -> pa.Array: return pa.array([self.start + i for i in range(num_rows)]) def bias_10() -> BiasedNumbers: @@ -655,35 +672,93 @@ def bias_10() -> BiasedNumbers: udwf2 = udwf(bias_10, pa.int64(), pa.int64(), "immutable") udwf3 = udwf(lambda: BiasedNumbers(20), pa.int64(), pa.int64(), "immutable") + ``` + + **Decorator example:** + ``` + @udwf(pa.int64(), pa.int64(), "immutable") + def biased_numbers() -> BiasedNumbers: + return BiasedNumbers(10) + ``` + Args: - func: A callable to create the window function. - input_types: The data types of the arguments to ``func``. + func: **Only needed when calling as a function. Skip this argument when + using `udwf` as a decorator.** + input_types: The data types of the arguments. return_type: The data type of the return value. volatility: See :py:class:`Volatility` for allowed values. - arguments: A list of arguments to pass in to the __init__ method for accum. name: A descriptive name for the function. Returns: - A user-defined window function. - """ # noqa: W505, E501 + A user-defined window function that can be used in window function calls. + """ + if args and callable(args[0]): + # Case 1: Used as a function, require the first parameter to be callable + return WindowUDF._create_window_udf(*args, **kwargs) + # Case 2: Used as a decorator with parameters + return WindowUDF._create_window_udf_decorator(*args, **kwargs) + + @staticmethod + def _create_window_udf( + func: Callable[[], WindowEvaluator], + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + volatility: Volatility | str, + name: Optional[str] = None, + ) -> WindowUDF: + """Create a WindowUDF instance from function arguments.""" if not callable(func): msg = "`func` must be callable." raise TypeError(msg) if not isinstance(func(), WindowEvaluator): msg = "`func` must implement the abstract base class WindowEvaluator" raise TypeError(msg) - if name is None: - name = func().__class__.__qualname__.lower() - if isinstance(input_types, pa.DataType): - input_types = [input_types] - return WindowUDF( - name=name, - func=func, - input_types=input_types, - return_type=return_type, - volatility=volatility, + + name = name or func.__qualname__.lower() + input_types = ( + [input_types] if isinstance(input_types, pa.DataType) else input_types ) + return WindowUDF(name, func, input_types, return_type, volatility) + + @staticmethod + def _get_default_name(func: Callable) -> str: + """Get the default name for a function based on its attributes.""" + if hasattr(func, "__qualname__"): + return func.__qualname__.lower() + return func.__class__.__name__.lower() + + @staticmethod + def _normalize_input_types( + input_types: pa.DataType | list[pa.DataType], + ) -> list[pa.DataType]: + """Convert a single DataType to a list if needed.""" + if isinstance(input_types, pa.DataType): + return [input_types] + return input_types + + @staticmethod + def _create_window_udf_decorator( + input_types: pa.DataType | list[pa.DataType], + return_type: pa.DataType, + volatility: Volatility | str, + name: Optional[str] = None, + ) -> Callable[[Callable[[], WindowEvaluator]], Callable[..., Expr]]: + """Create a decorator for a WindowUDF.""" + + def decorator(func: Callable[[], WindowEvaluator]) -> Callable[..., Expr]: + udwf_caller = WindowUDF._create_window_udf( + func, input_types, return_type, volatility, name + ) + + @functools.wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Expr: + return udwf_caller(*args, **kwargs) + + return wrapper + + return decorator + # Convenience exports so we can import instead of treating as # variables at the package root diff --git a/python/tests/test_udwf.py b/python/tests/test_udwf.py index 3d6dcf9d8..4190e7d64 100644 --- a/python/tests/test_udwf.py +++ b/python/tests/test_udwf.py @@ -162,14 +162,27 @@ def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: return pa.array(results) +class SimpleWindowCount(WindowEvaluator): + """A simple window evaluator that counts rows.""" + + def __init__(self, base: int = 0) -> None: + self.base = base + + def evaluate_all(self, values: list[pa.Array], num_rows: int) -> pa.Array: + return pa.array([self.base + i for i in range(num_rows)]) + + class NotSubclassOfWindowEvaluator: pass @pytest.fixture -def df(): - ctx = SessionContext() +def ctx(): + return SessionContext() + +@pytest.fixture +def complex_window_df(ctx): # create a RecordBatch and a new DataFrame from it batch = pa.RecordBatch.from_arrays( [ @@ -182,7 +195,17 @@ def df(): return ctx.create_dataframe([[batch]]) -def test_udwf_errors(df): +@pytest.fixture +def count_window_df(ctx): + # create a RecordBatch and a new DataFrame from it + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 4, 6])], + names=["a", "b"], + ) + return ctx.create_dataframe([[batch]], name="test_table") + + +def test_udwf_errors(complex_window_df): with pytest.raises(TypeError): udwf( NotSubclassOfWindowEvaluator, @@ -192,6 +215,103 @@ def test_udwf_errors(df): ) +def test_udwf_errors_with_message(): + """Test error cases for UDWF creation.""" + with pytest.raises( + TypeError, match="`func` must implement the abstract base class WindowEvaluator" + ): + udwf( + NotSubclassOfWindowEvaluator, pa.int64(), pa.int64(), volatility="immutable" + ) + + +def test_udwf_basic_usage(count_window_df): + """Test basic UDWF usage with a simple counting window function.""" + simple_count = udwf( + SimpleWindowCount, pa.int64(), pa.int64(), volatility="immutable" + ) + + df = count_window_df.select( + simple_count(column("a")) + .window_frame(WindowFrame("rows", None, None)) + .build() + .alias("count") + ) + result = df.collect()[0] + assert result.column(0) == pa.array([0, 1, 2]) + + +def test_udwf_with_args(count_window_df): + """Test UDWF with constructor arguments.""" + count_base10 = udwf( + lambda: SimpleWindowCount(10), pa.int64(), pa.int64(), volatility="immutable" + ) + + df = count_window_df.select( + count_base10(column("a")) + .window_frame(WindowFrame("rows", None, None)) + .build() + .alias("count") + ) + result = df.collect()[0] + assert result.column(0) == pa.array([10, 11, 12]) + + +def test_udwf_decorator_basic(count_window_df): + """Test UDWF used as a decorator.""" + + @udwf([pa.int64()], pa.int64(), "immutable") + def window_count() -> WindowEvaluator: + return SimpleWindowCount() + + df = count_window_df.select( + window_count(column("a")) + .window_frame(WindowFrame("rows", None, None)) + .build() + .alias("count") + ) + result = df.collect()[0] + assert result.column(0) == pa.array([0, 1, 2]) + + +def test_udwf_decorator_with_args(count_window_df): + """Test UDWF decorator with constructor arguments.""" + + @udwf([pa.int64()], pa.int64(), "immutable") + def window_count_base10() -> WindowEvaluator: + return SimpleWindowCount(10) + + df = count_window_df.select( + window_count_base10(column("a")) + .window_frame(WindowFrame("rows", None, None)) + .build() + .alias("count") + ) + result = df.collect()[0] + assert result.column(0) == pa.array([10, 11, 12]) + + +def test_register_udwf(ctx, count_window_df): + """Test registering and using UDWF in SQL context.""" + window_count = udwf( + SimpleWindowCount, + [pa.int64()], + pa.int64(), + volatility="immutable", + name="window_count", + ) + + ctx.register_udwf(window_count) + result = ctx.sql( + """ + SELECT window_count(a) + OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED + FOLLOWING) FROM test_table + """ + ).collect()[0] + assert result.column(0) == pa.array([0, 1, 2]) + + smooth_default = udwf( ExponentialSmoothDefault, pa.float64(), @@ -299,10 +419,50 @@ def test_udwf_errors(df): @pytest.mark.parametrize(("name", "expr", "expected"), data_test_udwf_functions) -def test_udwf_functions(df, name, expr, expected): - df = df.select("a", "b", f.round(expr, lit(3)).alias(name)) +def test_udwf_functions(complex_window_df, name, expr, expected): + df = complex_window_df.select("a", "b", f.round(expr, lit(3)).alias(name)) # execute and collect the first (and only) batch result = df.sort(column("a")).select(column(name)).collect()[0] assert result.column(0) == pa.array(expected) + + +@pytest.mark.parametrize( + "udwf_func", + [ + udwf(SimpleWindowCount, pa.int64(), pa.int64(), "immutable"), + udwf(SimpleWindowCount, [pa.int64()], pa.int64(), "immutable"), + udwf([pa.int64()], pa.int64(), "immutable")(lambda: SimpleWindowCount()), + udwf(pa.int64(), pa.int64(), "immutable")(lambda: SimpleWindowCount()), + ], +) +def test_udwf_overloads(udwf_func, count_window_df): + df = count_window_df.select( + udwf_func(column("a")) + .window_frame(WindowFrame("rows", None, None)) + .build() + .alias("count") + ) + result = df.collect()[0] + assert result.column(0) == pa.array([0, 1, 2]) + + +def test_udwf_named_function(ctx, count_window_df): + """Test UDWF with explicit name parameter.""" + window_count = udwf( + SimpleWindowCount, + pa.int64(), + pa.int64(), + volatility="immutable", + name="my_custom_counter", + ) + + ctx.register_udwf(window_count) + result = ctx.sql( + """ + SELECT my_custom_counter(a) + OVER (ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED + FOLLOWING) FROM test_table""" + ).collect()[0] + assert result.column(0) == pa.array([0, 1, 2]) From 7c1c08f8617ac97a2568eb0664e9d4ee30fceba9 Mon Sep 17 00:00:00 2001 From: Nirnay Roy <32942494+nirnayroy@users.noreply.github.com> Date: Sat, 15 Mar 2025 17:05:05 +0530 Subject: [PATCH 279/413] feat: expose regex_count function (#1066) * Added wrapper for regex_count function * fix comment --------- Co-authored-by: Nirnay Roy --- python/datafusion/functions.py | 18 ++++++++++++++++++ python/tests/test_functions.py | 4 ++++ src/functions.rs | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 0cc7434cf..26bac149c 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -217,6 +217,7 @@ "random", "range", "rank", + "regexp_count", "regexp_like", "regexp_match", "regexp_replace", @@ -779,6 +780,23 @@ def regexp_replace( return Expr(f.regexp_replace(string.expr, pattern.expr, replacement.expr, flags)) +def regexp_count( + string: Expr, pattern: Expr, start: Expr, flags: Expr | None = None +) -> Expr: + """Returns the number of matches in a string. + + Optional start position (the first position is 1) to search for the regular + expression. + """ + if flags is not None: + flags = flags.expr + if start is not None: + start = start.expr + else: + start = Expr.expr + return Expr(f.regexp_count(string.expr, pattern.expr, start, flags)) + + def repeat(string: Expr, n: Expr) -> Expr: """Repeats the ``string`` to ``n`` times.""" return Expr(f.repeat(string.expr, n.expr)) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index ed88a16e3..161e1e3bb 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -740,6 +740,10 @@ def test_array_function_obj_tests(stmt, py_expr): f.regexp_replace(column("a"), literal("(ell|orl)"), literal("-")), pa.array(["H-o", "W-d", "!"]), ), + ( + f.regexp_count(column("a"), literal("(ell|orl)"), literal(1)), + pa.array([1, 1, 0], type=pa.int64()), + ), ], ) def test_string_functions(df, function, expected_result): diff --git a/src/functions.rs b/src/functions.rs index 6a8abb18d..8fac239b4 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -173,6 +173,25 @@ fn regexp_replace( ) .into()) } + +#[pyfunction] +#[pyo3(signature = (string, pattern, start, flags=None))] +/// Returns the number of matches found in the string. +fn regexp_count( + string: PyExpr, + pattern: PyExpr, + start: Option, + flags: Option, +) -> PyResult { + Ok(functions::expr_fn::regexp_count( + string.expr, + pattern.expr, + start.map(|x| x.expr), + flags.map(|x| x.expr), + ) + .into()) +} + /// Creates a new Sort Expr #[pyfunction] fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult { @@ -943,6 +962,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(power))?; m.add_wrapped(wrap_pyfunction!(radians))?; m.add_wrapped(wrap_pyfunction!(random))?; + m.add_wrapped(wrap_pyfunction!(regexp_count))?; m.add_wrapped(wrap_pyfunction!(regexp_like))?; m.add_wrapped(wrap_pyfunction!(regexp_match))?; m.add_wrapped(wrap_pyfunction!(regexp_replace))?; From b8dd97bc8eefcfecfa8dcc864c4898c654b236a9 Mon Sep 17 00:00:00 2001 From: Spaarsh <67336892+Spaarsh@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:08:16 +0530 Subject: [PATCH 280/413] Add additional ruff suggestions (#1062) * Enabled ruff rule PT001 and ANN204 * Enabled ruff rule B008 * Enabled ruff rule EM101 * Enabled ruff rule PLR1714 * Enabled ruff rule ANN201 * Enabled ruff rule C400 * Enabled ruff rule B904 * Enabled ruff rule UP006 * Enabled ruff rule RUF012 * Enabled ruff rule FBT003 * Enabled ruff rule C416 * Enabled ruff rule SIM102 * Enabled ruff rule PGH003 * Enabled ruff rule PERF401 * Enabled ruff rule EM102 * Enabled ruff rule SIM108 * Enabled ruff rule ICN001 * Enabled ruff rule ICN001 * implemented reviews * Update pyproject.toml to ignore `SIM102` * Enabled ruff rule PLW2901 * Enabled ruff rule RET503 * Fixed failing ruff tests --- benchmarks/db-benchmark/groupby-datafusion.py | 24 ++-- benchmarks/db-benchmark/join-datafusion.py | 5 +- benchmarks/tpch/tpch.py | 7 +- dev/release/generate-changelog.py | 6 +- docs/source/conf.py | 4 +- examples/create-context.py | 12 +- examples/python-udaf.py | 36 +++-- examples/python-udf-comparisons.py | 9 +- examples/python-udf.py | 12 +- examples/query-pyarrow-data.py | 10 +- examples/sql-using-python-udaf.py | 2 +- examples/tpch/_tests.py | 4 +- examples/tpch/convert_data_to_parquet.py | 134 +++++++++--------- examples/tpch/q08_market_share.py | 2 +- examples/tpch/q19_discounted_revenue.py | 4 +- .../tpch/q21_suppliers_kept_orders_waiting.py | 2 +- pyproject.toml | 20 --- python/datafusion/__init__.py | 8 +- python/datafusion/catalog.py | 4 +- python/datafusion/context.py | 51 +++---- python/datafusion/dataframe.py | 55 +++---- python/datafusion/expr.py | 31 ++-- python/datafusion/functions.py | 9 +- python/tests/test_functions.py | 2 +- python/tests/test_wrapper_coverage.py | 7 +- 25 files changed, 213 insertions(+), 247 deletions(-) diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 04bf7a149..f9e8d638b 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -20,7 +20,7 @@ import timeit import datafusion as df -import pyarrow +import pyarrow as pa from datafusion import ( RuntimeEnvBuilder, SessionConfig, @@ -37,7 +37,7 @@ exec(open("./_helpers/helpers.py").read()) -def ans_shape(batches): +def ans_shape(batches) -> tuple[int, int]: rows, cols = 0, 0 for batch in batches: rows += batch.num_rows @@ -48,7 +48,7 @@ def ans_shape(batches): return rows, cols -def execute(df): +def execute(df) -> list: print(df.execution_plan().display_indent()) return df.collect() @@ -68,14 +68,14 @@ def execute(df): src_grp = os.path.join("data", data_name + ".csv") print("loading dataset %s" % src_grp, flush=True) -schema = pyarrow.schema( +schema = pa.schema( [ - ("id4", pyarrow.int32()), - ("id5", pyarrow.int32()), - ("id6", pyarrow.int32()), - ("v1", pyarrow.int32()), - ("v2", pyarrow.int32()), - ("v3", pyarrow.float64()), + ("id4", pa.int32()), + ("id5", pa.int32()), + ("id6", pa.int32()), + ("v1", pa.int32()), + ("v2", pa.int32()), + ("v3", pa.float64()), ] ) @@ -93,8 +93,8 @@ def execute(df): ) config = ( SessionConfig() - .with_repartition_joins(False) - .with_repartition_aggregations(False) + .with_repartition_joins(enabled=False) + .with_repartition_aggregations(enabled=False) .set("datafusion.execution.coalesce_batches", "false") ) ctx = SessionContext(config, runtime) diff --git a/benchmarks/db-benchmark/join-datafusion.py b/benchmarks/db-benchmark/join-datafusion.py index b45ebf632..039868031 100755 --- a/benchmarks/db-benchmark/join-datafusion.py +++ b/benchmarks/db-benchmark/join-datafusion.py @@ -29,7 +29,7 @@ exec(open("./_helpers/helpers.py").read()) -def ans_shape(batches): +def ans_shape(batches) -> tuple[int, int]: rows, cols = 0, 0 for batch in batches: rows += batch.num_rows @@ -57,7 +57,8 @@ def ans_shape(batches): os.path.join("data", y_data_name[2] + ".csv"), ] if len(src_jn_y) != 3: - raise Exception("Something went wrong in preparing files used for join") + error_msg = "Something went wrong in preparing files used for join" + raise Exception(error_msg) print( "loading datasets " diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py index bfb9ac398..2d1bbae5b 100644 --- a/benchmarks/tpch/tpch.py +++ b/benchmarks/tpch/tpch.py @@ -21,7 +21,7 @@ from datafusion import SessionContext -def bench(data_path, query_path): +def bench(data_path, query_path) -> None: with open("results.csv", "w") as results: # register tables start = time.time() @@ -68,10 +68,7 @@ def bench(data_path, query_path): with open(f"{query_path}/q{query}.sql") as f: text = f.read() tmp = text.split(";") - queries = [] - for str in tmp: - if len(str.strip()) > 0: - queries.append(str.strip()) + queries = [s.strip() for s in tmp if len(s.strip()) > 0] try: start = time.time() diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index e30e2def2..d86736773 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -24,7 +24,7 @@ from github import Github -def print_pulls(repo_name, title, pulls): +def print_pulls(repo_name, title, pulls) -> None: if len(pulls) > 0: print(f"**{title}:**") print() @@ -34,7 +34,7 @@ def print_pulls(repo_name, title, pulls): print() -def generate_changelog(repo, repo_name, tag1, tag2, version): +def generate_changelog(repo, repo_name, tag1, tag2, version) -> None: # get a list of commits between two tags print(f"Fetching list of commits between {tag1} and {tag2}", file=sys.stderr) comparison = repo.compare(tag1, tag2) @@ -154,7 +154,7 @@ def generate_changelog(repo, repo_name, tag1, tag2, version): ) -def cli(args=None): +def cli(args=None) -> None: """Process command line arguments.""" if not args: args = sys.argv[1:] diff --git a/docs/source/conf.py b/docs/source/conf.py index c82a189e0..0be03d81d 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -73,7 +73,7 @@ autoapi_python_class_content = "both" -def autoapi_skip_member_fn(app, what, name, obj, skip, options): # noqa: ARG001 +def autoapi_skip_member_fn(app, what, name, obj, skip, options) -> bool: # noqa: ARG001 skip_contents = [ # Re-exports ("class", "datafusion.DataFrame"), @@ -93,7 +93,7 @@ def autoapi_skip_member_fn(app, what, name, obj, skip, options): # noqa: ARG001 return skip -def setup(sphinx): +def setup(sphinx) -> None: sphinx.connect("autoapi-skip-member", autoapi_skip_member_fn) diff --git a/examples/create-context.py b/examples/create-context.py index 760c8513e..0026d6162 100644 --- a/examples/create-context.py +++ b/examples/create-context.py @@ -25,14 +25,14 @@ runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() - .with_create_default_catalog_and_schema(True) + .with_create_default_catalog_and_schema(enabled=True) .with_default_catalog_and_schema("foo", "bar") .with_target_partitions(8) - .with_information_schema(True) - .with_repartition_joins(False) - .with_repartition_aggregations(False) - .with_repartition_windows(False) - .with_parquet_pruning(False) + .with_information_schema(enabled=True) + .with_repartition_joins(enabled=False) + .with_repartition_aggregations(enabled=False) + .with_repartition_windows(enabled=False) + .with_parquet_pruning(enabled=False) .set("datafusion.execution.parquet.pushdown_filters", "true") ) ctx = SessionContext(config, runtime) diff --git a/examples/python-udaf.py b/examples/python-udaf.py index 538f69571..6655edb0a 100644 --- a/examples/python-udaf.py +++ b/examples/python-udaf.py @@ -16,7 +16,7 @@ # under the License. import datafusion -import pyarrow +import pyarrow as pa import pyarrow.compute from datafusion import Accumulator, col, udaf @@ -26,25 +26,21 @@ class MyAccumulator(Accumulator): Interface of a user-defined accumulation. """ - def __init__(self): - self._sum = pyarrow.scalar(0.0) + def __init__(self) -> None: + self._sum = pa.scalar(0.0) - def update(self, values: pyarrow.Array) -> None: + def update(self, values: pa.Array) -> None: # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar( - self._sum.as_py() + pyarrow.compute.sum(values).as_py() - ) + self._sum = pa.scalar(self._sum.as_py() + pa.compute.sum(values).as_py()) - def merge(self, states: pyarrow.Array) -> None: + def merge(self, states: pa.Array) -> None: # not nice since pyarrow scalars can't be summed yet. This breaks on `None` - self._sum = pyarrow.scalar( - self._sum.as_py() + pyarrow.compute.sum(states).as_py() - ) + self._sum = pa.scalar(self._sum.as_py() + pa.compute.sum(states).as_py()) - def state(self) -> pyarrow.Array: - return pyarrow.array([self._sum.as_py()]) + def state(self) -> pa.Array: + return pa.array([self._sum.as_py()]) - def evaluate(self) -> pyarrow.Scalar: + def evaluate(self) -> pa.Scalar: return self._sum @@ -52,17 +48,17 @@ def evaluate(self) -> pyarrow.Scalar: ctx = datafusion.SessionContext() # create a RecordBatch and a new DataFrame from it -batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], +batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], names=["a", "b"], ) df = ctx.create_dataframe([[batch]]) my_udaf = udaf( MyAccumulator, - pyarrow.float64(), - pyarrow.float64(), - [pyarrow.float64()], + pa.float64(), + pa.float64(), + [pa.float64()], "stable", ) @@ -70,4 +66,4 @@ def evaluate(self) -> pyarrow.Scalar: result = df.collect()[0] -assert result.column(0) == pyarrow.array([6.0]) +assert result.column(0) == pa.array([6.0]) diff --git a/examples/python-udf-comparisons.py b/examples/python-udf-comparisons.py index c5d5ec8dd..eb0825011 100644 --- a/examples/python-udf-comparisons.py +++ b/examples/python-udf-comparisons.py @@ -112,8 +112,8 @@ def is_of_interest_impl( returnflag_arr: pa.Array, ) -> pa.Array: result = [] - for idx, partkey in enumerate(partkey_arr): - partkey = partkey.as_py() + for idx, partkey_val in enumerate(partkey_arr): + partkey = partkey_val.as_py() suppkey = suppkey_arr[idx].as_py() returnflag = returnflag_arr[idx].as_py() value = (partkey, suppkey, returnflag) @@ -162,10 +162,7 @@ def udf_using_pyarrow_compute_impl( resultant_arr = pc.and_(filtered_partkey_arr, filtered_suppkey_arr) resultant_arr = pc.and_(resultant_arr, filtered_returnflag_arr) - if results is None: - results = resultant_arr - else: - results = pc.or_(results, resultant_arr) + results = resultant_arr if results is None else pc.or_(results, resultant_arr) return results diff --git a/examples/python-udf.py b/examples/python-udf.py index fb2bc253e..1c08acd1a 100644 --- a/examples/python-udf.py +++ b/examples/python-udf.py @@ -15,23 +15,23 @@ # specific language governing permissions and limitations # under the License. -import pyarrow +import pyarrow as pa from datafusion import SessionContext, udf from datafusion import functions as f -def is_null(array: pyarrow.Array) -> pyarrow.Array: +def is_null(array: pa.Array) -> pa.Array: return array.is_null() -is_null_arr = udf(is_null, [pyarrow.int64()], pyarrow.bool_(), "stable") +is_null_arr = udf(is_null, [pa.int64()], pa.bool_(), "stable") # create a context ctx = SessionContext() # create a RecordBatch and a new DataFrame from it -batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], +batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], names=["a", "b"], ) df = ctx.create_dataframe([[batch]]) @@ -40,4 +40,4 @@ def is_null(array: pyarrow.Array) -> pyarrow.Array: result = df.collect()[0] -assert result.column(0) == pyarrow.array([False] * 3) +assert result.column(0) == pa.array([False] * 3) diff --git a/examples/query-pyarrow-data.py b/examples/query-pyarrow-data.py index e3456fb5b..9cfe8a62b 100644 --- a/examples/query-pyarrow-data.py +++ b/examples/query-pyarrow-data.py @@ -16,15 +16,15 @@ # under the License. import datafusion -import pyarrow +import pyarrow as pa from datafusion import col # create a context ctx = datafusion.SessionContext() # create a RecordBatch and a new DataFrame from it -batch = pyarrow.RecordBatch.from_arrays( - [pyarrow.array([1, 2, 3]), pyarrow.array([4, 5, 6])], +batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], names=["a", "b"], ) df = ctx.create_dataframe([[batch]]) @@ -38,5 +38,5 @@ # execute and collect the first (and only) batch result = df.collect()[0] -assert result.column(0) == pyarrow.array([5, 7, 9]) -assert result.column(1) == pyarrow.array([-3, -3, -3]) +assert result.column(0) == pa.array([5, 7, 9]) +assert result.column(1) == pa.array([-3, -3, -3]) diff --git a/examples/sql-using-python-udaf.py b/examples/sql-using-python-udaf.py index 60ab8d134..32ce38900 100644 --- a/examples/sql-using-python-udaf.py +++ b/examples/sql-using-python-udaf.py @@ -25,7 +25,7 @@ class MyAccumulator(Accumulator): Interface of a user-defined accumulation. """ - def __init__(self): + def __init__(self) -> None: self._sum = pa.scalar(0.0) def update(self, values: pa.Array) -> None: diff --git a/examples/tpch/_tests.py b/examples/tpch/_tests.py index 2be4dfabd..80ff80244 100644 --- a/examples/tpch/_tests.py +++ b/examples/tpch/_tests.py @@ -91,7 +91,7 @@ def check_q17(df): ("q22_global_sales_opportunity", "q22"), ], ) -def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): +def test_tpch_query_vs_answer_file(query_code: str, answer_file: str) -> None: module = import_module(query_code) df: DataFrame = module.df @@ -122,3 +122,5 @@ def test_tpch_query_vs_answer_file(query_code: str, answer_file: str): assert df.join(df_expected, on=cols, how="anti").count() == 0 assert df.count() == df_expected.count() + + return None diff --git a/examples/tpch/convert_data_to_parquet.py b/examples/tpch/convert_data_to_parquet.py index 73097fac5..fd0fcca49 100644 --- a/examples/tpch/convert_data_to_parquet.py +++ b/examples/tpch/convert_data_to_parquet.py @@ -25,112 +25,112 @@ import os import datafusion -import pyarrow +import pyarrow as pa ctx = datafusion.SessionContext() all_schemas = {} all_schemas["customer"] = [ - ("C_CUSTKEY", pyarrow.int64()), - ("C_NAME", pyarrow.string()), - ("C_ADDRESS", pyarrow.string()), - ("C_NATIONKEY", pyarrow.int64()), - ("C_PHONE", pyarrow.string()), - ("C_ACCTBAL", pyarrow.decimal128(15, 2)), - ("C_MKTSEGMENT", pyarrow.string()), - ("C_COMMENT", pyarrow.string()), + ("C_CUSTKEY", pa.int64()), + ("C_NAME", pa.string()), + ("C_ADDRESS", pa.string()), + ("C_NATIONKEY", pa.int64()), + ("C_PHONE", pa.string()), + ("C_ACCTBAL", pa.decimal128(15, 2)), + ("C_MKTSEGMENT", pa.string()), + ("C_COMMENT", pa.string()), ] all_schemas["lineitem"] = [ - ("L_ORDERKEY", pyarrow.int64()), - ("L_PARTKEY", pyarrow.int64()), - ("L_SUPPKEY", pyarrow.int64()), - ("L_LINENUMBER", pyarrow.int32()), - ("L_QUANTITY", pyarrow.decimal128(15, 2)), - ("L_EXTENDEDPRICE", pyarrow.decimal128(15, 2)), - ("L_DISCOUNT", pyarrow.decimal128(15, 2)), - ("L_TAX", pyarrow.decimal128(15, 2)), - ("L_RETURNFLAG", pyarrow.string()), - ("L_LINESTATUS", pyarrow.string()), - ("L_SHIPDATE", pyarrow.date32()), - ("L_COMMITDATE", pyarrow.date32()), - ("L_RECEIPTDATE", pyarrow.date32()), - ("L_SHIPINSTRUCT", pyarrow.string()), - ("L_SHIPMODE", pyarrow.string()), - ("L_COMMENT", pyarrow.string()), + ("L_ORDERKEY", pa.int64()), + ("L_PARTKEY", pa.int64()), + ("L_SUPPKEY", pa.int64()), + ("L_LINENUMBER", pa.int32()), + ("L_QUANTITY", pa.decimal128(15, 2)), + ("L_EXTENDEDPRICE", pa.decimal128(15, 2)), + ("L_DISCOUNT", pa.decimal128(15, 2)), + ("L_TAX", pa.decimal128(15, 2)), + ("L_RETURNFLAG", pa.string()), + ("L_LINESTATUS", pa.string()), + ("L_SHIPDATE", pa.date32()), + ("L_COMMITDATE", pa.date32()), + ("L_RECEIPTDATE", pa.date32()), + ("L_SHIPINSTRUCT", pa.string()), + ("L_SHIPMODE", pa.string()), + ("L_COMMENT", pa.string()), ] all_schemas["nation"] = [ - ("N_NATIONKEY", pyarrow.int64()), - ("N_NAME", pyarrow.string()), - ("N_REGIONKEY", pyarrow.int64()), - ("N_COMMENT", pyarrow.string()), + ("N_NATIONKEY", pa.int64()), + ("N_NAME", pa.string()), + ("N_REGIONKEY", pa.int64()), + ("N_COMMENT", pa.string()), ] all_schemas["orders"] = [ - ("O_ORDERKEY", pyarrow.int64()), - ("O_CUSTKEY", pyarrow.int64()), - ("O_ORDERSTATUS", pyarrow.string()), - ("O_TOTALPRICE", pyarrow.decimal128(15, 2)), - ("O_ORDERDATE", pyarrow.date32()), - ("O_ORDERPRIORITY", pyarrow.string()), - ("O_CLERK", pyarrow.string()), - ("O_SHIPPRIORITY", pyarrow.int32()), - ("O_COMMENT", pyarrow.string()), + ("O_ORDERKEY", pa.int64()), + ("O_CUSTKEY", pa.int64()), + ("O_ORDERSTATUS", pa.string()), + ("O_TOTALPRICE", pa.decimal128(15, 2)), + ("O_ORDERDATE", pa.date32()), + ("O_ORDERPRIORITY", pa.string()), + ("O_CLERK", pa.string()), + ("O_SHIPPRIORITY", pa.int32()), + ("O_COMMENT", pa.string()), ] all_schemas["part"] = [ - ("P_PARTKEY", pyarrow.int64()), - ("P_NAME", pyarrow.string()), - ("P_MFGR", pyarrow.string()), - ("P_BRAND", pyarrow.string()), - ("P_TYPE", pyarrow.string()), - ("P_SIZE", pyarrow.int32()), - ("P_CONTAINER", pyarrow.string()), - ("P_RETAILPRICE", pyarrow.decimal128(15, 2)), - ("P_COMMENT", pyarrow.string()), + ("P_PARTKEY", pa.int64()), + ("P_NAME", pa.string()), + ("P_MFGR", pa.string()), + ("P_BRAND", pa.string()), + ("P_TYPE", pa.string()), + ("P_SIZE", pa.int32()), + ("P_CONTAINER", pa.string()), + ("P_RETAILPRICE", pa.decimal128(15, 2)), + ("P_COMMENT", pa.string()), ] all_schemas["partsupp"] = [ - ("PS_PARTKEY", pyarrow.int64()), - ("PS_SUPPKEY", pyarrow.int64()), - ("PS_AVAILQTY", pyarrow.int32()), - ("PS_SUPPLYCOST", pyarrow.decimal128(15, 2)), - ("PS_COMMENT", pyarrow.string()), + ("PS_PARTKEY", pa.int64()), + ("PS_SUPPKEY", pa.int64()), + ("PS_AVAILQTY", pa.int32()), + ("PS_SUPPLYCOST", pa.decimal128(15, 2)), + ("PS_COMMENT", pa.string()), ] all_schemas["region"] = [ - ("r_REGIONKEY", pyarrow.int64()), - ("r_NAME", pyarrow.string()), - ("r_COMMENT", pyarrow.string()), + ("r_REGIONKEY", pa.int64()), + ("r_NAME", pa.string()), + ("r_COMMENT", pa.string()), ] all_schemas["supplier"] = [ - ("S_SUPPKEY", pyarrow.int64()), - ("S_NAME", pyarrow.string()), - ("S_ADDRESS", pyarrow.string()), - ("S_NATIONKEY", pyarrow.int32()), - ("S_PHONE", pyarrow.string()), - ("S_ACCTBAL", pyarrow.decimal128(15, 2)), - ("S_COMMENT", pyarrow.string()), + ("S_SUPPKEY", pa.int64()), + ("S_NAME", pa.string()), + ("S_ADDRESS", pa.string()), + ("S_NATIONKEY", pa.int32()), + ("S_PHONE", pa.string()), + ("S_ACCTBAL", pa.decimal128(15, 2)), + ("S_COMMENT", pa.string()), ] curr_dir = os.path.dirname(os.path.abspath(__file__)) -for filename, curr_schema in all_schemas.items(): +for filename, curr_schema_val in all_schemas.items(): # For convenience, go ahead and convert the schema column names to lowercase - curr_schema = [(s[0].lower(), s[1]) for s in curr_schema] + curr_schema = [(s[0].lower(), s[1]) for s in curr_schema_val] # Pre-collect the output columns so we can ignore the null field we add # in to handle the trailing | in the file output_cols = [r[0] for r in curr_schema] - curr_schema = [pyarrow.field(r[0], r[1], nullable=False) for r in curr_schema] + curr_schema = [pa.field(r[0], r[1], nullable=False) for r in curr_schema] # Trailing | requires extra field for in processing - curr_schema.append(("some_null", pyarrow.null())) + curr_schema.append(("some_null", pa.null())) - schema = pyarrow.schema(curr_schema) + schema = pa.schema(curr_schema) source_file = os.path.abspath( os.path.join(curr_dir, f"../../benchmarks/tpch/data/{filename}.csv") diff --git a/examples/tpch/q08_market_share.py b/examples/tpch/q08_market_share.py index d46df30f2..4bf50efba 100644 --- a/examples/tpch/q08_market_share.py +++ b/examples/tpch/q08_market_share.py @@ -150,7 +150,7 @@ df = df.with_column( "national_volume", F.case(col("s_suppkey").is_null()) - .when(lit(False), col("volume")) + .when(lit(value=False), col("volume")) .otherwise(lit(0.0)), ) diff --git a/examples/tpch/q19_discounted_revenue.py b/examples/tpch/q19_discounted_revenue.py index 2b87e1120..bd492aac0 100644 --- a/examples/tpch/q19_discounted_revenue.py +++ b/examples/tpch/q19_discounted_revenue.py @@ -89,8 +89,8 @@ def is_of_interest( same number of rows in the output. """ result = [] - for idx, brand in enumerate(brand_arr): - brand = brand.as_py() + for idx, brand_val in enumerate(brand_arr): + brand = brand_val.as_py() if brand in items_of_interest: values_of_interest = items_of_interest[brand] diff --git a/examples/tpch/q21_suppliers_kept_orders_waiting.py b/examples/tpch/q21_suppliers_kept_orders_waiting.py index 9bbaad779..619c4406b 100644 --- a/examples/tpch/q21_suppliers_kept_orders_waiting.py +++ b/examples/tpch/q21_suppliers_kept_orders_waiting.py @@ -65,7 +65,7 @@ df = df.with_column( "failed_supp", F.case(col("l_receiptdate") > col("l_commitdate")) - .when(lit(True), col("l_suppkey")) + .when(lit(value=True), col("l_suppkey")) .end(), ) diff --git a/pyproject.toml b/pyproject.toml index a4ed18c4c..d86b657ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -80,37 +80,17 @@ ignore = [ "TD003", # Allow TODO lines "UP007", # Disallowing Union is pedantic # TODO: Enable all of the following, but this PR is getting too large already - "PT001", - "ANN204", - "B008", - "EM101", "PLR0913", - "PLR1714", - "ANN201", - "C400", "TRY003", - "B904", - "UP006", - "RUF012", - "FBT003", - "C416", - "SIM102", - "PGH003", "PLR2004", - "PERF401", "PD901", - "EM102", "ERA001", - "SIM108", - "ICN001", "ANN001", "ANN202", "PTH", "N812", "INP001", "DTZ007", - "PLW2901", - "RET503", "RUF015", "A005", "TC001", diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 286e5dc31..d871fdb71 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -92,17 +92,17 @@ ] -def column(value: str): +def column(value: str) -> Expr: """Create a column expression.""" return Expr.column(value) -def col(value: str): +def col(value: str) -> Expr: """Create a column expression.""" return Expr.column(value) -def literal(value): +def literal(value) -> Expr: """Create a literal expression.""" return Expr.literal(value) @@ -120,6 +120,6 @@ def str_lit(value): return string_literal(value) -def lit(value): +def lit(value) -> Expr: """Create a literal expression.""" return Expr.literal(value) diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py index 0560f4704..6c3f188cc 100644 --- a/python/datafusion/catalog.py +++ b/python/datafusion/catalog.py @@ -24,7 +24,7 @@ import datafusion._internal as df_internal if TYPE_CHECKING: - import pyarrow + import pyarrow as pa class Catalog: @@ -67,7 +67,7 @@ def __init__(self, table: df_internal.Table) -> None: self.table = table @property - def schema(self) -> pyarrow.Schema: + def schema(self) -> pa.Schema: """Returns the schema associated with this table.""" return self.table.schema diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 58ad9a943..1429a4975 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -40,9 +40,9 @@ if TYPE_CHECKING: import pathlib - import pandas - import polars - import pyarrow + import pandas as pd + import polars as pl + import pyarrow as pa from datafusion.plan import ExecutionPlan, LogicalPlan @@ -537,7 +537,7 @@ def register_listing_table( path: str | pathlib.Path, table_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = ".parquet", - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_sort_order: list[list[Expr | SortExpr]] | None = None, ) -> None: """Register multiple files as a single table. @@ -606,14 +606,14 @@ def sql_with_options(self, query: str, options: SQLOptions) -> DataFrame: def create_dataframe( self, - partitions: list[list[pyarrow.RecordBatch]], + partitions: list[list[pa.RecordBatch]], name: str | None = None, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, ) -> DataFrame: """Create and return a dataframe using the provided partitions. Args: - partitions: :py:class:`pyarrow.RecordBatch` partitions to register. + partitions: :py:class:`pa.RecordBatch` partitions to register. name: Resultant dataframe name. schema: Schema for the partitions. @@ -684,16 +684,14 @@ def from_arrow( return DataFrame(self.ctx.from_arrow(data, name)) @deprecated("Use ``from_arrow`` instead.") - def from_arrow_table( - self, data: pyarrow.Table, name: str | None = None - ) -> DataFrame: + def from_arrow_table(self, data: pa.Table, name: str | None = None) -> DataFrame: """Create a :py:class:`~datafusion.dataframe.DataFrame` from an Arrow table. This is an alias for :py:func:`from_arrow`. """ return self.from_arrow(data, name) - def from_pandas(self, data: pandas.DataFrame, name: str | None = None) -> DataFrame: + def from_pandas(self, data: pd.DataFrame, name: str | None = None) -> DataFrame: """Create a :py:class:`~datafusion.dataframe.DataFrame` from a Pandas DataFrame. Args: @@ -705,7 +703,7 @@ def from_pandas(self, data: pandas.DataFrame, name: str | None = None) -> DataFr """ return DataFrame(self.ctx.from_pandas(data, name)) - def from_polars(self, data: polars.DataFrame, name: str | None = None) -> DataFrame: + def from_polars(self, data: pl.DataFrame, name: str | None = None) -> DataFrame: """Create a :py:class:`~datafusion.dataframe.DataFrame` from a Polars DataFrame. Args: @@ -719,7 +717,7 @@ def from_polars(self, data: polars.DataFrame, name: str | None = None) -> DataFr # https://github.com/apache/datafusion-python/pull/1016#discussion_r1983239116 # is the discussion on how we arrived at adding register_view - def register_view(self, name: str, df: DataFrame): + def register_view(self, name: str, df: DataFrame) -> None: """Register a :py:class: `~datafusion.detaframe.DataFrame` as a view. Args: @@ -755,7 +753,7 @@ def register_table_provider( self.ctx.register_table_provider(name, provider) def register_record_batches( - self, name: str, partitions: list[list[pyarrow.RecordBatch]] + self, name: str, partitions: list[list[pa.RecordBatch]] ) -> None: """Register record batches as a table. @@ -776,7 +774,7 @@ def register_parquet( parquet_pruning: bool = True, file_extension: str = ".parquet", skip_metadata: bool = True, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_sort_order: list[list[SortExpr]] | None = None, ) -> None: """Register a Parquet file as a table. @@ -817,7 +815,7 @@ def register_csv( self, name: str, path: str | pathlib.Path | list[str | pathlib.Path], - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, has_header: bool = True, delimiter: str = ",", schema_infer_max_records: int = 1000, @@ -843,10 +841,7 @@ def register_csv( selected for data input. file_compression_type: File compression type. """ - if isinstance(path, list): - path = [str(p) for p in path] - else: - path = str(path) + path = [str(p) for p in path] if isinstance(path, list) else str(path) self.ctx.register_csv( name, @@ -863,7 +858,7 @@ def register_json( self, name: str, path: str | pathlib.Path, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = ".json", table_partition_cols: list[tuple[str, str]] | None = None, @@ -901,7 +896,7 @@ def register_avro( self, name: str, path: str | pathlib.Path, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_extension: str = ".avro", table_partition_cols: list[tuple[str, str]] | None = None, ) -> None: @@ -923,8 +918,8 @@ def register_avro( name, str(path), schema, file_extension, table_partition_cols ) - def register_dataset(self, name: str, dataset: pyarrow.dataset.Dataset) -> None: - """Register a :py:class:`pyarrow.dataset.Dataset` as a table. + def register_dataset(self, name: str, dataset: pa.dataset.Dataset) -> None: + """Register a :py:class:`pa.dataset.Dataset` as a table. Args: name: Name of the table to register. @@ -975,7 +970,7 @@ def session_id(self) -> str: def read_json( self, path: str | pathlib.Path, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, schema_infer_max_records: int = 1000, file_extension: str = ".json", table_partition_cols: list[tuple[str, str]] | None = None, @@ -1012,7 +1007,7 @@ def read_json( def read_csv( self, path: str | pathlib.Path | list[str] | list[pathlib.Path], - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, has_header: bool = True, delimiter: str = ",", schema_infer_max_records: int = 1000, @@ -1065,7 +1060,7 @@ def read_parquet( parquet_pruning: bool = True, file_extension: str = ".parquet", skip_metadata: bool = True, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_sort_order: list[list[Expr | SortExpr]] | None = None, ) -> DataFrame: """Read a Parquet source into a :py:class:`~datafusion.dataframe.Dataframe`. @@ -1110,7 +1105,7 @@ def read_parquet( def read_avro( self, path: str | pathlib.Path, - schema: pyarrow.Schema | None = None, + schema: pa.Schema | None = None, file_partition_cols: list[tuple[str, str]] | None = None, file_extension: str = ".avro", ) -> DataFrame: diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index d1c71c2bb..26fe8f453 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -26,10 +26,8 @@ TYPE_CHECKING, Any, Iterable, - List, Literal, Optional, - Type, Union, overload, ) @@ -75,7 +73,7 @@ class Compression(Enum): LZ4_RAW = "lz4_raw" @classmethod - def from_str(cls: Type[Compression], value: str) -> Compression: + def from_str(cls: type[Compression], value: str) -> Compression: """Convert a string to a Compression enum value. Args: @@ -89,11 +87,13 @@ def from_str(cls: Type[Compression], value: str) -> Compression: """ try: return cls(value.lower()) - except ValueError: + except ValueError as err: valid_values = str([item.value for item in Compression]) - raise ValueError( - f"{value} is not a valid Compression. Valid values are: {valid_values}" - ) + error_msg = f""" + {value} is not a valid Compression. + Valid values are: {valid_values} + """ + raise ValueError(error_msg) from err def get_default_level(self) -> Optional[int]: """Get the default compression level for the compression type. @@ -132,7 +132,7 @@ def into_view(self) -> pa.Table: """Convert DataFrame as a ViewTable which can be used in register_table.""" return self.df.into_view() - def __getitem__(self, key: str | List[str]) -> DataFrame: + def __getitem__(self, key: str | list[str]) -> DataFrame: """Return a new :py:class`DataFrame` with the specified column or columns. Args: @@ -287,8 +287,7 @@ def _simplify_expression( if isinstance(expr, Expr): expr_list.append(expr.expr) elif isinstance(expr, Iterable): - for inner_expr in expr: - expr_list.append(inner_expr.expr) + expr_list.extend(inner_expr.expr for inner_expr in expr) else: raise NotImplementedError if named_exprs: @@ -513,10 +512,15 @@ def join( # This check is to prevent breaking API changes where users prior to # DF 43.0.0 would pass the join_keys as a positional argument instead # of a keyword argument. - if isinstance(on, tuple) and len(on) == 2: - if isinstance(on[0], list) and isinstance(on[1], list): - join_keys = on # type: ignore - on = None + if ( + isinstance(on, tuple) + and len(on) == 2 + and isinstance(on[0], list) + and isinstance(on[1], list) + ): + # We know this is safe because we've checked the types + join_keys = on # type: ignore[assignment] + on = None if join_keys is not None: warnings.warn( @@ -529,18 +533,17 @@ def join( if on is not None: if left_on is not None or right_on is not None: - raise ValueError( - "`left_on` or `right_on` should not provided with `on`" - ) + error_msg = "`left_on` or `right_on` should not provided with `on`" + raise ValueError(error_msg) left_on = on right_on = on elif left_on is not None or right_on is not None: if left_on is None or right_on is None: - raise ValueError("`left_on` and `right_on` should both be provided.") + error_msg = "`left_on` and `right_on` should both be provided." + raise ValueError(error_msg) else: - raise ValueError( - "either `on` or `left_on` and `right_on` should be provided." - ) + error_msg = "either `on` or `left_on` and `right_on` should be provided." + raise ValueError(error_msg) if isinstance(left_on, str): left_on = [left_on] if isinstance(right_on, str): @@ -726,9 +729,11 @@ def write_parquet( if isinstance(compression, str): compression = Compression.from_str(compression) - if compression in {Compression.GZIP, Compression.BROTLI, Compression.ZSTD}: - if compression_level is None: - compression_level = compression.get_default_level() + if ( + compression in {Compression.GZIP, Compression.BROTLI, Compression.ZSTD} + and compression_level is None + ): + compression_level = compression.get_default_level() self.df.write_parquet(str(path), compression.value, compression_level) @@ -824,7 +829,7 @@ def unnest_columns(self, *columns: str, preserve_nulls: bool = True) -> DataFram Returns: A DataFrame with the columns expanded. """ - columns = [c for c in columns] + columns = list(columns) return DataFrame(self.df.unnest_columns(columns, preserve_nulls=preserve_nulls)) def __arrow_c_stream__(self, requested_schema: pa.Schema) -> Any: diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 77b6c272d..2697d8143 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -22,7 +22,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Type +from typing import TYPE_CHECKING, Any, ClassVar, Optional import pyarrow as pa @@ -176,7 +176,7 @@ def sort_or_default(e: Expr | SortExpr) -> expr_internal.SortExpr: """Helper function to return a default Sort if an Expr is provided.""" if isinstance(e, SortExpr): return e.raw_sort - return SortExpr(e, True, True).raw_sort + return SortExpr(e, ascending=True, nulls_first=True).raw_sort def sort_list_to_raw_sort_list( @@ -439,24 +439,21 @@ def fill_null(self, value: Any | Expr | None = None) -> Expr: value = Expr.literal(value) return Expr(functions_internal.nvl(self.expr, value.expr)) - _to_pyarrow_types = { + _to_pyarrow_types: ClassVar[dict[type, pa.DataType]] = { float: pa.float64(), int: pa.int64(), str: pa.string(), bool: pa.bool_(), } - def cast( - self, to: pa.DataType[Any] | Type[float] | Type[int] | Type[str] | Type[bool] - ) -> Expr: + def cast(self, to: pa.DataType[Any] | type[float | int | str | bool]) -> Expr: """Cast to a new data type.""" if not isinstance(to, pa.DataType): try: to = self._to_pyarrow_types[to] - except KeyError: - raise TypeError( - "Expected instance of pyarrow.DataType or builtins.type" - ) + except KeyError as err: + error_msg = "Expected instance of pyarrow.DataType or builtins.type" + raise TypeError(error_msg) from err return Expr(self.expr.cast(to)) @@ -565,9 +562,7 @@ def partition_by(self, *partition_by: Expr) -> ExprFuncBuilder: set parameters for either window or aggregate functions. If used on any other type of expression, an error will be generated when ``build()`` is called. """ - return ExprFuncBuilder( - self.expr.partition_by(list(e.expr for e in partition_by)) - ) + return ExprFuncBuilder(self.expr.partition_by([e.expr for e in partition_by])) def window_frame(self, window_frame: WindowFrame) -> ExprFuncBuilder: """Set the frame fora window function. @@ -610,7 +605,7 @@ def over(self, window: Window) -> Expr: class ExprFuncBuilder: - def __init__(self, builder: expr_internal.ExprFuncBuilder): + def __init__(self, builder: expr_internal.ExprFuncBuilder) -> None: self.builder = builder def order_by(self, *exprs: Expr) -> ExprFuncBuilder: @@ -638,7 +633,7 @@ def null_treatment(self, null_treatment: NullTreatment) -> ExprFuncBuilder: def partition_by(self, *partition_by: Expr) -> ExprFuncBuilder: """Set partitioning for window functions.""" return ExprFuncBuilder( - self.builder.partition_by(list(e.expr for e in partition_by)) + self.builder.partition_by([e.expr for e in partition_by]) ) def window_frame(self, window_frame: WindowFrame) -> ExprFuncBuilder: @@ -693,11 +688,11 @@ def __init__( """ if not isinstance(start_bound, pa.Scalar) and start_bound is not None: start_bound = pa.scalar(start_bound) - if units == "rows" or units == "groups": + if units in ("rows", "groups"): start_bound = start_bound.cast(pa.uint64()) if not isinstance(end_bound, pa.Scalar) and end_bound is not None: end_bound = pa.scalar(end_bound) - if units == "rows" or units == "groups": + if units in ("rows", "groups"): end_bound = end_bound.cast(pa.uint64()) self.window_frame = expr_internal.WindowFrame(units, start_bound, end_bound) @@ -709,7 +704,7 @@ def get_lower_bound(self) -> WindowFrameBound: """Returns starting bound.""" return WindowFrameBound(self.window_frame.get_lower_bound()) - def get_upper_bound(self): + def get_upper_bound(self) -> WindowFrameBound: """Returns end bound.""" return WindowFrameBound(self.window_frame.get_upper_bound()) diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 26bac149c..5cf914e16 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -790,10 +790,7 @@ def regexp_count( """ if flags is not None: flags = flags.expr - if start is not None: - start = start.expr - else: - start = Expr.expr + start = start.expr if start is not None else Expr.expr return Expr(f.regexp_count(string.expr, pattern.expr, start, flags)) @@ -817,13 +814,15 @@ def right(string: Expr, n: Expr) -> Expr: return Expr(f.right(string.expr, n.expr)) -def round(value: Expr, decimal_places: Expr = Expr.literal(0)) -> Expr: +def round(value: Expr, decimal_places: Expr | None = None) -> Expr: """Round the argument to the nearest integer. If the optional ``decimal_places`` is specified, round to the nearest number of decimal places. You can specify a negative number of decimal places. For example ``round(lit(125.2345), lit(-2))`` would yield a value of ``100.0``. """ + if decimal_places is None: + decimal_places = Expr.literal(0) return Expr(f.round(value.expr, decimal_places.expr)) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 161e1e3bb..37f2075f5 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -81,7 +81,7 @@ def test_literal(df): literal("1"), literal("OK"), literal(3.14), - literal(True), + literal(value=True), literal(b"hello world"), ) result = df.collect() diff --git a/python/tests/test_wrapper_coverage.py b/python/tests/test_wrapper_coverage.py index a2de2d32b..926a65961 100644 --- a/python/tests/test_wrapper_coverage.py +++ b/python/tests/test_wrapper_coverage.py @@ -28,7 +28,7 @@ from enum import EnumMeta as EnumType -def missing_exports(internal_obj, wrapped_obj) -> None: # noqa: C901 +def missing_exports(internal_obj, wrapped_obj) -> None: """ Identify if any of the rust exposted structs or functions do not have wrappers. @@ -56,9 +56,8 @@ def missing_exports(internal_obj, wrapped_obj) -> None: # noqa: C901 # __kwdefaults__ and __doc__. As long as these are None on the internal # object, it's okay to skip them. However if they do exist on the internal # object they must also exist on the wrapped object. - if internal_attr is not None: - if wrapped_attr is None: - pytest.fail(f"Missing attribute: {internal_attr_name}") + if internal_attr is not None and wrapped_attr is None: + pytest.fail(f"Missing attribute: {internal_attr_name}") if internal_attr_name in ["__self__", "__class__"]: continue From 42982dad27ad03e7e9395d4c3ae3064c2b489434 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 22 Mar 2025 10:14:55 -0400 Subject: [PATCH 281/413] Improve collection during repr and repr_html (#1036) * Improve table readout of a dataframe in jupyter notebooks by making the table scrollable and displaying the first record batch up to 2MB * Add option to only display a portion of a cell data and the user can click on a button to toggle showing more or less * We cannot expect that the first non-empy batch is sufficient for our 2MB limit, so switch over to collecting until we run out or use up the size * Update python unit test to allow the additional formatting data to exist and only check the table contents * Combining collection for repr and repr_html into one function * Small clippy suggestion * Collect was occuring twice on repr * Switch to execute_stream_partitioned --- python/tests/test_dataframe.py | 23 ++-- src/dataframe.rs | 240 ++++++++++++++++++++++++++++----- src/utils.rs | 2 +- 3 files changed, 225 insertions(+), 40 deletions(-) diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 384b17878..718ebf69d 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import os +import re from typing import Any import pyarrow as pa @@ -1245,13 +1246,17 @@ def add_with_parameter(df_internal, value: Any) -> DataFrame: def test_dataframe_repr_html(df) -> None: output = df._repr_html_() - ref_html = """
- - - - -
abc
148
255
368
- """ + # Since we've added a fair bit of processing to the html output, lets just verify + # the values we are expecting in the table exist. Use regex and ignore everything + # between the and . We also don't want the closing > on the + # td and th segments because that is where the formatting data is written. - # Ignore whitespace just to make this test look cleaner - assert output.replace(" ", "") == ref_html.replace(" ", "") + headers = ["a", "b", "c"] + headers = [f"{v}" for v in headers] + header_pattern = "(.*?)".join(headers) + assert len(re.findall(header_pattern, output, re.DOTALL)) == 1 + + body_data = [[1, 4, 8], [2, 5, 5], [3, 6, 8]] + body_lines = [f"{v}" for inner in body_data for v in inner] + body_pattern = "(.*?)".join(body_lines) + assert len(re.findall(body_pattern, output, re.DOTALL)) == 1 diff --git a/src/dataframe.rs b/src/dataframe.rs index 243e2e14f..be10b8c28 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -31,9 +31,11 @@ use datafusion::common::UnnestOptions; use datafusion::config::{CsvOptions, TableParquetOptions}; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::datasource::TableProvider; +use datafusion::error::DataFusionError; use datafusion::execution::SendableRecordBatchStream; use datafusion::parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use datafusion::prelude::*; +use futures::{StreamExt, TryStreamExt}; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use pyo3::pybacked::PyBackedStr; @@ -70,6 +72,9 @@ impl PyTableProvider { PyTable::new(table_provider) } } +const MAX_TABLE_BYTES_TO_DISPLAY: usize = 2 * 1024 * 1024; // 2 MB +const MIN_TABLE_ROWS_TO_DISPLAY: usize = 20; +const MAX_LENGTH_CELL_WITHOUT_MINIMIZE: usize = 25; /// A PyDataFrame is a representation of a logical plan and an API to compose statements. /// Use it to build a plan and `.collect()` to execute the plan and collect the result. @@ -111,56 +116,151 @@ impl PyDataFrame { } fn __repr__(&self, py: Python) -> PyDataFusionResult { - let df = self.df.as_ref().clone().limit(0, Some(10))?; - let batches = wait_for_future(py, df.collect())?; - let batches_as_string = pretty::pretty_format_batches(&batches); - match batches_as_string { - Ok(batch) => Ok(format!("DataFrame()\n{batch}")), - Err(err) => Ok(format!("Error: {:?}", err.to_string())), + let (batches, has_more) = wait_for_future( + py, + collect_record_batches_to_display(self.df.as_ref().clone(), 10, 10), + )?; + if batches.is_empty() { + // This should not be reached, but do it for safety since we index into the vector below + return Ok("No data to display".to_string()); } - } - fn _repr_html_(&self, py: Python) -> PyDataFusionResult { - let mut html_str = "\n".to_string(); + let batches_as_displ = + pretty::pretty_format_batches(&batches).map_err(py_datafusion_err)?; + + let additional_str = match has_more { + true => "\nData truncated.", + false => "", + }; - let df = self.df.as_ref().clone().limit(0, Some(10))?; - let batches = wait_for_future(py, df.collect())?; + Ok(format!("DataFrame()\n{batches_as_displ}{additional_str}")) + } + fn _repr_html_(&self, py: Python) -> PyDataFusionResult { + let (batches, has_more) = wait_for_future( + py, + collect_record_batches_to_display( + self.df.as_ref().clone(), + MIN_TABLE_ROWS_TO_DISPLAY, + usize::MAX, + ), + )?; if batches.is_empty() { - html_str.push_str("
\n"); - return Ok(html_str); + // This should not be reached, but do it for safety since we index into the vector below + return Ok("No data to display".to_string()); } + let table_uuid = uuid::Uuid::new_v4().to_string(); + + let mut html_str = " + + +

+ + \n".to_string(); + let schema = batches[0].schema(); let mut header = Vec::new(); for field in schema.fields() { - header.push(format!("", field.name())); } let header_str = header.join(""); - html_str.push_str(&format!("{}\n", header_str)); - - for batch in batches { - let formatters = batch - .columns() - .iter() - .map(|c| ArrayFormatter::try_new(c.as_ref(), &FormatOptions::default())) - .map(|c| { - c.map_err(|e| PyValueError::new_err(format!("Error: {:?}", e.to_string()))) - }) - .collect::, _>>()?; - - for row in 0..batch.num_rows() { + html_str.push_str(&format!("{}\n", header_str)); + + let batch_formatters = batches + .iter() + .map(|batch| { + batch + .columns() + .iter() + .map(|c| ArrayFormatter::try_new(c.as_ref(), &FormatOptions::default())) + .map(|c| { + c.map_err(|e| PyValueError::new_err(format!("Error: {:?}", e.to_string()))) + }) + .collect::, _>>() + }) + .collect::, _>>()?; + + let rows_per_batch = batches.iter().map(|batch| batch.num_rows()); + + // We need to build up row by row for html + let mut table_row = 0; + for (batch_formatter, num_rows_in_batch) in batch_formatters.iter().zip(rows_per_batch) { + for batch_row in 0..num_rows_in_batch { + table_row += 1; let mut cells = Vec::new(); - for formatter in &formatters { - cells.push(format!("", formatter.value(row))); + for (col, formatter) in batch_formatter.iter().enumerate() { + let cell_data = formatter.value(batch_row).to_string(); + // From testing, primitive data types do not typically get larger than 21 characters + if cell_data.len() > MAX_LENGTH_CELL_WITHOUT_MINIMIZE { + let short_cell_data = &cell_data[0..MAX_LENGTH_CELL_WITHOUT_MINIMIZE]; + cells.push(format!(" + ")); + } else { + cells.push(format!("", formatter.value(batch_row))); + } } let row_str = cells.join(""); html_str.push_str(&format!("{}\n", row_str)); } } + html_str.push_str("
{}", field.name())); + header.push(format!("{}
{} +
+ {short_cell_data} + {cell_data} + +
+ 		{}
\n"); + + html_str.push_str(" + + "); - html_str.push_str("\n"); + if has_more { + html_str.push_str("Data truncated due to size."); + } Ok(html_str) } @@ -771,3 +871,83 @@ fn record_batch_into_schema( RecordBatch::try_new(schema, data_arrays) } + +/// This is a helper function to return the first non-empty record batch from executing a DataFrame. +/// It additionally returns a bool, which indicates if there are more record batches available. +/// We do this so we can determine if we should indicate to the user that the data has been +/// truncated. This collects until we have achived both of these two conditions +/// +/// - We have collected our minimum number of rows +/// - We have reached our limit, either data size or maximum number of rows +/// +/// Otherwise it will return when the stream has exhausted. If you want a specific number of +/// rows, set min_rows == max_rows. +async fn collect_record_batches_to_display( + df: DataFrame, + min_rows: usize, + max_rows: usize, +) -> Result<(Vec, bool), DataFusionError> { + let partitioned_stream = df.execute_stream_partitioned().await?; + let mut stream = futures::stream::iter(partitioned_stream).flatten(); + let mut size_estimate_so_far = 0; + let mut rows_so_far = 0; + let mut record_batches = Vec::default(); + let mut has_more = false; + + while (size_estimate_so_far < MAX_TABLE_BYTES_TO_DISPLAY && rows_so_far < max_rows) + || rows_so_far < min_rows + { + let mut rb = match stream.next().await { + None => { + break; + } + Some(Ok(r)) => r, + Some(Err(e)) => return Err(e), + }; + + let mut rows_in_rb = rb.num_rows(); + if rows_in_rb > 0 { + size_estimate_so_far += rb.get_array_memory_size(); + + if size_estimate_so_far > MAX_TABLE_BYTES_TO_DISPLAY { + let ratio = MAX_TABLE_BYTES_TO_DISPLAY as f32 / size_estimate_so_far as f32; + let total_rows = rows_in_rb + rows_so_far; + + let mut reduced_row_num = (total_rows as f32 * ratio).round() as usize; + if reduced_row_num < min_rows { + reduced_row_num = min_rows.min(total_rows); + } + + let limited_rows_this_rb = reduced_row_num - rows_so_far; + if limited_rows_this_rb < rows_in_rb { + rows_in_rb = limited_rows_this_rb; + rb = rb.slice(0, limited_rows_this_rb); + has_more = true; + } + } + + if rows_in_rb + rows_so_far > max_rows { + rb = rb.slice(0, max_rows - rows_so_far); + has_more = true; + } + + rows_so_far += rb.num_rows(); + record_batches.push(rb); + } + } + + if record_batches.is_empty() { + return Ok((Vec::default(), false)); + } + + if !has_more { + // Data was not already truncated, so check to see if more record batches remain + has_more = match stream.try_next().await { + Ok(None) => false, // reached end + Ok(Some(_)) => true, + Err(_) => false, // Stream disconnected + }; + } + + Ok((record_batches, has_more)) +} diff --git a/src/utils.rs b/src/utils.rs index 999aad755..3487de21b 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -42,7 +42,7 @@ pub(crate) fn get_tokio_runtime() -> &'static TokioRuntime { #[inline] pub(crate) fn get_global_ctx() -> &'static SessionContext { static CTX: OnceLock = OnceLock::new(); - CTX.get_or_init(|| SessionContext::new()) + CTX.get_or_init(SessionContext::new) } /// Utility to collect rust futures with GIL released From d0315ffa704aba467f769f444208b7ce26d83037 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sat, 22 Mar 2025 14:37:24 -0400 Subject: [PATCH 282/413] feat: Update DataFusion dependency to 46 (#1079) * Update DataFusion dependency to 46 * There was an update upstream in the exec but it is not a breaking change and only needs unit test updates --- Cargo.lock | 296 +++++++++++++++++++-------------- Cargo.toml | 18 +- python/tests/test_dataframe.py | 3 +- src/expr.rs | 39 +++-- src/expr/aggregate.rs | 10 +- src/expr/aggregate_expr.rs | 11 +- src/expr/window.rs | 24 ++- src/functions.rs | 34 ++-- 8 files changed, 252 insertions(+), 183 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c7f2bf3c..3a4915f23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755b6da235ac356a869393c23668c663720b8749dd6f15e52b6c214b4b964cc7" +checksum = "84ef243634a39fb6e9d1710737e7a5ef96c9bacabd2326859ff889bc9ef755e5" dependencies = [ "arrow-arith", "arrow-array", @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64656a1e0b13ca766f8440752e9a93e11014eec7b67909986f83ed0ab1fe37b8" +checksum = "8f420c6aef51dad2e4a96ce29c0ec90ad84880bdb60b321c74c652a6be07b93f" dependencies = [ "arrow-array", "arrow-buffer", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a" +checksum = "24bda5ff6461a4ff9739959b3d57b377f45e3f878f7be1a4f28137c0a8f339fa" dependencies = [ "ahash", "arrow-buffer", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a" +checksum = "bc6ed265c73f134a583d02c3cab5e16afab9446d8048ede8707e31f85fad58a0" dependencies = [ "bytes", "half", @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee" +checksum = "01c648572391edcef10e5fd458db70ba27ed6f71bcaee04397d0cfb100b34f8b" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f12542b8164398fc9ec595ff783c4cf6044daa89622c5a7201be920e4c0d4c" +checksum = "a02fb265a6d8011a7d3ad1a36f25816ad0a3bb04cb8e9fe7929c165b98c0cbcd" dependencies = [ "arrow-array", "arrow-cast", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83" +checksum = "5f2cebf504bb6a92a134a87fff98f01b14fbb3a93ecf7aef90cd0f888c5fffa4" dependencies = [ "arrow-buffer", "arrow-schema", @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6" +checksum = "8e6405b287671c88846e7751f7291f717b164911474cabac6d3d8614d5aa7374" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9551d9400532f23a370cabbea1dc5a53c49230397d41f96c4c8eedf306199305" +checksum = "5329bf9e7390cbb6b117ddd4d82e94c5362ea4cab5095697139429f36a38350c" dependencies = [ "arrow-array", "arrow-buffer", @@ -319,16 +319,18 @@ dependencies = [ "half", "indexmap", "lexical-core", + "memchr", "num", "serde", "serde_json", + "simdutf8", ] [[package]] name = "arrow-ord" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c07223476f8219d1ace8cd8d85fa18c4ebd8d945013f25ef5c72e85085ca4ee" +checksum = "e103c13d4b80da28339c1d7aa23dd85bd59f42158acc45d39eeb6770627909ce" dependencies = [ "arrow-array", "arrow-buffer", @@ -339,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91b194b38bfd89feabc23e798238989c6648b2506ad639be42ec8eb1658d82c4" +checksum = "170549a11b8534f3097a0619cfe89c42812345dc998bcf81128fc700b84345b8" dependencies = [ "arrow-array", "arrow-buffer", @@ -352,18 +354,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735" +checksum = "a5c53775bba63f319189f366d2b86e9a8889373eb198f07d8544938fc9f8ed9a" dependencies = [ "bitflags 2.8.0", ] [[package]] name = "arrow-select" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539" +checksum = "0a99003b2eb562b8d9c99dfb672306f15e94b20d3734179d596895703e821dcf" dependencies = [ "ahash", "arrow-array", @@ -375,9 +377,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "54.2.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d44c8eed43be4ead49128370f7131f054839d3d6003e52aebf64322470b8fbd0" +checksum = "90fdb130ee8325f4cd8262e19bb6baa3cbcef2b2573c4bee8c6fda7ea08199d7" dependencies = [ "arrow-array", "arrow-buffer", @@ -535,9 +537,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.5" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" dependencies = [ "arrayref", "arrayvec", @@ -649,15 +651,15 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", "serde", - "windows-targets", + "windows-link", ] [[package]] @@ -864,30 +866,32 @@ dependencies = [ [[package]] name = "datafusion" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eae420e7a5b0b7f1c39364cc76cbcd0f5fdc416b2514ae3847c2676bbd60702a" +checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3" dependencies = [ "apache-avro", "arrow", - "arrow-array", "arrow-ipc", "arrow-schema", - "async-compression", "async-trait", "bytes", "bzip2 0.5.1", "chrono", "datafusion-catalog", + "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", + "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", + "datafusion-macros", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -896,7 +900,6 @@ dependencies = [ "datafusion-sql", "flate2", "futures", - "glob", "itertools 0.14.0", "log", "num-traits", @@ -908,7 +911,6 @@ dependencies = [ "sqlparser", "tempfile", "tokio", - "tokio-util", "url", "uuid", "xz2", @@ -917,9 +919,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f27987bc22b810939e8dfecc55571e9d50355d6ea8ec1c47af8383a76a6d0e1" +checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2" dependencies = [ "arrow", "async-trait", @@ -933,22 +935,40 @@ dependencies = [ "itertools 0.14.0", "log", "parking_lot", - "sqlparser", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "log", + "object_store", + "tokio", ] [[package]] name = "datafusion-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3f6d5b8c9408cc692f7c194b8aa0c0f9b253e065a8d960ad9cdc2a13e697602" +checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" dependencies = [ "ahash", "apache-avro", "arrow", - "arrow-array", - "arrow-buffer", "arrow-ipc", - "arrow-schema", "base64 0.22.1", "half", "hashbrown 0.14.5", @@ -966,25 +986,59 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4603c8e8a4baf77660ab7074cc66fc15cc8a18f2ce9dfadb755fc6ee294e48" +checksum = "e0fcf41523b22e14cc349b01526e8b9f59206653037f2949a4adbfde5f8cb668" dependencies = [ "log", "tokio", ] +[[package]] +name = "datafusion-datasource" +version = "46.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2 0.5.1", + "chrono", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "flate2", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand", + "tokio", + "tokio-util", + "url", + "xz2", + "zstd", +] + [[package]] name = "datafusion-doc" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bf4bc68623a5cf231eed601ed6eb41f46a37c4d15d11a0bff24cbc8396cd66" +checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256" [[package]] name = "datafusion-execution" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b491c012cdf8e051053426013429a76f74ee3c2db68496c79c323ca1084d27" +checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1" dependencies = [ "arrow", "dashmap", @@ -1001,9 +1055,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a181408d4fc5dc22f9252781a8f39f2d0e5d1b33ec9bde242844980a2689c1" +checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f" dependencies = [ "arrow", "chrono", @@ -1022,26 +1076,25 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1129b48e8534d8c03c6543bcdccef0b55c8ac0c1272a15a56c67068b6eb1885" +checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" dependencies = [ "arrow", "datafusion-common", + "indexmap", "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-ffi" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff47a79d442207c168c6e3e1d970c248589c148e4800e5b285ac1b2cb1a230f8" +checksum = "d740dd9f32a4f4ed1b907e6934201bb059efe6c877532512c661771d973c7b21" dependencies = [ "abi_stable", "arrow", - "arrow-array", - "arrow-schema", "async-ffi", "async-trait", "datafusion", @@ -1055,9 +1108,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6125874e4856dfb09b59886784fcb74cde5cfc5930b3a80a1a728ef7a010df6b" +checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40" dependencies = [ "arrow", "arrow-buffer", @@ -1071,7 +1124,6 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-macros", - "hashbrown 0.14.5", "hex", "itertools 0.14.0", "log", @@ -1085,14 +1137,12 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3add7b1d3888e05e7c95f2b281af900ca69ebdcb21069ba679b33bde8b3b9d6" +checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" dependencies = [ "ahash", "arrow", - "arrow-buffer", - "arrow-schema", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1108,9 +1158,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e18baa4cfc3d2f144f74148ed68a1f92337f5072b6dde204a0dbbdf3324989c" +checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" dependencies = [ "ahash", "arrow", @@ -1121,15 +1171,12 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec5ee8cecb0dc370291279673097ddabec03a011f73f30d7f1096457127e03e" +checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a" dependencies = [ "arrow", - "arrow-array", - "arrow-buffer", "arrow-ord", - "arrow-schema", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1145,9 +1192,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c403ddd473bbb0952ba880008428b3c7febf0ed3ce1eec35a205db20efb2a36" +checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5" dependencies = [ "arrow", "async-trait", @@ -1161,9 +1208,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab18c2fb835614d06a75f24a9e09136d3a8c12a92d97c95a6af316a1787a9c5" +checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6" dependencies = [ "datafusion-common", "datafusion-doc", @@ -1178,9 +1225,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77b73bc15e7d1967121fdc7a55d819bfb9d6c03766a6c322247dce9094a53a4" +checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1188,9 +1235,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09369b8d962291e808977cf94d495fd8b5b38647232d7ef562c27ac0f495b0af" +checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" dependencies = [ "datafusion-expr", "quote", @@ -1199,9 +1246,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2403a7e4a84637f3de7d8d4d7a9ccc0cc4be92d89b0161ba3ee5be82f0531c54" +checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b" dependencies = [ "arrow", "chrono", @@ -1218,15 +1265,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ff72ac702b62dbf2650c4e1d715ebd3e4aab14e3885e72e8549e250307347c" +checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" dependencies = [ "ahash", "arrow", - "arrow-array", - "arrow-buffer", - "arrow-schema", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1243,13 +1287,12 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60982b7d684e25579ee29754b4333057ed62e2cc925383c5f0bd8cab7962f435" +checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" dependencies = [ "ahash", "arrow", - "arrow-buffer", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", @@ -1258,12 +1301,11 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5e85c189d5238a5cf181a624e450c4cd4c66ac77ca551d6f3ff9080bac90bb" +checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr", @@ -1271,23 +1313,19 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "futures", "itertools 0.14.0", "log", "recursive", - "url", ] [[package]] name = "datafusion-physical-plan" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36bf163956d7e2542657c78b3383fdc78f791317ef358a359feffcdb968106f" +checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" dependencies = [ "ahash", "arrow", - "arrow-array", - "arrow-buffer", "arrow-ord", "arrow-schema", "async-trait", @@ -1312,9 +1350,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2db5d79f0c974041787b899d24dc91bdab2ff112d1942dd71356a4ce3b407e6c" +checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f" dependencies = [ "arrow", "chrono", @@ -1328,9 +1366,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de21bde1603aac0ff32cf478e47081be6e3583c6861fe8f57034da911efe7578" +checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f" dependencies = [ "arrow", "datafusion-common", @@ -1362,13 +1400,11 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13caa4daede211ecec53c78b13c503b592794d125f9a3cc3afe992edf9e7f43" +checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64" dependencies = [ "arrow", - "arrow-array", - "arrow-schema", "bigdecimal", "datafusion-common", "datafusion-expr", @@ -1381,11 +1417,10 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "45.0.0" +version = "46.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1634405abd8bd3c64c352f2da2f2aec6d80a815930257e0db0ce4ff5daf00944" +checksum = "2c2be3226a683e02cff65181e66e62eba9f812ed0e9b7ec8fe11ac8dabf1a73f" dependencies = [ - "arrow-buffer", "async-recursion", "async-trait", "chrono", @@ -1395,6 +1430,7 @@ dependencies = [ "pbjson-types", "prost", "substrait", + "tokio", "url", ] @@ -1472,9 +1508,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide", @@ -2117,9 +2153,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libflate" @@ -2447,9 +2483,9 @@ dependencies = [ [[package]] name = "parquet" -version = "54.1.0" +version = "54.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" +checksum = "94243778210509a5a5e9e012872127180c155d73a9cd6e2df9243d213e81e100" dependencies = [ "ahash", "arrow-array", @@ -2479,7 +2515,6 @@ dependencies = [ "tokio", "twox-hash", "zstd", - "zstd-sys", ] [[package]] @@ -3401,11 +3436,12 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.53.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" +checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" dependencies = [ "log", + "recursive", "sqlparser_derive", ] @@ -3466,9 +3502,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.52.3" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db15789cecbfdf6b1fcf2db807e767c92273bdc407ac057c2194b070c597756" +checksum = "6fac3d70185423235f37b889764e184b81a5af4bb7c95833396ee9bd92577e1b" dependencies = [ "heck", "pbjson", @@ -3922,12 +3958,14 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.13.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ "getrandom 0.3.1", + "js-sys", "serde", + "wasm-bindgen", ] [[package]] @@ -4114,6 +4152,12 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + [[package]] name = "windows-registry" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 50967a219..8afabdd82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,24 +34,24 @@ protoc = [ "datafusion-substrait/protoc" ] substrait = ["dep:datafusion-substrait"] [dependencies] -tokio = { version = "1.42", features = ["macros", "rt", "rt-multi-thread", "sync"] } +tokio = { version = "1.43", features = ["macros", "rt", "rt-multi-thread", "sync"] } pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"]} -arrow = { version = "54", features = ["pyarrow"] } -datafusion = { version = "45.0.0", features = ["avro", "unicode_expressions"] } -datafusion-substrait = { version = "45.0.0", optional = true } -datafusion-proto = { version = "45.0.0" } -datafusion-ffi = { version = "45.0.0" } -prost = "0.13" # keep in line with `datafusion-substrait` +arrow = { version = "54.2.1", features = ["pyarrow"] } +datafusion = { version = "46.0.1", features = ["avro", "unicode_expressions"] } +datafusion-substrait = { version = "46.0.1", optional = true } +datafusion-proto = { version = "46.0.1" } +datafusion-ffi = { version = "46.0.1" } +prost = "0.13.1" # keep in line with `datafusion-substrait` uuid = { version = "1.12", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } -async-trait = "0.1" +async-trait = "0.1.73" futures = "0.3" object_store = { version = "0.11.0", features = ["aws", "gcp", "azure", "http"] } url = "2" [build-dependencies] -prost-types = "0.13" # keep in line with `datafusion-substrait` +prost-types = "0.13.1" # keep in line with `datafusion-substrait` pyo3-build-config = "0.23" [lib] diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 718ebf69d..eda13930d 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -753,7 +753,8 @@ def test_execution_plan(aggregate_df): assert "AggregateExec:" in indent assert "CoalesceBatchesExec:" in indent assert "RepartitionExec:" in indent - assert "CsvExec:" in indent + assert "DataSourceExec:" in indent + assert "file_type=csv" in indent ctx = SessionContext() rows_returned = 0 diff --git a/src/expr.rs b/src/expr.rs index d3c528eb4..561170289 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::logical_expr::expr::{AggregateFunctionParams, WindowFunctionParams}; use datafusion::logical_expr::utils::exprlist_to_fields; use datafusion::logical_expr::{ ExprFuncBuilder, ExprFunctionExt, LogicalPlan, WindowFunctionDefinition, @@ -172,6 +173,7 @@ impl PyExpr { Expr::ScalarSubquery(value) => { Ok(scalar_subquery::PyScalarSubquery::from(value.clone()).into_bound_py_any(py)?) } + #[allow(deprecated)] Expr::Wildcard { qualifier, options } => Err(py_unsupported_variant_err(format!( "Converting Expr::Wildcard to a Python object is not implemented : {:?} {:?}", qualifier, options @@ -332,7 +334,6 @@ impl PyExpr { | Expr::AggregateFunction { .. } | Expr::WindowFunction { .. } | Expr::InList { .. } - | Expr::Wildcard { .. } | Expr::Exists { .. } | Expr::InSubquery { .. } | Expr::GroupingSet(..) @@ -346,6 +347,10 @@ impl PyExpr { | Expr::Unnest(_) | Expr::IsNotUnknown(_) => RexType::Call, Expr::ScalarSubquery(..) => RexType::ScalarSubquery, + #[allow(deprecated)] + Expr::Wildcard { .. } => { + return Err(py_unsupported_variant_err("Expr::Wildcard is unsupported")) + } }) } @@ -394,11 +399,15 @@ impl PyExpr { | Expr::InSubquery(InSubquery { expr, .. }) => Ok(vec![PyExpr::from(*expr.clone())]), // Expr variants containing a collection of Expr(s) for operands - Expr::AggregateFunction(AggregateFunction { args, .. }) + Expr::AggregateFunction(AggregateFunction { + params: AggregateFunctionParams { args, .. }, + .. + }) | Expr::ScalarFunction(ScalarFunction { args, .. }) - | Expr::WindowFunction(WindowFunction { args, .. }) => { - Ok(args.iter().map(|arg| PyExpr::from(arg.clone())).collect()) - } + | Expr::WindowFunction(WindowFunction { + params: WindowFunctionParams { args, .. }, + .. + }) => Ok(args.iter().map(|arg| PyExpr::from(arg.clone())).collect()), // Expr(s) that require more specific processing Expr::Case(Case { @@ -465,13 +474,17 @@ impl PyExpr { Expr::GroupingSet(..) | Expr::Unnest(_) | Expr::OuterReferenceColumn(_, _) - | Expr::Wildcard { .. } | Expr::ScalarSubquery(..) | Expr::Placeholder { .. } | Expr::Exists { .. } => Err(py_runtime_err(format!( "Unimplemented Expr type: {}", self.expr ))), + + #[allow(deprecated)] + Expr::Wildcard { .. } => { + Err(py_unsupported_variant_err("Expr::Wildcard is unsupported")) + } } } @@ -575,7 +588,7 @@ impl PyExpr { Expr::AggregateFunction(agg_fn) => { let window_fn = Expr::WindowFunction(WindowFunction::new( WindowFunctionDefinition::AggregateUDF(agg_fn.func.clone()), - agg_fn.args.clone(), + agg_fn.params.args.clone(), )); add_builder_fns_to_window( @@ -663,16 +676,8 @@ impl PyExpr { /// Create a [Field] representing an [Expr], given an input [LogicalPlan] to resolve against pub fn expr_to_field(expr: &Expr, input_plan: &LogicalPlan) -> PyDataFusionResult> { - match expr { - Expr::Wildcard { .. } => { - // Since * could be any of the valid column names just return the first one - Ok(Arc::new(input_plan.schema().field(0).clone())) - } - _ => { - let fields = exprlist_to_fields(&[expr.clone()], input_plan)?; - Ok(fields[0].1.clone()) - } - } + let fields = exprlist_to_fields(&[expr.clone()], input_plan)?; + Ok(fields[0].1.clone()) } fn _types(expr: &Expr) -> PyResult { match expr { diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index 8fc9da5b0..a99d83d23 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::common::DataFusionError; -use datafusion::logical_expr::expr::{AggregateFunction, Alias}; +use datafusion::logical_expr::expr::{AggregateFunction, AggregateFunctionParams, Alias}; use datafusion::logical_expr::logical_plan::Aggregate; use datafusion::logical_expr::Expr; use pyo3::{prelude::*, IntoPyObjectExt}; @@ -126,9 +126,11 @@ impl PyAggregate { match expr { // TODO: This Alias logic seems to be returning some strange results that we should investigate Expr::Alias(Alias { expr, .. }) => self._aggregation_arguments(expr.as_ref()), - Expr::AggregateFunction(AggregateFunction { func: _, args, .. }) => { - Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()) - } + Expr::AggregateFunction(AggregateFunction { + func: _, + params: AggregateFunctionParams { args, .. }, + .. + }) => Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()), _ => Err(py_type_err( "Encountered a non Aggregate type in aggregation_arguments", )), diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs index 09471097f..c09f116e3 100644 --- a/src/expr/aggregate_expr.rs +++ b/src/expr/aggregate_expr.rs @@ -40,7 +40,13 @@ impl From for PyAggregateFunction { impl Display for PyAggregateFunction { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - let args: Vec = self.aggr.args.iter().map(|expr| expr.to_string()).collect(); + let args: Vec = self + .aggr + .params + .args + .iter() + .map(|expr| expr.to_string()) + .collect(); write!(f, "{}({})", self.aggr.func.name(), args.join(", ")) } } @@ -54,12 +60,13 @@ impl PyAggregateFunction { /// is this a distinct aggregate such as `COUNT(DISTINCT expr)` fn is_distinct(&self) -> bool { - self.aggr.distinct + self.aggr.params.distinct } /// Get the arguments to the aggregate function fn args(&self) -> Vec { self.aggr + .params .args .iter() .map(|expr| PyExpr::from(expr.clone())) diff --git a/src/expr/window.rs b/src/expr/window.rs index 13deaec25..c5467bf94 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -16,7 +16,7 @@ // under the License. use datafusion::common::{DataFusionError, ScalarValue}; -use datafusion::logical_expr::expr::WindowFunction; +use datafusion::logical_expr::expr::{WindowFunction, WindowFunctionParams}; use datafusion::logical_expr::{Expr, Window, WindowFrame, WindowFrameBound, WindowFrameUnits}; use pyo3::{prelude::*, IntoPyObjectExt}; use std::fmt::{self, Display, Formatter}; @@ -118,7 +118,10 @@ impl PyWindowExpr { /// Returns order by columns in a window function expression pub fn get_sort_exprs(&self, expr: PyExpr) -> PyResult> { match expr.expr.unalias() { - Expr::WindowFunction(WindowFunction { order_by, .. }) => py_sort_expr_list(&order_by), + Expr::WindowFunction(WindowFunction { + params: WindowFunctionParams { order_by, .. }, + .. + }) => py_sort_expr_list(&order_by), other => Err(not_window_function_err(other)), } } @@ -126,9 +129,10 @@ impl PyWindowExpr { /// Return partition by columns in a window function expression pub fn get_partition_exprs(&self, expr: PyExpr) -> PyResult> { match expr.expr.unalias() { - Expr::WindowFunction(WindowFunction { partition_by, .. }) => { - py_expr_list(&partition_by) - } + Expr::WindowFunction(WindowFunction { + params: WindowFunctionParams { partition_by, .. }, + .. + }) => py_expr_list(&partition_by), other => Err(not_window_function_err(other)), } } @@ -136,7 +140,10 @@ impl PyWindowExpr { /// Return input args for window function pub fn get_args(&self, expr: PyExpr) -> PyResult> { match expr.expr.unalias() { - Expr::WindowFunction(WindowFunction { args, .. }) => py_expr_list(&args), + Expr::WindowFunction(WindowFunction { + params: WindowFunctionParams { args, .. }, + .. + }) => py_expr_list(&args), other => Err(not_window_function_err(other)), } } @@ -152,7 +159,10 @@ impl PyWindowExpr { /// Returns a Pywindow frame for a given window function expression pub fn get_frame(&self, expr: PyExpr) -> Option { match expr.expr.unalias() { - Expr::WindowFunction(WindowFunction { window_frame, .. }) => Some(window_frame.into()), + Expr::WindowFunction(WindowFunction { + params: WindowFunctionParams { window_frame, .. }, + .. + }) => Some(window_frame.into()), _ => None, } } diff --git a/src/functions.rs b/src/functions.rs index 8fac239b4..9c406b95a 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -17,6 +17,7 @@ use datafusion::functions_aggregate::all_default_aggregate_functions; use datafusion::functions_window::all_default_window_functions; +use datafusion::logical_expr::expr::WindowFunctionParams; use datafusion::logical_expr::ExprFunctionExt; use datafusion::logical_expr::WindowFrame; use pyo3::{prelude::*, wrap_pyfunction}; @@ -215,10 +216,7 @@ fn alias(expr: PyExpr, name: &str) -> PyResult { #[pyfunction] fn col(name: &str) -> PyResult { Ok(PyExpr { - expr: datafusion::logical_expr::Expr::Column(Column { - relation: None, - name: name.to_string(), - }), + expr: datafusion::logical_expr::Expr::Column(Column::new_unqualified(name)), }) } @@ -333,19 +331,21 @@ fn window( Ok(PyExpr { expr: datafusion::logical_expr::Expr::WindowFunction(WindowFunction { fun, - args: args.into_iter().map(|x| x.expr).collect::>(), - partition_by: partition_by - .unwrap_or_default() - .into_iter() - .map(|x| x.expr) - .collect::>(), - order_by: order_by - .unwrap_or_default() - .into_iter() - .map(|x| x.into()) - .collect::>(), - window_frame, - null_treatment: None, + params: WindowFunctionParams { + args: args.into_iter().map(|x| x.expr).collect::>(), + partition_by: partition_by + .unwrap_or_default() + .into_iter() + .map(|x| x.expr) + .collect::>(), + order_by: order_by + .unwrap_or_default() + .into_iter() + .map(|x| x.into()) + .collect::>(), + window_frame, + null_treatment: None, + }, }), }) } From 583e1e9420906c99b1fbdf57c0138f1e67548008 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Sun, 30 Mar 2025 08:44:55 -0400 Subject: [PATCH 283/413] Update changelog and version number (#1089) --- Cargo.lock | 2 +- Cargo.toml | 2 +- dev/changelog/46.0.0.md | 73 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 dev/changelog/46.0.0.md diff --git a/Cargo.lock b/Cargo.lock index 3a4915f23..f90038c50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1377,7 +1377,7 @@ dependencies = [ [[package]] name = "datafusion-python" -version = "45.2.0" +version = "46.0.0" dependencies = [ "arrow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 8afabdd82..bc8639d4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ [package] name = "datafusion-python" -version = "45.2.0" +version = "46.0.0" homepage = "https://datafusion.apache.org/python" repository = "https://github.com/apache/datafusion-python" authors = ["Apache DataFusion "] diff --git a/dev/changelog/46.0.0.md b/dev/changelog/46.0.0.md new file mode 100644 index 000000000..3e5768099 --- /dev/null +++ b/dev/changelog/46.0.0.md @@ -0,0 +1,73 @@ + + +# Apache DataFusion Python 46.0.0 Changelog + +This release consists of 21 commits from 11 contributors. See credits at the end of this changelog for more information. + +**Implemented enhancements:** + +- feat: reads using global ctx [#982](https://github.com/apache/datafusion-python/pull/982) (ion-elgreco) +- feat: Implementation of udf and udaf decorator [#1040](https://github.com/apache/datafusion-python/pull/1040) (CrystalZhou0529) +- feat: expose regex_count function [#1066](https://github.com/apache/datafusion-python/pull/1066) (nirnayroy) +- feat: Update DataFusion dependency to 46 [#1079](https://github.com/apache/datafusion-python/pull/1079) (timsaucer) + +**Fixed bugs:** + +- fix: add to_timestamp_nanos [#1020](https://github.com/apache/datafusion-python/pull/1020) (chenkovsky) +- fix: type checking [#993](https://github.com/apache/datafusion-python/pull/993) (chenkovsky) + +**Other:** + +- [infra] Fail Clippy on rust build warnings [#1029](https://github.com/apache/datafusion-python/pull/1029) (kevinjqliu) +- Add user documentation for the FFI approach [#1031](https://github.com/apache/datafusion-python/pull/1031) (timsaucer) +- build(deps): bump arrow from 54.1.0 to 54.2.0 [#1035](https://github.com/apache/datafusion-python/pull/1035) (dependabot[bot]) +- Chore: Release datafusion-python 45 [#1024](https://github.com/apache/datafusion-python/pull/1024) (timsaucer) +- Enable Dataframe to be converted into views which can be used in register_table [#1016](https://github.com/apache/datafusion-python/pull/1016) (kosiew) +- Add ruff check for missing futures import [#1052](https://github.com/apache/datafusion-python/pull/1052) (timsaucer) +- Enable take comments to assign issues to users [#1058](https://github.com/apache/datafusion-python/pull/1058) (timsaucer) +- Update python min version to 3.9 [#1043](https://github.com/apache/datafusion-python/pull/1043) (kevinjqliu) +- feat/improve ruff test coverage [#1055](https://github.com/apache/datafusion-python/pull/1055) (timsaucer) +- feat/making global context accessible for users [#1060](https://github.com/apache/datafusion-python/pull/1060) (jsai28) +- Renaming Internal Structs [#1059](https://github.com/apache/datafusion-python/pull/1059) (Spaarsh) +- test: add pytest asyncio tests [#1063](https://github.com/apache/datafusion-python/pull/1063) (jsai28) +- Add decorator for udwf [#1061](https://github.com/apache/datafusion-python/pull/1061) (kosiew) +- Add additional ruff suggestions [#1062](https://github.com/apache/datafusion-python/pull/1062) (Spaarsh) +- Improve collection during repr and repr_html [#1036](https://github.com/apache/datafusion-python/pull/1036) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 7 Tim Saucer + 2 Kevin Liu + 2 Spaarsh + 2 jsai28 + 2 kosiew + 1 Chen Chongchen + 1 Chongchen Chen + 1 Crystal Zhou + 1 Ion Koutsouris + 1 Nirnay Roy + 1 dependabot[bot] +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. + From ffafb59e1b1b7f49f4ba4507b28ba1cecfb0225a Mon Sep 17 00:00:00 2001 From: Chen Chongchen Date: Sun, 30 Mar 2025 20:45:15 +0800 Subject: [PATCH 284/413] feat: support unparser (#1088) * support unparser * add license * add export * format * format --- python/datafusion/__init__.py | 3 +- python/datafusion/unparser.py | 80 +++++++++++++++++++++++++++++++++++ python/tests/test_unparser.py | 33 +++++++++++++++ src/lib.rs | 5 +++ src/unparser/dialect.rs | 63 +++++++++++++++++++++++++++ src/unparser/mod.rs | 66 +++++++++++++++++++++++++++++ 6 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 python/datafusion/unparser.py create mode 100644 python/tests/test_unparser.py create mode 100644 src/unparser/dialect.rs create mode 100644 src/unparser/mod.rs diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index d871fdb71..ecf5545bc 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -26,7 +26,7 @@ except ImportError: import importlib_metadata -from . import functions, object_store, substrait +from . import functions, object_store, substrait, unparser # The following imports are okay to remain as opaque to the user. from ._internal import Config @@ -89,6 +89,7 @@ "udaf", "udf", "udwf", + "unparser", ] diff --git a/python/datafusion/unparser.py b/python/datafusion/unparser.py new file mode 100644 index 000000000..7ca5b9190 --- /dev/null +++ b/python/datafusion/unparser.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""This module provides support for unparsing datafusion plans to SQL. + +For additional information about unparsing, see https://docs.rs/datafusion-sql/latest/datafusion_sql/unparser/index.html +""" + +from ._internal import unparser as unparser_internal +from .plan import LogicalPlan + + +class Dialect: + """DataFusion data catalog.""" + + def __init__(self, dialect: unparser_internal.Dialect) -> None: + """This constructor is not typically called by the end user.""" + self.dialect = dialect + + @staticmethod + def default() -> "Dialect": + """Create a new default dialect.""" + return Dialect(unparser_internal.Dialect.default()) + + @staticmethod + def mysql() -> "Dialect": + """Create a new MySQL dialect.""" + return Dialect(unparser_internal.Dialect.mysql()) + + @staticmethod + def postgres() -> "Dialect": + """Create a new PostgreSQL dialect.""" + return Dialect(unparser_internal.Dialect.postgres()) + + @staticmethod + def sqlite() -> "Dialect": + """Create a new SQLite dialect.""" + return Dialect(unparser_internal.Dialect.sqlite()) + + @staticmethod + def duckdb() -> "Dialect": + """Create a new DuckDB dialect.""" + return Dialect(unparser_internal.Dialect.duckdb()) + + +class Unparser: + """DataFusion unparser.""" + + def __init__(self, dialect: Dialect) -> None: + """This constructor is not typically called by the end user.""" + self.unparser = unparser_internal.Unparser(dialect.dialect) + + def plan_to_sql(self, plan: LogicalPlan) -> str: + """Convert a logical plan to a SQL string.""" + return self.unparser.plan_to_sql(plan._raw_plan) + + def with_pretty(self, pretty: bool) -> "Unparser": + """Set the pretty flag.""" + self.unparser = self.unparser.with_pretty(pretty) + return self + + +__all__ = [ + "Dialect", + "Unparser", +] diff --git a/python/tests/test_unparser.py b/python/tests/test_unparser.py new file mode 100644 index 000000000..c4e05780c --- /dev/null +++ b/python/tests/test_unparser.py @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from datafusion.context import SessionContext +from datafusion.unparser import Dialect, Unparser + + +def test_unparser(): + ctx = SessionContext() + df = ctx.sql("SELECT 1") + for dialect in [ + Dialect.mysql(), + Dialect.postgres(), + Dialect.sqlite(), + Dialect.duckdb(), + ]: + unparser = Unparser(dialect) + sql = unparser.plan_to_sql(df.logical_plan()) + assert sql == "SELECT 1" diff --git a/src/lib.rs b/src/lib.rs index ce93ff0c3..6eeda0878 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -52,6 +52,7 @@ pub mod pyarrow_util; mod record_batch; pub mod sql; pub mod store; +pub mod unparser; #[cfg(feature = "substrait")] pub mod substrait; @@ -103,6 +104,10 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { expr::init_module(&expr)?; m.add_submodule(&expr)?; + let unparser = PyModule::new(py, "unparser")?; + unparser::init_module(&unparser)?; + m.add_submodule(&unparser)?; + // Register the functions as a submodule let funcs = PyModule::new(py, "functions")?; functions::init_module(&funcs)?; diff --git a/src/unparser/dialect.rs b/src/unparser/dialect.rs new file mode 100644 index 000000000..caeef9949 --- /dev/null +++ b/src/unparser/dialect.rs @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use datafusion::sql::unparser::dialect::{ + DefaultDialect, Dialect, DuckDBDialect, MySqlDialect, PostgreSqlDialect, SqliteDialect, +}; +use pyo3::prelude::*; + +#[pyclass(name = "Dialect", module = "datafusion.unparser", subclass)] +#[derive(Clone)] +pub struct PyDialect { + pub dialect: Arc, +} + +#[pymethods] +impl PyDialect { + #[staticmethod] + pub fn default() -> Self { + Self { + dialect: Arc::new(DefaultDialect {}), + } + } + #[staticmethod] + pub fn postgres() -> Self { + Self { + dialect: Arc::new(PostgreSqlDialect {}), + } + } + #[staticmethod] + pub fn mysql() -> Self { + Self { + dialect: Arc::new(MySqlDialect {}), + } + } + #[staticmethod] + pub fn sqlite() -> Self { + Self { + dialect: Arc::new(SqliteDialect {}), + } + } + #[staticmethod] + pub fn duckdb() -> Self { + Self { + dialect: Arc::new(DuckDBDialect::new()), + } + } +} diff --git a/src/unparser/mod.rs b/src/unparser/mod.rs new file mode 100644 index 000000000..b4b0fed10 --- /dev/null +++ b/src/unparser/mod.rs @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +mod dialect; + +use std::sync::Arc; + +use datafusion::sql::unparser::{dialect::Dialect, Unparser}; +use dialect::PyDialect; +use pyo3::{exceptions::PyValueError, prelude::*}; + +use crate::sql::logical::PyLogicalPlan; + +#[pyclass(name = "Unparser", module = "datafusion.unparser", subclass)] +#[derive(Clone)] +pub struct PyUnparser { + dialect: Arc, + pretty: bool, +} + +#[pymethods] +impl PyUnparser { + #[new] + pub fn new(dialect: PyDialect) -> Self { + Self { + dialect: dialect.dialect.clone(), + pretty: false, + } + } + + pub fn plan_to_sql(&self, plan: &PyLogicalPlan) -> PyResult { + let mut unparser = Unparser::new(self.dialect.as_ref()); + unparser = unparser.with_pretty(self.pretty); + let sql = unparser + .plan_to_sql(&plan.plan()) + .map_err(|e| PyValueError::new_err(e.to_string()))?; + Ok(sql.to_string()) + } + + pub fn with_pretty(&self, pretty: bool) -> Self { + Self { + dialect: self.dialect.clone(), + pretty, + } + } +} + +pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + Ok(()) +} From 09b929a65c27ce8c58563d4def8d79b426ae47e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=A4fer?= <33159547+floscha@users.noreply.github.com> Date: Sun, 30 Mar 2025 14:45:49 +0200 Subject: [PATCH 285/413] Documentation updates: mention correct dataset on basics page (#1081) * Documentation updates: mention correct dataset on basics page * Update docs/source/user-guide/basics.rst Co-authored-by: Kevin Liu * Make download hint more concise --------- Co-authored-by: Kevin Liu --- docs/source/user-guide/basics.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/basics.rst b/docs/source/user-guide/basics.rst index f37378a41..6636c0c6a 100644 --- a/docs/source/user-guide/basics.rst +++ b/docs/source/user-guide/basics.rst @@ -20,8 +20,8 @@ Concepts ======== -In this section, we will cover a basic example to introduce a few key concepts. We will use the same -source file as described in the :ref:`Introduction `, the Pokemon data set. +In this section, we will cover a basic example to introduce a few key concepts. We will use the +2021 Yellow Taxi Trip Records ([download](https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet)), from the [TLC Trip Record Data](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page). .. ipython:: python From 818975b5c43021fed109ebba3cb99d744e8f036a Mon Sep 17 00:00:00 2001 From: kosiew Date: Mon, 21 Apr 2025 19:51:25 +0800 Subject: [PATCH 286/413] Add Configurable HTML Table Formatter for DataFusion DataFrames in Python (#1100) * feat: add configurable HTML formatter for DataFrames * fix: update schema iteration in DataFrameHtmlFormatter to use correct format * refactor: remove unused constant MAX_LENGTH_CELL_WITHOUT_MINIMIZE in PyTableProvider * refactor: improve HTML rendering structure in DataFrameHtmlFormatter - Added List import to typing for type hints. - Refactored format_html method to modularize HTML component generation. - Created separate methods for building HTML header, table container, header, body, expandable cells, regular cells, and footer for better readability and maintainability. - Updated table_uuid generation to use f-string for consistency. - Ensured all HTML components are returned as lists for efficient joining. * doc: enhance docstrings for DataFrameHtmlFormatter methods to clarify usage * refactor: enhance DataFrameHtmlFormatter with customizable cell and header styles - Added methods `get_cell_style()` and `get_header_style()` to allow subclasses to customize the CSS styles for table cells and headers. - Updated `_build_table_header()` and `_build_regular_cell()` methods to utilize the new styling methods for improved maintainability. - Introduced a registry for custom type formatters in `DataFrameHtmlFormatter` to enable flexible formatting of cell values based on their types. - Enhanced `_format_cell_value()` to check for registered formatters before defaulting to string conversion, improving extensibility. * refactor: enhance DataFrameHtmlFormatter with custom cell and header builders - Introduced CellFormatter and StyleProvider protocols for better extensibility. - Added DefaultStyleProvider class with default CSS styles for cells and headers. - Updated DataFrameHtmlFormatter to support custom cell and header builders. - Refactored methods to utilize the new style provider for consistent styling. - Improved documentation for methods and classes to clarify usage and customization options. * doc: expand module docstring for DataFrameHtmlFormatter with usage examples and customization options * refactor: streamline HTML formatter by removing extensive docstring examples and enhancing cell formatting methods - Removed lengthy examples from the docstring of DataFrameHtmlFormatter to improve readability. - Added methods for extracting and formatting cell values, enhancing the clarity and maintainability of the code. - Updated cell building methods to utilize the new formatting logic, ensuring consistent application of styles and behaviors. - Introduced a reset fixture for tests to ensure the formatter is returned to default settings after each test case. - Added tests for HTML formatter configuration, custom style providers, type formatters, custom cell builders, and complex customizations to ensure robust functionality. * refactor: improve cell rendering logic in DataFrameHtmlFormatter by utilizing raw values for custom cell builders and optimizing expandable cell creation * refactor: enhance HTML representation in DataFrame by integrating latest formatter and improving cell value formatting logic * refactor: improve HTML formatting logic in DataFrame by separating data collection and schema retrieval for clarity refactor: enhance reset_formatter fixture to preserve original formatter configuration during tests * refactor: add debug utilities for HTML formatter integration testing and enhance debugging output in DataFrameHtmlFormatter * refactor: implement HTML formatter patch for DataFrame and enhance value retrieval in cell formatting * fix: correct typo in file extension check for parquet files in test_write_compressed_parquet * test: add test for DataFrame._repr_html_ to validate HTML output structure * refactor: remove monkeypatch for DataFrame._repr_html_ and associated logic * refactor: simplify _repr_html_ method in DataFrame to directly call internal representation * refactor: remove debug utilities for HTML formatter integration in DataFrame * refactor: remove debug print statements from DataFrameHtmlFormatter and add HTML formatter integration tests - Removed debug print statements from format_html, _build_table_body, and get_formatter methods in DataFrameHtmlFormatter to clean up the code. - Introduced a new debug_utils.py file containing a function to check HTML formatter integration. - Updated __init__.py to include configure_formatter for easier access. - Enhanced DataFrame class to include a docstring for _repr_html_ method. - Added comprehensive tests for HTML formatter configuration, custom style providers, type formatters, and cell/header builders in test_dataframe.py. * refactor: streamline imports and enhance HTML formatter integration in tests - Removed redundant import of `configure_formatter` in `__init__.py`. - Added `configure_formatter` to `__all__` in `__init__.py` for better module exposure. - Cleaned up import statements in `html_formatter.py` for clarity. - Consolidated import statements in `test_dataframe.py` for improved readability. - Simplified the `reset_formatter` fixture by removing unnecessary imports and comments. * refactor: remove redundant imports and debug print statements in HTML formatter tests * refactor: add reset_formatter function to reset global HTML formatter state - Implemented reset_formatter to create a new default DataFrame HTML formatter and update the global reference. - Added clean_formatter_state fixture in tests to ensure a fresh formatter state for each test case. - Updated test cases to use clean_formatter_state instead of the previous reset_formatter implementation. * refactor: enhance DataFrameHtmlFormatter initialization with parameter validation * test: add custom cell builder test for HTML formatter with value-based styling * test: enhance DataFrame HTML representation tests for structure and values * feat: enhance DataFrameHtmlFormatter with shared styles support and reset functionality - Added `use_shared_styles` parameter to control loading of styles/scripts. - Implemented logic to conditionally include styles based on `use_shared_styles`. - Updated the constructor to validate `use_shared_styles` as a boolean. - Introduced `reset_styles_loaded_state` function to reset the styles loaded state. - Modified `reset_formatter` to reset the `_styles_loaded` flag. * refactor: update footer comment in DataFrameHtmlFormatter to clarify content * test: enhance HTML representation test to accommodate span-wrapped values * docs: add usage examples to formatter functions in html_formatter.py * test: add HTML formatter tests for shared styles functionality * feat: add method to check if styles are loaded and enhance schema validation in DataFrameHtmlFormatter * refactor: streamline custom cell builder in HTML formatter tests for clarity and maintainability * fix ruff errors * chore: update license header in html_formatter.py for compliance * refactor: improve HTML formatter tests by updating import statements and enhancing regex patterns for body data * fix clippy errors --- python/datafusion/__init__.py | 2 + python/datafusion/html_formatter.py | 647 ++++++++++++++++++++++++++++ python/tests/test_dataframe.py | 396 ++++++++++++++++- src/dataframe.rs | 130 +----- 4 files changed, 1061 insertions(+), 114 deletions(-) create mode 100644 python/datafusion/html_formatter.py diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index ecf5545bc..60d0d61b4 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -45,6 +45,7 @@ Expr, WindowFrame, ) +from .html_formatter import configure_formatter from .io import read_avro, read_csv, read_json, read_parquet from .plan import ExecutionPlan, LogicalPlan from .record_batch import RecordBatch, RecordBatchStream @@ -76,6 +77,7 @@ "col", "column", "common", + "configure_formatter", "expr", "functions", "lit", diff --git a/python/datafusion/html_formatter.py b/python/datafusion/html_formatter.py new file mode 100644 index 000000000..a50e14fd5 --- /dev/null +++ b/python/datafusion/html_formatter.py @@ -0,0 +1,647 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""HTML formatting utilities for DataFusion DataFrames.""" + +from __future__ import annotations + +from typing import ( + Any, + Callable, + Optional, + Protocol, + runtime_checkable, +) + + +@runtime_checkable +class CellFormatter(Protocol): + """Protocol for cell value formatters.""" + + def __call__(self, value: Any) -> str: + """Format a cell value to string representation.""" + ... + + +@runtime_checkable +class StyleProvider(Protocol): + """Protocol for HTML style providers.""" + + def get_cell_style(self) -> str: + """Get the CSS style for table cells.""" + ... + + def get_header_style(self) -> str: + """Get the CSS style for header cells.""" + ... + + +class DefaultStyleProvider: + """Default implementation of StyleProvider.""" + + def get_cell_style(self) -> str: + """Get the CSS style for table cells. + + Returns: + CSS style string + """ + return ( + "border: 1px solid black; padding: 8px; text-align: left; " + "white-space: nowrap;" + ) + + def get_header_style(self) -> str: + """Get the CSS style for header cells. + + Returns: + CSS style string + """ + return ( + "border: 1px solid black; padding: 8px; text-align: left; " + "background-color: #f2f2f2; white-space: nowrap; min-width: fit-content; " + "max-width: fit-content;" + ) + + +class DataFrameHtmlFormatter: + """Configurable HTML formatter for DataFusion DataFrames. + + This class handles the HTML rendering of DataFrames for display in + Jupyter notebooks and other rich display contexts. + + This class supports extension through composition. Key extension points: + - Provide a custom StyleProvider for styling cells and headers + - Register custom formatters for specific types + - Provide custom cell builders for specialized cell rendering + + Args: + max_cell_length: Maximum characters to display in a cell before truncation + max_width: Maximum width of the HTML table in pixels + max_height: Maximum height of the HTML table in pixels + enable_cell_expansion: Whether to add expand/collapse buttons for long cell + values + custom_css: Additional CSS to include in the HTML output + show_truncation_message: Whether to display a message when data is truncated + style_provider: Custom provider for cell and header styles + use_shared_styles: Whether to load styles and scripts only once per notebook + session + """ + + # Class variable to track if styles have been loaded in the notebook + _styles_loaded = False + + def __init__( + self, + max_cell_length: int = 25, + max_width: int = 1000, + max_height: int = 300, + enable_cell_expansion: bool = True, + custom_css: Optional[str] = None, + show_truncation_message: bool = True, + style_provider: Optional[StyleProvider] = None, + use_shared_styles: bool = True, + ) -> None: + """Initialize the HTML formatter. + + Parameters + ---------- + max_cell_length : int, default 25 + Maximum length of cell content before truncation. + max_width : int, default 1000 + Maximum width of the displayed table in pixels. + max_height : int, default 300 + Maximum height of the displayed table in pixels. + enable_cell_expansion : bool, default True + Whether to allow cells to expand when clicked. + custom_css : str, optional + Custom CSS to apply to the HTML table. + show_truncation_message : bool, default True + Whether to show a message indicating that content has been truncated. + style_provider : StyleProvider, optional + Provider of CSS styles for the HTML table. If None, DefaultStyleProvider + is used. + use_shared_styles : bool, default True + Whether to use shared styles across multiple tables. + + Raises: + ------ + ValueError + If max_cell_length, max_width, or max_height is not a positive integer. + TypeError + If enable_cell_expansion, show_truncation_message, or use_shared_styles is + not a boolean, + or if custom_css is provided but is not a string, + or if style_provider is provided but does not implement the StyleProvider + protocol. + """ + # Validate numeric parameters + + if not isinstance(max_cell_length, int) or max_cell_length <= 0: + msg = "max_cell_length must be a positive integer" + raise ValueError(msg) + if not isinstance(max_width, int) or max_width <= 0: + msg = "max_width must be a positive integer" + raise ValueError(msg) + if not isinstance(max_height, int) or max_height <= 0: + msg = "max_height must be a positive integer" + raise ValueError(msg) + + # Validate boolean parameters + if not isinstance(enable_cell_expansion, bool): + msg = "enable_cell_expansion must be a boolean" + raise TypeError(msg) + if not isinstance(show_truncation_message, bool): + msg = "show_truncation_message must be a boolean" + raise TypeError(msg) + if not isinstance(use_shared_styles, bool): + msg = "use_shared_styles must be a boolean" + raise TypeError(msg) + + # Validate custom_css + if custom_css is not None and not isinstance(custom_css, str): + msg = "custom_css must be None or a string" + raise TypeError(msg) + + # Validate style_provider + if style_provider is not None and not isinstance(style_provider, StyleProvider): + msg = "style_provider must implement the StyleProvider protocol" + raise TypeError(msg) + + self.max_cell_length = max_cell_length + self.max_width = max_width + self.max_height = max_height + self.enable_cell_expansion = enable_cell_expansion + self.custom_css = custom_css + self.show_truncation_message = show_truncation_message + self.style_provider = style_provider or DefaultStyleProvider() + self.use_shared_styles = use_shared_styles + # Registry for custom type formatters + self._type_formatters: dict[type, CellFormatter] = {} + # Custom cell builders + self._custom_cell_builder: Optional[Callable[[Any, int, int, str], str]] = None + self._custom_header_builder: Optional[Callable[[Any], str]] = None + + def register_formatter(self, type_class: type, formatter: CellFormatter) -> None: + """Register a custom formatter for a specific data type. + + Args: + type_class: The type to register a formatter for + formatter: Function that takes a value of the given type and returns + a formatted string + """ + self._type_formatters[type_class] = formatter + + def set_custom_cell_builder( + self, builder: Callable[[Any, int, int, str], str] + ) -> None: + """Set a custom cell builder function. + + Args: + builder: Function that takes (value, row, col, table_id) and returns HTML + """ + self._custom_cell_builder = builder + + def set_custom_header_builder(self, builder: Callable[[Any], str]) -> None: + """Set a custom header builder function. + + Args: + builder: Function that takes a field and returns HTML + """ + self._custom_header_builder = builder + + @classmethod + def is_styles_loaded(cls) -> bool: + """Check if HTML styles have been loaded in the current session. + + This method is primarily intended for debugging UI rendering issues + related to style loading. + + Returns: + True if styles have been loaded, False otherwise + + Example: + >>> from datafusion.html_formatter import DataFrameHtmlFormatter + >>> DataFrameHtmlFormatter.is_styles_loaded() + False + """ + return cls._styles_loaded + + def format_html( + self, + batches: list, + schema: Any, + has_more: bool = False, + table_uuid: str | None = None, + ) -> str: + """Format record batches as HTML. + + This method is used by DataFrame's _repr_html_ implementation and can be + called directly when custom HTML rendering is needed. + + Args: + batches: List of Arrow RecordBatch objects + schema: Arrow Schema object + has_more: Whether there are more batches not shown + table_uuid: Unique ID for the table, used for JavaScript interactions + + Returns: + HTML string representation of the data + + Raises: + TypeError: If schema is invalid and no batches are provided + """ + if not batches: + return "No data to display" + + # Validate schema + if schema is None or not hasattr(schema, "__iter__"): + msg = "Schema must be provided" + raise TypeError(msg) + + # Generate a unique ID if none provided + table_uuid = table_uuid or f"df-{id(batches)}" + + # Build HTML components + html = [] + + # Only include styles and scripts if: + # 1. Not using shared styles, OR + # 2. Using shared styles but they haven't been loaded yet + include_styles = ( + not self.use_shared_styles or not DataFrameHtmlFormatter._styles_loaded + ) + + if include_styles: + html.extend(self._build_html_header()) + # If we're using shared styles, mark them as loaded + if self.use_shared_styles: + DataFrameHtmlFormatter._styles_loaded = True + + html.extend(self._build_table_container_start()) + + # Add table header and body + html.extend(self._build_table_header(schema)) + html.extend(self._build_table_body(batches, table_uuid)) + + html.append("") + html.append("") + + # Add footer (JavaScript and messages) + if include_styles and self.enable_cell_expansion: + html.append(self._get_javascript()) + + # Always add truncation message if needed (independent of styles) + if has_more and self.show_truncation_message: + html.append("
Data truncated due to size.
") + + return "\n".join(html) + + def _build_html_header(self) -> list[str]: + """Build the HTML header with CSS styles.""" + html = [] + html.append("") + return html + + def _build_table_container_start(self) -> list[str]: + """Build the opening tags for the table container.""" + html = [] + html.append( + f'
' + ) + html.append('') + return html + + def _build_table_header(self, schema: Any) -> list[str]: + """Build the HTML table header with column names.""" + html = [] + html.append("") + html.append("") + for field in schema: + if self._custom_header_builder: + html.append(self._custom_header_builder(field)) + else: + html.append( + f"" + ) + html.append("") + html.append("") + return html + + def _build_table_body(self, batches: list, table_uuid: str) -> list[str]: + """Build the HTML table body with data rows.""" + html = [] + html.append("") + + row_count = 0 + for batch in batches: + for row_idx in range(batch.num_rows): + row_count += 1 + html.append("") + + for col_idx, column in enumerate(batch.columns): + # Get the raw value from the column + raw_value = self._get_cell_value(column, row_idx) + + # Always check for type formatters first to format the value + formatted_value = self._format_cell_value(raw_value) + + # Then apply either custom cell builder or standard cell formatting + if self._custom_cell_builder: + # Pass both the raw value and formatted value to let the + # builder decide + cell_html = self._custom_cell_builder( + raw_value, row_count, col_idx, table_uuid + ) + html.append(cell_html) + else: + # Standard cell formatting with formatted value + if ( + len(str(raw_value)) > self.max_cell_length + and self.enable_cell_expansion + ): + cell_html = self._build_expandable_cell( + formatted_value, row_count, col_idx, table_uuid + ) + else: + cell_html = self._build_regular_cell(formatted_value) + html.append(cell_html) + + html.append("") + + html.append("") + return html + + def _get_cell_value(self, column: Any, row_idx: int) -> Any: + """Extract a cell value from a column. + + Args: + column: Arrow array + row_idx: Row index + + Returns: + The raw cell value + """ + try: + value = column[row_idx] + + if hasattr(value, "as_py"): + return value.as_py() + except (AttributeError, TypeError): + pass + else: + return value + + def _format_cell_value(self, value: Any) -> str: + """Format a cell value for display. + + Uses registered type formatters if available. + + Args: + value: The cell value to format + + Returns: + Formatted cell value as string + """ + # Check for custom type formatters + for type_cls, formatter in self._type_formatters.items(): + if isinstance(value, type_cls): + return formatter(value) + + # If no formatter matched, return string representation + return str(value) + + def _build_expandable_cell( + self, formatted_value: str, row_count: int, col_idx: int, table_uuid: str + ) -> str: + """Build an expandable cell for long content.""" + short_value = str(formatted_value)[: self.max_cell_length] + return ( + f"" + ) + + def _build_regular_cell(self, formatted_value: str) -> str: + """Build a regular table cell.""" + return ( + f"" + ) + + def _build_html_footer(self, has_more: bool) -> list[str]: + """Build the HTML footer with JavaScript and messages.""" + html = [] + + # Add JavaScript for interactivity only if cell expansion is enabled + # and we're not using the shared styles approach + if self.enable_cell_expansion and not self.use_shared_styles: + html.append(self._get_javascript()) + + # Add truncation message if needed + if has_more and self.show_truncation_message: + html.append("
Data truncated due to size.
") + + return html + + def _get_default_css(self) -> str: + """Get default CSS styles for the HTML table.""" + return """ + .expandable-container { + display: inline-block; + max-width: 200px; + } + .expandable { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + display: block; + } + .full-text { + display: none; + white-space: normal; + } + .expand-btn { + cursor: pointer; + color: blue; + text-decoration: underline; + border: none; + background: none; + font-size: inherit; + display: block; + margin-top: 5px; + } + """ + + def _get_javascript(self) -> str: + """Get JavaScript code for interactive elements.""" + return """ + + """ + + +class FormatterManager: + """Manager class for the global DataFrame HTML formatter instance.""" + + _default_formatter: DataFrameHtmlFormatter = DataFrameHtmlFormatter() + + @classmethod + def set_formatter(cls, formatter: DataFrameHtmlFormatter) -> None: + """Set the global DataFrame HTML formatter. + + Args: + formatter: The formatter instance to use globally + """ + cls._default_formatter = formatter + _refresh_formatter_reference() + + @classmethod + def get_formatter(cls) -> DataFrameHtmlFormatter: + """Get the current global DataFrame HTML formatter. + + Returns: + The global HTML formatter instance + """ + return cls._default_formatter + + +def get_formatter() -> DataFrameHtmlFormatter: + """Get the current global DataFrame HTML formatter. + + This function is used by the DataFrame._repr_html_ implementation to access + the shared formatter instance. It can also be used directly when custom + HTML rendering is needed. + + Returns: + The global HTML formatter instance + + Example: + >>> from datafusion.html_formatter import get_formatter + >>> formatter = get_formatter() + >>> formatter.max_cell_length = 50 # Increase cell length + """ + return FormatterManager.get_formatter() + + +def set_formatter(formatter: DataFrameHtmlFormatter) -> None: + """Set the global DataFrame HTML formatter. + + Args: + formatter: The formatter instance to use globally + + Example: + >>> from datafusion.html_formatter import get_formatter, set_formatter + >>> custom_formatter = DataFrameHtmlFormatter(max_cell_length=100) + >>> set_formatter(custom_formatter) + """ + FormatterManager.set_formatter(formatter) + + +def configure_formatter(**kwargs: Any) -> None: + """Configure the global DataFrame HTML formatter. + + This function creates a new formatter with the provided configuration + and sets it as the global formatter for all DataFrames. + + Args: + **kwargs: Formatter configuration parameters like max_cell_length, + max_width, max_height, enable_cell_expansion, etc. + + Example: + >>> from datafusion.html_formatter import configure_formatter + >>> configure_formatter( + ... max_cell_length=50, + ... max_height=500, + ... enable_cell_expansion=True, + ... use_shared_styles=True + ... ) + """ + set_formatter(DataFrameHtmlFormatter(**kwargs)) + + +def reset_formatter() -> None: + """Reset the global DataFrame HTML formatter to default settings. + + This function creates a new formatter with default configuration + and sets it as the global formatter for all DataFrames. + + Example: + >>> from datafusion.html_formatter import reset_formatter + >>> reset_formatter() # Reset formatter to default settings + """ + formatter = DataFrameHtmlFormatter() + # Reset the styles_loaded flag to ensure styles will be reloaded + DataFrameHtmlFormatter._styles_loaded = False + set_formatter(formatter) + + +def reset_styles_loaded_state() -> None: + """Reset the styles loaded state to force reloading of styles. + + This can be useful when switching between notebook sessions or + when styles need to be refreshed. + + Example: + >>> from datafusion.html_formatter import reset_styles_loaded_state + >>> reset_styles_loaded_state() # Force styles to reload in next render + """ + DataFrameHtmlFormatter._styles_loaded = False + + +def _refresh_formatter_reference() -> None: + """Refresh formatter reference in any modules using it. + + This helps ensure that changes to the formatter are reflected in existing + DataFrames that might be caching the formatter reference. + """ + # This is a no-op but signals modules to refresh their reference diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index eda13930d..464b884db 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -28,8 +28,17 @@ column, literal, ) -from datafusion import functions as f +from datafusion import ( + functions as f, +) from datafusion.expr import Window +from datafusion.html_formatter import ( + DataFrameHtmlFormatter, + configure_formatter, + get_formatter, + reset_formatter, + reset_styles_loaded_state, +) from pyarrow.csv import write_csv @@ -102,6 +111,12 @@ def partitioned_df(): return ctx.create_dataframe([[batch]]) +@pytest.fixture +def clean_formatter_state(): + """Reset the HTML formatter after each test.""" + reset_formatter() + + def test_select(df): df_1 = df.select( column("a") + column("b"), @@ -656,6 +671,252 @@ def test_window_frame_defaults_match_postgres(partitioned_df): assert df_2.sort(col_a).to_pydict() == expected +def test_html_formatter_configuration(df, clean_formatter_state): + """Test configuring the HTML formatter with different options.""" + # Configure with custom settings + configure_formatter( + max_cell_length=5, + max_width=500, + max_height=200, + enable_cell_expansion=False, + ) + + html_output = df._repr_html_() + + # Verify our configuration was applied + assert "max-height: 200px" in html_output + assert "max-width: 500px" in html_output + # With cell expansion disabled, we shouldn't see expandable-container elements + assert "expandable-container" not in html_output + + +def test_html_formatter_custom_style_provider(df, clean_formatter_state): + """Test using custom style providers with the HTML formatter.""" + + class CustomStyleProvider: + def get_cell_style(self) -> str: + return ( + "background-color: #f5f5f5; color: #333; padding: 8px; border: " + "1px solid #ddd;" + ) + + def get_header_style(self) -> str: + return ( + "background-color: #4285f4; color: white; font-weight: bold; " + "padding: 10px; border: 1px solid #3367d6;" + ) + + # Configure with custom style provider + configure_formatter(style_provider=CustomStyleProvider()) + + html_output = df._repr_html_() + + # Verify our custom styles were applied + assert "background-color: #4285f4" in html_output + assert "color: white" in html_output + assert "background-color: #f5f5f5" in html_output + + +def test_html_formatter_type_formatters(df, clean_formatter_state): + """Test registering custom type formatters for specific data types.""" + + # Get current formatter and register custom formatters + formatter = get_formatter() + + # Format integers with color based on value + # Using int as the type for the formatter will work since we convert + # Arrow scalar values to Python native types in _get_cell_value + def format_int(value): + return f' 2 else "blue"}">{value}' + + formatter.register_formatter(int, format_int) + + html_output = df._repr_html_() + + # Our test dataframe has values 1,2,3 so we should see: + assert '1' in html_output + + +def test_html_formatter_custom_cell_builder(df, clean_formatter_state): + """Test using a custom cell builder function.""" + + # Create a custom cell builder with distinct styling for different value ranges + def custom_cell_builder(value, row, col, table_id): + try: + num_value = int(value) + if num_value > 5: # Values > 5 get green background with indicator + return ( + '' + ) + if num_value < 3: # Values < 3 get blue background with indicator + return ( + '' + ) + except (ValueError, TypeError): + pass + + # Default styling for other cells (3, 4, 5) + return f'' + + # Set our custom cell builder + formatter = get_formatter() + formatter.set_custom_cell_builder(custom_cell_builder) + + html_output = df._repr_html_() + + # Extract cells with specific styling using regex + low_cells = re.findall( + r'', html_output + ) + mid_cells = re.findall( + r'', html_output + ) + high_cells = re.findall( + r'', html_output + ) + + # Sort the extracted values for consistent comparison + low_cells = sorted(map(int, low_cells)) + mid_cells = sorted(map(int, mid_cells)) + high_cells = sorted(map(int, high_cells)) + + # Verify specific values have the correct styling applied + assert low_cells == [1, 2] # Values < 3 + assert mid_cells == [3, 4, 5, 5] # Values 3-5 + assert high_cells == [6, 8, 8] # Values > 5 + + # Verify the exact content with styling appears in the output + assert ( + '' + in html_output + ) + assert ( + '' + in html_output + ) + assert ( + '' in html_output + ) + assert ( + '' in html_output + ) + assert ( + '' + in html_output + ) + assert ( + '' + in html_output + ) + + # Count occurrences to ensure all cells are properly styled + assert html_output.count("-low") == 2 # Two low values (1, 2) + assert html_output.count("-mid") == 4 # Four mid values (3, 4, 5, 5) + assert html_output.count("-high") == 3 # Three high values (6, 8, 8) + + # Create a custom cell builder that changes background color based on value + def custom_cell_builder(value, row, col, table_id): + # Handle numeric values regardless of their exact type + try: + num_value = int(value) + if num_value > 5: # Values > 5 get green background + return f'' + if num_value < 3: # Values < 3 get light blue background + return f'' + except (ValueError, TypeError): + pass + + # Default styling for other cells + return f'' + + # Set our custom cell builder + formatter = get_formatter() + formatter.set_custom_cell_builder(custom_cell_builder) + + html_output = df._repr_html_() + + # Verify our custom cell styling was applied + assert "background-color: #d3e9f0" in html_output # For values 1,2 + + +def test_html_formatter_custom_header_builder(df, clean_formatter_state): + """Test using a custom header builder function.""" + + # Create a custom header builder with tooltips + def custom_header_builder(field): + tooltips = { + "a": "Primary key column", + "b": "Secondary values", + "c": "Additional data", + } + tooltip = tooltips.get(field.name, "") + return ( + f'' + ) + + # Set our custom header builder + formatter = get_formatter() + formatter.set_custom_header_builder(custom_header_builder) + + html_output = df._repr_html_() + + # Verify our custom headers were applied + assert 'title="Primary key column"' in html_output + assert 'title="Secondary values"' in html_output + assert "background-color: #333; color: white" in html_output + + +def test_html_formatter_complex_customization(df, clean_formatter_state): + """Test combining multiple customization options together.""" + + # Create a dark mode style provider + class DarkModeStyleProvider: + def get_cell_style(self) -> str: + return ( + "background-color: #222; color: #eee; " + "padding: 8px; border: 1px solid #444;" + ) + + def get_header_style(self) -> str: + return ( + "background-color: #111; color: #fff; padding: 10px; " + "border: 1px solid #333;" + ) + + # Configure with dark mode style + configure_formatter( + max_cell_length=10, + style_provider=DarkModeStyleProvider(), + custom_css=""" + .datafusion-table { + font-family: monospace; + border-collapse: collapse; + } + .datafusion-table tr:hover td { + background-color: #444 !important; + } + """, + ) + + # Add type formatters for special formatting - now working with native int values + formatter = get_formatter() + formatter.register_formatter( + int, + lambda n: f'{n}', + ) + + html_output = df._repr_html_() + + # Verify our customizations were applied + assert "background-color: #222" in html_output + assert "background-color: #111" in html_output + assert ".datafusion-table" in html_output + assert "color: #5af" in html_output # Even numbers + + def test_get_dataframe(tmp_path): ctx = SessionContext() @@ -1244,7 +1505,10 @@ def add_with_parameter(df_internal, value: Any) -> DataFrame: assert result["new_col"] == [3 for _i in range(3)] -def test_dataframe_repr_html(df) -> None: +def test_dataframe_repr_html_structure(df) -> None: + """Test that DataFrame._repr_html_ produces expected HTML output structure.""" + import re + output = df._repr_html_() # Since we've added a fair bit of processing to the html output, lets just verify @@ -1255,9 +1519,131 @@ def test_dataframe_repr_html(df) -> None: headers = ["a", "b", "c"] headers = [f"{v}" for v in headers] header_pattern = "(.*?)".join(headers) - assert len(re.findall(header_pattern, output, re.DOTALL)) == 1 + header_matches = re.findall(header_pattern, output, re.DOTALL) + assert len(header_matches) == 1 + # Update the pattern to handle values that may be wrapped in spans body_data = [[1, 4, 8], [2, 5, 5], [3, 6, 8]] - body_lines = [f"{v}" for inner in body_data for v in inner] + + body_lines = [ + f"(?:]*?>)?{v}(?:)?" + for inner in body_data + for v in inner + ] body_pattern = "(.*?)".join(body_lines) - assert len(re.findall(body_pattern, output, re.DOTALL)) == 1 + + body_matches = re.findall(body_pattern, output, re.DOTALL) + + assert len(body_matches) == 1, "Expected pattern of values not found in HTML output" + + +def test_dataframe_repr_html_values(df): + """Test that DataFrame._repr_html_ contains the expected data values.""" + html = df._repr_html_() + assert html is not None + + # Create a more flexible pattern that handles values being wrapped in spans + # This pattern will match the sequence of values 1,4,8,2,5,5,3,6,8 regardless + # of formatting + pattern = re.compile( + r"]*?>(?:]*?>)?1(?:)?.*?" + r"]*?>(?:]*?>)?4(?:)?.*?" + r"]*?>(?:]*?>)?8(?:)?.*?" + r"]*?>(?:]*?>)?2(?:)?.*?" + r"]*?>(?:]*?>)?5(?:)?.*?" + r"]*?>(?:]*?>)?5(?:)?.*?" + r"]*?>(?:]*?>)?3(?:)?.*?" + r"]*?>(?:]*?>)?6(?:)?.*?" + r"]*?>(?:]*?>)?8(?:)?", + re.DOTALL, + ) + + # Print debug info if the test fails + matches = re.findall(pattern, html) + if not matches: + print(f"HTML output snippet: {html[:500]}...") # noqa: T201 + + assert len(matches) > 0, "Expected pattern of values not found in HTML output" + + +def test_html_formatter_shared_styles(df, clean_formatter_state): + """Test that shared styles work correctly across multiple tables.""" + + # First, ensure we're using shared styles + configure_formatter(use_shared_styles=True) + + # Get HTML output for first table - should include styles + html_first = df._repr_html_() + + # Verify styles are included in first render + assert " + // Convert record batches to PyObject list + let py_batches = batches + .into_iter() + .map(|rb| rb.to_pyarrow(py)) + .collect::>>()?; -
-
" + f"{field.name}
" + f"
" + "" + "" + f"{formatted_value}" + f"" + f"
" + f"		{formatted_value}{value}-high{value}-low{value}-mid]*>(\d+)-low]*>(\d+)-mid]*>(\d+)-high1-low2-low3-mid4-mid6-high8-high{value}{value}{value}{field.name}
- \n".to_string(); + let py_schema = self.schema().into_pyobject(py)?; - let schema = batches[0].schema(); + // Get the Python formatter module and call format_html + let formatter_module = py.import("datafusion.html_formatter")?; + let get_formatter = formatter_module.getattr("get_formatter")?; + let formatter = get_formatter.call0()?; - let mut header = Vec::new(); - for field in schema.fields() { - header.push(format!("", field.name())); - } - let header_str = header.join(""); - html_str.push_str(&format!("{}\n", header_str)); - - let batch_formatters = batches - .iter() - .map(|batch| { - batch - .columns() - .iter() - .map(|c| ArrayFormatter::try_new(c.as_ref(), &FormatOptions::default())) - .map(|c| { - c.map_err(|e| PyValueError::new_err(format!("Error: {:?}", e.to_string()))) - }) - .collect::, _>>() - }) - .collect::, _>>()?; - - let rows_per_batch = batches.iter().map(|batch| batch.num_rows()); - - // We need to build up row by row for html - let mut table_row = 0; - for (batch_formatter, num_rows_in_batch) in batch_formatters.iter().zip(rows_per_batch) { - for batch_row in 0..num_rows_in_batch { - table_row += 1; - let mut cells = Vec::new(); - for (col, formatter) in batch_formatter.iter().enumerate() { - let cell_data = formatter.value(batch_row).to_string(); - // From testing, primitive data types do not typically get larger than 21 characters - if cell_data.len() > MAX_LENGTH_CELL_WITHOUT_MINIMIZE { - let short_cell_data = &cell_data[0..MAX_LENGTH_CELL_WITHOUT_MINIMIZE]; - cells.push(format!(" - ")); - } else { - cells.push(format!("", formatter.value(batch_row))); - } - } - let row_str = cells.join(""); - html_str.push_str(&format!("{}\n", row_str)); - } - } - html_str.push_str("
{}
-
- {short_cell_data} - {cell_data} - -
- 		{}
\n"); - - html_str.push_str(" - - "); + // Call format_html method on the formatter + let kwargs = pyo3::types::PyDict::new(py); + let py_batches_list = PyList::new(py, py_batches.as_slice())?; + kwargs.set_item("batches", py_batches_list)?; + kwargs.set_item("schema", py_schema)?; + kwargs.set_item("has_more", has_more)?; + kwargs.set_item("table_uuid", table_uuid)?; - if has_more { - html_str.push_str("Data truncated due to size."); - } + let html_result = formatter.call_method("format_html", (), Some(&kwargs))?; + let html_str: String = html_result.extract()?; Ok(html_str) } @@ -835,7 +747,7 @@ fn record_batch_into_schema( ) -> Result { let schema = Arc::new(schema.clone()); let base_schema = record_batch.schema(); - if base_schema.fields().len() == 0 { + if base_schema.fields().is_empty() { // Nothing to project return Ok(RecordBatch::new_empty(schema)); } From d0d14f6e1584f9569cbf2e36c8a7abc7c70fd903 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 24 Apr 2025 09:38:38 -0400 Subject: [PATCH 287/413] feat: update datafusion dependency 47 (#1107) * Update cargo to use DF47 release candidate * Need to be explicit for collection of Expr due to change in dataframe API * Add missing enum variant * Add missing enum variants * The interface for last_value of aggregates upstream changed * Cargo fmt * last value aggregate without ordering is ill defined * Clippy warning * Set datafusion version to 47 now that it is released --- Cargo.lock | 600 +++++++++++++++++++------------ Cargo.toml | 24 +- python/tests/test_aggregation.py | 1 - src/dataframe.rs | 2 +- src/dataset_exec.rs | 4 +- src/expr.rs | 16 +- src/functions.rs | 37 +- 7 files changed, 415 insertions(+), 269 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f90038c50..b32d19d4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,9 +179,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ef243634a39fb6e9d1710737e7a5ef96c9bacabd2326859ff889bc9ef755e5" +checksum = "3095aaf545942ff5abd46654534f15b03a90fba78299d661e045e5d587222f0d" dependencies = [ "arrow-arith", "arrow-array", @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f420c6aef51dad2e4a96ce29c0ec90ad84880bdb60b321c74c652a6be07b93f" +checksum = "00752064ff47cee746e816ddb8450520c3a52cbad1e256f6fa861a35f86c45e7" dependencies = [ "arrow-array", "arrow-buffer", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24bda5ff6461a4ff9739959b3d57b377f45e3f878f7be1a4f28137c0a8f339fa" +checksum = "cebfe926794fbc1f49ddd0cdaf898956ca9f6e79541efce62dabccfd81380472" dependencies = [ "ahash", "arrow-buffer", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6ed265c73f134a583d02c3cab5e16afab9446d8048ede8707e31f85fad58a0" +checksum = "0303c7ec4cf1a2c60310fc4d6bbc3350cd051a17bf9e9c0a8e47b4db79277824" dependencies = [ "bytes", "half", @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c648572391edcef10e5fd458db70ba27ed6f71bcaee04397d0cfb100b34f8b" +checksum = "335f769c5a218ea823d3760a743feba1ef7857cba114c01399a891c2fff34285" dependencies = [ "arrow-array", "arrow-buffer", @@ -264,9 +264,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a02fb265a6d8011a7d3ad1a36f25816ad0a3bb04cb8e9fe7929c165b98c0cbcd" +checksum = "510db7dfbb4d5761826516cc611d97b3a68835d0ece95b034a052601109c0b1b" dependencies = [ "arrow-array", "arrow-cast", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f2cebf504bb6a92a134a87fff98f01b14fbb3a93ecf7aef90cd0f888c5fffa4" +checksum = "e8affacf3351a24039ea24adab06f316ded523b6f8c3dbe28fbac5f18743451b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -292,9 +292,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6405b287671c88846e7751f7291f717b164911474cabac6d3d8614d5aa7374" +checksum = "69880a9e6934d9cba2b8630dd08a3463a91db8693b16b499d54026b6137af284" dependencies = [ "arrow-array", "arrow-buffer", @@ -306,9 +306,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5329bf9e7390cbb6b117ddd4d82e94c5362ea4cab5095697139429f36a38350c" +checksum = "d8dafd17a05449e31e0114d740530e0ada7379d7cb9c338fd65b09a8130960b0" dependencies = [ "arrow-array", "arrow-buffer", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e103c13d4b80da28339c1d7aa23dd85bd59f42158acc45d39eeb6770627909ce" +checksum = "895644523af4e17502d42c3cb6b27cb820f0cb77954c22d75c23a85247c849e1" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170549a11b8534f3097a0619cfe89c42812345dc998bcf81128fc700b84345b8" +checksum = "9be8a2a4e5e7d9c822b2b8095ecd77010576d824f654d347817640acfc97d229" dependencies = [ "arrow-array", "arrow-buffer", @@ -354,18 +354,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c53775bba63f319189f366d2b86e9a8889373eb198f07d8544938fc9f8ed9a" +checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" dependencies = [ - "bitflags 2.8.0", + "bitflags", ] [[package]] name = "arrow-select" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a99003b2eb562b8d9c99dfb672306f15e94b20d3734179d596895703e821dcf" +checksum = "aa5f5a93c75f46ef48e4001535e7b6c922eeb0aa20b73cf58d09e13d057490d8" dependencies = [ "ahash", "arrow-array", @@ -377,9 +377,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fdb130ee8325f4cd8262e19bb6baa3cbcef2b2573c4bee8c6fda7ea08199d7" +checksum = "6e7005d858d84b56428ba2a98a107fe88c0132c61793cf6b8232a1f9bfc0452b" dependencies = [ "arrow-array", "arrow-buffer", @@ -406,11 +406,11 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2 0.4.4", + "bzip2 0.5.2", "flate2", "futures-core", "memchr", @@ -438,18 +438,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "async-trait" -version = "0.1.86" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -502,9 +502,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bigdecimal" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" dependencies = [ "autocfg", "libm", @@ -514,12 +514,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - [[package]] name = "bitflags" version = "2.8.0" @@ -537,9 +531,9 @@ dependencies = [ [[package]] name = "blake3" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" +checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" dependencies = [ "arrayref", "arrayvec", @@ -608,21 +602,20 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b89e7c29231c673a61a46e722602bcd138298f6b9e81e71119693534585f5c" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ "bzip2-sys", ] [[package]] name = "bzip2-sys" -version = "0.1.12+1.0.8" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ebc2f1a417f01e1da30ef264ee86ae31d2dcd2d603ea283d3c244a883ca2a9" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", - "libc", "pkg-config", ] @@ -866,23 +859,26 @@ dependencies = [ [[package]] name = "datafusion" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914e6f9525599579abbd90b0f7a55afcaaaa40350b9e9ed52563f126dfe45fd3" +checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" dependencies = [ - "apache-avro", "arrow", "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.5.1", + "bzip2 0.5.2", "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", + "datafusion-datasource-avro", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", @@ -897,12 +893,12 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", + "datafusion-session", "datafusion-sql", "flate2", "futures", "itertools 0.14.0", "log", - "num-traits", "object_store", "parking_lot", "parquet", @@ -919,29 +915,35 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998a6549e6ee4ee3980e05590b2960446a56b343ea30199ef38acd0e0b9036e2" +checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" dependencies = [ "arrow", "async-trait", "dashmap", "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", + "datafusion-session", "datafusion-sql", "futures", "itertools 0.14.0", "log", + "object_store", "parking_lot", + "tokio", ] [[package]] name = "datafusion-catalog-listing" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ac10096a5b3c0d8a227176c0e543606860842e943594ccddb45cf42a526e43" +checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" dependencies = [ "arrow", "async-trait", @@ -953,6 +955,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-session", "futures", "log", "object_store", @@ -961,9 +964,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f53d7ec508e1b3f68bd301cee3f649834fad51eff9240d898a4b2614cfd0a7a" +checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" dependencies = [ "ahash", "apache-avro", @@ -986,27 +989,27 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0fcf41523b22e14cc349b01526e8b9f59206653037f2949a4adbfde5f8cb668" +checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" dependencies = [ + "futures", "log", "tokio", ] [[package]] name = "datafusion-datasource" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf7f37ad8b6e88b46c7eeab3236147d32ea64b823544f498455a8d9042839c92" +checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.5.1", + "bzip2 0.5.2", "chrono", - "datafusion-catalog", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -1014,13 +1017,16 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-session", "flate2", "futures", "glob", "itertools 0.14.0", "log", "object_store", + "parquet", "rand", + "tempfile", "tokio", "tokio-util", "url", @@ -1028,17 +1034,123 @@ dependencies = [ "zstd", ] +[[package]] +name = "datafusion-datasource-avro" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ea5111aab9d3f2a8bff570343cccb03ce4c203875ef5a566b7d6f1eb72559e" +dependencies = [ + "apache-avro", + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "num-traits", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-catalog", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "rand", + "tokio", +] + [[package]] name = "datafusion-doc" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db7a0239fd060f359dc56c6e7db726abaa92babaed2fb2e91c3a8b2fff8b256" +checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" [[package]] name = "datafusion-execution" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0938f9e5b6bc5782be4111cdfb70c02b7b5451bf34fd57e4de062a7f7c4e31f1" +checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" dependencies = [ "arrow", "dashmap", @@ -1055,9 +1167,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b36c28b00b00019a8695ad7f1a53ee1673487b90322ecbd604e2cf32894eb14f" +checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" dependencies = [ "arrow", "chrono", @@ -1076,9 +1188,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18f0a851a436c5a2139189eb4617a54e6a9ccb9edc96c4b3c83b3bb7c58b950e" +checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" dependencies = [ "arrow", "datafusion-common", @@ -1089,12 +1201,13 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d740dd9f32a4f4ed1b907e6934201bb059efe6c877532512c661771d973c7b21" +checksum = "5cf3fe9ab492c56daeb7beed526690d33622d388b8870472e0b7b7f55490338c" dependencies = [ "abi_stable", "arrow", + "arrow-schema", "async-ffi", "async-trait", "datafusion", @@ -1108,9 +1221,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3196e37d7b65469fb79fee4f05e5bb58a456831035f9a38aa5919aeb3298d40" +checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" dependencies = [ "arrow", "arrow-buffer", @@ -1137,9 +1250,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adfc2d074d5ee4d9354fdcc9283d5b2b9037849237ddecb8942a29144b77ca05" +checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" dependencies = [ "ahash", "arrow", @@ -1158,9 +1271,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cbceba0f98d921309a9121b702bcd49289d383684cccabf9a92cda1602f3bbb" +checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" dependencies = [ "ahash", "arrow", @@ -1171,9 +1284,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "170e27ce4baa27113ddf5f77f1a7ec484b0dbeda0c7abbd4bad3fc609c8ab71a" +checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" dependencies = [ "arrow", "arrow-ord", @@ -1192,9 +1305,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3a06a7f0817ded87b026a437e7e51de7f59d48173b0a4e803aa896a7bd6bb5" +checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" dependencies = [ "arrow", "async-trait", @@ -1208,9 +1321,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6c608b66496a1e05e3d196131eb9bebea579eed1f59e88d962baf3dda853bc6" +checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" dependencies = [ "datafusion-common", "datafusion-doc", @@ -1225,9 +1338,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2f9d83348957b4ad0cd87b5cb9445f2651863a36592fe5484d43b49a5f8d82" +checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1235,20 +1348,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4800e1ff7ecf8f310887e9b54c9c444b8e215ccbc7b21c2f244cfae373b1ece7" +checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "datafusion-optimizer" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "971c51c54cd309001376fae752fb15a6b41750b6d1552345c46afbfb6458801b" +checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" dependencies = [ "arrow", "chrono", @@ -1265,9 +1378,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1447c2c6bc8674a16be4786b4abf528c302803fafa186aa6275692570e64d85" +checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" dependencies = [ "ahash", "arrow", @@ -1287,9 +1400,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f8c25dcd069073a75b3d2840a79d0f81e64bdd2c05f2d3d18939afb36a7dcb" +checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" dependencies = [ "ahash", "arrow", @@ -1301,9 +1414,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68da5266b5b9847c11d1b3404ee96b1d423814e1973e1ad3789131e5ec912763" +checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" dependencies = [ "arrow", "datafusion-common", @@ -1320,9 +1433,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cc160df00e413e370b3b259c8ea7bfbebc134d32de16325950e9e923846b7f" +checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" dependencies = [ "ahash", "arrow", @@ -1350,9 +1463,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f6ef4c6eb52370cb48639e25e2331a415aac0b2b0a0a472b36e26603bdf184f" +checksum = "a4a1afb2bdb05de7ff65be6883ebfd4ec027bd9f1f21c46aa3afd01927160a83" dependencies = [ "arrow", "chrono", @@ -1366,9 +1479,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5faf4a9bbb0d0a305fea8a6db21ba863286b53e53a212e687d2774028dd6f03f" +checksum = "35b7a5876ebd6b564fb9a1fd2c3a2a9686b787071a256b47e4708f0916f9e46f" dependencies = [ "arrow", "datafusion-common", @@ -1398,11 +1511,35 @@ dependencies = [ "uuid", ] +[[package]] +name = "datafusion-session" +version = "47.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + [[package]] name = "datafusion-sql" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325a212b67b677c0eb91447bf9a11b630f9fc4f62d8e5d145bf859f5a6b29e64" +checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" dependencies = [ "arrow", "bigdecimal", @@ -1417,9 +1554,9 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "46.0.1" +version = "47.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c2be3226a683e02cff65181e66e62eba9f812ed0e9b7ec8fe11ac8dabf1a73f" +checksum = "061efc0937f0ce3abb37ed0d56cfa01dd0e654b90e408656d05e846c8b7599fe" dependencies = [ "async-recursion", "async-trait", @@ -1453,7 +1590,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1498,21 +1635,22 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "24.12.23" +version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 1.3.2", + "bitflags", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -1593,7 +1731,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1703,9 +1841,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -1986,7 +2124,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2012,9 +2150,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -2207,6 +2345,15 @@ dependencies = [ "libc", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2241,7 +2388,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" dependencies = [ - "twox-hash", + "twox-hash 1.6.3", ] [[package]] @@ -2297,9 +2444,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.4" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] @@ -2407,19 +2554,22 @@ dependencies = [ [[package]] name = "object_store" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" +checksum = "e9ce831b09395f933addbc56d894d889e4b226eba304d4e7adbab591e26daf1e" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", + "form_urlencoded", "futures", + "http", + "http-body-util", "httparse", "humantime", "hyper", - "itertools 0.13.0", + "itertools 0.14.0", "md-5", "parking_lot", "percent-encoding", @@ -2430,7 +2580,8 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", - "snafu", + "serde_urlencoded", + "thiserror 2.0.11", "tokio", "tracing", "url", @@ -2483,9 +2634,9 @@ dependencies = [ [[package]] name = "parquet" -version = "54.3.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94243778210509a5a5e9e012872127180c155d73a9cd6e2df9243d213e81e100" +checksum = "cd31a8290ac5b19f09ad77ee7a1e6a541f1be7674ad410547d5f1eef6eef4a9c" dependencies = [ "ahash", "arrow-array", @@ -2513,7 +2664,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash", + "twox-hash 2.1.0", "zstd", ] @@ -2658,12 +2809,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.29" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2692,7 +2843,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "once_cell", @@ -2701,7 +2852,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.98", + "syn 2.0.100", "tempfile", ] @@ -2712,10 +2863,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2747,9 +2898,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" +checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" dependencies = [ "cfg-if", "indoc", @@ -2765,9 +2916,9 @@ dependencies = [ [[package]] name = "pyo3-async-runtimes" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977dc837525cfd22919ba6a831413854beb7c99a256c03bf8624ad707e45810e" +checksum = "dd0b83dc42f9d41f50d38180dad65f0c99763b65a3ff2a81bf351dd35a1df8bf" dependencies = [ "futures", "once_cell", @@ -2778,9 +2929,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" +checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" dependencies = [ "once_cell", "target-lexicon", @@ -2788,9 +2939,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" +checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" dependencies = [ "libc", "pyo3-build-config", @@ -2798,27 +2949,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" +checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "pyo3-macros-backend" -version = "0.23.4" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" +checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2891,9 +3042,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] @@ -2945,7 +3096,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2954,7 +3105,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.8.0", + "bitflags", ] [[package]] @@ -3104,7 +3255,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags", "errno", "libc", "linux-raw-sys", @@ -3198,9 +3349,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "schemars_derive", @@ -3210,14 +3361,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3232,7 +3383,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", + "bitflags", "core-foundation", "core-foundation-sys", "libc", @@ -3251,9 +3402,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" dependencies = [ "serde", ] @@ -3266,9 +3417,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] @@ -3284,13 +3435,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3301,14 +3452,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -3325,7 +3476,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3397,27 +3548,6 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" -[[package]] -name = "snafu" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" -dependencies = [ - "snafu-derive", -] - -[[package]] -name = "snafu-derive" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "snap" version = "1.1.1" @@ -3436,9 +3566,9 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.54.0" +version = "0.55.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66e3b7374ad4a6af849b08b3e7a6eda0edbd82f0fd59b57e22671bf16979899" +checksum = "c4521174166bac1ff04fe16ef4524c70144cd29682a45978978ca3d7f4e0be11" dependencies = [ "log", "recursive", @@ -3453,7 +3583,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3497,14 +3627,14 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "substrait" -version = "0.53.2" +version = "0.55.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac3d70185423235f37b889764e184b81a5af4bb7c95833396ee9bd92577e1b" +checksum = "048fe52a3664881ccdfdc9bdb0f4e8805f3444ee64abf299d365c54f6a2ffabb" dependencies = [ "heck", "pbjson", @@ -3521,7 +3651,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.98", + "syn 2.0.100", "typify", "walkdir", ] @@ -3545,9 +3675,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.98" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -3571,14 +3701,14 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "tempfile" @@ -3620,7 +3750,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3631,7 +3761,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3681,9 +3811,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.43.0" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -3703,7 +3833,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3718,9 +3848,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -3775,7 +3905,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3818,6 +3948,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "twox-hash" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" + [[package]] name = "typed-arena" version = "2.0.2" @@ -3841,7 +3977,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3875,7 +4011,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.98", + "syn 2.0.100", "thiserror 2.0.11", "unicode-ident", ] @@ -3893,7 +4029,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.98", + "syn 2.0.100", "typify-impl", ] @@ -4030,7 +4166,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -4065,7 +4201,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4276,7 +4412,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ - "bitflags 2.8.0", + "bitflags", ] [[package]] @@ -4320,7 +4456,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "synstructure", ] @@ -4342,7 +4478,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4362,7 +4498,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "synstructure", ] @@ -4391,9 +4527,15 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] +[[package]] +name = "zlib-rs" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index bc8639d4c..2c4188bb0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,25 +34,25 @@ protoc = [ "datafusion-substrait/protoc" ] substrait = ["dep:datafusion-substrait"] [dependencies] -tokio = { version = "1.43", features = ["macros", "rt", "rt-multi-thread", "sync"] } -pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } -pyo3-async-runtimes = { version = "0.23", features = ["tokio-runtime"]} -arrow = { version = "54.2.1", features = ["pyarrow"] } -datafusion = { version = "46.0.1", features = ["avro", "unicode_expressions"] } -datafusion-substrait = { version = "46.0.1", optional = true } -datafusion-proto = { version = "46.0.1" } -datafusion-ffi = { version = "46.0.1" } +tokio = { version = "1.44", features = ["macros", "rt", "rt-multi-thread", "sync"] } +pyo3 = { version = "0.24", features = ["extension-module", "abi3", "abi3-py39"] } +pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"]} +arrow = { version = "55.0.0", features = ["pyarrow"] } +datafusion = { version = "47.0.0", features = ["avro", "unicode_expressions"] } +datafusion-substrait = { version = "47.0.0", optional = true } +datafusion-proto = { version = "47.0.0" } +datafusion-ffi = { version = "47.0.0" } prost = "0.13.1" # keep in line with `datafusion-substrait` -uuid = { version = "1.12", features = ["v4"] } +uuid = { version = "1.16", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } -async-trait = "0.1.73" +async-trait = "0.1.88" futures = "0.3" -object_store = { version = "0.11.0", features = ["aws", "gcp", "azure", "http"] } +object_store = { version = "0.12.0", features = ["aws", "gcp", "azure", "http"] } url = "2" [build-dependencies] prost-types = "0.13.1" # keep in line with `datafusion-substrait` -pyo3-build-config = "0.23" +pyo3-build-config = "0.24" [lib] name = "datafusion_python" diff --git a/python/tests/test_aggregation.py b/python/tests/test_aggregation.py index 61b1c7d80..49dfb38cf 100644 --- a/python/tests/test_aggregation.py +++ b/python/tests/test_aggregation.py @@ -338,7 +338,6 @@ def test_bit_and_bool_fns(df, name, expr, result): ), [7, 9], ), - ("last_value", f.last_value(column("a")), [3, 6]), ( "last_value_ordered", f.last_value(column("a"), order_by=[column("a").sort(ascending=False)]), diff --git a/src/dataframe.rs b/src/dataframe.rs index 9b610b5d7..787f63520 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -216,7 +216,7 @@ impl PyDataFrame { #[pyo3(signature = (*args))] fn select(&self, args: Vec) -> PyDataFusionResult { - let expr = args.into_iter().map(|e| e.into()).collect(); + let expr: Vec = args.into_iter().map(|e| e.into()).collect(); let df = self.df.as_ref().clone().select(expr)?; Ok(Self::new(df)) } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 445e4fe74..aab8d7566 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -275,7 +275,9 @@ impl DisplayAs for DatasetExec { Python::with_gil(|py| { let number_of_fragments = self.fragments.bind(py).len(); match t { - DisplayFormatType::Default | DisplayFormatType::Verbose => { + DisplayFormatType::Default + | DisplayFormatType::Verbose + | DisplayFormatType::TreeRender => { let projected_columns: Vec = self .schema .fields() diff --git a/src/expr.rs b/src/expr.rs index 561170289..fe0e76daa 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -714,9 +714,19 @@ impl PyExpr { | Operator::BitwiseXor | Operator::BitwiseAnd | Operator::BitwiseOr => DataTypeMap::map_from_arrow_type(&DataType::Binary), - Operator::AtArrow | Operator::ArrowAt => { - Err(py_type_err(format!("Unsupported expr: ${op}"))) - } + Operator::AtArrow + | Operator::ArrowAt + | Operator::Arrow + | Operator::LongArrow + | Operator::HashArrow + | Operator::HashLongArrow + | Operator::AtAt + | Operator::IntegerDivide + | Operator::HashMinus + | Operator::AtQuestion + | Operator::Question + | Operator::QuestionAnd + | Operator::QuestionPipe => Err(py_type_err(format!("Unsupported expr: ${op}"))), }, Expr::Cast(Cast { expr: _, data_type }) => DataTypeMap::map_from_arrow_type(data_type), Expr::Literal(scalar_value) => DataTypeMap::map_from_scalar_value(scalar_value), diff --git a/src/functions.rs b/src/functions.rs index 9c406b95a..476c2b80e 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -375,27 +375,6 @@ macro_rules! aggregate_function { }; } -macro_rules! aggregate_function_vec_args { - ($NAME: ident) => { - aggregate_function_vec_args!($NAME, expr); - }; - ($NAME: ident, $($arg:ident)*) => { - #[pyfunction] - #[pyo3(signature = ($($arg),*, distinct=None, filter=None, order_by=None, null_treatment=None))] - fn $NAME( - $($arg: PyExpr),*, - distinct: Option, - filter: Option, - order_by: Option>, - null_treatment: Option - ) -> PyDataFusionResult { - let agg_fn = functions_aggregate::expr_fn::$NAME(vec![$($arg.into()),*]); - - add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) - } - }; -} - /// Generates a [pyo3] wrapper for [datafusion::functions::expr_fn] /// /// These functions have explicit named arguments. @@ -698,8 +677,22 @@ pub fn approx_percentile_cont_with_weight( add_builder_fns_to_aggregate(agg_fn, None, filter, None, None) } -aggregate_function_vec_args!(last_value); +// We handle first_value explicitly because the signature expects an order_by +// https://github.com/apache/datafusion/issues/12376 +#[pyfunction] +#[pyo3(signature = (expr, distinct=None, filter=None, order_by=None, null_treatment=None))] +pub fn last_value( + expr: PyExpr, + distinct: Option, + filter: Option, + order_by: Option>, + null_treatment: Option, +) -> PyDataFusionResult { + // If we initialize the UDAF with order_by directly, then it gets over-written by the builder + let agg_fn = functions_aggregate::expr_fn::last_value(expr.expr, None); + add_builder_fns_to_aggregate(agg_fn, distinct, filter, order_by, null_treatment) +} // We handle first_value explicitly because the signature expects an order_by // https://github.com/apache/datafusion/issues/12376 #[pyfunction] From c9f15547cb8019068bbf2dc8eaf148d6eb42bd48 Mon Sep 17 00:00:00 2001 From: Chen Chongchen Date: Fri, 25 Apr 2025 21:01:54 +0800 Subject: [PATCH 288/413] feat: alias with metadata (#1111) * feat: alias with metadata * fmt --- python/datafusion/expr.py | 14 +++++++++++--- python/datafusion/functions.py | 15 ++++++++++++--- python/tests/test_expr.py | 5 +++++ python/tests/test_functions.py | 5 +++++ src/expr.rs | 6 ++++-- src/functions.rs | 9 +++++++-- 6 files changed, 44 insertions(+), 10 deletions(-) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 2697d8143..01e1f3ded 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -406,9 +406,17 @@ def column(value: str) -> Expr: """Creates a new expression representing a column.""" return Expr(expr_internal.RawExpr.column(value)) - def alias(self, name: str) -> Expr: - """Assign a name to the expression.""" - return Expr(self.expr.alias(name)) + def alias(self, name: str, metadata: Optional[dict[str, str]] = None) -> Expr: + """Assign a name to the expression. + + Args: + name: The name to assign to the expression. + metadata: Optional metadata to attach to the expression. + + Returns: + A new expression with the assigned name. + """ + return Expr(self.expr.alias(name, metadata)) def sort(self, ascending: bool = True, nulls_first: bool = True) -> SortExpr: """Creates a sort :py:class:`Expr` from an existing :py:class:`Expr`. diff --git a/python/datafusion/functions.py b/python/datafusion/functions.py index 5cf914e16..f430cdf4b 100644 --- a/python/datafusion/functions.py +++ b/python/datafusion/functions.py @@ -372,9 +372,18 @@ def order_by(expr: Expr, ascending: bool = True, nulls_first: bool = True) -> So return SortExpr(expr, ascending=ascending, nulls_first=nulls_first) -def alias(expr: Expr, name: str) -> Expr: - """Creates an alias expression.""" - return Expr(f.alias(expr.expr, name)) +def alias(expr: Expr, name: str, metadata: Optional[dict[str, str]] = None) -> Expr: + """Creates an alias expression with an optional metadata dictionary. + + Args: + expr: The expression to alias + name: The alias name + metadata: Optional metadata to attach to the column + + Returns: + An expression with the given alias + """ + return Expr(f.alias(expr.expr, name, metadata)) def col(name: str) -> Expr: diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 926e69845..dcf75f021 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -247,3 +247,8 @@ def test_fill_null(df): assert result.column(0) == pa.array([1, 2, 100]) assert result.column(1) == pa.array([4, 25, 6]) assert result.column(2) == pa.array([1234, 1234, 8]) + + +def test_alias_with_metadata(df): + df = df.select(col("a").alias("b", {"key": "value"})) + assert df.schema().field("b").metadata == {b"key": b"value"} diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 37f2075f5..90cf01f7e 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -1231,3 +1231,8 @@ def test_between_default(df): actual = df.collect()[0].to_pydict() assert actual == expected + + +def test_alias_with_metadata(df): + df = df.select(f.alias(f.col("a"), "b", {"key": "value"})) + assert df.schema().field("b").metadata == {b"key": b"value"} diff --git a/src/expr.rs b/src/expr.rs index fe0e76daa..7d4aa8798 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -22,6 +22,7 @@ use datafusion::logical_expr::{ }; use pyo3::IntoPyObjectExt; use pyo3::{basic::CompareOp, prelude::*}; +use std::collections::HashMap; use std::convert::{From, Into}; use std::sync::Arc; use window::PyWindowFrame; @@ -275,8 +276,9 @@ impl PyExpr { } /// assign a name to the PyExpr - pub fn alias(&self, name: &str) -> PyExpr { - self.expr.clone().alias(name).into() + #[pyo3(signature = (name, metadata=None))] + pub fn alias(&self, name: &str, metadata: Option>) -> PyExpr { + self.expr.clone().alias_with_metadata(name, metadata).into() } /// Create a sort PyExpr from an existing PyExpr. diff --git a/src/functions.rs b/src/functions.rs index 476c2b80e..caa79b8ad 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; + use datafusion::functions_aggregate::all_default_aggregate_functions; use datafusion::functions_window::all_default_window_functions; use datafusion::logical_expr::expr::WindowFunctionParams; @@ -205,10 +207,13 @@ fn order_by(expr: PyExpr, asc: bool, nulls_first: bool) -> PyResult /// Creates a new Alias Expr #[pyfunction] -fn alias(expr: PyExpr, name: &str) -> PyResult { +#[pyo3(signature = (expr, name, metadata=None))] +fn alias(expr: PyExpr, name: &str, metadata: Option>) -> PyResult { let relation: Option = None; Ok(PyExpr { - expr: datafusion::logical_expr::Expr::Alias(Alias::new(expr.expr, relation, name)), + expr: datafusion::logical_expr::Expr::Alias( + Alias::new(expr.expr, relation, name).with_metadata(metadata), + ), }) } From 91b66351fb19d91b62e8db83444141743b106e43 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sun, 27 Apr 2025 21:41:01 +0800 Subject: [PATCH 289/413] Add DataFrame usage guide with HTML rendering customization options (#1108) * docs: enhance user guide with detailed DataFrame operations and examples * move /docs/source/api/dataframe.rst into user-guide * docs: remove DataFrame API documentation * docs: fix formatting inconsistencies in DataFrame user guide * Two minor corrections to documentation rendering --------- Co-authored-by: Tim Saucer --- docs/source/index.rst | 1 + docs/source/user-guide/basics.rst | 5 +- docs/source/user-guide/dataframe.rst | 179 +++++++++++++++++++++++++++ 3 files changed, 184 insertions(+), 1 deletion(-) create mode 100644 docs/source/user-guide/dataframe.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 558b2d572..c18793822 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -72,6 +72,7 @@ Example user-guide/introduction user-guide/basics user-guide/data-sources + user-guide/dataframe user-guide/common-operations/index user-guide/io/index user-guide/configuration diff --git a/docs/source/user-guide/basics.rst b/docs/source/user-guide/basics.rst index 6636c0c6a..2975d9a6b 100644 --- a/docs/source/user-guide/basics.rst +++ b/docs/source/user-guide/basics.rst @@ -21,7 +21,8 @@ Concepts ======== In this section, we will cover a basic example to introduce a few key concepts. We will use the -2021 Yellow Taxi Trip Records ([download](https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2021-01.parquet)), from the [TLC Trip Record Data](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page). +2021 Yellow Taxi Trip Records (`download `_), +from the `TLC Trip Record Data `_. .. ipython:: python @@ -72,6 +73,8 @@ DataFrames are typically created by calling a method on :py:class:`~datafusion.c calling the transformation methods, such as :py:func:`~datafusion.dataframe.DataFrame.filter`, :py:func:`~datafusion.dataframe.DataFrame.select`, :py:func:`~datafusion.dataframe.DataFrame.aggregate`, and :py:func:`~datafusion.dataframe.DataFrame.limit` to build up a query definition. +For more details on working with DataFrames, including visualization options and conversion to other formats, see :doc:`dataframe`. + Expressions ----------- diff --git a/docs/source/user-guide/dataframe.rst b/docs/source/user-guide/dataframe.rst new file mode 100644 index 000000000..a78fd8073 --- /dev/null +++ b/docs/source/user-guide/dataframe.rst @@ -0,0 +1,179 @@ +.. Licensed to the Apache Software Foundation (ASF) under one +.. or more contributor license agreements. See the NOTICE file +.. distributed with this work for additional information +.. regarding copyright ownership. The ASF licenses this file +.. to you under the Apache License, Version 2.0 (the +.. "License"); you may not use this file except in compliance +.. with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, +.. software distributed under the License is distributed on an +.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +.. KIND, either express or implied. See the License for the +.. specific language governing permissions and limitations +.. under the License. + +DataFrames +========== + +Overview +-------- + +DataFusion's DataFrame API provides a powerful interface for building and executing queries against data sources. +It offers a familiar API similar to pandas and other DataFrame libraries, but with the performance benefits of Rust +and Arrow. + +A DataFrame represents a logical plan that can be composed through operations like filtering, projection, and aggregation. +The actual execution happens when terminal operations like ``collect()`` or ``show()`` are called. + +Basic Usage +----------- + +.. code-block:: python + + import datafusion + from datafusion import col, lit + + # Create a context and register a data source + ctx = datafusion.SessionContext() + ctx.register_csv("my_table", "path/to/data.csv") + + # Create and manipulate a DataFrame + df = ctx.sql("SELECT * FROM my_table") + + # Or use the DataFrame API directly + df = (ctx.table("my_table") + .filter(col("age") > lit(25)) + .select([col("name"), col("age")])) + + # Execute and collect results + result = df.collect() + + # Display the first few rows + df.show() + +HTML Rendering +-------------- + +When working in Jupyter notebooks or other environments that support HTML rendering, DataFrames will +automatically display as formatted HTML tables, making it easier to visualize your data. + +The ``_repr_html_`` method is called automatically by Jupyter to render a DataFrame. This method +controls how DataFrames appear in notebook environments, providing a richer visualization than +plain text output. + +Customizing HTML Rendering +-------------------------- + +You can customize how DataFrames are rendered in HTML by configuring the formatter: + +.. code-block:: python + + from datafusion.html_formatter import configure_formatter + + # Change the default styling + configure_formatter( + max_rows=50, # Maximum number of rows to display + max_width=None, # Maximum width in pixels (None for auto) + theme="light", # Theme: "light" or "dark" + precision=2, # Floating point precision + thousands_separator=",", # Separator for thousands + date_format="%Y-%m-%d", # Date format + truncate_width=20 # Max width for string columns before truncating + ) + +The formatter settings affect all DataFrames displayed after configuration. + +Custom Style Providers +---------------------- + +For advanced styling needs, you can create a custom style provider: + +.. code-block:: python + + from datafusion.html_formatter import StyleProvider, configure_formatter + + class MyStyleProvider(StyleProvider): + def get_table_styles(self): + return { + "table": "border-collapse: collapse; width: 100%;", + "th": "background-color: #007bff; color: white; padding: 8px; text-align: left;", + "td": "border: 1px solid #ddd; padding: 8px;", + "tr:nth-child(even)": "background-color: #f2f2f2;", + } + + def get_value_styles(self, dtype, value): + """Return custom styles for specific values""" + if dtype == "float" and value < 0: + return "color: red;" + return None + + # Apply the custom style provider + configure_formatter(style_provider=MyStyleProvider()) + +Creating a Custom Formatter +--------------------------- + +For complete control over rendering, you can implement a custom formatter: + +.. code-block:: python + + from datafusion.html_formatter import Formatter, get_formatter + + class MyFormatter(Formatter): + def format_html(self, batches, schema, has_more=False, table_uuid=None): + # Create your custom HTML here + html = "
" + # ... formatting logic ... + html += "
" + return html + + # Set as the global formatter + configure_formatter(formatter_class=MyFormatter) + + # Or use the formatter just for specific operations + formatter = get_formatter() + custom_html = formatter.format_html(batches, schema) + +Managing Formatters +------------------- + +Reset to default formatting: + +.. code-block:: python + + from datafusion.html_formatter import reset_formatter + + # Reset to default settings + reset_formatter() + +Get the current formatter settings: + +.. code-block:: python + + from datafusion.html_formatter import get_formatter + + formatter = get_formatter() + print(formatter.max_rows) + print(formatter.theme) + +Contextual Formatting +--------------------- + +You can also use a context manager to temporarily change formatting settings: + +.. code-block:: python + + from datafusion.html_formatter import formatting_context + + # Default formatting + df.show() + + # Temporarily use different formatting + with formatting_context(max_rows=100, theme="dark"): + df.show() # Will use the temporary settings + + # Back to default formatting + df.show() From 00dea113eb85d54b758eb3451ea448c7b9263c1c Mon Sep 17 00:00:00 2001 From: deanm0000 <37878412+deanm0000@users.noreply.github.com> Date: Sun, 27 Apr 2025 10:14:54 -0400 Subject: [PATCH 290/413] Improve col class access using __getattr__ Co-authored-by: Tim Saucer --- python/datafusion/__init__.py | 12 ++-------- python/datafusion/col.py | 45 +++++++++++++++++++++++++++++++++++ python/tests/test_expr.py | 23 ++++++++++++++++++ 3 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 python/datafusion/col.py diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 60d0d61b4..15ceefbdb 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -26,6 +26,8 @@ except ImportError: import importlib_metadata +from datafusion.col import col, column + from . import functions, object_store, substrait, unparser # The following imports are okay to remain as opaque to the user. @@ -95,16 +97,6 @@ ] -def column(value: str) -> Expr: - """Create a column expression.""" - return Expr.column(value) - - -def col(value: str) -> Expr: - """Create a column expression.""" - return Expr.column(value) - - def literal(value) -> Expr: """Create a literal expression.""" return Expr.literal(value) diff --git a/python/datafusion/col.py b/python/datafusion/col.py new file mode 100644 index 000000000..1141dc092 --- /dev/null +++ b/python/datafusion/col.py @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Col class.""" + +from datafusion.expr import Expr + + +class Col: + """Create a column expression. + + This helper class allows an extra syntax of creating columns using the __getattr__ + method. + """ + + def __call__(self, value: str) -> Expr: + """Create a column expression.""" + return Expr.column(value) + + def __getattr__(self, value: str) -> Expr: + """Create a column using attribute syntax.""" + # For autocomplete to work with IPython + if value.startswith("__wrapped__"): + return getattr(type(self), value) + + return Expr.column(value) + + +col: Col = Col() +column: Col = Col() +__all__ = ["col", "column"] diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index dcf75f021..3651b60d6 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -249,6 +249,29 @@ def test_fill_null(df): assert result.column(2) == pa.array([1234, 1234, 8]) +def test_col_getattr(): + ctx = SessionContext() + data = { + "array_values": [[1, 2, 3], [4, 5], [6], []], + "struct_values": [ + {"name": "Alice", "age": 15}, + {"name": "Bob", "age": 14}, + {"name": "Charlie", "age": 13}, + {"name": None, "age": 12}, + ], + } + df = ctx.from_pydict(data, name="table1") + + names = df.select(col.struct_values["name"].alias("name")).collect() + names = [r.as_py() for rs in names for r in rs["name"]] + + array_values = df.select(col.array_values[1].alias("value")).collect() + array_values = [r.as_py() for rs in array_values for r in rs["value"]] + + assert names == ["Alice", "Bob", "Charlie", None] + assert array_values == [2, 5, None, None] + + def test_alias_with_metadata(df): df = df.select(col("a").alias("b", {"key": "value"})) assert df.schema().field("b").metadata == {b"key": b"value"} From 5a7f638286d2397bbce87e0e8197bebb46f26649 Mon Sep 17 00:00:00 2001 From: deanm0000 <37878412+deanm0000@users.noreply.github.com> Date: Sun, 27 Apr 2025 10:17:41 -0400 Subject: [PATCH 291/413] Add expression chaining of single parameter scalar functions --- python/datafusion/expr.py | 289 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 289 insertions(+) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 01e1f3ded..84e9d4ebb 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -24,6 +24,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional +import functions as F import pyarrow as pa try: @@ -611,6 +612,294 @@ def over(self, window: Window) -> Expr: ) ) + def asin(self) -> Expr: + """Returns the arc sine or inverse sine of a number.""" + return F.asin(self) + + def array_pop_back(self) -> Expr: + """Returns the array without the last element.""" + return F.array_pop_back(self) + + def reverse(self) -> Expr: + """Reverse the string argument.""" + return F.reverse(self) + + def bit_length(self) -> Expr: + """Returns the number of bits in the string argument.""" + return F.bit_length(self) + + def array_length(self) -> Expr: + """Returns the length of the array.""" + return F.array_length(self) + + def array_ndims(self) -> Expr: + """Returns the number of dimensions of the array.""" + return F.array_ndims(self) + + def to_hex(self) -> Expr: + """Converts an integer to a hexadecimal string.""" + return F.to_hex(self) + + def array_dims(self) -> Expr: + """Returns an array of the array's dimensions.""" + return F.array_dims(self) + + def from_unixtime(self) -> Expr: + """Converts an integer to RFC3339 timestamp format string.""" + return F.from_unixtime(self) + + def array_empty(self) -> Expr: + """Returns a boolean indicating whether the array is empty.""" + return F.array_empty(self) + + def sin(self) -> Expr: + """Returns the sine of the argument.""" + return F.sin(self) + + def log10(self) -> Expr: + """Base 10 logarithm of the argument.""" + return F.log10(self) + + def initcap(self) -> Expr: + """Set the initial letter of each word to capital. + + Converts the first letter of each word in ``string`` to uppercase and the remaining + characters to lowercase. + """ + return F.initcap(self) + + def list_distinct(self) -> Expr: + """Returns distinct values from the array after removing duplicates. + + This is an alias for :py:func:`array_distinct`. + """ + return F.list_distinct(self) + + def iszero(self) -> Expr: + """Returns true if a given number is +0.0 or -0.0 otherwise returns false.""" + return F.iszero(self) + + def array_distinct(self) -> Expr: + """Returns distinct values from the array after removing duplicates.""" + return F.array_distinct(self) + + def arrow_typeof(self) -> Expr: + """Returns the Arrow type of the expression.""" + return F.arrow_typeof(self) + + def length(self) -> Expr: + """The number of characters in the ``string``.""" + return F.length(self) + + def lower(self) -> Expr: + """Converts a string to lowercase.""" + return F.lower(self) + + def acos(self) -> Expr: + """Returns the arc cosine or inverse cosine of a number. + + Returns: + -------- + Expr + A new expression representing the arc cosine of the input expression. + """ + return F.acos(self) + + def ascii(self) -> Expr: + """Returns the numeric code of the first character of the argument.""" + return F.ascii(self) + + def sha384(self) -> Expr: + """Computes the SHA-384 hash of a binary string.""" + return F.sha384(self) + + def isnan(self) -> Expr: + """Returns true if a given number is +NaN or -NaN otherwise returns false.""" + return F.isnan(self) + + def degrees(self) -> Expr: + """Converts the argument from radians to degrees.""" + return F.degrees(self) + + def cardinality(self) -> Expr: + """Returns the total number of elements in the array.""" + return F.cardinality(self) + + def sha224(self) -> Expr: + """Computes the SHA-224 hash of a binary string.""" + return F.sha224(self) + + def asinh(self) -> Expr: + """Returns inverse hyperbolic sine.""" + return F.asinh(self) + + def flatten(self) -> Expr: + """Flattens an array of arrays into a single array.""" + return F.flatten(self) + + def exp(self) -> Expr: + """Returns the exponential of the argument.""" + return F.exp(self) + + def abs(self) -> Expr: + """Return the absolute value of a given number. + + Returns: + -------- + Expr + A new expression representing the absolute value of the input expression. + """ + return F.abs(self) + + def btrim(self) -> Expr: + """Removes all characters, spaces by default, from both sides of a string.""" + return F.btrim(self) + + def md5(self) -> Expr: + """Computes an MD5 128-bit checksum for a string expression.""" + return F.md5(self) + + def octet_length(self) -> Expr: + """Returns the number of bytes of a string.""" + return F.octet_length(self) + + def cosh(self) -> Expr: + """Returns the hyperbolic cosine of the argument.""" + return F.cosh(self) + + def radians(self) -> Expr: + """Converts the argument from degrees to radians.""" + return F.radians(self) + + def sqrt(self) -> Expr: + """Returns the square root of the argument.""" + return F.sqrt(self) + + def character_length(self) -> Expr: + """Returns the number of characters in the argument.""" + return F.character_length(self) + + def tanh(self) -> Expr: + """Returns the hyperbolic tangent of the argument.""" + return F.tanh(self) + + def atan(self) -> Expr: + """Returns inverse tangent of a number.""" + return F.atan(self) + + def rtrim(self) -> Expr: + """Removes all characters, spaces by default, from the end of a string.""" + return F.rtrim(self) + + def atanh(self) -> Expr: + """Returns inverse hyperbolic tangent.""" + return F.atanh(self) + + def list_dims(self) -> Expr: + """Returns an array of the array's dimensions. + + This is an alias for :py:func:`array_dims`. + """ + return F.list_dims(self) + + def sha256(self) -> Expr: + """Computes the SHA-256 hash of a binary string.""" + return F.sha256(self) + + def factorial(self) -> Expr: + """Returns the factorial of the argument.""" + return F.factorial(self) + + def acosh(self) -> Expr: + """Returns inverse hyperbolic cosine.""" + return F.acosh(self) + + def floor(self) -> Expr: + """Returns the nearest integer less than or equal to the argument.""" + return F.floor(self) + + def ceil(self) -> Expr: + """Returns the nearest integer greater than or equal to argument.""" + return F.ceil(self) + + def list_length(self) -> Expr: + """Returns the length of the array. + + This is an alias for :py:func:`array_length`. + """ + return F.list_length(self) + + def upper(self) -> Expr: + """Converts a string to uppercase.""" + return F.upper(self) + + def chr(self) -> Expr: + """Converts the Unicode code point to a UTF8 character.""" + return F.chr(self) + + def ln(self) -> Expr: + """Returns the natural logarithm (base e) of the argument.""" + return F.ln(self) + + def tan(self) -> Expr: + """Returns the tangent of the argument.""" + return F.tan(self) + + def array_pop_front(self) -> Expr: + """Returns the array without the first element.""" + return F.array_pop_front(self) + + def cbrt(self) -> Expr: + """Returns the cube root of a number.""" + return F.cbrt(self) + + def sha512(self) -> Expr: + """Computes the SHA-512 hash of a binary string.""" + return F.sha512(self) + + def char_length(self) -> Expr: + """The number of characters in the ``string``.""" + return F.char_length(self) + + def list_ndims(self) -> Expr: + """Returns the number of dimensions of the array. + + This is an alias for :py:func:`array_ndims`. + """ + return F.list_ndims(self) + + def trim(self) -> Expr: + """Removes all characters, spaces by default, from both sides of a string.""" + return F.trim(self) + + def cos(self) -> Expr: + """Returns the cosine of the argument.""" + return F.cos(self) + + def sinh(self) -> Expr: + """Returns the hyperbolic sine of the argument.""" + return F.sinh(self) + + def empty(self) -> Expr: + """This is an alias for :py:func:`array_empty`.""" + return F.empty(self) + + def ltrim(self) -> Expr: + """Removes all characters, spaces by default, from the beginning of a string.""" + return F.ltrim(self) + + def signum(self) -> Expr: + """Returns the sign of the argument (-1, 0, +1).""" + return F.signum(self) + + def log2(self) -> Expr: + """Base 2 logarithm of the argument.""" + return F.log2(self) + + def cot(self) -> Expr: + """Returns the cotangent of the argument.""" + return F.cot(self) + class ExprFuncBuilder: def __init__(self, builder: expr_internal.ExprFuncBuilder) -> None: From 10600fb8fc32eba43b0b0f198325b55c63f8223d Mon Sep 17 00:00:00 2001 From: Chen Chongchen Date: Mon, 28 Apr 2025 21:25:59 +0800 Subject: [PATCH 292/413] fix: recursive import (#1117) * fix: recursive import * format * format --- python/datafusion/expr.py | 135 +++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 3 deletions(-) diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 84e9d4ebb..3750eeb3f 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional -import functions as F import pyarrow as pa try: @@ -614,58 +613,84 @@ def over(self, window: Window) -> Expr: def asin(self) -> Expr: """Returns the arc sine or inverse sine of a number.""" + from . import functions as F + return F.asin(self) def array_pop_back(self) -> Expr: """Returns the array without the last element.""" + from . import functions as F + return F.array_pop_back(self) def reverse(self) -> Expr: """Reverse the string argument.""" + from . import functions as F + return F.reverse(self) def bit_length(self) -> Expr: """Returns the number of bits in the string argument.""" + from . import functions as F + return F.bit_length(self) def array_length(self) -> Expr: """Returns the length of the array.""" + from . import functions as F + return F.array_length(self) def array_ndims(self) -> Expr: """Returns the number of dimensions of the array.""" + from . import functions as F + return F.array_ndims(self) def to_hex(self) -> Expr: """Converts an integer to a hexadecimal string.""" + from . import functions as F + return F.to_hex(self) def array_dims(self) -> Expr: """Returns an array of the array's dimensions.""" + from . import functions as F + return F.array_dims(self) def from_unixtime(self) -> Expr: """Converts an integer to RFC3339 timestamp format string.""" + from . import functions as F + return F.from_unixtime(self) def array_empty(self) -> Expr: """Returns a boolean indicating whether the array is empty.""" + from . import functions as F + return F.array_empty(self) def sin(self) -> Expr: """Returns the sine of the argument.""" + from . import functions as F + return F.sin(self) def log10(self) -> Expr: """Base 10 logarithm of the argument.""" + from . import functions as F + return F.log10(self) def initcap(self) -> Expr: """Set the initial letter of each word to capital. - Converts the first letter of each word in ``string`` to uppercase and the remaining - characters to lowercase. + Converts the first letter of each word in ``string`` + to uppercase and the remaining characters to lowercase. """ + from . import functions as F + return F.initcap(self) def list_distinct(self) -> Expr: @@ -673,26 +698,38 @@ def list_distinct(self) -> Expr: This is an alias for :py:func:`array_distinct`. """ + from . import functions as F + return F.list_distinct(self) def iszero(self) -> Expr: """Returns true if a given number is +0.0 or -0.0 otherwise returns false.""" + from . import functions as F + return F.iszero(self) def array_distinct(self) -> Expr: """Returns distinct values from the array after removing duplicates.""" + from . import functions as F + return F.array_distinct(self) def arrow_typeof(self) -> Expr: """Returns the Arrow type of the expression.""" + from . import functions as F + return F.arrow_typeof(self) def length(self) -> Expr: """The number of characters in the ``string``.""" + from . import functions as F + return F.length(self) def lower(self) -> Expr: """Converts a string to lowercase.""" + from . import functions as F + return F.lower(self) def acos(self) -> Expr: @@ -703,42 +740,62 @@ def acos(self) -> Expr: Expr A new expression representing the arc cosine of the input expression. """ + from . import functions as F + return F.acos(self) def ascii(self) -> Expr: """Returns the numeric code of the first character of the argument.""" + from . import functions as F + return F.ascii(self) def sha384(self) -> Expr: """Computes the SHA-384 hash of a binary string.""" + from . import functions as F + return F.sha384(self) def isnan(self) -> Expr: """Returns true if a given number is +NaN or -NaN otherwise returns false.""" + from . import functions as F + return F.isnan(self) def degrees(self) -> Expr: """Converts the argument from radians to degrees.""" + from . import functions as F + return F.degrees(self) def cardinality(self) -> Expr: """Returns the total number of elements in the array.""" + from . import functions as F + return F.cardinality(self) def sha224(self) -> Expr: """Computes the SHA-224 hash of a binary string.""" + from . import functions as F + return F.sha224(self) def asinh(self) -> Expr: """Returns inverse hyperbolic sine.""" + from . import functions as F + return F.asinh(self) def flatten(self) -> Expr: """Flattens an array of arrays into a single array.""" + from . import functions as F + return F.flatten(self) def exp(self) -> Expr: """Returns the exponential of the argument.""" + from . import functions as F + return F.exp(self) def abs(self) -> Expr: @@ -749,50 +806,74 @@ def abs(self) -> Expr: Expr A new expression representing the absolute value of the input expression. """ + from . import functions as F + return F.abs(self) def btrim(self) -> Expr: """Removes all characters, spaces by default, from both sides of a string.""" + from . import functions as F + return F.btrim(self) def md5(self) -> Expr: """Computes an MD5 128-bit checksum for a string expression.""" + from . import functions as F + return F.md5(self) def octet_length(self) -> Expr: """Returns the number of bytes of a string.""" + from . import functions as F + return F.octet_length(self) def cosh(self) -> Expr: """Returns the hyperbolic cosine of the argument.""" + from . import functions as F + return F.cosh(self) def radians(self) -> Expr: """Converts the argument from degrees to radians.""" + from . import functions as F + return F.radians(self) def sqrt(self) -> Expr: """Returns the square root of the argument.""" + from . import functions as F + return F.sqrt(self) def character_length(self) -> Expr: """Returns the number of characters in the argument.""" + from . import functions as F + return F.character_length(self) def tanh(self) -> Expr: """Returns the hyperbolic tangent of the argument.""" + from . import functions as F + return F.tanh(self) def atan(self) -> Expr: """Returns inverse tangent of a number.""" + from . import functions as F + return F.atan(self) def rtrim(self) -> Expr: """Removes all characters, spaces by default, from the end of a string.""" + from . import functions as F + return F.rtrim(self) def atanh(self) -> Expr: """Returns inverse hyperbolic tangent.""" + from . import functions as F + return F.atanh(self) def list_dims(self) -> Expr: @@ -800,26 +881,38 @@ def list_dims(self) -> Expr: This is an alias for :py:func:`array_dims`. """ + from . import functions as F + return F.list_dims(self) def sha256(self) -> Expr: """Computes the SHA-256 hash of a binary string.""" + from . import functions as F + return F.sha256(self) def factorial(self) -> Expr: """Returns the factorial of the argument.""" + from . import functions as F + return F.factorial(self) def acosh(self) -> Expr: """Returns inverse hyperbolic cosine.""" + from . import functions as F + return F.acosh(self) def floor(self) -> Expr: """Returns the nearest integer less than or equal to the argument.""" + from . import functions as F + return F.floor(self) def ceil(self) -> Expr: """Returns the nearest integer greater than or equal to argument.""" + from . import functions as F + return F.ceil(self) def list_length(self) -> Expr: @@ -827,38 +920,56 @@ def list_length(self) -> Expr: This is an alias for :py:func:`array_length`. """ + from . import functions as F + return F.list_length(self) def upper(self) -> Expr: """Converts a string to uppercase.""" + from . import functions as F + return F.upper(self) def chr(self) -> Expr: """Converts the Unicode code point to a UTF8 character.""" + from . import functions as F + return F.chr(self) def ln(self) -> Expr: """Returns the natural logarithm (base e) of the argument.""" + from . import functions as F + return F.ln(self) def tan(self) -> Expr: """Returns the tangent of the argument.""" + from . import functions as F + return F.tan(self) def array_pop_front(self) -> Expr: """Returns the array without the first element.""" + from . import functions as F + return F.array_pop_front(self) def cbrt(self) -> Expr: """Returns the cube root of a number.""" + from . import functions as F + return F.cbrt(self) def sha512(self) -> Expr: """Computes the SHA-512 hash of a binary string.""" + from . import functions as F + return F.sha512(self) def char_length(self) -> Expr: """The number of characters in the ``string``.""" + from . import functions as F + return F.char_length(self) def list_ndims(self) -> Expr: @@ -866,38 +977,56 @@ def list_ndims(self) -> Expr: This is an alias for :py:func:`array_ndims`. """ + from . import functions as F + return F.list_ndims(self) def trim(self) -> Expr: """Removes all characters, spaces by default, from both sides of a string.""" + from . import functions as F + return F.trim(self) def cos(self) -> Expr: """Returns the cosine of the argument.""" + from . import functions as F + return F.cos(self) def sinh(self) -> Expr: """Returns the hyperbolic sine of the argument.""" + from . import functions as F + return F.sinh(self) def empty(self) -> Expr: """This is an alias for :py:func:`array_empty`.""" + from . import functions as F + return F.empty(self) def ltrim(self) -> Expr: """Removes all characters, spaces by default, from the beginning of a string.""" + from . import functions as F + return F.ltrim(self) def signum(self) -> Expr: """Returns the sign of the argument (-1, 0, +1).""" + from . import functions as F + return F.signum(self) def log2(self) -> Expr: """Base 2 logarithm of the argument.""" + from . import functions as F + return F.log2(self) def cot(self) -> Expr: """Returns the cotangent of the argument.""" + from . import functions as F + return F.cot(self) From 6fbeceff6091aee610273d9b27106483f9ce24ea Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 1 May 2025 12:10:40 -0400 Subject: [PATCH 293/413] Copy over protected branch rule from datafusion repo (#1122) --- .asf.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index e96b43cf0..75b2262de 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -29,6 +29,10 @@ github: rebase: false features: issues: true + protected_branches: + main: + required_pull_request_reviews: + required_approving_review_count: 1 staging: whoami: asf-staging From 15b96c48eb76ad8ea19022df427aa25b06c3012b Mon Sep 17 00:00:00 2001 From: Chen Chongchen Date: Mon, 5 May 2025 21:43:03 +0800 Subject: [PATCH 294/413] feat: add missing PyLogicalPlan to_variant (#1085) * add expr * format * clippy * add license * update * ruff * Update expr.py * add test * ruff * Minor ruff whitespace change * Minor format change --------- Co-authored-by: Tim Saucer --- python/datafusion/common.py | 6 + python/datafusion/expr.py | 54 +++- python/tests/test_expr.py | 86 ++++++ src/common.rs | 3 + src/common/schema.rs | 89 ++++++ src/expr.rs | 41 +++ src/expr/copy_to.rs | 138 +++++++++ src/expr/create_catalog.rs | 100 +++++++ src/expr/create_catalog_schema.rs | 100 +++++++ src/expr/create_external_table.rs | 183 ++++++++++++ src/expr/create_function.rs | 182 ++++++++++++ src/expr/create_index.rs | 129 +++++++++ src/expr/describe_table.rs | 92 ++++++ src/expr/dml.rs | 136 +++++++++ src/expr/drop_catalog_schema.rs | 116 ++++++++ src/expr/drop_function.rs | 95 +++++++ src/expr/drop_view.rs | 102 +++++++ src/expr/recursive_query.rs | 111 ++++++++ src/expr/statement.rs | 454 ++++++++++++++++++++++++++++++ src/expr/values.rs | 86 ++++++ src/sql/logical.rs | 85 +++++- 21 files changed, 2372 insertions(+), 16 deletions(-) create mode 100644 src/expr/copy_to.rs create mode 100644 src/expr/create_catalog.rs create mode 100644 src/expr/create_catalog_schema.rs create mode 100644 src/expr/create_external_table.rs create mode 100644 src/expr/create_function.rs create mode 100644 src/expr/create_index.rs create mode 100644 src/expr/describe_table.rs create mode 100644 src/expr/dml.rs create mode 100644 src/expr/drop_catalog_schema.rs create mode 100644 src/expr/drop_function.rs create mode 100644 src/expr/drop_view.rs create mode 100644 src/expr/recursive_query.rs create mode 100644 src/expr/statement.rs create mode 100644 src/expr/values.rs diff --git a/python/datafusion/common.py b/python/datafusion/common.py index e762a993b..c689a816d 100644 --- a/python/datafusion/common.py +++ b/python/datafusion/common.py @@ -33,8 +33,12 @@ SqlTable = common_internal.SqlTable SqlType = common_internal.SqlType SqlView = common_internal.SqlView +TableType = common_internal.TableType +TableSource = common_internal.TableSource +Constraints = common_internal.Constraints __all__ = [ + "Constraints", "DFSchema", "DataType", "DataTypeMap", @@ -47,6 +51,8 @@ "SqlTable", "SqlType", "SqlView", + "TableSource", + "TableType", ] diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index 3750eeb3f..9e58873d0 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -54,14 +54,29 @@ Case = expr_internal.Case Cast = expr_internal.Cast Column = expr_internal.Column +CopyTo = expr_internal.CopyTo +CreateCatalog = expr_internal.CreateCatalog +CreateCatalogSchema = expr_internal.CreateCatalogSchema +CreateExternalTable = expr_internal.CreateExternalTable +CreateFunction = expr_internal.CreateFunction +CreateFunctionBody = expr_internal.CreateFunctionBody +CreateIndex = expr_internal.CreateIndex CreateMemoryTable = expr_internal.CreateMemoryTable CreateView = expr_internal.CreateView +Deallocate = expr_internal.Deallocate +DescribeTable = expr_internal.DescribeTable Distinct = expr_internal.Distinct +DmlStatement = expr_internal.DmlStatement +DropCatalogSchema = expr_internal.DropCatalogSchema +DropFunction = expr_internal.DropFunction DropTable = expr_internal.DropTable +DropView = expr_internal.DropView EmptyRelation = expr_internal.EmptyRelation +Execute = expr_internal.Execute Exists = expr_internal.Exists Explain = expr_internal.Explain Extension = expr_internal.Extension +FileType = expr_internal.FileType Filter = expr_internal.Filter GroupingSet = expr_internal.GroupingSet Join = expr_internal.Join @@ -83,21 +98,31 @@ Literal = expr_internal.Literal Negative = expr_internal.Negative Not = expr_internal.Not +OperateFunctionArg = expr_internal.OperateFunctionArg Partitioning = expr_internal.Partitioning Placeholder = expr_internal.Placeholder +Prepare = expr_internal.Prepare Projection = expr_internal.Projection +RecursiveQuery = expr_internal.RecursiveQuery Repartition = expr_internal.Repartition ScalarSubquery = expr_internal.ScalarSubquery ScalarVariable = expr_internal.ScalarVariable +SetVariable = expr_internal.SetVariable SimilarTo = expr_internal.SimilarTo Sort = expr_internal.Sort Subquery = expr_internal.Subquery SubqueryAlias = expr_internal.SubqueryAlias TableScan = expr_internal.TableScan +TransactionAccessMode = expr_internal.TransactionAccessMode +TransactionConclusion = expr_internal.TransactionConclusion +TransactionEnd = expr_internal.TransactionEnd +TransactionIsolationLevel = expr_internal.TransactionIsolationLevel +TransactionStart = expr_internal.TransactionStart TryCast = expr_internal.TryCast Union = expr_internal.Union Unnest = expr_internal.Unnest UnnestExpr = expr_internal.UnnestExpr +Values = expr_internal.Values WindowExpr = expr_internal.WindowExpr __all__ = [ @@ -111,15 +136,30 @@ "CaseBuilder", "Cast", "Column", + "CopyTo", + "CreateCatalog", + "CreateCatalogSchema", + "CreateExternalTable", + "CreateFunction", + "CreateFunctionBody", + "CreateIndex", "CreateMemoryTable", "CreateView", + "Deallocate", + "DescribeTable", "Distinct", + "DmlStatement", + "DropCatalogSchema", + "DropFunction", "DropTable", + "DropView", "EmptyRelation", + "Execute", "Exists", "Explain", "Expr", "Extension", + "FileType", "Filter", "GroupingSet", "ILike", @@ -142,22 +182,32 @@ "Literal", "Negative", "Not", + "OperateFunctionArg", "Partitioning", "Placeholder", + "Prepare", "Projection", + "RecursiveQuery", "Repartition", "ScalarSubquery", "ScalarVariable", + "SetVariable", "SimilarTo", "Sort", "SortExpr", "Subquery", "SubqueryAlias", "TableScan", + "TransactionAccessMode", + "TransactionConclusion", + "TransactionEnd", + "TransactionIsolationLevel", + "TransactionStart", "TryCast", "Union", "Unnest", "UnnestExpr", + "Values", "Window", "WindowExpr", "WindowFrame", @@ -686,8 +736,8 @@ def log10(self) -> Expr: def initcap(self) -> Expr: """Set the initial letter of each word to capital. - Converts the first letter of each word in ``string`` - to uppercase and the remaining characters to lowercase. + Converts the first letter of each word in ``string`` to uppercase and the + remaining characters to lowercase. """ from . import functions as F diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index 3651b60d6..58a202724 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -23,12 +23,21 @@ AggregateFunction, BinaryExpr, Column, + CopyTo, + CreateIndex, + DescribeTable, + DmlStatement, + DropCatalogSchema, Filter, Limit, Literal, Projection, + RecursiveQuery, Sort, TableScan, + TransactionEnd, + TransactionStart, + Values, ) @@ -249,6 +258,83 @@ def test_fill_null(df): assert result.column(2) == pa.array([1234, 1234, 8]) +def test_copy_to(): + ctx = SessionContext() + ctx.sql("CREATE TABLE foo (a int, b int)").collect() + df = ctx.sql("COPY foo TO bar STORED AS CSV") + plan = df.logical_plan() + plan = plan.to_variant() + assert isinstance(plan, CopyTo) + + +def test_create_index(): + ctx = SessionContext() + ctx.sql("CREATE TABLE foo (a int, b int)").collect() + plan = ctx.sql("create index idx on foo (a)").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, CreateIndex) + + +def test_describe_table(): + ctx = SessionContext() + ctx.sql("CREATE TABLE foo (a int, b int)").collect() + plan = ctx.sql("describe foo").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, DescribeTable) + + +def test_dml_statement(): + ctx = SessionContext() + ctx.sql("CREATE TABLE foo (a int, b int)").collect() + plan = ctx.sql("insert into foo values (1, 2)").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, DmlStatement) + + +def drop_catalog_schema(): + ctx = SessionContext() + plan = ctx.sql("drop schema cat").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, DropCatalogSchema) + + +def test_recursive_query(): + ctx = SessionContext() + plan = ctx.sql( + """ + WITH RECURSIVE cte AS ( + SELECT 1 as n + UNION ALL + SELECT n + 1 FROM cte WHERE n < 5 + ) + SELECT * FROM cte; + """ + ).logical_plan() + plan = plan.inputs()[0].inputs()[0].to_variant() + assert isinstance(plan, RecursiveQuery) + + +def test_values(): + ctx = SessionContext() + plan = ctx.sql("values (1, 'foo'), (2, 'bar')").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, Values) + + +def test_transaction_start(): + ctx = SessionContext() + plan = ctx.sql("START TRANSACTION").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, TransactionStart) + + +def test_transaction_end(): + ctx = SessionContext() + plan = ctx.sql("COMMIT").logical_plan() + plan = plan.to_variant() + assert isinstance(plan, TransactionEnd) + + def test_col_getattr(): ctx = SessionContext() data = { diff --git a/src/common.rs b/src/common.rs index 453bf67a4..88d2fdd5f 100644 --- a/src/common.rs +++ b/src/common.rs @@ -36,5 +36,8 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/src/common/schema.rs b/src/common/schema.rs index 66ce925ae..5a54fe333 100644 --- a/src/common/schema.rs +++ b/src/common/schema.rs @@ -15,14 +15,22 @@ // specific language governing permissions and limitations // under the License. +use std::fmt::{self, Display, Formatter}; +use std::sync::Arc; use std::{any::Any, borrow::Cow}; +use arrow::datatypes::Schema; +use arrow::pyarrow::PyArrowType; use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::Constraints; +use datafusion::datasource::TableType; use datafusion::logical_expr::{Expr, TableProviderFilterPushDown, TableSource}; use pyo3::prelude::*; use datafusion::logical_expr::utils::split_conjunction; +use crate::sql::logical::PyLogicalPlan; + use super::{data_type::DataTypeMap, function::SqlFunction}; #[pyclass(name = "SqlSchema", module = "datafusion.common", subclass)] @@ -218,3 +226,84 @@ impl SqlStatistics { self.row_count } } + +#[pyclass(name = "Constraints", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyConstraints { + pub constraints: Constraints, +} + +impl From for Constraints { + fn from(constraints: PyConstraints) -> Self { + constraints.constraints + } +} + +impl From for PyConstraints { + fn from(constraints: Constraints) -> Self { + PyConstraints { constraints } + } +} + +impl Display for PyConstraints { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "Constraints: {:?}", self.constraints) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(eq, eq_int, name = "TableType", module = "datafusion.common")] +pub enum PyTableType { + Base, + View, + Temporary, +} + +impl From for datafusion::logical_expr::TableType { + fn from(table_type: PyTableType) -> Self { + match table_type { + PyTableType::Base => datafusion::logical_expr::TableType::Base, + PyTableType::View => datafusion::logical_expr::TableType::View, + PyTableType::Temporary => datafusion::logical_expr::TableType::Temporary, + } + } +} + +impl From for PyTableType { + fn from(table_type: TableType) -> Self { + match table_type { + datafusion::logical_expr::TableType::Base => PyTableType::Base, + datafusion::logical_expr::TableType::View => PyTableType::View, + datafusion::logical_expr::TableType::Temporary => PyTableType::Temporary, + } + } +} + +#[pyclass(name = "TableSource", module = "datafusion.common", subclass)] +#[derive(Clone)] +pub struct PyTableSource { + pub table_source: Arc, +} + +#[pymethods] +impl PyTableSource { + pub fn schema(&self) -> PyArrowType { + (*self.table_source.schema()).clone().into() + } + + pub fn constraints(&self) -> Option { + self.table_source.constraints().map(|c| PyConstraints { + constraints: c.clone(), + }) + } + + pub fn table_type(&self) -> PyTableType { + self.table_source.table_type().into() + } + + pub fn get_logical_plan(&self) -> Option { + self.table_source + .get_logical_plan() + .map(|plan| PyLogicalPlan::new(plan.into_owned())) + } +} diff --git a/src/expr.rs b/src/expr.rs index 7d4aa8798..404e575f8 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -67,10 +67,21 @@ pub mod case; pub mod cast; pub mod column; pub mod conditional_expr; +pub mod copy_to; +pub mod create_catalog; +pub mod create_catalog_schema; +pub mod create_external_table; +pub mod create_function; +pub mod create_index; pub mod create_memory_table; pub mod create_view; +pub mod describe_table; pub mod distinct; +pub mod dml; +pub mod drop_catalog_schema; +pub mod drop_function; pub mod drop_table; +pub mod drop_view; pub mod empty_relation; pub mod exists; pub mod explain; @@ -86,18 +97,21 @@ pub mod literal; pub mod logical_node; pub mod placeholder; pub mod projection; +pub mod recursive_query; pub mod repartition; pub mod scalar_subquery; pub mod scalar_variable; pub mod signature; pub mod sort; pub mod sort_expr; +pub mod statement; pub mod subquery; pub mod subquery_alias; pub mod table_scan; pub mod union; pub mod unnest; pub mod unnest_expr; +pub mod values; pub mod window; use sort_expr::{to_sort_expressions, PySortExpr}; @@ -802,5 +816,32 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + Ok(()) } diff --git a/src/expr/copy_to.rs b/src/expr/copy_to.rs new file mode 100644 index 000000000..ebfcb8ebc --- /dev/null +++ b/src/expr/copy_to.rs @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + collections::HashMap, + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::{common::file_options::file_type::FileType, logical_expr::dml::CopyTo}; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::sql::logical::PyLogicalPlan; + +use super::logical_node::LogicalNode; + +#[pyclass(name = "CopyTo", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCopyTo { + copy: CopyTo, +} + +impl From for CopyTo { + fn from(copy: PyCopyTo) -> Self { + copy.copy + } +} + +impl From for PyCopyTo { + fn from(copy: CopyTo) -> PyCopyTo { + PyCopyTo { copy } + } +} + +impl Display for PyCopyTo { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "CopyTo: {:?}", self.copy.output_url) + } +} + +impl LogicalNode for PyCopyTo { + fn inputs(&self) -> Vec { + vec![PyLogicalPlan::from((*self.copy.input).clone())] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PyCopyTo { + #[new] + pub fn new( + input: PyLogicalPlan, + output_url: String, + partition_by: Vec, + file_type: PyFileType, + options: HashMap, + ) -> Self { + PyCopyTo { + copy: CopyTo { + input: input.plan(), + output_url, + partition_by, + file_type: file_type.file_type, + options, + }, + } + } + + fn input(&self) -> PyLogicalPlan { + PyLogicalPlan::from((*self.copy.input).clone()) + } + + fn output_url(&self) -> String { + self.copy.output_url.clone() + } + + fn partition_by(&self) -> Vec { + self.copy.partition_by.clone() + } + + fn file_type(&self) -> PyFileType { + PyFileType { + file_type: self.copy.file_type.clone(), + } + } + + fn options(&self) -> HashMap { + self.copy.options.clone() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CopyTo({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("CopyTo".to_string()) + } +} + +#[pyclass(name = "FileType", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyFileType { + file_type: Arc, +} + +impl Display for PyFileType { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "FileType: {}", self.file_type) + } +} + +#[pymethods] +impl PyFileType { + fn __repr__(&self) -> PyResult { + Ok(format!("FileType({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("FileType".to_string()) + } +} diff --git a/src/expr/create_catalog.rs b/src/expr/create_catalog.rs new file mode 100644 index 000000000..f4ea0f517 --- /dev/null +++ b/src/expr/create_catalog.rs @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::CreateCatalog; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; + +use super::logical_node::LogicalNode; + +#[pyclass(name = "CreateCatalog", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCreateCatalog { + create: CreateCatalog, +} + +impl From for CreateCatalog { + fn from(create: PyCreateCatalog) -> Self { + create.create + } +} + +impl From for PyCreateCatalog { + fn from(create: CreateCatalog) -> PyCreateCatalog { + PyCreateCatalog { create } + } +} + +impl Display for PyCreateCatalog { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "CreateCatalog: {:?}", self.create.catalog_name) + } +} + +#[pymethods] +impl PyCreateCatalog { + #[new] + pub fn new( + catalog_name: String, + if_not_exists: bool, + schema: PyDFSchema, + ) -> PyResult { + Ok(PyCreateCatalog { + create: CreateCatalog { + catalog_name, + if_not_exists, + schema: Arc::new(schema.into()), + }, + }) + } + + pub fn catalog_name(&self) -> String { + self.create.catalog_name.clone() + } + + pub fn if_not_exists(&self) -> bool { + self.create.if_not_exists + } + + pub fn schema(&self) -> PyDFSchema { + (*self.create.schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CreateCatalog({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("CreateCatalog".to_string()) + } +} + +impl LogicalNode for PyCreateCatalog { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/create_catalog_schema.rs b/src/expr/create_catalog_schema.rs new file mode 100644 index 000000000..85f447e1e --- /dev/null +++ b/src/expr/create_catalog_schema.rs @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::CreateCatalogSchema; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; + +use super::logical_node::LogicalNode; + +#[pyclass(name = "CreateCatalogSchema", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCreateCatalogSchema { + create: CreateCatalogSchema, +} + +impl From for CreateCatalogSchema { + fn from(create: PyCreateCatalogSchema) -> Self { + create.create + } +} + +impl From for PyCreateCatalogSchema { + fn from(create: CreateCatalogSchema) -> PyCreateCatalogSchema { + PyCreateCatalogSchema { create } + } +} + +impl Display for PyCreateCatalogSchema { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "CreateCatalogSchema: {:?}", self.create.schema_name) + } +} + +#[pymethods] +impl PyCreateCatalogSchema { + #[new] + pub fn new( + schema_name: String, + if_not_exists: bool, + schema: PyDFSchema, + ) -> PyResult { + Ok(PyCreateCatalogSchema { + create: CreateCatalogSchema { + schema_name, + if_not_exists, + schema: Arc::new(schema.into()), + }, + }) + } + + pub fn schema_name(&self) -> String { + self.create.schema_name.clone() + } + + pub fn if_not_exists(&self) -> bool { + self.create.if_not_exists + } + + pub fn schema(&self) -> PyDFSchema { + (*self.create.schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CreateCatalogSchema({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("CreateCatalogSchema".to_string()) + } +} + +impl LogicalNode for PyCreateCatalogSchema { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/create_external_table.rs b/src/expr/create_external_table.rs new file mode 100644 index 000000000..01ce7d0ca --- /dev/null +++ b/src/expr/create_external_table.rs @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::{common::schema::PyConstraints, expr::PyExpr, sql::logical::PyLogicalPlan}; +use std::{ + collections::HashMap, + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::CreateExternalTable; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::common::df_schema::PyDFSchema; + +use super::{logical_node::LogicalNode, sort_expr::PySortExpr}; + +#[pyclass(name = "CreateExternalTable", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCreateExternalTable { + create: CreateExternalTable, +} + +impl From for CreateExternalTable { + fn from(create: PyCreateExternalTable) -> Self { + create.create + } +} + +impl From for PyCreateExternalTable { + fn from(create: CreateExternalTable) -> PyCreateExternalTable { + PyCreateExternalTable { create } + } +} + +impl Display for PyCreateExternalTable { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreateExternalTable: {:?}{}", + self.create.name, self.create.constraints + ) + } +} + +#[pymethods] +impl PyCreateExternalTable { + #[allow(clippy::too_many_arguments)] + #[new] + #[pyo3(signature = (schema, name, location, file_type, table_partition_cols, if_not_exists, temporary, order_exprs, unbounded, options, constraints, column_defaults, definition=None))] + pub fn new( + schema: PyDFSchema, + name: String, + location: String, + file_type: String, + table_partition_cols: Vec, + if_not_exists: bool, + temporary: bool, + order_exprs: Vec>, + unbounded: bool, + options: HashMap, + constraints: PyConstraints, + column_defaults: HashMap, + definition: Option, + ) -> Self { + let create = CreateExternalTable { + schema: Arc::new(schema.into()), + name: name.into(), + location, + file_type, + table_partition_cols, + if_not_exists, + temporary, + definition, + order_exprs: order_exprs + .into_iter() + .map(|vec| vec.into_iter().map(|s| s.into()).collect::>()) + .collect::>(), + unbounded, + options, + constraints: constraints.constraints, + column_defaults: column_defaults + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect(), + }; + PyCreateExternalTable { create } + } + + pub fn schema(&self) -> PyDFSchema { + (*self.create.schema).clone().into() + } + + pub fn name(&self) -> PyResult { + Ok(self.create.name.to_string()) + } + + pub fn location(&self) -> String { + self.create.location.clone() + } + + pub fn file_type(&self) -> String { + self.create.file_type.clone() + } + + pub fn table_partition_cols(&self) -> Vec { + self.create.table_partition_cols.clone() + } + + pub fn if_not_exists(&self) -> bool { + self.create.if_not_exists + } + + pub fn temporary(&self) -> bool { + self.create.temporary + } + + pub fn definition(&self) -> Option { + self.create.definition.clone() + } + + pub fn order_exprs(&self) -> Vec> { + self.create + .order_exprs + .iter() + .map(|vec| vec.iter().map(|s| s.clone().into()).collect()) + .collect() + } + + pub fn unbounded(&self) -> bool { + self.create.unbounded + } + + pub fn options(&self) -> HashMap { + self.create.options.clone() + } + + pub fn constraints(&self) -> PyConstraints { + PyConstraints { + constraints: self.create.constraints.clone(), + } + } + + pub fn column_defaults(&self) -> HashMap { + self.create + .column_defaults + .iter() + .map(|(k, v)| (k.clone(), v.clone().into())) + .collect() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CreateExternalTable({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("CreateExternalTable".to_string()) + } +} + +impl LogicalNode for PyCreateExternalTable { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/create_function.rs b/src/expr/create_function.rs new file mode 100644 index 000000000..6f3c3f0ff --- /dev/null +++ b/src/expr/create_function.rs @@ -0,0 +1,182 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::{ + CreateFunction, CreateFunctionBody, OperateFunctionArg, Volatility, +}; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use super::logical_node::LogicalNode; +use super::PyExpr; +use crate::common::{data_type::PyDataType, df_schema::PyDFSchema}; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass(name = "CreateFunction", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCreateFunction { + create: CreateFunction, +} + +impl From for CreateFunction { + fn from(create: PyCreateFunction) -> Self { + create.create + } +} + +impl From for PyCreateFunction { + fn from(create: CreateFunction) -> PyCreateFunction { + PyCreateFunction { create } + } +} + +impl Display for PyCreateFunction { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "CreateFunction: name {:?}", self.create.name) + } +} + +#[pyclass(name = "OperateFunctionArg", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyOperateFunctionArg { + arg: OperateFunctionArg, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(eq, eq_int, name = "Volatility", module = "datafusion.expr")] +pub enum PyVolatility { + Immutable, + Stable, + Volatile, +} + +#[pyclass(name = "CreateFunctionBody", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCreateFunctionBody { + body: CreateFunctionBody, +} + +#[pymethods] +impl PyCreateFunctionBody { + pub fn language(&self) -> Option { + self.body + .language + .as_ref() + .map(|language| language.to_string()) + } + + pub fn behavior(&self) -> Option { + self.body.behavior.as_ref().map(|behavior| match behavior { + Volatility::Immutable => PyVolatility::Immutable, + Volatility::Stable => PyVolatility::Stable, + Volatility::Volatile => PyVolatility::Volatile, + }) + } + + pub fn function_body(&self) -> Option { + self.body + .function_body + .as_ref() + .map(|function_body| function_body.clone().into()) + } +} + +#[pymethods] +impl PyCreateFunction { + #[new] + #[pyo3(signature = (or_replace, temporary, name, params, schema, return_type=None, args=None))] + pub fn new( + or_replace: bool, + temporary: bool, + name: String, + params: PyCreateFunctionBody, + schema: PyDFSchema, + return_type: Option, + args: Option>, + ) -> Self { + PyCreateFunction { + create: CreateFunction { + or_replace, + temporary, + name, + args: args.map(|args| args.into_iter().map(|arg| arg.arg).collect()), + return_type: return_type.map(|return_type| return_type.data_type), + params: params.body, + schema: Arc::new(schema.into()), + }, + } + } + + pub fn or_replace(&self) -> bool { + self.create.or_replace + } + + pub fn temporary(&self) -> bool { + self.create.temporary + } + + pub fn name(&self) -> String { + self.create.name.clone() + } + + pub fn params(&self) -> PyCreateFunctionBody { + PyCreateFunctionBody { + body: self.create.params.clone(), + } + } + + pub fn schema(&self) -> PyDFSchema { + (*self.create.schema).clone().into() + } + + pub fn return_type(&self) -> Option { + self.create + .return_type + .as_ref() + .map(|return_type| return_type.clone().into()) + } + + pub fn args(&self) -> Option> { + self.create.args.as_ref().map(|args| { + args.iter() + .map(|arg| PyOperateFunctionArg { arg: arg.clone() }) + .collect() + }) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CreateFunction({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("CreateFunction".to_string()) + } +} + +impl LogicalNode for PyCreateFunction { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/create_index.rs b/src/expr/create_index.rs new file mode 100644 index 000000000..13dadbc3f --- /dev/null +++ b/src/expr/create_index.rs @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::CreateIndex; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; + +use super::{logical_node::LogicalNode, sort_expr::PySortExpr}; + +#[pyclass(name = "CreateIndex", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyCreateIndex { + create: CreateIndex, +} + +impl From for CreateIndex { + fn from(create: PyCreateIndex) -> Self { + create.create + } +} + +impl From for PyCreateIndex { + fn from(create: CreateIndex) -> PyCreateIndex { + PyCreateIndex { create } + } +} + +impl Display for PyCreateIndex { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "CreateIndex: {:?}", self.create.name) + } +} + +#[pymethods] +impl PyCreateIndex { + #[new] + #[pyo3(signature = (table, columns, unique, if_not_exists, schema, name=None, using=None))] + pub fn new( + table: String, + columns: Vec, + unique: bool, + if_not_exists: bool, + schema: PyDFSchema, + name: Option, + using: Option, + ) -> PyResult { + Ok(PyCreateIndex { + create: CreateIndex { + name, + table: table.into(), + using, + columns: columns.iter().map(|c| c.clone().into()).collect(), + unique, + if_not_exists, + schema: Arc::new(schema.into()), + }, + }) + } + + pub fn name(&self) -> Option { + self.create.name.clone() + } + + pub fn table(&self) -> PyResult { + Ok(self.create.table.to_string()) + } + + pub fn using(&self) -> Option { + self.create.using.clone() + } + + pub fn columns(&self) -> Vec { + self.create + .columns + .iter() + .map(|c| c.clone().into()) + .collect() + } + + pub fn unique(&self) -> bool { + self.create.unique + } + + pub fn if_not_exists(&self) -> bool { + self.create.if_not_exists + } + + pub fn schema(&self) -> PyDFSchema { + (*self.create.schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CreateIndex({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("CreateIndex".to_string()) + } +} + +impl LogicalNode for PyCreateIndex { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/describe_table.rs b/src/expr/describe_table.rs new file mode 100644 index 000000000..5658a13f2 --- /dev/null +++ b/src/expr/describe_table.rs @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use arrow::{datatypes::Schema, pyarrow::PyArrowType}; +use datafusion::logical_expr::DescribeTable; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; + +use super::logical_node::LogicalNode; + +#[pyclass(name = "DescribeTable", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyDescribeTable { + describe: DescribeTable, +} + +impl Display for PyDescribeTable { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "DescribeTable") + } +} + +#[pymethods] +impl PyDescribeTable { + #[new] + fn new(schema: PyArrowType, output_schema: PyDFSchema) -> Self { + Self { + describe: DescribeTable { + schema: Arc::new(schema.0), + output_schema: Arc::new(output_schema.into()), + }, + } + } + + pub fn schema(&self) -> PyArrowType { + (*self.describe.schema).clone().into() + } + + pub fn output_schema(&self) -> PyDFSchema { + (*self.describe.output_schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("DescribeTable({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("DescribeTable".to_string()) + } +} + +impl From for DescribeTable { + fn from(describe: PyDescribeTable) -> Self { + describe.describe + } +} + +impl From for PyDescribeTable { + fn from(describe: DescribeTable) -> PyDescribeTable { + PyDescribeTable { describe } + } +} + +impl LogicalNode for PyDescribeTable { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/dml.rs b/src/expr/dml.rs new file mode 100644 index 000000000..251e336cc --- /dev/null +++ b/src/expr/dml.rs @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion::logical_expr::dml::InsertOp; +use datafusion::logical_expr::{DmlStatement, WriteOp}; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::common::schema::PyTableSource; +use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; + +use super::logical_node::LogicalNode; + +#[pyclass(name = "DmlStatement", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyDmlStatement { + dml: DmlStatement, +} + +impl From for DmlStatement { + fn from(dml: PyDmlStatement) -> Self { + dml.dml + } +} + +impl From for PyDmlStatement { + fn from(dml: DmlStatement) -> PyDmlStatement { + PyDmlStatement { dml } + } +} + +impl LogicalNode for PyDmlStatement { + fn inputs(&self) -> Vec { + vec![PyLogicalPlan::from((*self.dml.input).clone())] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PyDmlStatement { + pub fn table_name(&self) -> PyResult { + Ok(self.dml.table_name.to_string()) + } + + pub fn target(&self) -> PyResult { + Ok(PyTableSource { + table_source: self.dml.target.clone(), + }) + } + + pub fn op(&self) -> PyWriteOp { + self.dml.op.clone().into() + } + + pub fn input(&self) -> PyLogicalPlan { + PyLogicalPlan { + plan: self.dml.input.clone(), + } + } + + pub fn output_schema(&self) -> PyDFSchema { + (*self.dml.output_schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok("DmlStatement".to_string()) + } + + fn __name__(&self) -> PyResult { + Ok("DmlStatement".to_string()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(eq, eq_int, name = "WriteOp", module = "datafusion.expr")] +pub enum PyWriteOp { + Append, + Overwrite, + Replace, + + Update, + Delete, + Ctas, +} + +impl From for PyWriteOp { + fn from(write_op: WriteOp) -> Self { + match write_op { + WriteOp::Insert(InsertOp::Append) => PyWriteOp::Append, + WriteOp::Insert(InsertOp::Overwrite) => PyWriteOp::Overwrite, + WriteOp::Insert(InsertOp::Replace) => PyWriteOp::Replace, + + WriteOp::Update => PyWriteOp::Update, + WriteOp::Delete => PyWriteOp::Delete, + WriteOp::Ctas => PyWriteOp::Ctas, + } + } +} + +impl From for WriteOp { + fn from(py: PyWriteOp) -> Self { + match py { + PyWriteOp::Append => WriteOp::Insert(InsertOp::Append), + PyWriteOp::Overwrite => WriteOp::Insert(InsertOp::Overwrite), + PyWriteOp::Replace => WriteOp::Insert(InsertOp::Replace), + + PyWriteOp::Update => WriteOp::Update, + PyWriteOp::Delete => WriteOp::Delete, + PyWriteOp::Ctas => WriteOp::Ctas, + } + } +} + +#[pymethods] +impl PyWriteOp { + fn name(&self) -> String { + let write_op: WriteOp = self.clone().into(); + write_op.name().to_string() + } +} diff --git a/src/expr/drop_catalog_schema.rs b/src/expr/drop_catalog_schema.rs new file mode 100644 index 000000000..b7420a99c --- /dev/null +++ b/src/expr/drop_catalog_schema.rs @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::{common::SchemaReference, logical_expr::DropCatalogSchema, sql::TableReference}; +use pyo3::{exceptions::PyValueError, prelude::*, IntoPyObjectExt}; + +use crate::common::df_schema::PyDFSchema; + +use super::logical_node::LogicalNode; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass(name = "DropCatalogSchema", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyDropCatalogSchema { + drop: DropCatalogSchema, +} + +impl From for DropCatalogSchema { + fn from(drop: PyDropCatalogSchema) -> Self { + drop.drop + } +} + +impl From for PyDropCatalogSchema { + fn from(drop: DropCatalogSchema) -> PyDropCatalogSchema { + PyDropCatalogSchema { drop } + } +} + +impl Display for PyDropCatalogSchema { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "DropCatalogSchema") + } +} + +fn parse_schema_reference(name: String) -> PyResult { + match name.into() { + TableReference::Bare { table } => Ok(SchemaReference::Bare { schema: table }), + TableReference::Partial { schema, table } => Ok(SchemaReference::Full { + schema: table, + catalog: schema, + }), + TableReference::Full { + catalog: _, + schema: _, + table: _, + } => Err(PyErr::new::( + "Invalid schema specifier (has 3 parts)".to_string(), + )), + } +} + +#[pymethods] +impl PyDropCatalogSchema { + #[new] + fn new(name: String, schema: PyDFSchema, if_exists: bool, cascade: bool) -> PyResult { + let name = parse_schema_reference(name)?; + Ok(PyDropCatalogSchema { + drop: DropCatalogSchema { + name, + schema: Arc::new(schema.into()), + if_exists, + cascade, + }, + }) + } + + fn name(&self) -> PyResult { + Ok(self.drop.name.to_string()) + } + + fn schema(&self) -> PyDFSchema { + (*self.drop.schema).clone().into() + } + + fn if_exists(&self) -> PyResult { + Ok(self.drop.if_exists) + } + + fn cascade(&self) -> PyResult { + Ok(self.drop.cascade) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("DropCatalogSchema({})", self)) + } +} + +impl LogicalNode for PyDropCatalogSchema { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/drop_function.rs b/src/expr/drop_function.rs new file mode 100644 index 000000000..9fbd78fdc --- /dev/null +++ b/src/expr/drop_function.rs @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::DropFunction; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use super::logical_node::LogicalNode; +use crate::common::df_schema::PyDFSchema; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass(name = "DropFunction", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyDropFunction { + drop: DropFunction, +} + +impl From for DropFunction { + fn from(drop: PyDropFunction) -> Self { + drop.drop + } +} + +impl From for PyDropFunction { + fn from(drop: DropFunction) -> PyDropFunction { + PyDropFunction { drop } + } +} + +impl Display for PyDropFunction { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "DropFunction") + } +} + +#[pymethods] +impl PyDropFunction { + #[new] + fn new(name: String, schema: PyDFSchema, if_exists: bool) -> PyResult { + Ok(PyDropFunction { + drop: DropFunction { + name, + schema: Arc::new(schema.into()), + if_exists, + }, + }) + } + fn name(&self) -> PyResult { + Ok(self.drop.name.clone()) + } + + fn schema(&self) -> PyDFSchema { + (*self.drop.schema).clone().into() + } + + fn if_exists(&self) -> PyResult { + Ok(self.drop.if_exists) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("DropFunction({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("DropFunction".to_string()) + } +} + +impl LogicalNode for PyDropFunction { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/drop_view.rs b/src/expr/drop_view.rs new file mode 100644 index 000000000..1d1ab1e59 --- /dev/null +++ b/src/expr/drop_view.rs @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::{ + fmt::{self, Display, Formatter}, + sync::Arc, +}; + +use datafusion::logical_expr::DropView; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::common::df_schema::PyDFSchema; + +use super::logical_node::LogicalNode; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass(name = "DropView", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyDropView { + drop: DropView, +} + +impl From for DropView { + fn from(drop: PyDropView) -> Self { + drop.drop + } +} + +impl From for PyDropView { + fn from(drop: DropView) -> PyDropView { + PyDropView { drop } + } +} + +impl Display for PyDropView { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "DropView: {name:?} if not exist:={if_exists}", + name = self.drop.name, + if_exists = self.drop.if_exists + ) + } +} + +#[pymethods] +impl PyDropView { + #[new] + fn new(name: String, schema: PyDFSchema, if_exists: bool) -> PyResult { + Ok(PyDropView { + drop: DropView { + name: name.into(), + schema: Arc::new(schema.into()), + if_exists, + }, + }) + } + + fn name(&self) -> PyResult { + Ok(self.drop.name.to_string()) + } + + fn schema(&self) -> PyDFSchema { + (*self.drop.schema).clone().into() + } + + fn if_exists(&self) -> PyResult { + Ok(self.drop.if_exists) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("DropView({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("DropView".to_string()) + } +} + +impl LogicalNode for PyDropView { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/recursive_query.rs b/src/expr/recursive_query.rs new file mode 100644 index 000000000..65181f7d3 --- /dev/null +++ b/src/expr/recursive_query.rs @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{self, Display, Formatter}; + +use datafusion::logical_expr::RecursiveQuery; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::sql::logical::PyLogicalPlan; + +use super::logical_node::LogicalNode; + +#[pyclass(name = "RecursiveQuery", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyRecursiveQuery { + query: RecursiveQuery, +} + +impl From for RecursiveQuery { + fn from(query: PyRecursiveQuery) -> Self { + query.query + } +} + +impl From for PyRecursiveQuery { + fn from(query: RecursiveQuery) -> PyRecursiveQuery { + PyRecursiveQuery { query } + } +} + +impl Display for PyRecursiveQuery { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "RecursiveQuery {name:?} is_distinct:={is_distinct}", + name = self.query.name, + is_distinct = self.query.is_distinct + ) + } +} + +#[pymethods] +impl PyRecursiveQuery { + #[new] + fn new( + name: String, + static_term: PyLogicalPlan, + recursive_term: PyLogicalPlan, + is_distinct: bool, + ) -> Self { + Self { + query: RecursiveQuery { + name, + static_term: static_term.plan(), + recursive_term: recursive_term.plan(), + is_distinct, + }, + } + } + + fn name(&self) -> PyResult { + Ok(self.query.name.clone()) + } + + fn static_term(&self) -> PyLogicalPlan { + PyLogicalPlan::from((*self.query.static_term).clone()) + } + + fn recursive_term(&self) -> PyLogicalPlan { + PyLogicalPlan::from((*self.query.recursive_term).clone()) + } + + fn is_distinct(&self) -> PyResult { + Ok(self.query.is_distinct) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("RecursiveQuery({})", self)) + } + + fn __name__(&self) -> PyResult { + Ok("RecursiveQuery".to_string()) + } +} + +impl LogicalNode for PyRecursiveQuery { + fn inputs(&self) -> Vec { + vec![ + PyLogicalPlan::from((*self.query.static_term).clone()), + PyLogicalPlan::from((*self.query.recursive_term).clone()), + ] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/src/expr/statement.rs b/src/expr/statement.rs new file mode 100644 index 000000000..83774cda1 --- /dev/null +++ b/src/expr/statement.rs @@ -0,0 +1,454 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion::logical_expr::{ + Deallocate, Execute, Prepare, SetVariable, TransactionAccessMode, TransactionConclusion, + TransactionEnd, TransactionIsolationLevel, TransactionStart, +}; +use pyo3::{prelude::*, IntoPyObjectExt}; + +use crate::{common::data_type::PyDataType, sql::logical::PyLogicalPlan}; + +use super::{logical_node::LogicalNode, PyExpr}; + +#[pyclass(name = "TransactionStart", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyTransactionStart { + transaction_start: TransactionStart, +} + +impl From for PyTransactionStart { + fn from(transaction_start: TransactionStart) -> PyTransactionStart { + PyTransactionStart { transaction_start } + } +} + +impl TryFrom for TransactionStart { + type Error = PyErr; + + fn try_from(py: PyTransactionStart) -> Result { + Ok(py.transaction_start) + } +} + +impl LogicalNode for PyTransactionStart { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(eq, eq_int, name = "TransactionAccessMode", module = "datafusion.expr")] +pub enum PyTransactionAccessMode { + ReadOnly, + ReadWrite, +} + +impl From for PyTransactionAccessMode { + fn from(access_mode: TransactionAccessMode) -> PyTransactionAccessMode { + match access_mode { + TransactionAccessMode::ReadOnly => PyTransactionAccessMode::ReadOnly, + TransactionAccessMode::ReadWrite => PyTransactionAccessMode::ReadWrite, + } + } +} + +impl TryFrom for TransactionAccessMode { + type Error = PyErr; + + fn try_from(py: PyTransactionAccessMode) -> Result { + match py { + PyTransactionAccessMode::ReadOnly => Ok(TransactionAccessMode::ReadOnly), + PyTransactionAccessMode::ReadWrite => Ok(TransactionAccessMode::ReadWrite), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass( + eq, + eq_int, + name = "TransactionIsolationLevel", + module = "datafusion.expr" +)] +pub enum PyTransactionIsolationLevel { + ReadUncommitted, + ReadCommitted, + RepeatableRead, + Serializable, + Snapshot, +} + +impl From for PyTransactionIsolationLevel { + fn from(isolation_level: TransactionIsolationLevel) -> PyTransactionIsolationLevel { + match isolation_level { + TransactionIsolationLevel::ReadUncommitted => { + PyTransactionIsolationLevel::ReadUncommitted + } + TransactionIsolationLevel::ReadCommitted => PyTransactionIsolationLevel::ReadCommitted, + TransactionIsolationLevel::RepeatableRead => { + PyTransactionIsolationLevel::RepeatableRead + } + TransactionIsolationLevel::Serializable => PyTransactionIsolationLevel::Serializable, + TransactionIsolationLevel::Snapshot => PyTransactionIsolationLevel::Snapshot, + } + } +} + +impl TryFrom for TransactionIsolationLevel { + type Error = PyErr; + + fn try_from(value: PyTransactionIsolationLevel) -> Result { + match value { + PyTransactionIsolationLevel::ReadUncommitted => { + Ok(TransactionIsolationLevel::ReadUncommitted) + } + PyTransactionIsolationLevel::ReadCommitted => { + Ok(TransactionIsolationLevel::ReadCommitted) + } + PyTransactionIsolationLevel::RepeatableRead => { + Ok(TransactionIsolationLevel::RepeatableRead) + } + PyTransactionIsolationLevel::Serializable => { + Ok(TransactionIsolationLevel::Serializable) + } + PyTransactionIsolationLevel::Snapshot => Ok(TransactionIsolationLevel::Snapshot), + } + } +} + +#[pymethods] +impl PyTransactionStart { + #[new] + pub fn new( + access_mode: PyTransactionAccessMode, + isolation_level: PyTransactionIsolationLevel, + ) -> PyResult { + let access_mode = access_mode.try_into()?; + let isolation_level = isolation_level.try_into()?; + Ok(PyTransactionStart { + transaction_start: TransactionStart { + access_mode, + isolation_level, + }, + }) + } + + pub fn access_mode(&self) -> PyResult { + Ok(self.transaction_start.access_mode.clone().into()) + } + + pub fn isolation_level(&self) -> PyResult { + Ok(self.transaction_start.isolation_level.clone().into()) + } +} + +#[pyclass(name = "TransactionEnd", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyTransactionEnd { + transaction_end: TransactionEnd, +} + +impl From for PyTransactionEnd { + fn from(transaction_end: TransactionEnd) -> PyTransactionEnd { + PyTransactionEnd { transaction_end } + } +} + +impl TryFrom for TransactionEnd { + type Error = PyErr; + + fn try_from(py: PyTransactionEnd) -> Result { + Ok(py.transaction_end) + } +} + +impl LogicalNode for PyTransactionEnd { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[pyclass(eq, eq_int, name = "TransactionConclusion", module = "datafusion.expr")] +pub enum PyTransactionConclusion { + Commit, + Rollback, +} + +impl From for PyTransactionConclusion { + fn from(value: TransactionConclusion) -> Self { + match value { + TransactionConclusion::Commit => PyTransactionConclusion::Commit, + TransactionConclusion::Rollback => PyTransactionConclusion::Rollback, + } + } +} + +impl TryFrom for TransactionConclusion { + type Error = PyErr; + + fn try_from(value: PyTransactionConclusion) -> Result { + match value { + PyTransactionConclusion::Commit => Ok(TransactionConclusion::Commit), + PyTransactionConclusion::Rollback => Ok(TransactionConclusion::Rollback), + } + } +} +#[pymethods] +impl PyTransactionEnd { + #[new] + pub fn new(conclusion: PyTransactionConclusion, chain: bool) -> PyResult { + let conclusion = conclusion.try_into()?; + Ok(PyTransactionEnd { + transaction_end: TransactionEnd { conclusion, chain }, + }) + } + + pub fn conclusion(&self) -> PyResult { + Ok(self.transaction_end.conclusion.clone().into()) + } + + pub fn chain(&self) -> bool { + self.transaction_end.chain + } +} + +#[pyclass(name = "SetVariable", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PySetVariable { + set_variable: SetVariable, +} + +impl From for PySetVariable { + fn from(set_variable: SetVariable) -> PySetVariable { + PySetVariable { set_variable } + } +} + +impl TryFrom for SetVariable { + type Error = PyErr; + + fn try_from(py: PySetVariable) -> Result { + Ok(py.set_variable) + } +} + +impl LogicalNode for PySetVariable { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PySetVariable { + #[new] + pub fn new(variable: String, value: String) -> Self { + PySetVariable { + set_variable: SetVariable { variable, value }, + } + } + + pub fn variable(&self) -> String { + self.set_variable.variable.clone() + } + + pub fn value(&self) -> String { + self.set_variable.value.clone() + } +} + +#[pyclass(name = "Prepare", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyPrepare { + prepare: Prepare, +} + +impl From for PyPrepare { + fn from(prepare: Prepare) -> PyPrepare { + PyPrepare { prepare } + } +} + +impl TryFrom for Prepare { + type Error = PyErr; + + fn try_from(py: PyPrepare) -> Result { + Ok(py.prepare) + } +} + +impl LogicalNode for PyPrepare { + fn inputs(&self) -> Vec { + vec![PyLogicalPlan::from((*self.prepare.input).clone())] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PyPrepare { + #[new] + pub fn new(name: String, data_types: Vec, input: PyLogicalPlan) -> Self { + let input = input.plan().clone(); + let data_types = data_types + .into_iter() + .map(|data_type| data_type.into()) + .collect(); + PyPrepare { + prepare: Prepare { + name, + data_types, + input, + }, + } + } + + pub fn name(&self) -> String { + self.prepare.name.clone() + } + + pub fn data_types(&self) -> Vec { + self.prepare + .data_types + .clone() + .into_iter() + .map(|t| t.into()) + .collect() + } + + pub fn input(&self) -> PyLogicalPlan { + PyLogicalPlan { + plan: self.prepare.input.clone(), + } + } +} + +#[pyclass(name = "Execute", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyExecute { + execute: Execute, +} + +impl From for PyExecute { + fn from(execute: Execute) -> PyExecute { + PyExecute { execute } + } +} + +impl TryFrom for Execute { + type Error = PyErr; + + fn try_from(py: PyExecute) -> Result { + Ok(py.execute) + } +} + +impl LogicalNode for PyExecute { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PyExecute { + #[new] + pub fn new(name: String, parameters: Vec) -> Self { + let parameters = parameters + .into_iter() + .map(|parameter| parameter.into()) + .collect(); + PyExecute { + execute: Execute { name, parameters }, + } + } + + pub fn name(&self) -> String { + self.execute.name.clone() + } + + pub fn parameters(&self) -> Vec { + self.execute + .parameters + .clone() + .into_iter() + .map(|t| t.into()) + .collect() + } +} + +#[pyclass(name = "Deallocate", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyDeallocate { + deallocate: Deallocate, +} + +impl From for PyDeallocate { + fn from(deallocate: Deallocate) -> PyDeallocate { + PyDeallocate { deallocate } + } +} + +impl TryFrom for Deallocate { + type Error = PyErr; + + fn try_from(py: PyDeallocate) -> Result { + Ok(py.deallocate) + } +} + +impl LogicalNode for PyDeallocate { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PyDeallocate { + #[new] + pub fn new(name: String) -> Self { + PyDeallocate { + deallocate: Deallocate { name }, + } + } + + pub fn name(&self) -> String { + self.deallocate.name.clone() + } +} diff --git a/src/expr/values.rs b/src/expr/values.rs new file mode 100644 index 000000000..fb2692230 --- /dev/null +++ b/src/expr/values.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use datafusion::logical_expr::Values; +use pyo3::{prelude::*, IntoPyObjectExt}; +use pyo3::{pyclass, PyErr, PyResult, Python}; + +use crate::{common::df_schema::PyDFSchema, sql::logical::PyLogicalPlan}; + +use super::{logical_node::LogicalNode, PyExpr}; + +#[pyclass(name = "Values", module = "datafusion.expr", subclass)] +#[derive(Clone)] +pub struct PyValues { + values: Values, +} + +impl From for PyValues { + fn from(values: Values) -> PyValues { + PyValues { values } + } +} + +impl TryFrom for Values { + type Error = PyErr; + + fn try_from(py: PyValues) -> Result { + Ok(py.values) + } +} + +impl LogicalNode for PyValues { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pymethods] +impl PyValues { + #[new] + pub fn new(schema: PyDFSchema, values: Vec>) -> PyResult { + let values = values + .into_iter() + .map(|row| row.into_iter().map(|expr| expr.into()).collect()) + .collect(); + Ok(PyValues { + values: Values { + schema: Arc::new(schema.into()), + values, + }, + }) + } + + pub fn schema(&self) -> PyResult { + Ok((*self.values.schema).clone().into()) + } + + pub fn values(&self) -> Vec> { + self.values + .values + .clone() + .into_iter() + .map(|row| row.into_iter().map(|expr| expr.into()).collect()) + .collect() + } +} diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 96561c434..198d68bdc 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -17,10 +17,25 @@ use std::sync::Arc; +use crate::context::PySessionContext; use crate::errors::PyDataFusionResult; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; +use crate::expr::copy_to::PyCopyTo; +use crate::expr::create_catalog::PyCreateCatalog; +use crate::expr::create_catalog_schema::PyCreateCatalogSchema; +use crate::expr::create_external_table::PyCreateExternalTable; +use crate::expr::create_function::PyCreateFunction; +use crate::expr::create_index::PyCreateIndex; +use crate::expr::create_memory_table::PyCreateMemoryTable; +use crate::expr::create_view::PyCreateView; +use crate::expr::describe_table::PyDescribeTable; use crate::expr::distinct::PyDistinct; +use crate::expr::dml::PyDmlStatement; +use crate::expr::drop_catalog_schema::PyDropCatalogSchema; +use crate::expr::drop_function::PyDropFunction; +use crate::expr::drop_table::PyDropTable; +use crate::expr::drop_view::PyDropView; use crate::expr::empty_relation::PyEmptyRelation; use crate::expr::explain::PyExplain; use crate::expr::extension::PyExtension; @@ -28,14 +43,20 @@ use crate::expr::filter::PyFilter; use crate::expr::join::PyJoin; use crate::expr::limit::PyLimit; use crate::expr::projection::PyProjection; +use crate::expr::recursive_query::PyRecursiveQuery; +use crate::expr::repartition::PyRepartition; use crate::expr::sort::PySort; +use crate::expr::statement::{ + PyDeallocate, PyExecute, PyPrepare, PySetVariable, PyTransactionEnd, PyTransactionStart, +}; use crate::expr::subquery::PySubquery; use crate::expr::subquery_alias::PySubqueryAlias; use crate::expr::table_scan::PyTableScan; +use crate::expr::union::PyUnion; use crate::expr::unnest::PyUnnest; +use crate::expr::values::PyValues; use crate::expr::window::PyWindowExpr; -use crate::{context::PySessionContext, errors::py_unsupported_variant_err}; -use datafusion::logical_expr::LogicalPlan; +use datafusion::logical_expr::{DdlStatement, LogicalPlan, Statement}; use datafusion_proto::logical_plan::{AsLogicalPlan, DefaultLogicalExtensionCodec}; use prost::Message; use pyo3::{exceptions::PyRuntimeError, prelude::*, types::PyBytes}; @@ -82,18 +103,54 @@ impl PyLogicalPlan { LogicalPlan::SubqueryAlias(plan) => PySubqueryAlias::from(plan.clone()).to_variant(py), LogicalPlan::Unnest(plan) => PyUnnest::from(plan.clone()).to_variant(py), LogicalPlan::Window(plan) => PyWindowExpr::from(plan.clone()).to_variant(py), - LogicalPlan::Repartition(_) - | LogicalPlan::Union(_) - | LogicalPlan::Statement(_) - | LogicalPlan::Values(_) - | LogicalPlan::Dml(_) - | LogicalPlan::Ddl(_) - | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) - | LogicalPlan::RecursiveQuery(_) => Err(py_unsupported_variant_err(format!( - "Conversion of variant not implemented: {:?}", - self.plan - ))), + LogicalPlan::Repartition(plan) => PyRepartition::from(plan.clone()).to_variant(py), + LogicalPlan::Union(plan) => PyUnion::from(plan.clone()).to_variant(py), + LogicalPlan::Statement(plan) => match plan { + Statement::TransactionStart(plan) => { + PyTransactionStart::from(plan.clone()).to_variant(py) + } + Statement::TransactionEnd(plan) => { + PyTransactionEnd::from(plan.clone()).to_variant(py) + } + Statement::SetVariable(plan) => PySetVariable::from(plan.clone()).to_variant(py), + Statement::Prepare(plan) => PyPrepare::from(plan.clone()).to_variant(py), + Statement::Execute(plan) => PyExecute::from(plan.clone()).to_variant(py), + Statement::Deallocate(plan) => PyDeallocate::from(plan.clone()).to_variant(py), + }, + LogicalPlan::Values(plan) => PyValues::from(plan.clone()).to_variant(py), + LogicalPlan::Dml(plan) => PyDmlStatement::from(plan.clone()).to_variant(py), + LogicalPlan::Ddl(plan) => match plan { + DdlStatement::CreateExternalTable(plan) => { + PyCreateExternalTable::from(plan.clone()).to_variant(py) + } + DdlStatement::CreateMemoryTable(plan) => { + PyCreateMemoryTable::from(plan.clone()).to_variant(py) + } + DdlStatement::CreateView(plan) => PyCreateView::from(plan.clone()).to_variant(py), + DdlStatement::CreateCatalogSchema(plan) => { + PyCreateCatalogSchema::from(plan.clone()).to_variant(py) + } + DdlStatement::CreateCatalog(plan) => { + PyCreateCatalog::from(plan.clone()).to_variant(py) + } + DdlStatement::CreateIndex(plan) => PyCreateIndex::from(plan.clone()).to_variant(py), + DdlStatement::DropTable(plan) => PyDropTable::from(plan.clone()).to_variant(py), + DdlStatement::DropView(plan) => PyDropView::from(plan.clone()).to_variant(py), + DdlStatement::DropCatalogSchema(plan) => { + PyDropCatalogSchema::from(plan.clone()).to_variant(py) + } + DdlStatement::CreateFunction(plan) => { + PyCreateFunction::from(plan.clone()).to_variant(py) + } + DdlStatement::DropFunction(plan) => { + PyDropFunction::from(plan.clone()).to_variant(py) + } + }, + LogicalPlan::Copy(plan) => PyCopyTo::from(plan.clone()).to_variant(py), + LogicalPlan::DescribeTable(plan) => PyDescribeTable::from(plan.clone()).to_variant(py), + LogicalPlan::RecursiveQuery(plan) => { + PyRecursiveQuery::from(plan.clone()).to_variant(py) + } } } From 7d8bcd8d20623beb76a397eb4fddfb18781589eb Mon Sep 17 00:00:00 2001 From: kosiew Date: Mon, 5 May 2025 21:50:52 +0800 Subject: [PATCH 295/413] Partial fix for 1078: Enhance DataFrame Formatter Configuration with Memory and Display Controls (#1119) * feat: add configurable max table bytes and min table rows for DataFrame display * Revert "feat: add configurable max table bytes and min table rows for DataFrame display" This reverts commit f9b78fa3180c5d6c20eaa3b6d0af7426d7084093. * feat: add FormatterConfig for configurable DataFrame display options * refactor: simplify attribute extraction in get_formatter_config function * refactor: remove hardcoded constants and use FormatterConfig for display options * refactor: simplify record batch collection by using FormatterConfig for display options * feat: add max_memory_bytes, min_rows_display, and repr_rows parameters to DataFrameHtmlFormatter * feat: add tests for HTML formatter row display settings and memory limit * refactor: extract Python formatter retrieval into a separate function * Revert "feat: add tests for HTML formatter row display settings and memory limit" This reverts commit e089d7b282e53e587116b11d92760e6d292ec871. * feat: add tests for HTML formatter row and memory limit configurations * Revert "feat: add tests for HTML formatter row and memory limit configurations" This reverts commit 4090fd2f7378855b045d6bfd1368d088cc9ada75. * feat: add tests for new parameters and validation in DataFrameHtmlFormatter * Reorganize tests * refactor: rename and restructure formatter functions for clarity and maintainability * feat: implement PythonFormatter struct and refactor formatter retrieval for improved clarity * refactor: improve comments and restructure FormatterConfig usage in PyDataFrame * Add DataFrame usage guide with HTML rendering customization options (#1108) * docs: enhance user guide with detailed DataFrame operations and examples * move /docs/source/api/dataframe.rst into user-guide * docs: remove DataFrame API documentation * docs: fix formatting inconsistencies in DataFrame user guide * Two minor corrections to documentation rendering --------- Co-authored-by: Tim Saucer * Update documentation * refactor: streamline HTML rendering documentation * refactor: extract validation logic into separate functions for clarity * Implement feature X to enhance user experience and optimize performance * feat: add validation method for FormatterConfig to ensure positive integer values * add comment - ensure minimum rows are collected even if memory or row limits are hit * Update html_formatter documentation * update tests * remove unused type hints from imports in html_formatter.py * remove redundant tests for DataFrameHtmlFormatter and clean up assertions * refactor get_attr function to support generic default values * build_formatter_config_from_python return PyResult * fix ruff errors * trigger ci * fix: remove redundant newline in test_custom_style_provider_html_formatter * add more tests * trigger ci * Fix ruff errors * fix clippy error * feat: add validation for parameters in configure_formatter * test: add tests for invalid parameters in configure_formatter * Fix ruff errors --------- Co-authored-by: Tim Saucer --- docs/source/user-guide/dataframe.rst | 52 +++++++- python/datafusion/html_formatter.py | 104 ++++++++++++--- python/tests/test_dataframe.py | 183 ++++++++++++++++++++++++--- src/dataframe.rs | 142 +++++++++++++++++---- 4 files changed, 413 insertions(+), 68 deletions(-) diff --git a/docs/source/user-guide/dataframe.rst b/docs/source/user-guide/dataframe.rst index a78fd8073..11e3d7e72 100644 --- a/docs/source/user-guide/dataframe.rst +++ b/docs/source/user-guide/dataframe.rst @@ -75,13 +75,17 @@ You can customize how DataFrames are rendered in HTML by configuring the formatt # Change the default styling configure_formatter( - max_rows=50, # Maximum number of rows to display - max_width=None, # Maximum width in pixels (None for auto) - theme="light", # Theme: "light" or "dark" - precision=2, # Floating point precision - thousands_separator=",", # Separator for thousands - date_format="%Y-%m-%d", # Date format - truncate_width=20 # Max width for string columns before truncating + max_cell_length=25, # Maximum characters in a cell before truncation + max_width=1000, # Maximum width in pixels + max_height=300, # Maximum height in pixels + max_memory_bytes=2097152, # Maximum memory for rendering (2MB) + min_rows_display=20, # Minimum number of rows to display + repr_rows=10, # Number of rows to display in __repr__ + enable_cell_expansion=True,# Allow expanding truncated cells + custom_css=None, # Additional custom CSS + show_truncation_message=True, # Show message when data is truncated + style_provider=None, # Custom styling provider + use_shared_styles=True # Share styles across tables ) The formatter settings affect all DataFrames displayed after configuration. @@ -113,6 +117,25 @@ For advanced styling needs, you can create a custom style provider: # Apply the custom style provider configure_formatter(style_provider=MyStyleProvider()) +Performance Optimization with Shared Styles +------------------------------------------- +The ``use_shared_styles`` parameter (enabled by default) optimizes performance when displaying +multiple DataFrames in notebook environments: + + .. code-block:: python + from datafusion.html_formatter import StyleProvider, configure_formatter + # Default: Use shared styles (recommended for notebooks) + configure_formatter(use_shared_styles=True) + + # Disable shared styles (each DataFrame includes its own styles) + configure_formatter(use_shared_styles=False) + +When ``use_shared_styles=True``: +- CSS styles and JavaScript are included only once per notebook session +- This reduces HTML output size and prevents style duplication +- Improves rendering performance with many DataFrames +- Applies consistent styling across all DataFrames + Creating a Custom Formatter --------------------------- @@ -177,3 +200,18 @@ You can also use a context manager to temporarily change formatting settings: # Back to default formatting df.show() + +Memory and Display Controls +--------------------------- + +You can control how much data is displayed and how much memory is used for rendering: + + .. code-block:: python + + configure_formatter( + max_memory_bytes=4 * 1024 * 1024, # 4MB maximum memory for display + min_rows_display=50, # Always show at least 50 rows + repr_rows=20 # Show 20 rows in __repr__ output + ) + +These parameters help balance comprehensive data display against performance considerations. \ No newline at end of file diff --git a/python/datafusion/html_formatter.py b/python/datafusion/html_formatter.py index a50e14fd5..12a7e4553 100644 --- a/python/datafusion/html_formatter.py +++ b/python/datafusion/html_formatter.py @@ -27,6 +27,36 @@ ) +def _validate_positive_int(value: Any, param_name: str) -> None: + """Validate that a parameter is a positive integer. + + Args: + value: The value to validate + param_name: Name of the parameter (used in error message) + + Raises: + ValueError: If the value is not a positive integer + """ + if not isinstance(value, int) or value <= 0: + msg = f"{param_name} must be a positive integer" + raise ValueError(msg) + + +def _validate_bool(value: Any, param_name: str) -> None: + """Validate that a parameter is a boolean. + + Args: + value: The value to validate + param_name: Name of the parameter (used in error message) + + Raises: + TypeError: If the value is not a boolean + """ + if not isinstance(value, bool): + msg = f"{param_name} must be a boolean" + raise TypeError(msg) + + @runtime_checkable class CellFormatter(Protocol): """Protocol for cell value formatters.""" @@ -91,6 +121,9 @@ class DataFrameHtmlFormatter: max_cell_length: Maximum characters to display in a cell before truncation max_width: Maximum width of the HTML table in pixels max_height: Maximum height of the HTML table in pixels + max_memory_bytes: Maximum memory in bytes for rendered data (default: 2MB) + min_rows_display: Minimum number of rows to display + repr_rows: Default number of rows to display in repr output enable_cell_expansion: Whether to add expand/collapse buttons for long cell values custom_css: Additional CSS to include in the HTML output @@ -108,6 +141,9 @@ def __init__( max_cell_length: int = 25, max_width: int = 1000, max_height: int = 300, + max_memory_bytes: int = 2 * 1024 * 1024, # 2 MB + min_rows_display: int = 20, + repr_rows: int = 10, enable_cell_expansion: bool = True, custom_css: Optional[str] = None, show_truncation_message: bool = True, @@ -124,6 +160,12 @@ def __init__( Maximum width of the displayed table in pixels. max_height : int, default 300 Maximum height of the displayed table in pixels. + max_memory_bytes : int, default 2097152 (2MB) + Maximum memory in bytes for rendered data. + min_rows_display : int, default 20 + Minimum number of rows to display. + repr_rows : int, default 10 + Default number of rows to display in repr output. enable_cell_expansion : bool, default True Whether to allow cells to expand when clicked. custom_css : str, optional @@ -139,7 +181,8 @@ def __init__( Raises: ------ ValueError - If max_cell_length, max_width, or max_height is not a positive integer. + If max_cell_length, max_width, max_height, max_memory_bytes, + min_rows_display, or repr_rows is not a positive integer. TypeError If enable_cell_expansion, show_truncation_message, or use_shared_styles is not a boolean, @@ -148,27 +191,17 @@ def __init__( protocol. """ # Validate numeric parameters - - if not isinstance(max_cell_length, int) or max_cell_length <= 0: - msg = "max_cell_length must be a positive integer" - raise ValueError(msg) - if not isinstance(max_width, int) or max_width <= 0: - msg = "max_width must be a positive integer" - raise ValueError(msg) - if not isinstance(max_height, int) or max_height <= 0: - msg = "max_height must be a positive integer" - raise ValueError(msg) + _validate_positive_int(max_cell_length, "max_cell_length") + _validate_positive_int(max_width, "max_width") + _validate_positive_int(max_height, "max_height") + _validate_positive_int(max_memory_bytes, "max_memory_bytes") + _validate_positive_int(min_rows_display, "min_rows_display") + _validate_positive_int(repr_rows, "repr_rows") # Validate boolean parameters - if not isinstance(enable_cell_expansion, bool): - msg = "enable_cell_expansion must be a boolean" - raise TypeError(msg) - if not isinstance(show_truncation_message, bool): - msg = "show_truncation_message must be a boolean" - raise TypeError(msg) - if not isinstance(use_shared_styles, bool): - msg = "use_shared_styles must be a boolean" - raise TypeError(msg) + _validate_bool(enable_cell_expansion, "enable_cell_expansion") + _validate_bool(show_truncation_message, "show_truncation_message") + _validate_bool(use_shared_styles, "use_shared_styles") # Validate custom_css if custom_css is not None and not isinstance(custom_css, str): @@ -183,6 +216,9 @@ def __init__( self.max_cell_length = max_cell_length self.max_width = max_width self.max_height = max_height + self.max_memory_bytes = max_memory_bytes + self.min_rows_display = min_rows_display + self.repr_rows = repr_rows self.enable_cell_expansion = enable_cell_expansion self.custom_css = custom_css self.show_truncation_message = show_truncation_message @@ -597,6 +633,9 @@ def configure_formatter(**kwargs: Any) -> None: **kwargs: Formatter configuration parameters like max_cell_length, max_width, max_height, enable_cell_expansion, etc. + Raises: + ValueError: If any invalid parameters are provided + Example: >>> from datafusion.html_formatter import configure_formatter >>> configure_formatter( @@ -606,6 +645,31 @@ def configure_formatter(**kwargs: Any) -> None: ... use_shared_styles=True ... ) """ + # Valid parameters accepted by DataFrameHtmlFormatter + valid_params = { + "max_cell_length", + "max_width", + "max_height", + "max_memory_bytes", + "min_rows_display", + "repr_rows", + "enable_cell_expansion", + "custom_css", + "show_truncation_message", + "style_provider", + "use_shared_styles", + } + + # Check for invalid parameters + invalid_params = set(kwargs) - valid_params + if invalid_params: + msg = ( + f"Invalid formatter parameters: {', '.join(invalid_params)}. " + f"Valid parameters are: {', '.join(valid_params)}" + ) + raise ValueError(msg) + + # Create and set formatter with validated parameters set_formatter(DataFrameHtmlFormatter(**kwargs)) diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 464b884db..e01308c86 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -41,6 +41,8 @@ ) from pyarrow.csv import write_csv +MB = 1024 * 1024 + @pytest.fixture def ctx(): @@ -117,6 +119,31 @@ def clean_formatter_state(): reset_formatter() +# custom style for testing with html formatter +class CustomStyleProvider: + def get_cell_style(self) -> str: + return ( + "background-color: #f5f5f5; color: #333; padding: 8px; border: " + "1px solid #ddd;" + ) + + def get_header_style(self) -> str: + return ( + "background-color: #4285f4; color: white; font-weight: bold; " + "padding: 10px; border: 1px solid #3367d6;" + ) + + +def count_table_rows(html_content: str) -> int: + """Count the number of table rows in HTML content. + Args: + html_content: HTML string to analyze + Returns: + Number of table rows found (number of tags) + """ + return len(re.findall(r" str: - return ( - "background-color: #f5f5f5; color: #333; padding: 8px; border: " - "1px solid #ddd;" - ) - - def get_header_style(self) -> str: - return ( - "background-color: #4285f4; color: white; font-weight: bold; " - "padding: 10px; border: 1px solid #3367d6;" - ) - # Configure with custom style provider configure_formatter(style_provider=CustomStyleProvider()) @@ -917,6 +930,141 @@ def get_header_style(self) -> str: assert "color: #5af" in html_output # Even numbers +def test_html_formatter_memory(df, clean_formatter_state): + """Test the memory and row control parameters in DataFrameHtmlFormatter.""" + configure_formatter(max_memory_bytes=10, min_rows_display=1) + html_output = df._repr_html_() + + # Count the number of table rows in the output + tr_count = count_table_rows(html_output) + # With a tiny memory limit of 10 bytes, the formatter should display + # the minimum number of rows (1) plus a message about truncation + assert tr_count == 2 # 1 for header row, 1 for data row + assert "data truncated" in html_output.lower() + + configure_formatter(max_memory_bytes=10 * MB, min_rows_display=1) + html_output = df._repr_html_() + # With larger memory limit and min_rows=2, should display all rows + tr_count = count_table_rows(html_output) + # Table should have header row (1) + 3 data rows = 4 rows + assert tr_count == 4 + # No truncation message should appear + assert "data truncated" not in html_output.lower() + + +def test_html_formatter_repr_rows(df, clean_formatter_state): + configure_formatter(min_rows_display=2, repr_rows=2) + html_output = df._repr_html_() + + tr_count = count_table_rows(html_output) + # Tabe should have header row (1) + 2 data rows = 3 rows + assert tr_count == 3 + + configure_formatter(min_rows_display=2, repr_rows=3) + html_output = df._repr_html_() + + tr_count = count_table_rows(html_output) + # Tabe should have header row (1) + 3 data rows = 4 rows + assert tr_count == 4 + + +def test_html_formatter_validation(): + # Test validation for invalid parameters + + with pytest.raises(ValueError, match="max_cell_length must be a positive integer"): + DataFrameHtmlFormatter(max_cell_length=0) + + with pytest.raises(ValueError, match="max_width must be a positive integer"): + DataFrameHtmlFormatter(max_width=0) + + with pytest.raises(ValueError, match="max_height must be a positive integer"): + DataFrameHtmlFormatter(max_height=0) + + with pytest.raises(ValueError, match="max_memory_bytes must be a positive integer"): + DataFrameHtmlFormatter(max_memory_bytes=0) + + with pytest.raises(ValueError, match="max_memory_bytes must be a positive integer"): + DataFrameHtmlFormatter(max_memory_bytes=-100) + + with pytest.raises(ValueError, match="min_rows_display must be a positive integer"): + DataFrameHtmlFormatter(min_rows_display=0) + + with pytest.raises(ValueError, match="min_rows_display must be a positive integer"): + DataFrameHtmlFormatter(min_rows_display=-5) + + with pytest.raises(ValueError, match="repr_rows must be a positive integer"): + DataFrameHtmlFormatter(repr_rows=0) + + with pytest.raises(ValueError, match="repr_rows must be a positive integer"): + DataFrameHtmlFormatter(repr_rows=-10) + + +def test_configure_formatter(df, clean_formatter_state): + """Test using custom style providers with the HTML formatter and configured + parameters.""" + + # these are non-default values + max_cell_length = 10 + max_width = 500 + max_height = 30 + max_memory_bytes = 3 * MB + min_rows_display = 2 + repr_rows = 2 + enable_cell_expansion = False + show_truncation_message = False + use_shared_styles = False + + reset_formatter() + formatter_default = get_formatter() + + assert formatter_default.max_cell_length != max_cell_length + assert formatter_default.max_width != max_width + assert formatter_default.max_height != max_height + assert formatter_default.max_memory_bytes != max_memory_bytes + assert formatter_default.min_rows_display != min_rows_display + assert formatter_default.repr_rows != repr_rows + assert formatter_default.enable_cell_expansion != enable_cell_expansion + assert formatter_default.show_truncation_message != show_truncation_message + assert formatter_default.use_shared_styles != use_shared_styles + + # Configure with custom style provider and additional parameters + configure_formatter( + max_cell_length=max_cell_length, + max_width=max_width, + max_height=max_height, + max_memory_bytes=max_memory_bytes, + min_rows_display=min_rows_display, + repr_rows=repr_rows, + enable_cell_expansion=enable_cell_expansion, + show_truncation_message=show_truncation_message, + use_shared_styles=use_shared_styles, + ) + formatter_custom = get_formatter() + assert formatter_custom.max_cell_length == max_cell_length + assert formatter_custom.max_width == max_width + assert formatter_custom.max_height == max_height + assert formatter_custom.max_memory_bytes == max_memory_bytes + assert formatter_custom.min_rows_display == min_rows_display + assert formatter_custom.repr_rows == repr_rows + assert formatter_custom.enable_cell_expansion == enable_cell_expansion + assert formatter_custom.show_truncation_message == show_truncation_message + assert formatter_custom.use_shared_styles == use_shared_styles + + +def test_configure_formatter_invalid_params(clean_formatter_state): + """Test that configure_formatter rejects invalid parameters.""" + with pytest.raises(ValueError, match="Invalid formatter parameters"): + configure_formatter(invalid_param=123) + + # Test with multiple parameters, one valid and one invalid + with pytest.raises(ValueError, match="Invalid formatter parameters"): + configure_formatter(max_width=500, not_a_real_param="test") + + # Test with multiple invalid parameters + with pytest.raises(ValueError, match="Invalid formatter parameters"): + configure_formatter(fake_param1="test", fake_param2=456) + + def test_get_dataframe(tmp_path): ctx = SessionContext() @@ -1505,9 +1653,8 @@ def add_with_parameter(df_internal, value: Any) -> DataFrame: assert result["new_col"] == [3 for _i in range(3)] -def test_dataframe_repr_html_structure(df) -> None: +def test_dataframe_repr_html_structure(df, clean_formatter_state) -> None: """Test that DataFrame._repr_html_ produces expected HTML output structure.""" - import re output = df._repr_html_() @@ -1537,7 +1684,7 @@ def test_dataframe_repr_html_structure(df) -> None: assert len(body_matches) == 1, "Expected pattern of values not found in HTML output" -def test_dataframe_repr_html_values(df): +def test_dataframe_repr_html_values(df, clean_formatter_state): """Test that DataFrame._repr_html_ contains the expected data values.""" html = df._repr_html_() assert html is not None diff --git a/src/dataframe.rs b/src/dataframe.rs index 787f63520..211e31bd1 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -71,8 +71,103 @@ impl PyTableProvider { PyTable::new(table_provider) } } -const MAX_TABLE_BYTES_TO_DISPLAY: usize = 2 * 1024 * 1024; // 2 MB -const MIN_TABLE_ROWS_TO_DISPLAY: usize = 20; + +/// Configuration for DataFrame display formatting +#[derive(Debug, Clone)] +pub struct FormatterConfig { + /// Maximum memory in bytes to use for display (default: 2MB) + pub max_bytes: usize, + /// Minimum number of rows to display (default: 20) + pub min_rows: usize, + /// Number of rows to include in __repr__ output (default: 10) + pub repr_rows: usize, +} + +impl Default for FormatterConfig { + fn default() -> Self { + Self { + max_bytes: 2 * 1024 * 1024, // 2MB + min_rows: 20, + repr_rows: 10, + } + } +} + +impl FormatterConfig { + /// Validates that all configuration values are positive integers. + /// + /// # Returns + /// + /// `Ok(())` if all values are valid, or an `Err` with a descriptive error message. + pub fn validate(&self) -> Result<(), String> { + if self.max_bytes == 0 { + return Err("max_bytes must be a positive integer".to_string()); + } + + if self.min_rows == 0 { + return Err("min_rows must be a positive integer".to_string()); + } + + if self.repr_rows == 0 { + return Err("repr_rows must be a positive integer".to_string()); + } + + Ok(()) + } +} + +/// Holds the Python formatter and its configuration +struct PythonFormatter<'py> { + /// The Python formatter object + formatter: Bound<'py, PyAny>, + /// The formatter configuration + config: FormatterConfig, +} + +/// Get the Python formatter and its configuration +fn get_python_formatter_with_config(py: Python) -> PyResult { + let formatter = import_python_formatter(py)?; + let config = build_formatter_config_from_python(&formatter)?; + Ok(PythonFormatter { formatter, config }) +} + +/// Get the Python formatter from the datafusion.html_formatter module +fn import_python_formatter(py: Python) -> PyResult> { + let formatter_module = py.import("datafusion.html_formatter")?; + let get_formatter = formatter_module.getattr("get_formatter")?; + get_formatter.call0() +} + +// Helper function to extract attributes with fallback to default +fn get_attr<'a, T>(py_object: &'a Bound<'a, PyAny>, attr_name: &str, default_value: T) -> T +where + T: for<'py> pyo3::FromPyObject<'py> + Clone, +{ + py_object + .getattr(attr_name) + .and_then(|v| v.extract::()) + .unwrap_or_else(|_| default_value.clone()) +} + +/// Helper function to create a FormatterConfig from a Python formatter object +fn build_formatter_config_from_python(formatter: &Bound<'_, PyAny>) -> PyResult { + let default_config = FormatterConfig::default(); + let max_bytes = get_attr(formatter, "max_memory_bytes", default_config.max_bytes); + let min_rows = get_attr(formatter, "min_rows_display", default_config.min_rows); + let repr_rows = get_attr(formatter, "repr_rows", default_config.repr_rows); + + let config = FormatterConfig { + max_bytes, + min_rows, + repr_rows, + }; + + // Return the validated config, converting String error to PyErr + config + .validate() + .map_err(|e| pyo3::exceptions::PyValueError::new_err(e))?; + Ok(config) +} /// A PyDataFrame is a representation of a logical plan and an API to compose statements. /// Use it to build a plan and `.collect()` to execute the plan and collect the result. @@ -114,9 +209,14 @@ impl PyDataFrame { } fn __repr__(&self, py: Python) -> PyDataFusionResult { + // Get the Python formatter config + let PythonFormatter { + formatter: _, + config, + } = get_python_formatter_with_config(py)?; let (batches, has_more) = wait_for_future( py, - collect_record_batches_to_display(self.df.as_ref().clone(), 10, 10), + collect_record_batches_to_display(self.df.as_ref().clone(), config), )?; if batches.is_empty() { // This should not be reached, but do it for safety since we index into the vector below @@ -135,13 +235,11 @@ impl PyDataFrame { } fn _repr_html_(&self, py: Python) -> PyDataFusionResult { + // Get the Python formatter and config + let PythonFormatter { formatter, config } = get_python_formatter_with_config(py)?; let (batches, has_more) = wait_for_future( py, - collect_record_batches_to_display( - self.df.as_ref().clone(), - MIN_TABLE_ROWS_TO_DISPLAY, - usize::MAX, - ), + collect_record_batches_to_display(self.df.as_ref().clone(), config), )?; if batches.is_empty() { // This should not be reached, but do it for safety since we index into the vector below @@ -158,12 +256,6 @@ impl PyDataFrame { let py_schema = self.schema().into_pyobject(py)?; - // Get the Python formatter module and call format_html - let formatter_module = py.import("datafusion.html_formatter")?; - let get_formatter = formatter_module.getattr("get_formatter")?; - let formatter = get_formatter.call0()?; - - // Call format_html method on the formatter let kwargs = pyo3::types::PyDict::new(py); let py_batches_list = PyList::new(py, py_batches.as_slice())?; kwargs.set_item("batches", py_batches_list)?; @@ -796,9 +888,14 @@ fn record_batch_into_schema( /// rows, set min_rows == max_rows. async fn collect_record_batches_to_display( df: DataFrame, - min_rows: usize, - max_rows: usize, + config: FormatterConfig, ) -> Result<(Vec, bool), DataFusionError> { + let FormatterConfig { + max_bytes, + min_rows, + repr_rows, + } = config; + let partitioned_stream = df.execute_stream_partitioned().await?; let mut stream = futures::stream::iter(partitioned_stream).flatten(); let mut size_estimate_so_far = 0; @@ -806,9 +903,8 @@ async fn collect_record_batches_to_display( let mut record_batches = Vec::default(); let mut has_more = false; - while (size_estimate_so_far < MAX_TABLE_BYTES_TO_DISPLAY && rows_so_far < max_rows) - || rows_so_far < min_rows - { + // ensure minimum rows even if memory/row limits are hit + while (size_estimate_so_far < max_bytes && rows_so_far < repr_rows) || rows_so_far < min_rows { let mut rb = match stream.next().await { None => { break; @@ -821,8 +917,8 @@ async fn collect_record_batches_to_display( if rows_in_rb > 0 { size_estimate_so_far += rb.get_array_memory_size(); - if size_estimate_so_far > MAX_TABLE_BYTES_TO_DISPLAY { - let ratio = MAX_TABLE_BYTES_TO_DISPLAY as f32 / size_estimate_so_far as f32; + if size_estimate_so_far > max_bytes { + let ratio = max_bytes as f32 / size_estimate_so_far as f32; let total_rows = rows_in_rb + rows_so_far; let mut reduced_row_num = (total_rows as f32 * ratio).round() as usize; @@ -838,8 +934,8 @@ async fn collect_record_batches_to_display( } } - if rows_in_rb + rows_so_far > max_rows { - rb = rb.slice(0, max_rows - rows_so_far); + if rows_in_rb + rows_so_far > repr_rows { + rb = rb.slice(0, repr_rows - rows_so_far); has_more = true; } From f3c98ec7a2eb325041530b1ae8d6de41aa558037 Mon Sep 17 00:00:00 2001 From: kosiew Date: Fri, 16 May 2025 14:34:19 +0800 Subject: [PATCH 296/413] Add fill_null method to DataFrame API for handling missing values (#1019) * feat: add fill_null method to DataFrame for handling null values * test: add coalesce function tests for handling default values * Resolve test cases for fill_null * feat: add fill_nan method to DataFrame for handling NaN values * move imports out of functions * docs: add documentation for fill_null and fill_nan methods in DataFrame * Add more tests * fix ruff errors * amend def fill_null to invoke PyDataFrame's fill_null - Implemented `fill_null` method in `dataframe.rs` to allow filling null values with a specified value for specific columns or all columns. - Added a helper function `python_value_to_scalar_value` to convert Python values to DataFusion ScalarValues, supporting various types including integers, floats, booleans, strings, and timestamps. - Updated the `count` method in `PyDataFrame` to maintain functionality. * refactor: remove fill_nan method documentation from functions.rst * refactor: remove unused import of Enum from dataframe.py * refactor: improve error handling and type extraction in python_value_to_scalar_value function * refactor: enhance datetime and date conversion logic in python_value_to_scalar_value function * refactor: streamline type extraction in python_value_to_scalar_value function * fix try_convert_to_string * refactor: improve type handling in python_value_to_scalar_value function * refactor: move py_obj_to_scalar_value function to utils module * refactor: update fill_null to use py_obj_to_scalar_value from utils * Remove python_object_to_scalar_value code * refactor: enhance py_obj_to_scalar_value to utilize PyArrow for complex type conversion * refactor: update py_obj_to_scalar_value to handle errors and use extract_bound for PyArrow scalar conversion * refactor: modify py_obj_to_scalar_value to return ScalarValue directly and streamline error handling * refactor: update py_obj_to_scalar_value to return a Result for better error handling * test: add tests for fill_null functionality in DataFrame with null values * test: enhance null DataFrame tests to include date32 and date64 columns * refactor: simplify py_obj_to_scalar_value by removing direct extraction of basic types * refactor: remove unnecessary documentation from py_obj_to_scalar_value function * Fix ruff errors * test: update datetime handling in coalesce tests to include timezone information * Fix ruff errors * trigger ci --- .../common-operations/functions.rst | 21 ++ python/datafusion/dataframe.py | 26 +- python/tests/test_dataframe.py | 266 ++++++++++++++++++ python/tests/test_functions.py | 61 ++++ src/config.rs | 21 +- src/dataframe.rs | 23 +- src/utils.rs | 18 ++ 7 files changed, 414 insertions(+), 22 deletions(-) diff --git a/docs/source/user-guide/common-operations/functions.rst b/docs/source/user-guide/common-operations/functions.rst index 12097be8f..d458d3eb0 100644 --- a/docs/source/user-guide/common-operations/functions.rst +++ b/docs/source/user-guide/common-operations/functions.rst @@ -129,3 +129,24 @@ The function :py:func:`~datafusion.functions.in_list` allows to check a column f .limit(20) .to_pandas() ) + + +Handling Missing Values +===================== + +DataFusion provides methods to handle missing values in DataFrames: + +fill_null +--------- + +The ``fill_null()`` method replaces NULL values in specified columns with a provided value: + +.. code-block:: python + + # Fill all NULL values with 0 where possible + df = df.fill_null(0) + + # Fill NULL values only in specific string columns + df = df.fill_null("missing", subset=["name", "category"]) + +The fill value will be cast to match each column's type. If casting fails for a column, that column remains unchanged. diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 26fe8f453..a1df7e080 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -37,6 +37,8 @@ except ImportError: from typing_extensions import deprecated # Python 3.12 +from datafusion._internal import DataFrame as DataFrameInternal +from datafusion.expr import Expr, SortExpr, sort_or_default from datafusion.plan import ExecutionPlan, LogicalPlan from datafusion.record_batch import RecordBatchStream @@ -53,8 +55,6 @@ from enum import Enum -from datafusion.expr import Expr, SortExpr, sort_or_default - # excerpt from deltalake # https://github.com/apache/datafusion-python/pull/981#discussion_r1905619163 @@ -869,3 +869,25 @@ def within_limit(df: DataFrame, limit: int) -> DataFrame: DataFrame: After applying func to the original dataframe. """ return func(self, *args) + + def fill_null(self, value: Any, subset: list[str] | None = None) -> DataFrame: + """Fill null values in specified columns with a value. + + Args: + value: Value to replace nulls with. Will be cast to match column type. + subset: Optional list of column names to fill. If None, fills all columns. + + Returns: + DataFrame with null values replaced where type casting is possible + + Examples: + >>> df = df.fill_null(0) # Fill all nulls with 0 where possible + >>> # Fill nulls in specific string columns + >>> df = df.fill_null("missing", subset=["name", "category"]) + + Notes: + - Only fills nulls in columns where the value can be cast to the column type + - For columns where casting fails, the original column is kept unchanged + - For columns not in subset, the original column is kept unchanged + """ + return DataFrame(self.df.fill_null(value, subset)) diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index e01308c86..dd5f962b2 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import datetime import os import re from typing import Any @@ -119,6 +120,38 @@ def clean_formatter_state(): reset_formatter() +@pytest.fixture +def null_df(): + """Create a DataFrame with null values of different types.""" + ctx = SessionContext() + + # Create a RecordBatch with nulls across different types + batch = pa.RecordBatch.from_arrays( + [ + pa.array([1, None, 3, None], type=pa.int64()), + pa.array([4.5, 6.7, None, None], type=pa.float64()), + pa.array(["a", None, "c", None], type=pa.string()), + pa.array([True, None, False, None], type=pa.bool_()), + pa.array( + [10957, None, 18993, None], type=pa.date32() + ), # 2000-01-01, null, 2022-01-01, null + pa.array( + [946684800000, None, 1640995200000, None], type=pa.date64() + ), # 2000-01-01, null, 2022-01-01, null + ], + names=[ + "int_col", + "float_col", + "str_col", + "bool_col", + "date32_col", + "date64_col", + ], + ) + + return ctx.create_dataframe([[batch]]) + + # custom style for testing with html formatter class CustomStyleProvider: def get_cell_style(self) -> str: @@ -1794,3 +1827,236 @@ def test_html_formatter_manual_format_html(clean_formatter_state): assert "") + return html + + def _build_table_container_start(self) -> list[str]: + """Build the opening tags for the table container.""" + html = [] + html.append( + f'
' + ) + html.append('') + return html + + def _build_table_header(self, schema: Any) -> list[str]: + """Build the HTML table header with column names.""" + html = [] + html.append("") + html.append("") + for field in schema: + if self._custom_header_builder: + html.append(self._custom_header_builder(field)) + else: + html.append( + f"" + ) + html.append("") + html.append("") + return html + + def _build_table_body(self, batches: list, table_uuid: str) -> list[str]: + """Build the HTML table body with data rows.""" + html = [] + html.append("") + + row_count = 0 + for batch in batches: + for row_idx in range(batch.num_rows): + row_count += 1 + html.append("") + + for col_idx, column in enumerate(batch.columns): + # Get the raw value from the column + raw_value = self._get_cell_value(column, row_idx) + + # Always check for type formatters first to format the value + formatted_value = self._format_cell_value(raw_value) + + # Then apply either custom cell builder or standard cell formatting + if self._custom_cell_builder: + # Pass both the raw value and formatted value to let the + # builder decide + cell_html = self._custom_cell_builder( + raw_value, row_count, col_idx, table_uuid + ) + html.append(cell_html) + else: + # Standard cell formatting with formatted value + if ( + len(str(raw_value)) > self.max_cell_length + and self.enable_cell_expansion + ): + cell_html = self._build_expandable_cell( + formatted_value, row_count, col_idx, table_uuid + ) + else: + cell_html = self._build_regular_cell(formatted_value) + html.append(cell_html) + + html.append("") + + html.append("") + return html + + def _get_cell_value(self, column: Any, row_idx: int) -> Any: + """Extract a cell value from a column. + + Args: + column: Arrow array + row_idx: Row index + + Returns: + The raw cell value + """ + try: + value = column[row_idx] + + if hasattr(value, "as_py"): + return value.as_py() + except (AttributeError, TypeError): + pass + else: + return value + + def _format_cell_value(self, value: Any) -> str: + """Format a cell value for display. + + Uses registered type formatters if available. + + Args: + value: The cell value to format + + Returns: + Formatted cell value as string + """ + # Check for custom type formatters + for type_cls, formatter in self._type_formatters.items(): + if isinstance(value, type_cls): + return formatter(value) + + # If no formatter matched, return string representation + return str(value) + + def _build_expandable_cell( + self, formatted_value: str, row_count: int, col_idx: int, table_uuid: str + ) -> str: + """Build an expandable cell for long content.""" + short_value = str(formatted_value)[: self.max_cell_length] + return ( + f"" + ) + + def _build_regular_cell(self, formatted_value: str) -> str: + """Build a regular table cell.""" + return ( + f"" + ) + + def _build_html_footer(self, has_more: bool) -> list[str]: + """Build the HTML footer with JavaScript and messages.""" + html = [] + + # Add JavaScript for interactivity only if cell expansion is enabled + # and we're not using the shared styles approach + if self.enable_cell_expansion and not self.use_shared_styles: + html.append(self._get_javascript()) + + # Add truncation message if needed + if has_more and self.show_truncation_message: + html.append("
Data truncated due to size.
") + + return html + + def _get_default_css(self) -> str: + """Get default CSS styles for the HTML table.""" + return """ + .expandable-container { + display: inline-block; + max-width: 200px; + } + .expandable { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + display: block; + } + .full-text { + display: none; + white-space: normal; + } + .expand-btn { + cursor: pointer; + color: blue; + text-decoration: underline; + border: none; + background: none; + font-size: inherit; + display: block; + margin-top: 5px; + } + """ + + def _get_javascript(self) -> str: + """Get JavaScript code for interactive elements.""" + return """ + + """ + + +class FormatterManager: + """Manager class for the global DataFrame HTML formatter instance.""" + + _default_formatter: DataFrameHtmlFormatter = DataFrameHtmlFormatter() + + @classmethod + def set_formatter(cls, formatter: DataFrameHtmlFormatter) -> None: + """Set the global DataFrame HTML formatter. + + Args: + formatter: The formatter instance to use globally + """ + cls._default_formatter = formatter + _refresh_formatter_reference() + + @classmethod + def get_formatter(cls) -> DataFrameHtmlFormatter: + """Get the current global DataFrame HTML formatter. + + Returns: + The global HTML formatter instance + """ + return cls._default_formatter + + +def get_formatter() -> DataFrameHtmlFormatter: + """Get the current global DataFrame HTML formatter. + + This function is used by the DataFrame._repr_html_ implementation to access + the shared formatter instance. It can also be used directly when custom + HTML rendering is needed. + + Returns: + The global HTML formatter instance + + Example: + >>> from datafusion.html_formatter import get_formatter + >>> formatter = get_formatter() + >>> formatter.max_cell_length = 50 # Increase cell length + """ + return FormatterManager.get_formatter() + + +def set_formatter(formatter: DataFrameHtmlFormatter) -> None: + """Set the global DataFrame HTML formatter. + + Args: + formatter: The formatter instance to use globally + + Example: + >>> from datafusion.html_formatter import get_formatter, set_formatter + >>> custom_formatter = DataFrameHtmlFormatter(max_cell_length=100) + >>> set_formatter(custom_formatter) + """ + FormatterManager.set_formatter(formatter) + + +def configure_formatter(**kwargs: Any) -> None: + """Configure the global DataFrame HTML formatter. + + This function creates a new formatter with the provided configuration + and sets it as the global formatter for all DataFrames. + + Args: + **kwargs: Formatter configuration parameters like max_cell_length, + max_width, max_height, enable_cell_expansion, etc. + + Raises: + ValueError: If any invalid parameters are provided + + Example: + >>> from datafusion.html_formatter import configure_formatter + >>> configure_formatter( + ... max_cell_length=50, + ... max_height=500, + ... enable_cell_expansion=True, + ... use_shared_styles=True + ... ) + """ + # Valid parameters accepted by DataFrameHtmlFormatter + valid_params = { + "max_cell_length", + "max_width", + "max_height", + "max_memory_bytes", + "min_rows_display", + "repr_rows", + "enable_cell_expansion", + "custom_css", + "show_truncation_message", + "style_provider", + "use_shared_styles", + } + + # Check for invalid parameters + invalid_params = set(kwargs) - valid_params + if invalid_params: + msg = ( + f"Invalid formatter parameters: {', '.join(invalid_params)}. " + f"Valid parameters are: {', '.join(valid_params)}" + ) + raise ValueError(msg) + + # Create and set formatter with validated parameters + set_formatter(DataFrameHtmlFormatter(**kwargs)) + + +def reset_formatter() -> None: + """Reset the global DataFrame HTML formatter to default settings. + + This function creates a new formatter with default configuration + and sets it as the global formatter for all DataFrames. + + Example: + >>> from datafusion.html_formatter import reset_formatter + >>> reset_formatter() # Reset formatter to default settings + """ + formatter = DataFrameHtmlFormatter() + # Reset the styles_loaded flag to ensure styles will be reloaded + DataFrameHtmlFormatter._styles_loaded = False + set_formatter(formatter) + + +def reset_styles_loaded_state() -> None: + """Reset the styles loaded state to force reloading of styles. + + This can be useful when switching between notebook sessions or + when styles need to be refreshed. + + Example: + >>> from datafusion.html_formatter import reset_styles_loaded_state + >>> reset_styles_loaded_state() # Force styles to reload in next render + """ + DataFrameHtmlFormatter._styles_loaded = False + + +def _refresh_formatter_reference() -> None: + """Refresh formatter reference in any modules using it. + + This helps ensure that changes to the formatter are reflected in existing + DataFrames that might be caching the formatter reference. + """ + # This is a no-op but signals modules to refresh their reference diff --git a/python/datafusion/html_formatter.py b/python/datafusion/html_formatter.py index 12a7e4553..65eb1f042 100644 --- a/python/datafusion/html_formatter.py +++ b/python/datafusion/html_formatter.py @@ -14,698 +14,16 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""HTML formatting utilities for DataFusion DataFrames.""" -from __future__ import annotations +"""Deprecated module for dataframe formatting.""" -from typing import ( - Any, - Callable, - Optional, - Protocol, - runtime_checkable, -) - - -def _validate_positive_int(value: Any, param_name: str) -> None: - """Validate that a parameter is a positive integer. - - Args: - value: The value to validate - param_name: Name of the parameter (used in error message) - - Raises: - ValueError: If the value is not a positive integer - """ - if not isinstance(value, int) or value <= 0: - msg = f"{param_name} must be a positive integer" - raise ValueError(msg) - - -def _validate_bool(value: Any, param_name: str) -> None: - """Validate that a parameter is a boolean. - - Args: - value: The value to validate - param_name: Name of the parameter (used in error message) - - Raises: - TypeError: If the value is not a boolean - """ - if not isinstance(value, bool): - msg = f"{param_name} must be a boolean" - raise TypeError(msg) - - -@runtime_checkable -class CellFormatter(Protocol): - """Protocol for cell value formatters.""" - - def __call__(self, value: Any) -> str: - """Format a cell value to string representation.""" - ... - - -@runtime_checkable -class StyleProvider(Protocol): - """Protocol for HTML style providers.""" - - def get_cell_style(self) -> str: - """Get the CSS style for table cells.""" - ... - - def get_header_style(self) -> str: - """Get the CSS style for header cells.""" - ... - - -class DefaultStyleProvider: - """Default implementation of StyleProvider.""" - - def get_cell_style(self) -> str: - """Get the CSS style for table cells. - - Returns: - CSS style string - """ - return ( - "border: 1px solid black; padding: 8px; text-align: left; " - "white-space: nowrap;" - ) - - def get_header_style(self) -> str: - """Get the CSS style for header cells. - - Returns: - CSS style string - """ - return ( - "border: 1px solid black; padding: 8px; text-align: left; " - "background-color: #f2f2f2; white-space: nowrap; min-width: fit-content; " - "max-width: fit-content;" - ) - - -class DataFrameHtmlFormatter: - """Configurable HTML formatter for DataFusion DataFrames. - - This class handles the HTML rendering of DataFrames for display in - Jupyter notebooks and other rich display contexts. - - This class supports extension through composition. Key extension points: - - Provide a custom StyleProvider for styling cells and headers - - Register custom formatters for specific types - - Provide custom cell builders for specialized cell rendering - - Args: - max_cell_length: Maximum characters to display in a cell before truncation - max_width: Maximum width of the HTML table in pixels - max_height: Maximum height of the HTML table in pixels - max_memory_bytes: Maximum memory in bytes for rendered data (default: 2MB) - min_rows_display: Minimum number of rows to display - repr_rows: Default number of rows to display in repr output - enable_cell_expansion: Whether to add expand/collapse buttons for long cell - values - custom_css: Additional CSS to include in the HTML output - show_truncation_message: Whether to display a message when data is truncated - style_provider: Custom provider for cell and header styles - use_shared_styles: Whether to load styles and scripts only once per notebook - session - """ - - # Class variable to track if styles have been loaded in the notebook - _styles_loaded = False - - def __init__( - self, - max_cell_length: int = 25, - max_width: int = 1000, - max_height: int = 300, - max_memory_bytes: int = 2 * 1024 * 1024, # 2 MB - min_rows_display: int = 20, - repr_rows: int = 10, - enable_cell_expansion: bool = True, - custom_css: Optional[str] = None, - show_truncation_message: bool = True, - style_provider: Optional[StyleProvider] = None, - use_shared_styles: bool = True, - ) -> None: - """Initialize the HTML formatter. - - Parameters - ---------- - max_cell_length : int, default 25 - Maximum length of cell content before truncation. - max_width : int, default 1000 - Maximum width of the displayed table in pixels. - max_height : int, default 300 - Maximum height of the displayed table in pixels. - max_memory_bytes : int, default 2097152 (2MB) - Maximum memory in bytes for rendered data. - min_rows_display : int, default 20 - Minimum number of rows to display. - repr_rows : int, default 10 - Default number of rows to display in repr output. - enable_cell_expansion : bool, default True - Whether to allow cells to expand when clicked. - custom_css : str, optional - Custom CSS to apply to the HTML table. - show_truncation_message : bool, default True - Whether to show a message indicating that content has been truncated. - style_provider : StyleProvider, optional - Provider of CSS styles for the HTML table. If None, DefaultStyleProvider - is used. - use_shared_styles : bool, default True - Whether to use shared styles across multiple tables. - - Raises: - ------ - ValueError - If max_cell_length, max_width, max_height, max_memory_bytes, - min_rows_display, or repr_rows is not a positive integer. - TypeError - If enable_cell_expansion, show_truncation_message, or use_shared_styles is - not a boolean, - or if custom_css is provided but is not a string, - or if style_provider is provided but does not implement the StyleProvider - protocol. - """ - # Validate numeric parameters - _validate_positive_int(max_cell_length, "max_cell_length") - _validate_positive_int(max_width, "max_width") - _validate_positive_int(max_height, "max_height") - _validate_positive_int(max_memory_bytes, "max_memory_bytes") - _validate_positive_int(min_rows_display, "min_rows_display") - _validate_positive_int(repr_rows, "repr_rows") - - # Validate boolean parameters - _validate_bool(enable_cell_expansion, "enable_cell_expansion") - _validate_bool(show_truncation_message, "show_truncation_message") - _validate_bool(use_shared_styles, "use_shared_styles") - - # Validate custom_css - if custom_css is not None and not isinstance(custom_css, str): - msg = "custom_css must be None or a string" - raise TypeError(msg) - - # Validate style_provider - if style_provider is not None and not isinstance(style_provider, StyleProvider): - msg = "style_provider must implement the StyleProvider protocol" - raise TypeError(msg) - - self.max_cell_length = max_cell_length - self.max_width = max_width - self.max_height = max_height - self.max_memory_bytes = max_memory_bytes - self.min_rows_display = min_rows_display - self.repr_rows = repr_rows - self.enable_cell_expansion = enable_cell_expansion - self.custom_css = custom_css - self.show_truncation_message = show_truncation_message - self.style_provider = style_provider or DefaultStyleProvider() - self.use_shared_styles = use_shared_styles - # Registry for custom type formatters - self._type_formatters: dict[type, CellFormatter] = {} - # Custom cell builders - self._custom_cell_builder: Optional[Callable[[Any, int, int, str], str]] = None - self._custom_header_builder: Optional[Callable[[Any], str]] = None - - def register_formatter(self, type_class: type, formatter: CellFormatter) -> None: - """Register a custom formatter for a specific data type. - - Args: - type_class: The type to register a formatter for - formatter: Function that takes a value of the given type and returns - a formatted string - """ - self._type_formatters[type_class] = formatter - - def set_custom_cell_builder( - self, builder: Callable[[Any, int, int, str], str] - ) -> None: - """Set a custom cell builder function. - - Args: - builder: Function that takes (value, row, col, table_id) and returns HTML - """ - self._custom_cell_builder = builder - - def set_custom_header_builder(self, builder: Callable[[Any], str]) -> None: - """Set a custom header builder function. - - Args: - builder: Function that takes a field and returns HTML - """ - self._custom_header_builder = builder - - @classmethod - def is_styles_loaded(cls) -> bool: - """Check if HTML styles have been loaded in the current session. - - This method is primarily intended for debugging UI rendering issues - related to style loading. - - Returns: - True if styles have been loaded, False otherwise - - Example: - >>> from datafusion.html_formatter import DataFrameHtmlFormatter - >>> DataFrameHtmlFormatter.is_styles_loaded() - False - """ - return cls._styles_loaded - - def format_html( - self, - batches: list, - schema: Any, - has_more: bool = False, - table_uuid: str | None = None, - ) -> str: - """Format record batches as HTML. - - This method is used by DataFrame's _repr_html_ implementation and can be - called directly when custom HTML rendering is needed. - - Args: - batches: List of Arrow RecordBatch objects - schema: Arrow Schema object - has_more: Whether there are more batches not shown - table_uuid: Unique ID for the table, used for JavaScript interactions - - Returns: - HTML string representation of the data - - Raises: - TypeError: If schema is invalid and no batches are provided - """ - if not batches: - return "No data to display" - - # Validate schema - if schema is None or not hasattr(schema, "__iter__"): - msg = "Schema must be provided" - raise TypeError(msg) - - # Generate a unique ID if none provided - table_uuid = table_uuid or f"df-{id(batches)}" - - # Build HTML components - html = [] - - # Only include styles and scripts if: - # 1. Not using shared styles, OR - # 2. Using shared styles but they haven't been loaded yet - include_styles = ( - not self.use_shared_styles or not DataFrameHtmlFormatter._styles_loaded - ) - - if include_styles: - html.extend(self._build_html_header()) - # If we're using shared styles, mark them as loaded - if self.use_shared_styles: - DataFrameHtmlFormatter._styles_loaded = True - - html.extend(self._build_table_container_start()) - - # Add table header and body - html.extend(self._build_table_header(schema)) - html.extend(self._build_table_body(batches, table_uuid)) - - html.append("
" + f"{field.name}
" + f"
" + "" + "" + f"{formatted_value}" + f"" + f"
" + f"		{formatted_value}
") - html.append("
") - - # Add footer (JavaScript and messages) - if include_styles and self.enable_cell_expansion: - html.append(self._get_javascript()) - - # Always add truncation message if needed (independent of styles) - if has_more and self.show_truncation_message: - html.append("
Data truncated due to size.
") - - return "\n".join(html) - - def _build_html_header(self) -> list[str]: - """Build the HTML header with CSS styles.""" - html = [] - html.append("") - return html +import warnings - def _build_table_container_start(self) -> list[str]: - """Build the opening tags for the table container.""" - html = [] - html.append( - f'
' - ) - html.append('') - return html +from datafusion.dataframe_formatter import * # noqa: F403 - def _build_table_header(self, schema: Any) -> list[str]: - """Build the HTML table header with column names.""" - html = [] - html.append("") - html.append("") - for field in schema: - if self._custom_header_builder: - html.append(self._custom_header_builder(field)) - else: - html.append( - f"" - ) - html.append("") - html.append("") - return html - - def _build_table_body(self, batches: list, table_uuid: str) -> list[str]: - """Build the HTML table body with data rows.""" - html = [] - html.append("") - - row_count = 0 - for batch in batches: - for row_idx in range(batch.num_rows): - row_count += 1 - html.append("") - - for col_idx, column in enumerate(batch.columns): - # Get the raw value from the column - raw_value = self._get_cell_value(column, row_idx) - - # Always check for type formatters first to format the value - formatted_value = self._format_cell_value(raw_value) - - # Then apply either custom cell builder or standard cell formatting - if self._custom_cell_builder: - # Pass both the raw value and formatted value to let the - # builder decide - cell_html = self._custom_cell_builder( - raw_value, row_count, col_idx, table_uuid - ) - html.append(cell_html) - else: - # Standard cell formatting with formatted value - if ( - len(str(raw_value)) > self.max_cell_length - and self.enable_cell_expansion - ): - cell_html = self._build_expandable_cell( - formatted_value, row_count, col_idx, table_uuid - ) - else: - cell_html = self._build_regular_cell(formatted_value) - html.append(cell_html) - - html.append("") - - html.append("") - return html - - def _get_cell_value(self, column: Any, row_idx: int) -> Any: - """Extract a cell value from a column. - - Args: - column: Arrow array - row_idx: Row index - - Returns: - The raw cell value - """ - try: - value = column[row_idx] - - if hasattr(value, "as_py"): - return value.as_py() - except (AttributeError, TypeError): - pass - else: - return value - - def _format_cell_value(self, value: Any) -> str: - """Format a cell value for display. - - Uses registered type formatters if available. - - Args: - value: The cell value to format - - Returns: - Formatted cell value as string - """ - # Check for custom type formatters - for type_cls, formatter in self._type_formatters.items(): - if isinstance(value, type_cls): - return formatter(value) - - # If no formatter matched, return string representation - return str(value) - - def _build_expandable_cell( - self, formatted_value: str, row_count: int, col_idx: int, table_uuid: str - ) -> str: - """Build an expandable cell for long content.""" - short_value = str(formatted_value)[: self.max_cell_length] - return ( - f"" - ) - - def _build_regular_cell(self, formatted_value: str) -> str: - """Build a regular table cell.""" - return ( - f"" - ) - - def _build_html_footer(self, has_more: bool) -> list[str]: - """Build the HTML footer with JavaScript and messages.""" - html = [] - - # Add JavaScript for interactivity only if cell expansion is enabled - # and we're not using the shared styles approach - if self.enable_cell_expansion and not self.use_shared_styles: - html.append(self._get_javascript()) - - # Add truncation message if needed - if has_more and self.show_truncation_message: - html.append("
Data truncated due to size.
") - - return html - - def _get_default_css(self) -> str: - """Get default CSS styles for the HTML table.""" - return """ - .expandable-container { - display: inline-block; - max-width: 200px; - } - .expandable { - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; - display: block; - } - .full-text { - display: none; - white-space: normal; - } - .expand-btn { - cursor: pointer; - color: blue; - text-decoration: underline; - border: none; - background: none; - font-size: inherit; - display: block; - margin-top: 5px; - } - """ - - def _get_javascript(self) -> str: - """Get JavaScript code for interactive elements.""" - return """ - - """ - - -class FormatterManager: - """Manager class for the global DataFrame HTML formatter instance.""" - - _default_formatter: DataFrameHtmlFormatter = DataFrameHtmlFormatter() - - @classmethod - def set_formatter(cls, formatter: DataFrameHtmlFormatter) -> None: - """Set the global DataFrame HTML formatter. - - Args: - formatter: The formatter instance to use globally - """ - cls._default_formatter = formatter - _refresh_formatter_reference() - - @classmethod - def get_formatter(cls) -> DataFrameHtmlFormatter: - """Get the current global DataFrame HTML formatter. - - Returns: - The global HTML formatter instance - """ - return cls._default_formatter - - -def get_formatter() -> DataFrameHtmlFormatter: - """Get the current global DataFrame HTML formatter. - - This function is used by the DataFrame._repr_html_ implementation to access - the shared formatter instance. It can also be used directly when custom - HTML rendering is needed. - - Returns: - The global HTML formatter instance - - Example: - >>> from datafusion.html_formatter import get_formatter - >>> formatter = get_formatter() - >>> formatter.max_cell_length = 50 # Increase cell length - """ - return FormatterManager.get_formatter() - - -def set_formatter(formatter: DataFrameHtmlFormatter) -> None: - """Set the global DataFrame HTML formatter. - - Args: - formatter: The formatter instance to use globally - - Example: - >>> from datafusion.html_formatter import get_formatter, set_formatter - >>> custom_formatter = DataFrameHtmlFormatter(max_cell_length=100) - >>> set_formatter(custom_formatter) - """ - FormatterManager.set_formatter(formatter) - - -def configure_formatter(**kwargs: Any) -> None: - """Configure the global DataFrame HTML formatter. - - This function creates a new formatter with the provided configuration - and sets it as the global formatter for all DataFrames. - - Args: - **kwargs: Formatter configuration parameters like max_cell_length, - max_width, max_height, enable_cell_expansion, etc. - - Raises: - ValueError: If any invalid parameters are provided - - Example: - >>> from datafusion.html_formatter import configure_formatter - >>> configure_formatter( - ... max_cell_length=50, - ... max_height=500, - ... enable_cell_expansion=True, - ... use_shared_styles=True - ... ) - """ - # Valid parameters accepted by DataFrameHtmlFormatter - valid_params = { - "max_cell_length", - "max_width", - "max_height", - "max_memory_bytes", - "min_rows_display", - "repr_rows", - "enable_cell_expansion", - "custom_css", - "show_truncation_message", - "style_provider", - "use_shared_styles", - } - - # Check for invalid parameters - invalid_params = set(kwargs) - valid_params - if invalid_params: - msg = ( - f"Invalid formatter parameters: {', '.join(invalid_params)}. " - f"Valid parameters are: {', '.join(valid_params)}" - ) - raise ValueError(msg) - - # Create and set formatter with validated parameters - set_formatter(DataFrameHtmlFormatter(**kwargs)) - - -def reset_formatter() -> None: - """Reset the global DataFrame HTML formatter to default settings. - - This function creates a new formatter with default configuration - and sets it as the global formatter for all DataFrames. - - Example: - >>> from datafusion.html_formatter import reset_formatter - >>> reset_formatter() # Reset formatter to default settings - """ - formatter = DataFrameHtmlFormatter() - # Reset the styles_loaded flag to ensure styles will be reloaded - DataFrameHtmlFormatter._styles_loaded = False - set_formatter(formatter) - - -def reset_styles_loaded_state() -> None: - """Reset the styles loaded state to force reloading of styles. - - This can be useful when switching between notebook sessions or - when styles need to be refreshed. - - Example: - >>> from datafusion.html_formatter import reset_styles_loaded_state - >>> reset_styles_loaded_state() # Force styles to reload in next render - """ - DataFrameHtmlFormatter._styles_loaded = False - - -def _refresh_formatter_reference() -> None: - """Refresh formatter reference in any modules using it. - - This helps ensure that changes to the formatter are reflected in existing - DataFrames that might be caching the formatter reference. - """ - # This is a no-op but signals modules to refresh their reference +warnings.warn( + "The module 'html_formatter' is deprecated and will be removed in the next release." + "Please use 'dataframe_formatter' instead.", + DeprecationWarning, + stacklevel=3, +) diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index deaa30b3d..c9ae38d8e 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -37,14 +37,14 @@ from datafusion import ( functions as f, ) -from datafusion.expr import Window -from datafusion.html_formatter import ( +from datafusion.dataframe_formatter import ( DataFrameHtmlFormatter, configure_formatter, get_formatter, reset_formatter, reset_styles_loaded_state, ) +from datafusion.expr import Window from pyarrow.csv import write_csv MB = 1024 * 1024 diff --git a/src/dataframe.rs b/src/dataframe.rs index 3d68db279..c2ad4771e 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -24,6 +24,7 @@ use arrow::compute::can_cast_types; use arrow::error::ArrowError; use arrow::ffi::FFI_ArrowSchema; use arrow::ffi_stream::FFI_ArrowArrayStream; +use arrow::pyarrow::FromPyArrow; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; @@ -150,9 +151,9 @@ fn get_python_formatter_with_config(py: Python) -> PyResult { Ok(PythonFormatter { formatter, config }) } -/// Get the Python formatter from the datafusion.html_formatter module +/// Get the Python formatter from the datafusion.dataframe_formatter module fn import_python_formatter(py: Python) -> PyResult> { - let formatter_module = py.import("datafusion.html_formatter")?; + let formatter_module = py.import("datafusion.dataframe_formatter")?; let get_formatter = formatter_module.getattr("get_formatter")?; get_formatter.call0() } @@ -295,6 +296,46 @@ impl PyDataFrame { pub fn new(df: DataFrame) -> Self { Self { df: Arc::new(df) } } + + fn prepare_repr_string(&self, py: Python, as_html: bool) -> PyDataFusionResult { + // Get the Python formatter and config + let PythonFormatter { formatter, config } = get_python_formatter_with_config(py)?; + let (batches, has_more) = wait_for_future( + py, + collect_record_batches_to_display(self.df.as_ref().clone(), config), + )??; + if batches.is_empty() { + // This should not be reached, but do it for safety since we index into the vector below + return Ok("No data to display".to_string()); + } + + let table_uuid = uuid::Uuid::new_v4().to_string(); + + // Convert record batches to PyObject list + let py_batches = batches + .into_iter() + .map(|rb| rb.to_pyarrow(py)) + .collect::>>()?; + + let py_schema = self.schema().into_pyobject(py)?; + + let kwargs = pyo3::types::PyDict::new(py); + let py_batches_list = PyList::new(py, py_batches.as_slice())?; + kwargs.set_item("batches", py_batches_list)?; + kwargs.set_item("schema", py_schema)?; + kwargs.set_item("has_more", has_more)?; + kwargs.set_item("table_uuid", table_uuid)?; + + let method_name = match as_html { + true => "format_html", + false => "format_str", + }; + + let html_result = formatter.call_method(method_name, (), Some(&kwargs))?; + let html_str: String = html_result.extract()?; + + Ok(html_str) + } } #[pymethods] @@ -321,18 +362,27 @@ impl PyDataFrame { } fn __repr__(&self, py: Python) -> PyDataFusionResult { - // Get the Python formatter config - let PythonFormatter { - formatter: _, - config, - } = get_python_formatter_with_config(py)?; - let (batches, has_more) = wait_for_future( - py, - collect_record_batches_to_display(self.df.as_ref().clone(), config), - )??; + self.prepare_repr_string(py, false) + } + + #[staticmethod] + #[expect(unused_variables)] + fn default_str_repr<'py>( + batches: Vec>, + schema: &Bound<'py, PyAny>, + has_more: bool, + table_uuid: &str, + ) -> PyResult { + let batches = batches + .into_iter() + .map(|batch| RecordBatch::from_pyarrow_bound(&batch)) + .collect::>>()? + .into_iter() + .filter(|batch| batch.num_rows() > 0) + .collect::>(); + if batches.is_empty() { - // This should not be reached, but do it for safety since we index into the vector below - return Ok("No data to display".to_string()); + return Ok("No data to display".to_owned()); } let batches_as_displ = @@ -347,38 +397,7 @@ impl PyDataFrame { } fn _repr_html_(&self, py: Python) -> PyDataFusionResult { - // Get the Python formatter and config - let PythonFormatter { formatter, config } = get_python_formatter_with_config(py)?; - let (batches, has_more) = wait_for_future( - py, - collect_record_batches_to_display(self.df.as_ref().clone(), config), - )??; - if batches.is_empty() { - // This should not be reached, but do it for safety since we index into the vector below - return Ok("No data to display".to_string()); - } - - let table_uuid = uuid::Uuid::new_v4().to_string(); - - // Convert record batches to PyObject list - let py_batches = batches - .into_iter() - .map(|rb| rb.to_pyarrow(py)) - .collect::>>()?; - - let py_schema = self.schema().into_pyobject(py)?; - - let kwargs = pyo3::types::PyDict::new(py); - let py_batches_list = PyList::new(py, py_batches.as_slice())?; - kwargs.set_item("batches", py_batches_list)?; - kwargs.set_item("schema", py_schema)?; - kwargs.set_item("has_more", has_more)?; - kwargs.set_item("table_uuid", table_uuid)?; - - let html_result = formatter.call_method("format_html", (), Some(&kwargs))?; - let html_str: String = html_result.extract()?; - - Ok(html_str) + self.prepare_repr_string(py, true) } /// Calculate summary statistics for a DataFrame From 954563429384078a9e85c56ad553c7e3be7ac52a Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 25 Jun 2025 11:29:35 -0400 Subject: [PATCH 312/413] feat: collect once during display() in jupyter notebooks (#1167) * Only collect one time during display() in jupyter notebooks * Check for juypter notebook environment specifically * Remove approach of checking environment which could not differentiate between jupyter console and notebook * Instead of trying to detect notebook vs console, collect one time when we have any kind if ipython environment. --- src/dataframe.rs | 36 ++++++++++++++++++++++++++---------- src/utils.rs | 11 +++++++++++ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/dataframe.rs b/src/dataframe.rs index c2ad4771e..ab4749e35 100644 --- a/src/dataframe.rs +++ b/src/dataframe.rs @@ -51,7 +51,7 @@ use crate::physical_plan::PyExecutionPlan; use crate::record_batch::PyRecordBatchStream; use crate::sql::logical::PyLogicalPlan; use crate::utils::{ - get_tokio_runtime, py_obj_to_scalar_value, validate_pycapsule, wait_for_future, + get_tokio_runtime, is_ipython_env, py_obj_to_scalar_value, validate_pycapsule, wait_for_future, }; use crate::{ errors::PyDataFusionResult, @@ -289,21 +289,33 @@ impl PyParquetColumnOptions { #[derive(Clone)] pub struct PyDataFrame { df: Arc, + + // In IPython environment cache batches between __repr__ and _repr_html_ calls. + batches: Option<(Vec, bool)>, } impl PyDataFrame { /// creates a new PyDataFrame pub fn new(df: DataFrame) -> Self { - Self { df: Arc::new(df) } + Self { + df: Arc::new(df), + batches: None, + } } - fn prepare_repr_string(&self, py: Python, as_html: bool) -> PyDataFusionResult { + fn prepare_repr_string(&mut self, py: Python, as_html: bool) -> PyDataFusionResult { // Get the Python formatter and config let PythonFormatter { formatter, config } = get_python_formatter_with_config(py)?; - let (batches, has_more) = wait_for_future( - py, - collect_record_batches_to_display(self.df.as_ref().clone(), config), - )??; + + let should_cache = *is_ipython_env(py) && self.batches.is_none(); + let (batches, has_more) = match self.batches.take() { + Some(b) => b, + None => wait_for_future( + py, + collect_record_batches_to_display(self.df.as_ref().clone(), config), + )??, + }; + if batches.is_empty() { // This should not be reached, but do it for safety since we index into the vector below return Ok("No data to display".to_string()); @@ -313,7 +325,7 @@ impl PyDataFrame { // Convert record batches to PyObject list let py_batches = batches - .into_iter() + .iter() .map(|rb| rb.to_pyarrow(py)) .collect::>>()?; @@ -334,6 +346,10 @@ impl PyDataFrame { let html_result = formatter.call_method(method_name, (), Some(&kwargs))?; let html_str: String = html_result.extract()?; + if should_cache { + self.batches = Some((batches, has_more)); + } + Ok(html_str) } } @@ -361,7 +377,7 @@ impl PyDataFrame { } } - fn __repr__(&self, py: Python) -> PyDataFusionResult { + fn __repr__(&mut self, py: Python) -> PyDataFusionResult { self.prepare_repr_string(py, false) } @@ -396,7 +412,7 @@ impl PyDataFrame { Ok(format!("DataFrame()\n{batches_as_displ}{additional_str}")) } - fn _repr_html_(&self, py: Python) -> PyDataFusionResult { + fn _repr_html_(&mut self, py: Python) -> PyDataFusionResult { self.prepare_repr_string(py, true) } diff --git a/src/utils.rs b/src/utils.rs index 90d654385..f4e121fd5 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -39,6 +39,17 @@ pub(crate) fn get_tokio_runtime() -> &'static TokioRuntime { RUNTIME.get_or_init(|| TokioRuntime(tokio::runtime::Runtime::new().unwrap())) } +#[inline] +pub(crate) fn is_ipython_env(py: Python) -> &'static bool { + static IS_IPYTHON_ENV: OnceLock = OnceLock::new(); + IS_IPYTHON_ENV.get_or_init(|| { + py.import("IPython") + .and_then(|ipython| ipython.call_method0("get_ipython")) + .map(|ipython| !ipython.is_none()) + .unwrap_or(false) + }) +} + /// Utility to get the Global Datafussion CTX #[inline] pub(crate) fn get_global_ctx() -> &'static SessionContext { From 9362f53150e5423581757ed56883b3ca2c95b8a2 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Wed, 2 Jul 2025 08:08:53 -0400 Subject: [PATCH 313/413] feat: python based catalog and schema provider (#1156) * Exposing FFI to python * Exposing FFI to python * Workin progress on python catalog * Flushing out schema and catalog providers * Adding implementation of python based catalog and schema providers * Small updates after rebase * Add default in memory options for adding schema and catalogs * Add support for creating in memory catalog and schema * Update from database to schema in unit tests * xfailed label no longer applies to these unit tests * Defining abstract methods for catalog and schema providers * Working through issues between custom catalog and build in schema * Check types on schema provider to return * Add docstring * Add documentation about how to use catalog and schema providers * Re-add module to all after rebase * Minor bugfix * Clippy updates from the new rust version --------- Co-authored-by: renato2099 --- Cargo.lock | 19 + Cargo.toml | 2 + docs/source/user-guide/data-sources.rst | 56 ++ examples/datafusion-ffi-example/Cargo.lock | 1 + examples/datafusion-ffi-example/Cargo.toml | 1 + .../python/tests/_test_catalog_provider.py | 60 +++ .../src/catalog_provider.rs | 179 +++++++ examples/datafusion-ffi-example/src/lib.rs | 3 + python/datafusion/__init__.py | 1 + python/datafusion/catalog.py | 195 ++++++- python/datafusion/context.py | 24 +- python/datafusion/dataframe.py | 9 +- python/tests/test_catalog.py | 173 ++++++- python/tests/test_context.py | 40 +- python/tests/test_sql.py | 30 +- python/tests/test_substrait.py | 4 +- src/catalog.rs | 490 ++++++++++++++++-- src/common/data_type.rs | 120 ++--- src/context.rs | 61 ++- src/expr.rs | 15 +- src/expr/aggregate.rs | 2 +- src/expr/aggregate_expr.rs | 2 +- src/expr/alias.rs | 2 +- src/expr/analyze.rs | 2 +- src/expr/between.rs | 2 +- src/expr/column.rs | 2 +- src/expr/copy_to.rs | 4 +- src/expr/create_catalog.rs | 2 +- src/expr/create_catalog_schema.rs | 2 +- src/expr/create_external_table.rs | 2 +- src/expr/create_function.rs | 2 +- src/expr/create_index.rs | 2 +- src/expr/create_memory_table.rs | 2 +- src/expr/create_view.rs | 2 +- src/expr/describe_table.rs | 2 +- src/expr/distinct.rs | 5 +- src/expr/drop_catalog_schema.rs | 2 +- src/expr/drop_function.rs | 2 +- src/expr/drop_table.rs | 2 +- src/expr/drop_view.rs | 2 +- src/expr/empty_relation.rs | 2 +- src/expr/filter.rs | 2 +- src/expr/join.rs | 2 +- src/expr/like.rs | 6 +- src/expr/limit.rs | 2 +- src/expr/projection.rs | 2 +- src/expr/recursive_query.rs | 2 +- src/expr/repartition.rs | 2 +- src/expr/sort.rs | 2 +- src/expr/sort_expr.rs | 2 +- src/expr/subquery.rs | 2 +- src/expr/subquery_alias.rs | 2 +- src/expr/table_scan.rs | 2 +- src/expr/union.rs | 2 +- src/expr/unnest.rs | 2 +- src/expr/unnest_expr.rs | 2 +- src/expr/window.rs | 11 +- src/functions.rs | 2 +- src/lib.rs | 10 +- src/physical_plan.rs | 3 +- src/sql/logical.rs | 3 +- src/utils.rs | 5 +- 62 files changed, 1340 insertions(+), 258 deletions(-) create mode 100644 examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py create mode 100644 examples/datafusion-ffi-example/src/catalog_provider.rs diff --git a/Cargo.lock b/Cargo.lock index 112167cb4..a3e9336cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,6 +165,12 @@ dependencies = [ "zstd", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayref" version = "0.3.9" @@ -1503,6 +1509,7 @@ dependencies = [ "datafusion-proto", "datafusion-substrait", "futures", + "log", "mimalloc", "object_store", "prost", @@ -1510,6 +1517,7 @@ dependencies = [ "pyo3", "pyo3-async-runtimes", "pyo3-build-config", + "pyo3-log", "tokio", "url", "uuid", @@ -2953,6 +2961,17 @@ dependencies = [ "pyo3-build-config", ] +[[package]] +name = "pyo3-log" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45192e5e4a4d2505587e27806c7b710c231c40c56f3bfc19535d0bb25df52264" +dependencies = [ + "arc-swap", + "log", + "pyo3", +] + [[package]] name = "pyo3-macros" version = "0.24.2" diff --git a/Cargo.toml b/Cargo.toml index 4135e64e2..1f7895a50 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ substrait = ["dep:datafusion-substrait"] tokio = { version = "1.45", features = ["macros", "rt", "rt-multi-thread", "sync"] } pyo3 = { version = "0.24", features = ["extension-module", "abi3", "abi3-py39"] } pyo3-async-runtimes = { version = "0.24", features = ["tokio-runtime"]} +pyo3-log = "0.12.4" arrow = { version = "55.1.0", features = ["pyarrow"] } datafusion = { version = "48.0.0", features = ["avro", "unicode_expressions"] } datafusion-substrait = { version = "48.0.0", optional = true } @@ -49,6 +50,7 @@ async-trait = "0.1.88" futures = "0.3" object_store = { version = "0.12.1", features = ["aws", "gcp", "azure", "http"] } url = "2" +log = "0.4.27" [build-dependencies] prost-types = "0.13.1" # keep in line with `datafusion-substrait` diff --git a/docs/source/user-guide/data-sources.rst b/docs/source/user-guide/data-sources.rst index ba5967c97..9c95d58e0 100644 --- a/docs/source/user-guide/data-sources.rst +++ b/docs/source/user-guide/data-sources.rst @@ -185,3 +185,59 @@ the interface as describe in the :ref:`Custom Table Provider `_ is provided in the DataFusion repository. + +Catalog +======= + +A common technique for organizing tables is using a three level hierarchical approach. DataFusion +supports this form of organizing using the :py:class:`~datafusion.catalog.Catalog`, +:py:class:`~datafusion.catalog.Schema`, and :py:class:`~datafusion.catalog.Table`. By default, +a :py:class:`~datafusion.context.SessionContext` comes with a single Catalog and a single Schema +with the names ``datafusion`` and ``default``, respectively. + +The default implementation uses an in-memory approach to the catalog and schema. We have support +for adding additional in-memory catalogs and schemas. This can be done like in the following +example: + +.. code-block:: python + + from datafusion.catalog import Catalog, Schema + + my_catalog = Catalog.memory_catalog() + my_schema = Schema.memory_schema() + + my_catalog.register_schema("my_schema_name", my_schema) + + ctx.register_catalog("my_catalog_name", my_catalog) + +You could then register tables in ``my_schema`` and access them either through the DataFrame +API or via sql commands such as ``"SELECT * from my_catalog_name.my_schema_name.my_table"``. + +User Defined Catalog and Schema +------------------------------- + +If the in-memory catalogs are insufficient for your uses, there are two approaches you can take +to implementing a custom catalog and/or schema. In the below discussion, we describe how to +implement these for a Catalog, but the approach to implementing for a Schema is nearly +identical. + +DataFusion supports Catalogs written in either Rust or Python. If you write a Catalog in Rust, +you will need to export it as a Python library via PyO3. There is a complete example of a +catalog implemented this way in the +`examples folder `_ +of our repository. Writing catalog providers in Rust provides typically can lead to significant +performance improvements over the Python based approach. + +To implement a Catalog in Python, you will need to inherit from the abstract base class +:py:class:`~datafusion.catalog.CatalogProvider`. There are examples in the +`unit tests `_ of +implementing a basic Catalog in Python where we simply keep a dictionary of the +registered Schemas. + +One important note for developers is that when we have a Catalog defined in Python, we have +two different ways of accessing this Catalog. First, we register the catalog with a Rust +wrapper. This allows for any rust based code to call the Python functions as necessary. +Second, if the user access the Catalog via the Python API, we identify this and return back +the original Python object that implements the Catalog. This is an important distinction +for developers because we do *not* return a Python wrapper around the Rust wrapper of the +original Python object. diff --git a/examples/datafusion-ffi-example/Cargo.lock b/examples/datafusion-ffi-example/Cargo.lock index 075ebd5a1..e5a1ca8d1 100644 --- a/examples/datafusion-ffi-example/Cargo.lock +++ b/examples/datafusion-ffi-example/Cargo.lock @@ -1448,6 +1448,7 @@ dependencies = [ "arrow", "arrow-array", "arrow-schema", + "async-trait", "datafusion", "datafusion-ffi", "pyo3", diff --git a/examples/datafusion-ffi-example/Cargo.toml b/examples/datafusion-ffi-example/Cargo.toml index 0e17567b9..319163554 100644 --- a/examples/datafusion-ffi-example/Cargo.toml +++ b/examples/datafusion-ffi-example/Cargo.toml @@ -27,6 +27,7 @@ pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] arrow = { version = "55.0.0" } arrow-array = { version = "55.0.0" } arrow-schema = { version = "55.0.0" } +async-trait = "0.1.88" [build-dependencies] pyo3-build-config = "0.23" diff --git a/examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py b/examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py new file mode 100644 index 000000000..72aadf64c --- /dev/null +++ b/examples/datafusion-ffi-example/python/tests/_test_catalog_provider.py @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import pyarrow as pa +from datafusion import SessionContext +from datafusion_ffi_example import MyCatalogProvider + + +def test_catalog_provider(): + ctx = SessionContext() + + my_catalog_name = "my_catalog" + expected_schema_name = "my_schema" + expected_table_name = "my_table" + expected_table_columns = ["units", "price"] + + catalog_provider = MyCatalogProvider() + ctx.register_catalog_provider(my_catalog_name, catalog_provider) + my_catalog = ctx.catalog(my_catalog_name) + + my_catalog_schemas = my_catalog.names() + assert expected_schema_name in my_catalog_schemas + my_database = my_catalog.database(expected_schema_name) + assert expected_table_name in my_database.names() + my_table = my_database.table(expected_table_name) + assert expected_table_columns == my_table.schema.names + + result = ctx.table( + f"{my_catalog_name}.{expected_schema_name}.{expected_table_name}" + ).collect() + assert len(result) == 2 + + col0_result = [r.column(0) for r in result] + col1_result = [r.column(1) for r in result] + expected_col0 = [ + pa.array([10, 20, 30], type=pa.int32()), + pa.array([5, 7], type=pa.int32()), + ] + expected_col1 = [ + pa.array([1, 2, 5], type=pa.float64()), + pa.array([1.5, 2.5], type=pa.float64()), + ] + assert col0_result == expected_col0 + assert col1_result == expected_col1 diff --git a/examples/datafusion-ffi-example/src/catalog_provider.rs b/examples/datafusion-ffi-example/src/catalog_provider.rs new file mode 100644 index 000000000..54e61cf3e --- /dev/null +++ b/examples/datafusion-ffi-example/src/catalog_provider.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use pyo3::{pyclass, pymethods, Bound, PyResult, Python}; +use std::{any::Any, fmt::Debug, sync::Arc}; + +use arrow::datatypes::Schema; +use async_trait::async_trait; +use datafusion::{ + catalog::{ + CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider, TableProvider, + }, + common::exec_err, + datasource::MemTable, + error::{DataFusionError, Result}, +}; +use datafusion_ffi::catalog_provider::FFI_CatalogProvider; +use pyo3::types::PyCapsule; + +pub fn my_table() -> Arc { + use arrow::datatypes::{DataType, Field}; + use datafusion::common::record_batch; + + let schema = Arc::new(Schema::new(vec![ + Field::new("units", DataType::Int32, true), + Field::new("price", DataType::Float64, true), + ])); + + let partitions = vec![ + record_batch!( + ("units", Int32, vec![10, 20, 30]), + ("price", Float64, vec![1.0, 2.0, 5.0]) + ) + .unwrap(), + record_batch!( + ("units", Int32, vec![5, 7]), + ("price", Float64, vec![1.5, 2.5]) + ) + .unwrap(), + ]; + + Arc::new(MemTable::try_new(schema, vec![partitions]).unwrap()) +} + +#[derive(Debug)] +pub struct FixedSchemaProvider { + inner: MemorySchemaProvider, +} + +impl Default for FixedSchemaProvider { + fn default() -> Self { + let inner = MemorySchemaProvider::new(); + + let table = my_table(); + + let _ = inner.register_table("my_table".to_string(), table).unwrap(); + + Self { inner } + } +} + +#[async_trait] +impl SchemaProvider for FixedSchemaProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn table_names(&self) -> Vec { + self.inner.table_names() + } + + async fn table(&self, name: &str) -> Result>, DataFusionError> { + self.inner.table(name).await + } + + fn register_table( + &self, + name: String, + table: Arc, + ) -> Result>> { + self.inner.register_table(name, table) + } + + fn deregister_table(&self, name: &str) -> Result>> { + self.inner.deregister_table(name) + } + + fn table_exist(&self, name: &str) -> bool { + self.inner.table_exist(name) + } +} + +/// This catalog provider is intended only for unit tests. It prepopulates with one +/// schema and only allows for schemas named after four types of fruit. +#[pyclass( + name = "MyCatalogProvider", + module = "datafusion_ffi_example", + subclass +)] +#[derive(Debug)] +pub(crate) struct MyCatalogProvider { + inner: MemoryCatalogProvider, +} + +impl Default for MyCatalogProvider { + fn default() -> Self { + let inner = MemoryCatalogProvider::new(); + + let schema_name: &str = "my_schema"; + let _ = inner.register_schema(schema_name, Arc::new(FixedSchemaProvider::default())); + + Self { inner } + } +} + +impl CatalogProvider for MyCatalogProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema_names(&self) -> Vec { + self.inner.schema_names() + } + + fn schema(&self, name: &str) -> Option> { + self.inner.schema(name) + } + + fn register_schema( + &self, + name: &str, + schema: Arc, + ) -> Result>> { + self.inner.register_schema(name, schema) + } + + fn deregister_schema( + &self, + name: &str, + cascade: bool, + ) -> Result>> { + self.inner.deregister_schema(name, cascade) + } +} + +#[pymethods] +impl MyCatalogProvider { + #[new] + pub fn new() -> Self { + Self { + inner: Default::default(), + } + } + + pub fn __datafusion_catalog_provider__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let name = cr"datafusion_catalog_provider".into(); + let catalog_provider = + FFI_CatalogProvider::new(Arc::new(MyCatalogProvider::default()), None); + + PyCapsule::new(py, catalog_provider, Some(name)) + } +} diff --git a/examples/datafusion-ffi-example/src/lib.rs b/examples/datafusion-ffi-example/src/lib.rs index ae08c3b65..3a4cf2247 100644 --- a/examples/datafusion-ffi-example/src/lib.rs +++ b/examples/datafusion-ffi-example/src/lib.rs @@ -15,10 +15,12 @@ // specific language governing permissions and limitations // under the License. +use crate::catalog_provider::MyCatalogProvider; use crate::table_function::MyTableFunction; use crate::table_provider::MyTableProvider; use pyo3::prelude::*; +pub(crate) mod catalog_provider; pub(crate) mod table_function; pub(crate) mod table_provider; @@ -26,5 +28,6 @@ pub(crate) mod table_provider; fn datafusion_ffi_example(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index fd7f4fc06..e9d2dba75 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -92,6 +92,7 @@ "TableFunction", "WindowFrame", "WindowUDF", + "catalog", "col", "column", "common", diff --git a/python/datafusion/catalog.py b/python/datafusion/catalog.py index 67ab3ead2..536b3a790 100644 --- a/python/datafusion/catalog.py +++ b/python/datafusion/catalog.py @@ -19,18 +19,33 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Protocol import datafusion._internal as df_internal if TYPE_CHECKING: import pyarrow as pa +try: + from warnings import deprecated # Python 3.13+ +except ImportError: + from typing_extensions import deprecated # Python 3.12 + + +__all__ = [ + "Catalog", + "CatalogProvider", + "Schema", + "SchemaProvider", + "Table", +] + class Catalog: """DataFusion data catalog.""" - def __init__(self, catalog: df_internal.Catalog) -> None: + def __init__(self, catalog: df_internal.catalog.RawCatalog) -> None: """This constructor is not typically called by the end user.""" self.catalog = catalog @@ -38,39 +53,95 @@ def __repr__(self) -> str: """Print a string representation of the catalog.""" return self.catalog.__repr__() - def names(self) -> list[str]: - """Returns the list of databases in this catalog.""" - return self.catalog.names() + def names(self) -> set[str]: + """This is an alias for `schema_names`.""" + return self.schema_names() + + def schema_names(self) -> set[str]: + """Returns the list of schemas in this catalog.""" + return self.catalog.schema_names() + + @staticmethod + def memory_catalog() -> Catalog: + """Create an in-memory catalog provider.""" + catalog = df_internal.catalog.RawCatalog.memory_catalog() + return Catalog(catalog) - def database(self, name: str = "public") -> Database: + def schema(self, name: str = "public") -> Schema: """Returns the database with the given ``name`` from this catalog.""" - return Database(self.catalog.database(name)) + schema = self.catalog.schema(name) + + return ( + Schema(schema) + if isinstance(schema, df_internal.catalog.RawSchema) + else schema + ) + + @deprecated("Use `schema` instead.") + def database(self, name: str = "public") -> Schema: + """Returns the database with the given ``name`` from this catalog.""" + return self.schema(name) + + def register_schema(self, name, schema) -> Schema | None: + """Register a schema with this catalog.""" + if isinstance(schema, Schema): + return self.catalog.register_schema(name, schema._raw_schema) + return self.catalog.register_schema(name, schema) + + def deregister_schema(self, name: str, cascade: bool = True) -> Schema | None: + """Deregister a schema from this catalog.""" + return self.catalog.deregister_schema(name, cascade) -class Database: - """DataFusion Database.""" +class Schema: + """DataFusion Schema.""" - def __init__(self, db: df_internal.Database) -> None: + def __init__(self, schema: df_internal.catalog.RawSchema) -> None: """This constructor is not typically called by the end user.""" - self.db = db + self._raw_schema = schema def __repr__(self) -> str: - """Print a string representation of the database.""" - return self.db.__repr__() + """Print a string representation of the schema.""" + return self._raw_schema.__repr__() + + @staticmethod + def memory_schema() -> Schema: + """Create an in-memory schema provider.""" + schema = df_internal.catalog.RawSchema.memory_schema() + return Schema(schema) def names(self) -> set[str]: - """Returns the list of all tables in this database.""" - return self.db.names() + """This is an alias for `table_names`.""" + return self.table_names() + + def table_names(self) -> set[str]: + """Returns the list of all tables in this schema.""" + return self._raw_schema.table_names def table(self, name: str) -> Table: - """Return the table with the given ``name`` from this database.""" - return Table(self.db.table(name)) + """Return the table with the given ``name`` from this schema.""" + return Table(self._raw_schema.table(name)) + + def register_table(self, name, table) -> None: + """Register a table provider in this schema.""" + if isinstance(table, Table): + return self._raw_schema.register_table(name, table.table) + return self._raw_schema.register_table(name, table) + + def deregister_table(self, name: str) -> None: + """Deregister a table provider from this schema.""" + return self._raw_schema.deregister_table(name) + + +@deprecated("Use `Schema` instead.") +class Database(Schema): + """See `Schema`.""" class Table: """DataFusion table.""" - def __init__(self, table: df_internal.Table) -> None: + def __init__(self, table: df_internal.catalog.RawTable) -> None: """This constructor is not typically called by the end user.""" self.table = table @@ -78,6 +149,11 @@ def __repr__(self) -> str: """Print a string representation of the table.""" return self.table.__repr__() + @staticmethod + def from_dataset(dataset: pa.dataset.Dataset) -> Table: + """Turn a pyarrow Dataset into a Table.""" + return Table(df_internal.catalog.RawTable.from_dataset(dataset)) + @property def schema(self) -> pa.Schema: """Returns the schema associated with this table.""" @@ -87,3 +163,86 @@ def schema(self) -> pa.Schema: def kind(self) -> str: """Returns the kind of table.""" return self.table.kind + + +class CatalogProvider(ABC): + """Abstract class for defining a Python based Catalog Provider.""" + + @abstractmethod + def schema_names(self) -> set[str]: + """Set of the names of all schemas in this catalog.""" + ... + + @abstractmethod + def schema(self, name: str) -> Schema | None: + """Retrieve a specific schema from this catalog.""" + ... + + def register_schema( # noqa: B027 + self, name: str, schema: SchemaProviderExportable | SchemaProvider | Schema + ) -> None: + """Add a schema to this catalog. + + This method is optional. If your catalog provides a fixed list of schemas, you + do not need to implement this method. + """ + + def deregister_schema(self, name: str, cascade: bool) -> None: # noqa: B027 + """Remove a schema from this catalog. + + This method is optional. If your catalog provides a fixed list of schemas, you + do not need to implement this method. + + Args: + name: The name of the schema to remove. + cascade: If true, deregister the tables within the schema. + """ + + +class SchemaProvider(ABC): + """Abstract class for defining a Python based Schema Provider.""" + + def owner_name(self) -> str | None: + """Returns the owner of the schema. + + This is an optional method. The default return is None. + """ + return None + + @abstractmethod + def table_names(self) -> set[str]: + """Set of the names of all tables in this schema.""" + ... + + @abstractmethod + def table(self, name: str) -> Table | None: + """Retrieve a specific table from this schema.""" + ... + + def register_table(self, name: str, table: Table) -> None: # noqa: B027 + """Add a table from this schema. + + This method is optional. If your schema provides a fixed list of tables, you do + not need to implement this method. + """ + + def deregister_table(self, name, cascade: bool) -> None: # noqa: B027 + """Remove a table from this schema. + + This method is optional. If your schema provides a fixed list of tables, you do + not need to implement this method. + """ + + @abstractmethod + def table_exist(self, name: str) -> bool: + """Returns true if the table exists in this schema.""" + ... + + +class SchemaProviderExportable(Protocol): + """Type hint for object that has __datafusion_schema_provider__ PyCapsule. + + https://docs.rs/datafusion/latest/datafusion/catalog/trait.SchemaProvider.html + """ + + def __datafusion_schema_provider__(self) -> object: ... diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 5b99b0d26..bce51d644 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -29,7 +29,7 @@ except ImportError: from typing_extensions import deprecated # Python 3.12 -from datafusion.catalog import Catalog, Table +from datafusion.catalog import Catalog, CatalogProvider, Table from datafusion.dataframe import DataFrame from datafusion.expr import Expr, SortExpr, sort_list_to_raw_sort_list from datafusion.record_batch import RecordBatchStream @@ -80,6 +80,15 @@ class TableProviderExportable(Protocol): def __datafusion_table_provider__(self) -> object: ... # noqa: D105 +class CatalogProviderExportable(Protocol): + """Type hint for object that has __datafusion_catalog_provider__ PyCapsule. + + https://docs.rs/datafusion/latest/datafusion/catalog/trait.CatalogProvider.html + """ + + def __datafusion_catalog_provider__(self) -> object: ... # noqa: D105 + + class SessionConfig: """Session configuration options.""" @@ -749,6 +758,19 @@ def deregister_table(self, name: str) -> None: """Remove a table from the session.""" self.ctx.deregister_table(name) + def catalog_names(self) -> set[str]: + """Returns the list of catalogs in this context.""" + return self.ctx.catalog_names() + + def register_catalog_provider( + self, name: str, provider: CatalogProviderExportable | CatalogProvider | Catalog + ) -> None: + """Register a catalog provider.""" + if isinstance(provider, Catalog): + self.ctx.register_catalog_provider(name, provider.catalog) + else: + self.ctx.register_catalog_provider(name, provider) + def register_table_provider( self, name: str, provider: TableProviderExportable ) -> None: diff --git a/python/datafusion/dataframe.py b/python/datafusion/dataframe.py index 991e6875a..61cb09438 100644 --- a/python/datafusion/dataframe.py +++ b/python/datafusion/dataframe.py @@ -760,19 +760,16 @@ def join_on( exprs = [expr.expr for expr in on_exprs] return DataFrame(self.df.join_on(right.df, exprs, how)) - def explain(self, verbose: bool = False, analyze: bool = False) -> DataFrame: - """Return a DataFrame with the explanation of its plan so far. + def explain(self, verbose: bool = False, analyze: bool = False) -> None: + """Print an explanation of the DataFrame's plan so far. If ``analyze`` is specified, runs the plan and reports metrics. Args: verbose: If ``True``, more details will be included. analyze: If ``Tru`e``, the plan will run and metrics reported. - - Returns: - DataFrame with the explanation of its plan. """ - return DataFrame(self.df.explain(verbose, analyze)) + self.df.explain(verbose, analyze) def logical_plan(self) -> LogicalPlan: """Return the unoptimized ``LogicalPlan``. diff --git a/python/tests/test_catalog.py b/python/tests/test_catalog.py index 23b328458..1f9ecbfc3 100644 --- a/python/tests/test_catalog.py +++ b/python/tests/test_catalog.py @@ -14,9 +14,13 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from __future__ import annotations +import datafusion as dfn import pyarrow as pa +import pyarrow.dataset as ds import pytest +from datafusion import SessionContext, Table # Note we take in `database` as a variable even though we don't use @@ -27,9 +31,9 @@ def test_basic(ctx, database): ctx.catalog("non-existent") default = ctx.catalog() - assert default.names() == ["public"] + assert default.names() == {"public"} - for db in [default.database("public"), default.database()]: + for db in [default.schema("public"), default.schema()]: assert db.names() == {"csv1", "csv", "csv2"} table = db.table("csv") @@ -41,3 +45,168 @@ def test_basic(ctx, database): pa.field("float", pa.float64(), nullable=True), ] ) + + +def create_dataset() -> Table: + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3]), pa.array([4, 5, 6])], + names=["a", "b"], + ) + dataset = ds.dataset([batch]) + return Table.from_dataset(dataset) + + +class CustomSchemaProvider(dfn.catalog.SchemaProvider): + def __init__(self): + self.tables = {"table1": create_dataset()} + + def table_names(self) -> set[str]: + return set(self.tables.keys()) + + def register_table(self, name: str, table: Table): + self.tables[name] = table + + def deregister_table(self, name, cascade: bool = True): + del self.tables[name] + + def table(self, name: str) -> Table | None: + return self.tables[name] + + def table_exist(self, name: str) -> bool: + return name in self.tables + + +class CustomCatalogProvider(dfn.catalog.CatalogProvider): + def __init__(self): + self.schemas = {"my_schema": CustomSchemaProvider()} + + def schema_names(self) -> set[str]: + return set(self.schemas.keys()) + + def schema(self, name: str): + return self.schemas[name] + + def register_schema(self, name: str, schema: dfn.catalog.Schema): + self.schemas[name] = schema + + def deregister_schema(self, name, cascade: bool): + del self.schemas[name] + + +def test_python_catalog_provider(ctx: SessionContext): + ctx.register_catalog_provider("my_catalog", CustomCatalogProvider()) + + # Check the default catalog provider + assert ctx.catalog("datafusion").names() == {"public"} + + my_catalog = ctx.catalog("my_catalog") + assert my_catalog.names() == {"my_schema"} + + my_catalog.register_schema("second_schema", CustomSchemaProvider()) + assert my_catalog.schema_names() == {"my_schema", "second_schema"} + + my_catalog.deregister_schema("my_schema") + assert my_catalog.schema_names() == {"second_schema"} + + +def test_in_memory_providers(ctx: SessionContext): + catalog = dfn.catalog.Catalog.memory_catalog() + ctx.register_catalog_provider("in_mem_catalog", catalog) + + assert ctx.catalog_names() == {"datafusion", "in_mem_catalog"} + + schema = dfn.catalog.Schema.memory_schema() + catalog.register_schema("in_mem_schema", schema) + + schema.register_table("my_table", create_dataset()) + + batches = ctx.sql("select * from in_mem_catalog.in_mem_schema.my_table").collect() + + assert len(batches) == 1 + assert batches[0].column(0) == pa.array([1, 2, 3]) + assert batches[0].column(1) == pa.array([4, 5, 6]) + + +def test_python_schema_provider(ctx: SessionContext): + catalog = ctx.catalog() + + catalog.deregister_schema("public") + + catalog.register_schema("test_schema1", CustomSchemaProvider()) + assert catalog.names() == {"test_schema1"} + + catalog.register_schema("test_schema2", CustomSchemaProvider()) + catalog.deregister_schema("test_schema1") + assert catalog.names() == {"test_schema2"} + + +def test_python_table_provider(ctx: SessionContext): + catalog = ctx.catalog() + + catalog.register_schema("custom_schema", CustomSchemaProvider()) + schema = catalog.schema("custom_schema") + + assert schema.table_names() == {"table1"} + + schema.deregister_table("table1") + schema.register_table("table2", create_dataset()) + assert schema.table_names() == {"table2"} + + # Use the default schema instead of our custom schema + + schema = catalog.schema() + + schema.register_table("table3", create_dataset()) + assert schema.table_names() == {"table3"} + + schema.deregister_table("table3") + schema.register_table("table4", create_dataset()) + assert schema.table_names() == {"table4"} + + +def test_in_end_to_end_python_providers(ctx: SessionContext): + """Test registering all python providers and running a query against them.""" + + all_catalog_names = [ + "datafusion", + "custom_catalog", + "in_mem_catalog", + ] + + all_schema_names = [ + "custom_schema", + "in_mem_schema", + ] + + ctx.register_catalog_provider(all_catalog_names[1], CustomCatalogProvider()) + ctx.register_catalog_provider( + all_catalog_names[2], dfn.catalog.Catalog.memory_catalog() + ) + + for catalog_name in all_catalog_names: + catalog = ctx.catalog(catalog_name) + + # Clean out previous schemas if they exist so we can start clean + for schema_name in catalog.schema_names(): + catalog.deregister_schema(schema_name, cascade=False) + + catalog.register_schema(all_schema_names[0], CustomSchemaProvider()) + catalog.register_schema(all_schema_names[1], dfn.catalog.Schema.memory_schema()) + + for schema_name in all_schema_names: + schema = catalog.schema(schema_name) + + for table_name in schema.table_names(): + schema.deregister_table(table_name) + + schema.register_table("test_table", create_dataset()) + + for catalog_name in all_catalog_names: + for schema_name in all_schema_names: + table_full_name = f"{catalog_name}.{schema_name}.test_table" + + batches = ctx.sql(f"select * from {table_full_name}").collect() + + assert len(batches) == 1 + assert batches[0].column(0) == pa.array([1, 2, 3]) + assert batches[0].column(1) == pa.array([4, 5, 6]) diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 4a15ac9cf..6dbcc0d5e 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -57,7 +57,7 @@ def test_runtime_configs(tmp_path, path_to_str): ctx = SessionContext(config, runtime) assert ctx is not None - db = ctx.catalog("foo").database("bar") + db = ctx.catalog("foo").schema("bar") assert db is not None @@ -70,7 +70,7 @@ def test_temporary_files(tmp_path, path_to_str): ctx = SessionContext(config, runtime) assert ctx is not None - db = ctx.catalog("foo").database("bar") + db = ctx.catalog("foo").schema("bar") assert db is not None @@ -91,7 +91,7 @@ def test_create_context_with_all_valid_args(): ctx = SessionContext(config, runtime) # verify that at least some of the arguments worked - ctx.catalog("foo").database("bar") + ctx.catalog("foo").schema("bar") with pytest.raises(KeyError): ctx.catalog("datafusion") @@ -105,7 +105,7 @@ def test_register_record_batches(ctx): ctx.register_record_batches("t", [[batch]]) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} result = ctx.sql("SELECT a+b, a-b FROM t").collect() @@ -121,7 +121,7 @@ def test_create_dataframe_registers_unique_table_name(ctx): ) df = ctx.create_dataframe([[batch]]) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -141,7 +141,7 @@ def test_create_dataframe_registers_with_defined_table_name(ctx): ) df = ctx.create_dataframe([[batch]], name="tbl") - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -155,7 +155,7 @@ def test_from_arrow_table(ctx): # convert to DataFrame df = ctx.from_arrow(table) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -200,7 +200,7 @@ def test_from_arrow_table_with_name(ctx): # convert to DataFrame with optional name df = ctx.from_arrow(table, name="tbl") - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert tables[0] == "tbl" @@ -213,7 +213,7 @@ def test_from_arrow_table_empty(ctx): # convert to DataFrame df = ctx.from_arrow(table) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -228,7 +228,7 @@ def test_from_arrow_table_empty_no_schema(ctx): # convert to DataFrame df = ctx.from_arrow(table) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -246,7 +246,7 @@ def test_from_pylist(ctx): ] df = ctx.from_pylist(data) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -260,7 +260,7 @@ def test_from_pydict(ctx): data = {"a": [1, 2, 3], "b": [4, 5, 6]} df = ctx.from_pydict(data) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -276,7 +276,7 @@ def test_from_pandas(ctx): pandas_df = pd.DataFrame(data) df = ctx.from_pandas(pandas_df) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -292,7 +292,7 @@ def test_from_polars(ctx): polars_df = pd.DataFrame(data) df = ctx.from_polars(polars_df) - tables = list(ctx.catalog().database().names()) + tables = list(ctx.catalog().schema().names()) assert df assert len(tables) == 1 @@ -303,7 +303,7 @@ def test_from_polars(ctx): def test_register_table(ctx, database): default = ctx.catalog() - public = default.database("public") + public = default.schema("public") assert public.names() == {"csv", "csv1", "csv2"} table = public.table("csv") @@ -313,7 +313,7 @@ def test_register_table(ctx, database): def test_read_table(ctx, database): default = ctx.catalog() - public = default.database("public") + public = default.schema("public") assert public.names() == {"csv", "csv1", "csv2"} table = public.table("csv") @@ -323,7 +323,7 @@ def test_read_table(ctx, database): def test_deregister_table(ctx, database): default = ctx.catalog() - public = default.database("public") + public = default.schema("public") assert public.names() == {"csv", "csv1", "csv2"} ctx.deregister_table("csv") @@ -339,7 +339,7 @@ def test_register_dataset(ctx): dataset = ds.dataset([batch]) ctx.register_dataset("t", dataset) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} result = ctx.sql("SELECT a+b, a-b FROM t").collect() @@ -356,7 +356,7 @@ def test_dataset_filter(ctx, capfd): dataset = ds.dataset([batch]) ctx.register_dataset("t", dataset) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} df = ctx.sql("SELECT a+b, a-b FROM t WHERE a BETWEEN 2 and 3 AND b > 5") # Make sure the filter was pushed down in Physical Plan @@ -455,7 +455,7 @@ def test_dataset_filter_nested_data(ctx): dataset = ds.dataset([batch]) ctx.register_dataset("t", dataset) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} df = ctx.table("t") diff --git a/python/tests/test_sql.py b/python/tests/test_sql.py index 41cee4ef3..c383edc60 100644 --- a/python/tests/test_sql.py +++ b/python/tests/test_sql.py @@ -75,7 +75,7 @@ def test_register_csv(ctx, tmp_path): ) ctx.register_csv("csv3", path, schema=alternative_schema) - assert ctx.catalog().database().names() == { + assert ctx.catalog().schema().names() == { "csv", "csv1", "csv2", @@ -150,7 +150,7 @@ def test_register_parquet(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", helpers.data()) ctx.register_parquet("t", path) ctx.register_parquet("t1", str(path)) - assert ctx.catalog().database().names() == {"t", "t1"} + assert ctx.catalog().schema().names() == {"t", "t1"} result = ctx.sql("SELECT COUNT(a) AS cnt FROM t").collect() result = pa.Table.from_batches(result) @@ -188,7 +188,7 @@ def test_register_parquet_partitioned(ctx, tmp_path, path_to_str, legacy_data_ty parquet_pruning=True, file_extension=".parquet", ) - assert ctx.catalog().database().names() == {"datapp"} + assert ctx.catalog().schema().names() == {"datapp"} result = ctx.sql("SELECT grp, COUNT(*) AS cnt FROM datapp GROUP BY grp").collect() result = pa.Table.from_batches(result) @@ -204,7 +204,7 @@ def test_register_dataset(ctx, tmp_path, path_to_str): dataset = ds.dataset(path, format="parquet") ctx.register_dataset("t", dataset) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} result = ctx.sql("SELECT COUNT(a) AS cnt FROM t").collect() result = pa.Table.from_batches(result) @@ -251,7 +251,7 @@ def test_register_json(ctx, tmp_path): ) ctx.register_json("json3", path, schema=alternative_schema) - assert ctx.catalog().database().names() == { + assert ctx.catalog().schema().names() == { "json", "json1", "json2", @@ -308,7 +308,7 @@ def test_execute(ctx, tmp_path): path = helpers.write_parquet(tmp_path / "a.parquet", pa.array(data)) ctx.register_parquet("t", path) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} # count result = ctx.sql("SELECT COUNT(a) AS cnt FROM t WHERE a IS NOT NULL").collect() @@ -451,18 +451,10 @@ def test_udf( id="datetime_ns", ), # Not writtable to parquet - pytest.param( - helpers.data_timedelta("s"), id="timedelta_s", marks=pytest.mark.xfail - ), - pytest.param( - helpers.data_timedelta("ms"), id="timedelta_ms", marks=pytest.mark.xfail - ), - pytest.param( - helpers.data_timedelta("us"), id="timedelta_us", marks=pytest.mark.xfail - ), - pytest.param( - helpers.data_timedelta("ns"), id="timedelta_ns", marks=pytest.mark.xfail - ), + pytest.param(helpers.data_timedelta("s"), id="timedelta_s"), + pytest.param(helpers.data_timedelta("ms"), id="timedelta_ms"), + pytest.param(helpers.data_timedelta("us"), id="timedelta_us"), + pytest.param(helpers.data_timedelta("ns"), id="timedelta_ns"), ], ) def test_simple_select(ctx, tmp_path, arr): @@ -524,7 +516,7 @@ def test_register_listing_table( schema=table.schema if pass_schema else None, file_sort_order=file_sort_order, ) - assert ctx.catalog().database().names() == {"my_table"} + assert ctx.catalog().schema().names() == {"my_table"} result = ctx.sql( "SELECT grp, COUNT(*) AS count FROM my_table GROUP BY grp" diff --git a/python/tests/test_substrait.py b/python/tests/test_substrait.py index f367a447d..43aa327d4 100644 --- a/python/tests/test_substrait.py +++ b/python/tests/test_substrait.py @@ -34,7 +34,7 @@ def test_substrait_serialization(ctx): ctx.register_record_batches("t", [[batch]]) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} # For now just make sure the method calls blow up substrait_plan = ss.Serde.serialize_to_plan("SELECT * FROM t", ctx) @@ -59,7 +59,7 @@ def test_substrait_file_serialization(ctx, tmp_path, path_to_str): ctx.register_record_batches("t", [[batch]]) - assert ctx.catalog().database().names() == {"t"} + assert ctx.catalog().schema().names() == {"t"} path = tmp_path / "substrait_plan" path = str(path) if path_to_str else path diff --git a/src/catalog.rs b/src/catalog.rs index 83f8d08cb..17d4ec3b8 100644 --- a/src/catalog.rs +++ b/src/catalog.rs @@ -15,44 +15,54 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashSet; -use std::sync::Arc; - -use pyo3::exceptions::PyKeyError; -use pyo3::prelude::*; - -use crate::errors::{PyDataFusionError, PyDataFusionResult}; -use crate::utils::wait_for_future; +use crate::dataset::Dataset; +use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionError, PyDataFusionResult}; +use crate::utils::{validate_pycapsule, wait_for_future}; +use async_trait::async_trait; +use datafusion::catalog::{MemoryCatalogProvider, MemorySchemaProvider}; +use datafusion::common::DataFusionError; use datafusion::{ arrow::pyarrow::ToPyArrow, catalog::{CatalogProvider, SchemaProvider}, datasource::{TableProvider, TableType}, }; +use datafusion_ffi::schema_provider::{FFI_SchemaProvider, ForeignSchemaProvider}; +use datafusion_ffi::table_provider::{FFI_TableProvider, ForeignTableProvider}; +use pyo3::exceptions::PyKeyError; +use pyo3::prelude::*; +use pyo3::types::PyCapsule; +use pyo3::IntoPyObjectExt; +use std::any::Any; +use std::collections::HashSet; +use std::sync::Arc; -#[pyclass(name = "Catalog", module = "datafusion", subclass)] +#[pyclass(name = "RawCatalog", module = "datafusion.catalog", subclass)] +#[derive(Clone)] pub struct PyCatalog { pub catalog: Arc, } -#[pyclass(name = "Database", module = "datafusion", subclass)] -pub struct PyDatabase { - pub database: Arc, +#[pyclass(name = "RawSchema", module = "datafusion.catalog", subclass)] +#[derive(Clone)] +pub struct PySchema { + pub schema: Arc, } -#[pyclass(name = "Table", module = "datafusion", subclass)] +#[pyclass(name = "RawTable", module = "datafusion.catalog", subclass)] +#[derive(Clone)] pub struct PyTable { pub table: Arc, } -impl PyCatalog { - pub fn new(catalog: Arc) -> Self { +impl From> for PyCatalog { + fn from(catalog: Arc) -> Self { Self { catalog } } } -impl PyDatabase { - pub fn new(database: Arc) -> Self { - Self { database } +impl From> for PySchema { + fn from(schema: Arc) -> Self { + Self { schema } } } @@ -68,36 +78,109 @@ impl PyTable { #[pymethods] impl PyCatalog { - fn names(&self) -> Vec { - self.catalog.schema_names() + #[new] + fn new(catalog: PyObject) -> Self { + let catalog_provider = + Arc::new(RustWrappedPyCatalogProvider::new(catalog)) as Arc; + catalog_provider.into() + } + + #[staticmethod] + fn memory_catalog() -> Self { + let catalog_provider = + Arc::new(MemoryCatalogProvider::default()) as Arc; + catalog_provider.into() + } + + fn schema_names(&self) -> HashSet { + self.catalog.schema_names().into_iter().collect() } #[pyo3(signature = (name="public"))] - fn database(&self, name: &str) -> PyResult { - match self.catalog.schema(name) { - Some(database) => Ok(PyDatabase::new(database)), - None => Err(PyKeyError::new_err(format!( - "Database with name {name} doesn't exist." - ))), - } + fn schema(&self, name: &str) -> PyResult { + let schema = self + .catalog + .schema(name) + .ok_or(PyKeyError::new_err(format!( + "Schema with name {name} doesn't exist." + )))?; + + Python::with_gil(|py| { + match schema + .as_any() + .downcast_ref::() + { + Some(wrapped_schema) => Ok(wrapped_schema.schema_provider.clone_ref(py)), + None => PySchema::from(schema).into_py_any(py), + } + }) + } + + fn register_schema(&self, name: &str, schema_provider: Bound<'_, PyAny>) -> PyResult<()> { + let provider = if schema_provider.hasattr("__datafusion_schema_provider__")? { + let capsule = schema_provider + .getattr("__datafusion_schema_provider__")? + .call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_schema_provider")?; + + let provider = unsafe { capsule.reference::() }; + let provider: ForeignSchemaProvider = provider.into(); + Arc::new(provider) as Arc + } else { + match schema_provider.extract::() { + Ok(py_schema) => py_schema.schema, + Err(_) => Arc::new(RustWrappedPySchemaProvider::new(schema_provider.into())) + as Arc, + } + }; + + let _ = self + .catalog + .register_schema(name, provider) + .map_err(py_datafusion_err)?; + + Ok(()) + } + + fn deregister_schema(&self, name: &str, cascade: bool) -> PyResult<()> { + let _ = self + .catalog + .deregister_schema(name, cascade) + .map_err(py_datafusion_err)?; + + Ok(()) } fn __repr__(&self) -> PyResult { - Ok(format!( - "Catalog(schema_names=[{}])", - self.names().join(";") - )) + let mut names: Vec = self.schema_names().into_iter().collect(); + names.sort(); + Ok(format!("Catalog(schema_names=[{}])", names.join(", "))) } } #[pymethods] -impl PyDatabase { - fn names(&self) -> HashSet { - self.database.table_names().into_iter().collect() +impl PySchema { + #[new] + fn new(schema_provider: PyObject) -> Self { + let schema_provider = + Arc::new(RustWrappedPySchemaProvider::new(schema_provider)) as Arc; + schema_provider.into() + } + + #[staticmethod] + fn memory_schema() -> Self { + let schema_provider = Arc::new(MemorySchemaProvider::default()) as Arc; + schema_provider.into() + } + + #[getter] + fn table_names(&self) -> HashSet { + self.schema.table_names().into_iter().collect() } fn table(&self, name: &str, py: Python) -> PyDataFusionResult { - if let Some(table) = wait_for_future(py, self.database.table(name))?? { + if let Some(table) = wait_for_future(py, self.schema.table(name))?? { Ok(PyTable::new(table)) } else { Err(PyDataFusionError::Common(format!( @@ -107,14 +190,49 @@ impl PyDatabase { } fn __repr__(&self) -> PyResult { - Ok(format!( - "Database(table_names=[{}])", - Vec::from_iter(self.names()).join(";") - )) + let mut names: Vec = self.table_names().into_iter().collect(); + names.sort(); + Ok(format!("Schema(table_names=[{}])", names.join(";"))) } - // register_table - // deregister_table + fn register_table(&self, name: &str, table_provider: Bound<'_, PyAny>) -> PyResult<()> { + let provider = if table_provider.hasattr("__datafusion_table_provider__")? { + let capsule = table_provider + .getattr("__datafusion_table_provider__")? + .call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_table_provider")?; + + let provider = unsafe { capsule.reference::() }; + let provider: ForeignTableProvider = provider.into(); + Arc::new(provider) as Arc + } else { + match table_provider.extract::() { + Ok(py_table) => py_table.table, + Err(_) => { + let py = table_provider.py(); + let provider = Dataset::new(&table_provider, py)?; + Arc::new(provider) as Arc + } + } + }; + + let _ = self + .schema + .register_table(name.to_string(), provider) + .map_err(py_datafusion_err)?; + + Ok(()) + } + + fn deregister_table(&self, name: &str) -> PyResult<()> { + let _ = self + .schema + .deregister_table(name) + .map_err(py_datafusion_err)?; + + Ok(()) + } } #[pymethods] @@ -125,6 +243,14 @@ impl PyTable { self.table.schema().to_pyarrow(py) } + #[staticmethod] + fn from_dataset(py: Python<'_>, dataset: &Bound<'_, PyAny>) -> PyResult { + let ds = Arc::new(Dataset::new(dataset, py).map_err(py_datafusion_err)?) + as Arc; + + Ok(Self::new(ds)) + } + /// Get the type of this table for metadata/catalog purposes. #[getter] fn kind(&self) -> &str { @@ -145,3 +271,285 @@ impl PyTable { // fn has_exact_statistics // fn supports_filter_pushdown } + +#[derive(Debug)] +pub(crate) struct RustWrappedPySchemaProvider { + schema_provider: PyObject, + owner_name: Option, +} + +impl RustWrappedPySchemaProvider { + pub fn new(schema_provider: PyObject) -> Self { + let owner_name = Python::with_gil(|py| { + schema_provider + .bind(py) + .getattr("owner_name") + .ok() + .map(|name| name.to_string()) + }); + + Self { + schema_provider, + owner_name, + } + } + + fn table_inner(&self, name: &str) -> PyResult>> { + Python::with_gil(|py| { + let provider = self.schema_provider.bind(py); + let py_table_method = provider.getattr("table")?; + + let py_table = py_table_method.call((name,), None)?; + if py_table.is_none() { + return Ok(None); + } + + if py_table.hasattr("__datafusion_table_provider__")? { + let capsule = provider.getattr("__datafusion_table_provider__")?.call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_table_provider")?; + + let provider = unsafe { capsule.reference::() }; + let provider: ForeignTableProvider = provider.into(); + + Ok(Some(Arc::new(provider) as Arc)) + } else { + if let Ok(inner_table) = py_table.getattr("table") { + if let Ok(inner_table) = inner_table.extract::() { + return Ok(Some(inner_table.table)); + } + } + + match py_table.extract::() { + Ok(py_table) => Ok(Some(py_table.table)), + Err(_) => { + let ds = Dataset::new(&py_table, py).map_err(py_datafusion_err)?; + Ok(Some(Arc::new(ds) as Arc)) + } + } + } + }) + } +} + +#[async_trait] +impl SchemaProvider for RustWrappedPySchemaProvider { + fn owner_name(&self) -> Option<&str> { + self.owner_name.as_deref() + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn table_names(&self) -> Vec { + Python::with_gil(|py| { + let provider = self.schema_provider.bind(py); + + provider + .getattr("table_names") + .and_then(|names| names.extract::>()) + .unwrap_or_else(|err| { + log::error!("Unable to get table_names: {err}"); + Vec::default() + }) + }) + } + + async fn table( + &self, + name: &str, + ) -> datafusion::common::Result>, DataFusionError> { + self.table_inner(name).map_err(to_datafusion_err) + } + + fn register_table( + &self, + name: String, + table: Arc, + ) -> datafusion::common::Result>> { + let py_table = PyTable::new(table); + Python::with_gil(|py| { + let provider = self.schema_provider.bind(py); + let _ = provider + .call_method1("register_table", (name, py_table)) + .map_err(to_datafusion_err)?; + // Since the definition of `register_table` says that an error + // will be returned if the table already exists, there is no + // case where we want to return a table provider as output. + Ok(None) + }) + } + + fn deregister_table( + &self, + name: &str, + ) -> datafusion::common::Result>> { + Python::with_gil(|py| { + let provider = self.schema_provider.bind(py); + let table = provider + .call_method1("deregister_table", (name,)) + .map_err(to_datafusion_err)?; + if table.is_none() { + return Ok(None); + } + + // If we can turn this table provider into a `Dataset`, return it. + // Otherwise, return None. + let dataset = match Dataset::new(&table, py) { + Ok(dataset) => Some(Arc::new(dataset) as Arc), + Err(_) => None, + }; + + Ok(dataset) + }) + } + + fn table_exist(&self, name: &str) -> bool { + Python::with_gil(|py| { + let provider = self.schema_provider.bind(py); + provider + .call_method1("table_exist", (name,)) + .and_then(|pyobj| pyobj.extract()) + .unwrap_or(false) + }) + } +} + +#[derive(Debug)] +pub(crate) struct RustWrappedPyCatalogProvider { + pub(crate) catalog_provider: PyObject, +} + +impl RustWrappedPyCatalogProvider { + pub fn new(catalog_provider: PyObject) -> Self { + Self { catalog_provider } + } + + fn schema_inner(&self, name: &str) -> PyResult>> { + Python::with_gil(|py| { + let provider = self.catalog_provider.bind(py); + + let py_schema = provider.call_method1("schema", (name,))?; + if py_schema.is_none() { + return Ok(None); + } + + if py_schema.hasattr("__datafusion_schema_provider__")? { + let capsule = provider + .getattr("__datafusion_schema_provider__")? + .call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_schema_provider")?; + + let provider = unsafe { capsule.reference::() }; + let provider: ForeignSchemaProvider = provider.into(); + + Ok(Some(Arc::new(provider) as Arc)) + } else { + if let Ok(inner_schema) = py_schema.getattr("schema") { + if let Ok(inner_schema) = inner_schema.extract::() { + return Ok(Some(inner_schema.schema)); + } + } + match py_schema.extract::() { + Ok(inner_schema) => Ok(Some(inner_schema.schema)), + Err(_) => { + let py_schema = RustWrappedPySchemaProvider::new(py_schema.into()); + + Ok(Some(Arc::new(py_schema) as Arc)) + } + } + } + }) + } +} + +#[async_trait] +impl CatalogProvider for RustWrappedPyCatalogProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema_names(&self) -> Vec { + Python::with_gil(|py| { + let provider = self.catalog_provider.bind(py); + provider + .getattr("schema_names") + .and_then(|names| names.extract::>()) + .unwrap_or_else(|err| { + log::error!("Unable to get schema_names: {err}"); + Vec::default() + }) + }) + } + + fn schema(&self, name: &str) -> Option> { + self.schema_inner(name).unwrap_or_else(|err| { + log::error!("CatalogProvider schema returned error: {err}"); + None + }) + } + + fn register_schema( + &self, + name: &str, + schema: Arc, + ) -> datafusion::common::Result>> { + // JRIGHT HERE + // let py_schema: PySchema = schema.into(); + Python::with_gil(|py| { + let py_schema = match schema + .as_any() + .downcast_ref::() + { + Some(wrapped_schema) => wrapped_schema.schema_provider.as_any(), + None => &PySchema::from(schema) + .into_py_any(py) + .map_err(to_datafusion_err)?, + }; + + let provider = self.catalog_provider.bind(py); + let schema = provider + .call_method1("register_schema", (name, py_schema)) + .map_err(to_datafusion_err)?; + if schema.is_none() { + return Ok(None); + } + + let schema = Arc::new(RustWrappedPySchemaProvider::new(schema.into())) + as Arc; + + Ok(Some(schema)) + }) + } + + fn deregister_schema( + &self, + name: &str, + cascade: bool, + ) -> datafusion::common::Result>> { + Python::with_gil(|py| { + let provider = self.catalog_provider.bind(py); + let schema = provider + .call_method1("deregister_schema", (name, cascade)) + .map_err(to_datafusion_err)?; + if schema.is_none() { + return Ok(None); + } + + let schema = Arc::new(RustWrappedPySchemaProvider::new(schema.into())) + as Arc; + + Ok(Some(schema)) + }) + } +} + +pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + + Ok(()) +} diff --git a/src/common/data_type.rs b/src/common/data_type.rs index f5f8a6b06..5cf9d6e9f 100644 --- a/src/common/data_type.rs +++ b/src/common/data_type.rs @@ -172,7 +172,7 @@ impl DataTypeMap { SqlType::DATE, )), DataType::Duration(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::Interval(interval_unit) => Ok(DataTypeMap::new( DataType::Interval(*interval_unit), @@ -189,7 +189,7 @@ impl DataTypeMap { SqlType::BINARY, )), DataType::FixedSizeBinary(_) => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", arrow_type)), + DataFusionError::NotImplemented(format!("{arrow_type:?}")), )), DataType::LargeBinary => Ok(DataTypeMap::new( DataType::LargeBinary, @@ -207,23 +207,22 @@ impl DataTypeMap { SqlType::VARCHAR, )), DataType::List(_) => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - arrow_type + "{arrow_type:?}" )))), DataType::FixedSizeList(_, _) => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", arrow_type)), + DataFusionError::NotImplemented(format!("{arrow_type:?}")), )), DataType::LargeList(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::Struct(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::Union(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::Dictionary(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::Decimal128(precision, scale) => Ok(DataTypeMap::new( DataType::Decimal128(*precision, *scale), @@ -236,23 +235,22 @@ impl DataTypeMap { SqlType::DECIMAL, )), DataType::Map(_, _) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::RunEndEncoded(_, _) => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", arrow_type)), + DataFusionError::NotImplemented(format!("{arrow_type:?}")), )), DataType::BinaryView => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::Utf8View => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - arrow_type + "{arrow_type:?}" )))), DataType::ListView(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), DataType::LargeListView(_) => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", arrow_type), + format!("{arrow_type:?}"), ))), } } @@ -379,8 +377,7 @@ impl DataTypeMap { "double" => Ok(DataType::Float64), "byte_array" => Ok(DataType::Utf8), _ => Err(PyValueError::new_err(format!( - "Unable to determine Arrow Data Type from Parquet String type: {:?}", - parquet_str_type + "Unable to determine Arrow Data Type from Parquet String type: {parquet_str_type:?}" ))), }; DataTypeMap::map_from_arrow_type(&arrow_dtype?) @@ -404,12 +401,10 @@ impl DataTypeMap { pub fn py_map_from_sql_type(sql_type: &SqlType) -> PyResult { match sql_type { SqlType::ANY => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::ARRAY => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::BIGINT => Ok(DataTypeMap::new( DataType::Int64, @@ -432,11 +427,10 @@ impl DataTypeMap { SqlType::CHAR, )), SqlType::COLUMN_LIST => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::CURSOR => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::DATE => Ok(DataTypeMap::new( DataType::Date64, @@ -449,8 +443,7 @@ impl DataTypeMap { SqlType::DECIMAL, )), SqlType::DISTINCT => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::DOUBLE => Ok(DataTypeMap::new( DataType::Decimal256(1, 1), @@ -458,7 +451,7 @@ impl DataTypeMap { SqlType::DOUBLE, )), SqlType::DYNAMIC_STAR => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::FLOAT => Ok(DataTypeMap::new( DataType::Decimal128(1, 1), @@ -466,8 +459,7 @@ impl DataTypeMap { SqlType::FLOAT, )), SqlType::GEOMETRY => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::INTEGER => Ok(DataTypeMap::new( DataType::Int8, @@ -475,55 +467,52 @@ impl DataTypeMap { SqlType::INTEGER, )), SqlType::INTERVAL => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::INTERVAL_DAY => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_DAY_HOUR => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_DAY_MINUTE => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::INTERVAL_DAY_SECOND => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::INTERVAL_HOUR => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_HOUR_MINUTE => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::INTERVAL_HOUR_SECOND => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::INTERVAL_MINUTE => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_MINUTE_SECOND => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::INTERVAL_MONTH => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_SECOND => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_YEAR => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::INTERVAL_YEAR_MONTH => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::MAP => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::MULTISET => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::NULL => Ok(DataTypeMap::new( DataType::Null, @@ -531,20 +520,16 @@ impl DataTypeMap { SqlType::NULL, )), SqlType::OTHER => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::REAL => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::ROW => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::SARG => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::SMALLINT => Ok(DataTypeMap::new( DataType::Int16, @@ -552,25 +537,22 @@ impl DataTypeMap { SqlType::SMALLINT, )), SqlType::STRUCTURED => Err(py_datafusion_err(DataFusionError::NotImplemented( - format!("{:?}", sql_type), + format!("{sql_type:?}"), ))), SqlType::SYMBOL => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::TIME => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::TIME_WITH_LOCAL_TIME_ZONE => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::TIMESTAMP => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::TIMESTAMP_WITH_LOCAL_TIME_ZONE => Err(py_datafusion_err( - DataFusionError::NotImplemented(format!("{:?}", sql_type)), + DataFusionError::NotImplemented(format!("{sql_type:?}")), )), SqlType::TINYINT => Ok(DataTypeMap::new( DataType::Int8, @@ -578,8 +560,7 @@ impl DataTypeMap { SqlType::TINYINT, )), SqlType::UNKNOWN => Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - sql_type + "{sql_type:?}" )))), SqlType::VARBINARY => Ok(DataTypeMap::new( DataType::LargeBinary, @@ -682,8 +663,7 @@ impl PyDataType { "datetime64" => Ok(DataType::Date64), "object" => Ok(DataType::Utf8), _ => Err(PyValueError::new_err(format!( - "Unable to determine Arrow Data Type from Arrow String type: {:?}", - arrow_str_type + "Unable to determine Arrow Data Type from Arrow String type: {arrow_str_type:?}" ))), }; Ok(PyDataType { diff --git a/src/context.rs b/src/context.rs index 6ce1f12bc..36133a33d 100644 --- a/src/context.rs +++ b/src/context.rs @@ -31,7 +31,7 @@ use uuid::Uuid; use pyo3::exceptions::{PyKeyError, PyValueError}; use pyo3::prelude::*; -use crate::catalog::{PyCatalog, PyTable}; +use crate::catalog::{PyCatalog, PyTable, RustWrappedPyCatalogProvider}; use crate::dataframe::PyDataFrame; use crate::dataset::Dataset; use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionResult}; @@ -49,6 +49,7 @@ use crate::utils::{get_global_ctx, get_tokio_runtime, validate_pycapsule, wait_f use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; use datafusion::arrow::record_batch::RecordBatch; +use datafusion::catalog::CatalogProvider; use datafusion::common::TableReference; use datafusion::common::{exec_err, ScalarValue}; use datafusion::datasource::file_format::file_compression_type::FileCompressionType; @@ -69,8 +70,10 @@ use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, }; +use datafusion_ffi::catalog_provider::{FFI_CatalogProvider, ForeignCatalogProvider}; use datafusion_ffi::table_provider::{FFI_TableProvider, ForeignTableProvider}; use pyo3::types::{PyCapsule, PyDict, PyList, PyTuple, PyType}; +use pyo3::IntoPyObjectExt; use tokio::task::JoinHandle; /// Configuration options for a SessionContext @@ -365,7 +368,7 @@ impl PySessionContext { } else { &upstream_host }; - let url_string = format!("{}{}", scheme, derived_host); + let url_string = format!("{scheme}{derived_host}"); let url = Url::parse(&url_string).unwrap(); self.ctx.runtime_env().register_object_store(&url, store); Ok(()) @@ -614,6 +617,34 @@ impl PySessionContext { Ok(()) } + pub fn register_catalog_provider( + &mut self, + name: &str, + provider: Bound<'_, PyAny>, + ) -> PyDataFusionResult<()> { + let provider = if provider.hasattr("__datafusion_catalog_provider__")? { + let capsule = provider + .getattr("__datafusion_catalog_provider__")? + .call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_catalog_provider")?; + + let provider = unsafe { capsule.reference::() }; + let provider: ForeignCatalogProvider = provider.into(); + Arc::new(provider) as Arc + } else { + match provider.extract::() { + Ok(py_catalog) => py_catalog.catalog, + Err(_) => Arc::new(RustWrappedPyCatalogProvider::new(provider.into())) + as Arc, + } + }; + + let _ = self.ctx.register_catalog(name, provider); + + Ok(()) + } + /// Construct datafusion dataframe from Arrow Table pub fn register_table_provider( &mut self, @@ -845,14 +876,24 @@ impl PySessionContext { } #[pyo3(signature = (name="datafusion"))] - pub fn catalog(&self, name: &str) -> PyResult { - match self.ctx.catalog(name) { - Some(catalog) => Ok(PyCatalog::new(catalog)), - None => Err(PyKeyError::new_err(format!( - "Catalog with name {} doesn't exist.", - &name, - ))), - } + pub fn catalog(&self, name: &str) -> PyResult { + let catalog = self.ctx.catalog(name).ok_or(PyKeyError::new_err(format!( + "Catalog with name {name} doesn't exist." + )))?; + + Python::with_gil(|py| { + match catalog + .as_any() + .downcast_ref::() + { + Some(wrapped_schema) => Ok(wrapped_schema.catalog_provider.clone_ref(py)), + None => PyCatalog::from(catalog).into_py_any(py), + } + }) + } + + pub fn catalog_names(&self) -> HashSet { + self.ctx.catalog_names().into_iter().collect() } pub fn tables(&self) -> HashSet { diff --git a/src/expr.rs b/src/expr.rs index 6b1d01d65..f1e002367 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -171,12 +171,10 @@ impl PyExpr { Expr::Cast(value) => Ok(cast::PyCast::from(value.clone()).into_bound_py_any(py)?), Expr::TryCast(value) => Ok(cast::PyTryCast::from(value.clone()).into_bound_py_any(py)?), Expr::ScalarFunction(value) => Err(py_unsupported_variant_err(format!( - "Converting Expr::ScalarFunction to a Python object is not implemented: {:?}", - value + "Converting Expr::ScalarFunction to a Python object is not implemented: {value:?}" ))), Expr::WindowFunction(value) => Err(py_unsupported_variant_err(format!( - "Converting Expr::WindowFunction to a Python object is not implemented: {:?}", - value + "Converting Expr::WindowFunction to a Python object is not implemented: {value:?}" ))), Expr::InList(value) => Ok(in_list::PyInList::from(value.clone()).into_bound_py_any(py)?), Expr::Exists(value) => Ok(exists::PyExists::from(value.clone()).into_bound_py_any(py)?), @@ -188,8 +186,7 @@ impl PyExpr { } #[allow(deprecated)] Expr::Wildcard { qualifier, options } => Err(py_unsupported_variant_err(format!( - "Converting Expr::Wildcard to a Python object is not implemented : {:?} {:?}", - qualifier, options + "Converting Expr::Wildcard to a Python object is not implemented : {qualifier:?} {options:?}" ))), Expr::GroupingSet(value) => { Ok(grouping_set::PyGroupingSet::from(value.clone()).into_bound_py_any(py)?) @@ -198,8 +195,7 @@ impl PyExpr { Ok(placeholder::PyPlaceholder::from(value.clone()).into_bound_py_any(py)?) } Expr::OuterReferenceColumn(data_type, column) => Err(py_unsupported_variant_err(format!( - "Converting Expr::OuterReferenceColumn to a Python object is not implemented: {:?} - {:?}", - data_type, column + "Converting Expr::OuterReferenceColumn to a Python object is not implemented: {data_type:?} - {column:?}" ))), Expr::Unnest(value) => Ok(unnest_expr::PyUnnestExpr::from(value.clone()).into_bound_py_any(py)?), } @@ -755,8 +751,7 @@ impl PyExpr { Expr::Cast(Cast { expr: _, data_type }) => DataTypeMap::map_from_arrow_type(data_type), Expr::Literal(scalar_value, _) => DataTypeMap::map_from_scalar_value(scalar_value), _ => Err(py_type_err(format!( - "Non Expr::Literal encountered in types: {:?}", - expr + "Non Expr::Literal encountered in types: {expr:?}" ))), } } diff --git a/src/expr/aggregate.rs b/src/expr/aggregate.rs index a99d83d23..fd4393271 100644 --- a/src/expr/aggregate.rs +++ b/src/expr/aggregate.rs @@ -116,7 +116,7 @@ impl PyAggregate { } fn __repr__(&self) -> PyResult { - Ok(format!("Aggregate({})", self)) + Ok(format!("Aggregate({self})")) } } diff --git a/src/expr/aggregate_expr.rs b/src/expr/aggregate_expr.rs index c09f116e3..7c5d3d31f 100644 --- a/src/expr/aggregate_expr.rs +++ b/src/expr/aggregate_expr.rs @@ -75,6 +75,6 @@ impl PyAggregateFunction { /// Get a String representation of this column fn __repr__(&self) -> String { - format!("{}", self) + format!("{self}") } } diff --git a/src/expr/alias.rs b/src/expr/alias.rs index e8e03cfad..40746f200 100644 --- a/src/expr/alias.rs +++ b/src/expr/alias.rs @@ -64,6 +64,6 @@ impl PyAlias { /// Get a String representation of this column fn __repr__(&self) -> String { - format!("{}", self) + format!("{self}") } } diff --git a/src/expr/analyze.rs b/src/expr/analyze.rs index 62f93cd26..e8081e95b 100644 --- a/src/expr/analyze.rs +++ b/src/expr/analyze.rs @@ -69,7 +69,7 @@ impl PyAnalyze { } fn __repr__(&self) -> PyResult { - Ok(format!("Analyze({})", self)) + Ok(format!("Analyze({self})")) } } diff --git a/src/expr/between.rs b/src/expr/between.rs index a2cac1442..817f1baae 100644 --- a/src/expr/between.rs +++ b/src/expr/between.rs @@ -71,6 +71,6 @@ impl PyBetween { } fn __repr__(&self) -> String { - format!("{}", self) + format!("{self}") } } diff --git a/src/expr/column.rs b/src/expr/column.rs index 365dbc0d2..50f316f1c 100644 --- a/src/expr/column.rs +++ b/src/expr/column.rs @@ -45,7 +45,7 @@ impl PyColumn { /// Get the column relation fn relation(&self) -> Option { - self.col.relation.as_ref().map(|r| format!("{}", r)) + self.col.relation.as_ref().map(|r| format!("{r}")) } /// Get the fully-qualified column name diff --git a/src/expr/copy_to.rs b/src/expr/copy_to.rs index ebfcb8ebc..473dabfed 100644 --- a/src/expr/copy_to.rs +++ b/src/expr/copy_to.rs @@ -106,7 +106,7 @@ impl PyCopyTo { } fn __repr__(&self) -> PyResult { - Ok(format!("CopyTo({})", self)) + Ok(format!("CopyTo({self})")) } fn __name__(&self) -> PyResult { @@ -129,7 +129,7 @@ impl Display for PyFileType { #[pymethods] impl PyFileType { fn __repr__(&self) -> PyResult { - Ok(format!("FileType({})", self)) + Ok(format!("FileType({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_catalog.rs b/src/expr/create_catalog.rs index f4ea0f517..d2d2ee8f6 100644 --- a/src/expr/create_catalog.rs +++ b/src/expr/create_catalog.rs @@ -81,7 +81,7 @@ impl PyCreateCatalog { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateCatalog({})", self)) + Ok(format!("CreateCatalog({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_catalog_schema.rs b/src/expr/create_catalog_schema.rs index 85f447e1e..e794962f5 100644 --- a/src/expr/create_catalog_schema.rs +++ b/src/expr/create_catalog_schema.rs @@ -81,7 +81,7 @@ impl PyCreateCatalogSchema { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateCatalogSchema({})", self)) + Ok(format!("CreateCatalogSchema({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_external_table.rs b/src/expr/create_external_table.rs index 01ce7d0ca..3e35af006 100644 --- a/src/expr/create_external_table.rs +++ b/src/expr/create_external_table.rs @@ -164,7 +164,7 @@ impl PyCreateExternalTable { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateExternalTable({})", self)) + Ok(format!("CreateExternalTable({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_function.rs b/src/expr/create_function.rs index 6f3c3f0ff..c02ceebb1 100644 --- a/src/expr/create_function.rs +++ b/src/expr/create_function.rs @@ -163,7 +163,7 @@ impl PyCreateFunction { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateFunction({})", self)) + Ok(format!("CreateFunction({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_index.rs b/src/expr/create_index.rs index 13dadbc3f..0f4b5011a 100644 --- a/src/expr/create_index.rs +++ b/src/expr/create_index.rs @@ -110,7 +110,7 @@ impl PyCreateIndex { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateIndex({})", self)) + Ok(format!("CreateIndex({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_memory_table.rs b/src/expr/create_memory_table.rs index 8872b2d47..37f4d3420 100644 --- a/src/expr/create_memory_table.rs +++ b/src/expr/create_memory_table.rs @@ -78,7 +78,7 @@ impl PyCreateMemoryTable { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateMemoryTable({})", self)) + Ok(format!("CreateMemoryTable({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/create_view.rs b/src/expr/create_view.rs index 87bb76876..718e404d0 100644 --- a/src/expr/create_view.rs +++ b/src/expr/create_view.rs @@ -75,7 +75,7 @@ impl PyCreateView { } fn __repr__(&self) -> PyResult { - Ok(format!("CreateView({})", self)) + Ok(format!("CreateView({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/describe_table.rs b/src/expr/describe_table.rs index 5658a13f2..6c48f3c77 100644 --- a/src/expr/describe_table.rs +++ b/src/expr/describe_table.rs @@ -61,7 +61,7 @@ impl PyDescribeTable { } fn __repr__(&self) -> PyResult { - Ok(format!("DescribeTable({})", self)) + Ok(format!("DescribeTable({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/distinct.rs b/src/expr/distinct.rs index b62b776f8..889e7099d 100644 --- a/src/expr/distinct.rs +++ b/src/expr/distinct.rs @@ -48,8 +48,7 @@ impl Display for PyDistinct { Distinct::All(input) => write!( f, "Distinct ALL - \nInput: {:?}", - input, + \nInput: {input:?}", ), Distinct::On(distinct_on) => { write!( @@ -71,7 +70,7 @@ impl PyDistinct { } fn __repr__(&self) -> PyResult { - Ok(format!("Distinct({})", self)) + Ok(format!("Distinct({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/drop_catalog_schema.rs b/src/expr/drop_catalog_schema.rs index b7420a99c..b4a4c521c 100644 --- a/src/expr/drop_catalog_schema.rs +++ b/src/expr/drop_catalog_schema.rs @@ -101,7 +101,7 @@ impl PyDropCatalogSchema { } fn __repr__(&self) -> PyResult { - Ok(format!("DropCatalogSchema({})", self)) + Ok(format!("DropCatalogSchema({self})")) } } diff --git a/src/expr/drop_function.rs b/src/expr/drop_function.rs index 9fbd78fdc..fca9eb94b 100644 --- a/src/expr/drop_function.rs +++ b/src/expr/drop_function.rs @@ -76,7 +76,7 @@ impl PyDropFunction { } fn __repr__(&self) -> PyResult { - Ok(format!("DropFunction({})", self)) + Ok(format!("DropFunction({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/drop_table.rs b/src/expr/drop_table.rs index 96983c1cf..3f442539a 100644 --- a/src/expr/drop_table.rs +++ b/src/expr/drop_table.rs @@ -70,7 +70,7 @@ impl PyDropTable { } fn __repr__(&self) -> PyResult { - Ok(format!("DropTable({})", self)) + Ok(format!("DropTable({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/drop_view.rs b/src/expr/drop_view.rs index 1d1ab1e59..6196c8bb5 100644 --- a/src/expr/drop_view.rs +++ b/src/expr/drop_view.rs @@ -83,7 +83,7 @@ impl PyDropView { } fn __repr__(&self) -> PyResult { - Ok(format!("DropView({})", self)) + Ok(format!("DropView({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/empty_relation.rs b/src/expr/empty_relation.rs index a1534ac15..758213423 100644 --- a/src/expr/empty_relation.rs +++ b/src/expr/empty_relation.rs @@ -65,7 +65,7 @@ impl PyEmptyRelation { /// Get a String representation of this column fn __repr__(&self) -> String { - format!("{}", self) + format!("{self}") } fn __name__(&self) -> PyResult { diff --git a/src/expr/filter.rs b/src/expr/filter.rs index 9bdb667cd..4fcb600cd 100644 --- a/src/expr/filter.rs +++ b/src/expr/filter.rs @@ -72,7 +72,7 @@ impl PyFilter { } fn __repr__(&self) -> String { - format!("Filter({})", self) + format!("Filter({self})") } } diff --git a/src/expr/join.rs b/src/expr/join.rs index 76ec532e7..b8d1d9da7 100644 --- a/src/expr/join.rs +++ b/src/expr/join.rs @@ -177,7 +177,7 @@ impl PyJoin { } fn __repr__(&self) -> PyResult { - Ok(format!("Join({})", self)) + Ok(format!("Join({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/like.rs b/src/expr/like.rs index 2e1f060bd..f180f5d4c 100644 --- a/src/expr/like.rs +++ b/src/expr/like.rs @@ -75,7 +75,7 @@ impl PyLike { } fn __repr__(&self) -> String { - format!("Like({})", self) + format!("Like({self})") } } @@ -133,7 +133,7 @@ impl PyILike { } fn __repr__(&self) -> String { - format!("Like({})", self) + format!("Like({self})") } } @@ -191,6 +191,6 @@ impl PySimilarTo { } fn __repr__(&self) -> String { - format!("Like({})", self) + format!("Like({self})") } } diff --git a/src/expr/limit.rs b/src/expr/limit.rs index c2a33ff89..92552814e 100644 --- a/src/expr/limit.rs +++ b/src/expr/limit.rs @@ -81,7 +81,7 @@ impl PyLimit { } fn __repr__(&self) -> PyResult { - Ok(format!("Limit({})", self)) + Ok(format!("Limit({self})")) } } diff --git a/src/expr/projection.rs b/src/expr/projection.rs index dc7e5e3c1..b5a9ef34a 100644 --- a/src/expr/projection.rs +++ b/src/expr/projection.rs @@ -85,7 +85,7 @@ impl PyProjection { } fn __repr__(&self) -> PyResult { - Ok(format!("Projection({})", self)) + Ok(format!("Projection({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/recursive_query.rs b/src/expr/recursive_query.rs index 65181f7d3..2517b7417 100644 --- a/src/expr/recursive_query.rs +++ b/src/expr/recursive_query.rs @@ -89,7 +89,7 @@ impl PyRecursiveQuery { } fn __repr__(&self) -> PyResult { - Ok(format!("RecursiveQuery({})", self)) + Ok(format!("RecursiveQuery({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/repartition.rs b/src/expr/repartition.rs index 3e782d6af..48b5e7041 100644 --- a/src/expr/repartition.rs +++ b/src/expr/repartition.rs @@ -108,7 +108,7 @@ impl PyRepartition { } fn __repr__(&self) -> PyResult { - Ok(format!("Repartition({})", self)) + Ok(format!("Repartition({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/sort.rs b/src/expr/sort.rs index ed4947591..79a8aee50 100644 --- a/src/expr/sort.rs +++ b/src/expr/sort.rs @@ -87,7 +87,7 @@ impl PySort { } fn __repr__(&self) -> PyResult { - Ok(format!("Sort({})", self)) + Ok(format!("Sort({self})")) } } diff --git a/src/expr/sort_expr.rs b/src/expr/sort_expr.rs index 12f74e4d8..79e35d978 100644 --- a/src/expr/sort_expr.rs +++ b/src/expr/sort_expr.rs @@ -85,6 +85,6 @@ impl PySortExpr { } fn __repr__(&self) -> String { - format!("{}", self) + format!("{self}") } } diff --git a/src/expr/subquery.rs b/src/expr/subquery.rs index 5ebfe6927..77f56f9a9 100644 --- a/src/expr/subquery.rs +++ b/src/expr/subquery.rs @@ -62,7 +62,7 @@ impl PySubquery { } fn __repr__(&self) -> PyResult { - Ok(format!("Subquery({})", self)) + Ok(format!("Subquery({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/subquery_alias.rs b/src/expr/subquery_alias.rs index 267a4d485..3302e7f23 100644 --- a/src/expr/subquery_alias.rs +++ b/src/expr/subquery_alias.rs @@ -72,7 +72,7 @@ impl PySubqueryAlias { } fn __repr__(&self) -> PyResult { - Ok(format!("SubqueryAlias({})", self)) + Ok(format!("SubqueryAlias({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/table_scan.rs b/src/expr/table_scan.rs index 6a0d53f0f..329964687 100644 --- a/src/expr/table_scan.rs +++ b/src/expr/table_scan.rs @@ -136,7 +136,7 @@ impl PyTableScan { } fn __repr__(&self) -> PyResult { - Ok(format!("TableScan({})", self)) + Ok(format!("TableScan({self})")) } } diff --git a/src/expr/union.rs b/src/expr/union.rs index 5a08ccc13..e0b221398 100644 --- a/src/expr/union.rs +++ b/src/expr/union.rs @@ -66,7 +66,7 @@ impl PyUnion { } fn __repr__(&self) -> PyResult { - Ok(format!("Union({})", self)) + Ok(format!("Union({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/unnest.rs b/src/expr/unnest.rs index 8e70e0990..c8833347f 100644 --- a/src/expr/unnest.rs +++ b/src/expr/unnest.rs @@ -66,7 +66,7 @@ impl PyUnnest { } fn __repr__(&self) -> PyResult { - Ok(format!("Unnest({})", self)) + Ok(format!("Unnest({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/unnest_expr.rs b/src/expr/unnest_expr.rs index 2234d24b1..634186ed8 100644 --- a/src/expr/unnest_expr.rs +++ b/src/expr/unnest_expr.rs @@ -58,7 +58,7 @@ impl PyUnnestExpr { } fn __repr__(&self) -> PyResult { - Ok(format!("UnnestExpr({})", self)) + Ok(format!("UnnestExpr({self})")) } fn __name__(&self) -> PyResult { diff --git a/src/expr/window.rs b/src/expr/window.rs index 052d9eeb4..a408731c2 100644 --- a/src/expr/window.rs +++ b/src/expr/window.rs @@ -185,8 +185,7 @@ impl PyWindowFrame { "groups" => WindowFrameUnits::Groups, _ => { return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - units, + "{units:?}", )))); } }; @@ -197,8 +196,7 @@ impl PyWindowFrame { WindowFrameUnits::Rows => WindowFrameBound::Preceding(ScalarValue::UInt64(None)), WindowFrameUnits::Groups => { return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - units, + "{units:?}", )))); } }, @@ -210,8 +208,7 @@ impl PyWindowFrame { WindowFrameUnits::Range => WindowFrameBound::Following(ScalarValue::UInt64(None)), WindowFrameUnits::Groups => { return Err(py_datafusion_err(DataFusionError::NotImplemented(format!( - "{:?}", - units, + "{units:?}", )))); } }, @@ -236,7 +233,7 @@ impl PyWindowFrame { /// Get a String representation of this window frame fn __repr__(&self) -> String { - format!("{}", self) + format!("{self}") } } diff --git a/src/functions.rs b/src/functions.rs index b2bafcb65..b40500b8b 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -937,7 +937,7 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(left))?; m.add_wrapped(wrap_pyfunction!(length))?; m.add_wrapped(wrap_pyfunction!(ln))?; - m.add_wrapped(wrap_pyfunction!(log))?; + m.add_wrapped(wrap_pyfunction!(self::log))?; m.add_wrapped(wrap_pyfunction!(log10))?; m.add_wrapped(wrap_pyfunction!(log2))?; m.add_wrapped(wrap_pyfunction!(lower))?; diff --git a/src/lib.rs b/src/lib.rs index 1293eee3c..29d3f41da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -77,10 +77,10 @@ pub(crate) struct TokioRuntime(tokio::runtime::Runtime); /// datafusion directory. #[pymodule] fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { + // Initialize logging + pyo3_log::init(); + // Register the python classes - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; @@ -98,6 +98,10 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; + let catalog = PyModule::new(py, "catalog")?; + catalog::init_module(&catalog)?; + m.add_submodule(&catalog)?; + // Register `common` as a submodule. Matching `datafusion-common` https://docs.rs/datafusion-common/latest/datafusion_common/ let common = PyModule::new(py, "common")?; common::init_module(&common)?; diff --git a/src/physical_plan.rs b/src/physical_plan.rs index f0be45c6a..49db643e1 100644 --- a/src/physical_plan.rs +++ b/src/physical_plan.rs @@ -78,8 +78,7 @@ impl PyExecutionPlan { let proto_plan = datafusion_proto::protobuf::PhysicalPlanNode::decode(bytes).map_err(|e| { PyRuntimeError::new_err(format!( - "Unable to decode logical node from serialized bytes: {}", - e + "Unable to decode logical node from serialized bytes: {e}" )) })?; diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 198d68bdc..97d320470 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -201,8 +201,7 @@ impl PyLogicalPlan { let proto_plan = datafusion_proto::protobuf::LogicalPlanNode::decode(bytes).map_err(|e| { PyRuntimeError::new_err(format!( - "Unable to decode logical node from serialized bytes: {}", - e + "Unable to decode logical node from serialized bytes: {e}" )) })?; diff --git a/src/utils.rs b/src/utils.rs index f4e121fd5..3b30de5de 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -109,8 +109,7 @@ pub(crate) fn validate_pycapsule(capsule: &Bound, name: &str) -> PyRe let capsule_name = capsule_name.unwrap().to_str()?; if capsule_name != name { return Err(PyValueError::new_err(format!( - "Expected name '{}' in PyCapsule, instead got '{}'", - name, capsule_name + "Expected name '{name}' in PyCapsule, instead got '{capsule_name}'" ))); } @@ -127,7 +126,7 @@ pub(crate) fn py_obj_to_scalar_value(py: Python, obj: PyObject) -> PyResult Date: Wed, 2 Jul 2025 09:59:02 -0400 Subject: [PATCH 314/413] feat: add FFI support for user defined functions (#1145) * Intermediate work adding ffi scalar udf * Add scalar UDF and example * Add aggregate udf via ffi * Initial commit for window ffi integration * Remove unused import --- docs/source/contributor-guide/ffi.rst | 2 +- examples/datafusion-ffi-example/Cargo.lock | 217 ++++++++++-------- examples/datafusion-ffi-example/Cargo.toml | 8 +- .../python/tests/_test_aggregate_udf.py | 77 +++++++ .../python/tests/_test_scalar_udf.py | 70 ++++++ .../python/tests/_test_window_udf.py | 89 +++++++ .../src/aggregate_udf.rs | 81 +++++++ .../src/catalog_provider.rs | 1 - examples/datafusion-ffi-example/src/lib.rs | 9 + .../datafusion-ffi-example/src/scalar_udf.rs | 91 ++++++++ .../datafusion-ffi-example/src/window_udf.rs | 81 +++++++ python/datafusion/user_defined.py | 107 ++++++++- src/functions.rs | 2 +- src/udaf.rs | 31 ++- src/udf.rs | 25 +- src/udwf.rs | 27 ++- 16 files changed, 805 insertions(+), 113 deletions(-) create mode 100644 examples/datafusion-ffi-example/python/tests/_test_aggregate_udf.py create mode 100644 examples/datafusion-ffi-example/python/tests/_test_scalar_udf.py create mode 100644 examples/datafusion-ffi-example/python/tests/_test_window_udf.py create mode 100644 examples/datafusion-ffi-example/src/aggregate_udf.rs create mode 100644 examples/datafusion-ffi-example/src/scalar_udf.rs create mode 100644 examples/datafusion-ffi-example/src/window_udf.rs diff --git a/docs/source/contributor-guide/ffi.rst b/docs/source/contributor-guide/ffi.rst index c1f9806b3..a40af1234 100644 --- a/docs/source/contributor-guide/ffi.rst +++ b/docs/source/contributor-guide/ffi.rst @@ -176,7 +176,7 @@ By convention the ``datafusion-python`` library expects a Python object that has ``TableProvider`` PyCapsule to have this capsule accessible by calling a function named ``__datafusion_table_provider__``. You can see a complete working example of how to share a ``TableProvider`` from one python library to DataFusion Python in the -`repository examples folder `_. +`repository examples folder `_. This section has been written using ``TableProvider`` as an example. It is the first extension that has been written using this approach and the most thoroughly implemented. diff --git a/examples/datafusion-ffi-example/Cargo.lock b/examples/datafusion-ffi-example/Cargo.lock index e5a1ca8d1..1b4ca6bee 100644 --- a/examples/datafusion-ffi-example/Cargo.lock +++ b/examples/datafusion-ffi-example/Cargo.lock @@ -323,6 +323,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" dependencies = [ "bitflags", + "serde", + "serde_json", ] [[package]] @@ -748,9 +750,9 @@ dependencies = [ [[package]] name = "datafusion" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe060b978f74ab446be722adb8a274e052e005bf6dfd171caadc3abaad10080" +checksum = "cc6cb8c2c81eada072059983657d6c9caf3fddefc43b4a65551d243253254a96" dependencies = [ "arrow", "arrow-ipc", @@ -775,7 +777,6 @@ dependencies = [ "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", - "datafusion-macros", "datafusion-optimizer", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -790,7 +791,7 @@ dependencies = [ "object_store", "parking_lot", "parquet", - "rand", + "rand 0.9.1", "regex", "sqlparser", "tempfile", @@ -803,9 +804,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fe34f401bd03724a1f96d12108144f8cd495a3cdda2bf5e091822fb80b7e66" +checksum = "b7be8d1b627843af62e447396db08fe1372d882c0eb8d0ea655fd1fbc33120ee" dependencies = [ "arrow", "async-trait", @@ -829,9 +830,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4411b8e3bce5e0fc7521e44f201def2e2d5d1b5f176fb56e8cdc9942c890f00" +checksum = "38ab16c5ae43f65ee525fc493ceffbc41f40dee38b01f643dfcfc12959e92038" dependencies = [ "arrow", "async-trait", @@ -852,9 +853,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0734015d81c8375eb5d4869b7f7ecccc2ee8d6cb81948ef737cd0e7b743bd69c" +checksum = "d3d56b2ac9f476b93ca82e4ef5fb00769c8a3f248d12b4965af7e27635fa7e12" dependencies = [ "ahash", "arrow", @@ -876,9 +877,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5167bb1d2ccbb87c6bc36c295274d7a0519b14afcfdaf401d53cbcaa4ef4968b" +checksum = "16015071202d6133bc84d72756176467e3e46029f3ce9ad2cb788f9b1ff139b2" dependencies = [ "futures", "log", @@ -887,9 +888,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e602dcdf2f50c2abf297cc2203c73531e6f48b29516af7695d338cf2a778b1" +checksum = "b77523c95c89d2a7eb99df14ed31390e04ab29b43ff793e562bdc1716b07e17b" dependencies = [ "arrow", "async-compression", @@ -912,7 +913,7 @@ dependencies = [ "log", "object_store", "parquet", - "rand", + "rand 0.9.1", "tempfile", "tokio", "tokio-util", @@ -923,9 +924,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bb2253952dc32296ed5b84077cb2e0257fea4be6373e1c376426e17ead4ef6" +checksum = "40d25c5e2c0ebe8434beeea997b8e88d55b3ccc0d19344293f2373f65bc524fc" dependencies = [ "arrow", "async-trait", @@ -948,9 +949,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8c7f47a5d2fe03bfa521ec9bafdb8a5c82de8377f60967c3663f00c8790352" +checksum = "3dc6959e1155741ab35369e1dc7673ba30fc45ed568fad34c01b7cb1daeb4d4c" dependencies = [ "arrow", "async-trait", @@ -973,9 +974,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27d15868ea39ed2dc266728b554f6304acd473de2142281ecfa1294bb7415923" +checksum = "b7a6afdfe358d70f4237f60eaef26ae5a1ce7cb2c469d02d5fc6c7fd5d84e58b" dependencies = [ "arrow", "async-trait", @@ -998,21 +999,21 @@ dependencies = [ "object_store", "parking_lot", "parquet", - "rand", + "rand 0.9.1", "tokio", ] [[package]] name = "datafusion-doc" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a91f8c2c5788ef32f48ff56c68e5b545527b744822a284373ac79bba1ba47292" +checksum = "9bcd8a3e3e3d02ea642541be23d44376b5d5c37c2938cce39b3873cdf7186eea" [[package]] name = "datafusion-execution" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06f004d100f49a3658c9da6fb0c3a9b760062d96cd4ad82ccc3b7b69a9fb2f84" +checksum = "670da1d45d045eee4c2319b8c7ea57b26cf48ab77b630aaa50b779e406da476a" dependencies = [ "arrow", "dashmap", @@ -1022,16 +1023,16 @@ dependencies = [ "log", "object_store", "parking_lot", - "rand", + "rand 0.9.1", "tempfile", "url", ] [[package]] name = "datafusion-expr" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a4e4ce3802609be38eeb607ee72f6fe86c3091460de9dbfae9e18db423b3964" +checksum = "b3a577f64bdb7e2cc4043cd97f8901d8c504711fde2dbcb0887645b00d7c660b" dependencies = [ "arrow", "chrono", @@ -1050,9 +1051,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422ac9cf3b22bbbae8cdf8ceb33039107fde1b5492693168f13bd566b1bcc839" +checksum = "51b7916806ace3e9f41884f230f7f38ebf0e955dfbd88266da1826f29a0b9a6a" dependencies = [ "arrow", "datafusion-common", @@ -1063,9 +1064,9 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cf3fe9ab492c56daeb7beed526690d33622d388b8870472e0b7b7f55490338c" +checksum = "980cca31de37f5dadf7ea18e4ffc2b6833611f45bed5ef9de0831d2abb50f1ef" dependencies = [ "abi_stable", "arrow", @@ -1073,7 +1074,9 @@ dependencies = [ "async-ffi", "async-trait", "datafusion", + "datafusion-functions-aggregate-common", "datafusion-proto", + "datafusion-proto-common", "futures", "log", "prost", @@ -1081,11 +1084,25 @@ dependencies = [ "tokio", ] +[[package]] +name = "datafusion-ffi-example" +version = "0.2.0" +dependencies = [ + "arrow", + "arrow-array", + "arrow-schema", + "async-trait", + "datafusion", + "datafusion-ffi", + "pyo3", + "pyo3-build-config", +] + [[package]] name = "datafusion-functions" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ddf0a0a2db5d2918349c978d42d80926c6aa2459cd8a3c533a84ec4bb63479e" +checksum = "7fb31c9dc73d3e0c365063f91139dc273308f8a8e124adda9898db8085d68357" dependencies = [ "arrow", "arrow-buffer", @@ -1103,7 +1120,7 @@ dependencies = [ "itertools", "log", "md-5", - "rand", + "rand 0.9.1", "regex", "sha2", "unicode-segmentation", @@ -1112,9 +1129,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "408a05dafdc70d05a38a29005b8b15e21b0238734dab1e98483fcb58038c5aba" +checksum = "ebb72c6940697eaaba9bd1f746a697a07819de952b817e3fb841fb75331ad5d4" dependencies = [ "ahash", "arrow", @@ -1133,9 +1150,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "756d21da2dd6c9bef97af1504970ff56cbf35d03fbd4ffd62827f02f4d2279d4" +checksum = "d7fdc54656659e5ecd49bf341061f4156ab230052611f4f3609612a0da259696" dependencies = [ "ahash", "arrow", @@ -1146,9 +1163,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8d50f6334b378930d992d801a10ac5b3e93b846b39e4a05085742572844537" +checksum = "fad94598e3374938ca43bca6b675febe557e7a14eb627d617db427d70d65118b" dependencies = [ "arrow", "arrow-ord", @@ -1167,9 +1184,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc9a97220736c8fff1446e936be90d57216c06f28969f9ffd3b72ac93c958c8a" +checksum = "de2fc6c2946da5cab8364fb28b5cac3115f0f3a87960b235ed031c3f7e2e639b" dependencies = [ "arrow", "async-trait", @@ -1183,10 +1200,11 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cefc2d77646e1aadd1d6a9c40088937aedec04e68c5f0465939912e1291f8193" +checksum = "3e5746548a8544870a119f556543adcd88fe0ba6b93723fe78ad0439e0fbb8b4" dependencies = [ + "arrow", "datafusion-common", "datafusion-doc", "datafusion-expr", @@ -1200,9 +1218,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd4aff082c42fa6da99ce0698c85addd5252928c908eb087ca3cfa64ff16b313" +checksum = "dcbe9404382cda257c434f22e13577bee7047031dfdb6216dd5e841b9465e6fe" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1210,9 +1228,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" +checksum = "8dce50e3b637dab0d25d04d2fe79dfdca2b257eabd76790bffd22c7f90d700c8" dependencies = [ "datafusion-expr", "quote", @@ -1221,9 +1239,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084d9f979c4b155346d3c34b18f4256e6904ded508e9554d90fed416415c3515" +checksum = "03cfaacf06445dc3bbc1e901242d2a44f2cae99a744f49f3fefddcee46240058" dependencies = [ "arrow", "chrono", @@ -1240,9 +1258,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c536062b0076f4e30084065d805f389f9fe38af0ca75bcbac86bc5e9fbab65" +checksum = "1908034a89d7b2630898e06863583ae4c00a0dd310c1589ca284195ee3f7f8a6" dependencies = [ "ahash", "arrow", @@ -1262,9 +1280,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8a92b53b3193fac1916a1c5b8e3f4347c526f6822e56b71faa5fb372327a863" +checksum = "47b7a12dd59ea07614b67dbb01d85254fbd93df45bcffa63495e11d3bdf847df" dependencies = [ "ahash", "arrow", @@ -1276,9 +1294,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa0a5ac94c7cf3da97bedabd69d6bbca12aef84b9b37e6e9e8c25286511b5e2" +checksum = "4371cc4ad33978cc2a8be93bd54a232d3f2857b50401a14631c0705f3f910aae" dependencies = [ "arrow", "datafusion-common", @@ -1295,9 +1313,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "690c615db468c2e5fe5085b232d8b1c088299a6c63d87fd960a354a71f7acb55" +checksum = "dc47bc33025757a5c11f2cd094c5b6b5ed87f46fa33c023e6fdfa25fcbfade23" dependencies = [ "ahash", "arrow", @@ -1325,9 +1343,9 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a1afb2bdb05de7ff65be6883ebfd4ec027bd9f1f21c46aa3afd01927160a83" +checksum = "d8f5d9acd7d96e3bf2a7bb04818373cab6e51de0356e3694b94905fee7b4e8b6" dependencies = [ "arrow", "chrono", @@ -1341,9 +1359,9 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b7a5876ebd6b564fb9a1fd2c3a2a9686b787071a256b47e4708f0916f9e46f" +checksum = "09ecb5ec152c4353b60f7a5635489834391f7a291d2b39a4820cd469e318b78e" dependencies = [ "arrow", "datafusion-common", @@ -1352,9 +1370,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad229a134c7406c057ece00c8743c0c34b97f4e72f78b475fe17b66c5e14fa4f" +checksum = "d7485da32283985d6b45bd7d13a65169dcbe8c869e25d01b2cfbc425254b4b49" dependencies = [ "arrow", "async-trait", @@ -1376,9 +1394,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "47.0.0" +version = "48.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f6ab28b72b664c21a27b22a2ff815fd390ed224c26e89a93b5a8154a4e8607" +checksum = "a466b15632befddfeac68c125f0260f569ff315c6831538cbb40db754134e0df" dependencies = [ "arrow", "bigdecimal", @@ -1441,20 +1459,6 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" -[[package]] -name = "ffi-table-provider" -version = "0.1.0" -dependencies = [ - "arrow", - "arrow-array", - "arrow-schema", - "async-trait", - "datafusion", - "datafusion-ffi", - "pyo3", - "pyo3-build-config", -] - [[package]] name = "fixedbitset" version = "0.5.7" @@ -1488,6 +1492,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1666,6 +1676,11 @@ name = "hashbrown" version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -2271,12 +2286,14 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "petgraph" -version = "0.7.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", + "hashbrown 0.15.3", "indexmap", + "serde", ] [[package]] @@ -2305,7 +2322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -2484,19 +2501,27 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "libc", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ "rand_chacha", - "rand_core", + "rand_core 0.9.3", ] [[package]] name = "rand_chacha" -version = "0.3.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.9.3", ] [[package]] @@ -2504,8 +2529,14 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.3.3", ] [[package]] @@ -3032,9 +3063,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "getrandom 0.3.3", "js-sys", diff --git a/examples/datafusion-ffi-example/Cargo.toml b/examples/datafusion-ffi-example/Cargo.toml index 319163554..b26ab48e3 100644 --- a/examples/datafusion-ffi-example/Cargo.toml +++ b/examples/datafusion-ffi-example/Cargo.toml @@ -16,13 +16,13 @@ # under the License. [package] -name = "ffi-table-provider" -version = "0.1.0" +name = "datafusion-ffi-example" +version = "0.2.0" edition = "2021" [dependencies] -datafusion = { version = "47.0.0" } -datafusion-ffi = { version = "47.0.0" } +datafusion = { version = "48.0.0" } +datafusion-ffi = { version = "48.0.0" } pyo3 = { version = "0.23", features = ["extension-module", "abi3", "abi3-py39"] } arrow = { version = "55.0.0" } arrow-array = { version = "55.0.0" } diff --git a/examples/datafusion-ffi-example/python/tests/_test_aggregate_udf.py b/examples/datafusion-ffi-example/python/tests/_test_aggregate_udf.py new file mode 100644 index 000000000..7ea6b295c --- /dev/null +++ b/examples/datafusion-ffi-example/python/tests/_test_aggregate_udf.py @@ -0,0 +1,77 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import pyarrow as pa +from datafusion import SessionContext, col, udaf +from datafusion_ffi_example import MySumUDF + + +def setup_context_with_table(): + ctx = SessionContext() + + # Pick numbers here so we get the same value in both groups + # since we cannot be certain of the output order of batches + batch = pa.RecordBatch.from_arrays( + [ + pa.array([1, 2, 3, None], type=pa.int64()), + pa.array([1, 1, 2, 2], type=pa.int64()), + ], + names=["a", "b"], + ) + ctx.register_record_batches("test_table", [[batch]]) + return ctx + + +def test_ffi_aggregate_register(): + ctx = setup_context_with_table() + my_udaf = udaf(MySumUDF()) + ctx.register_udaf(my_udaf) + + result = ctx.sql("select my_custom_sum(a) from test_table group by b").collect() + + assert len(result) == 2 + assert result[0].num_columns == 1 + + result = [r.column(0) for r in result] + expected = [ + pa.array([3], type=pa.int64()), + pa.array([3], type=pa.int64()), + ] + + assert result == expected + + +def test_ffi_aggregate_call_directly(): + ctx = setup_context_with_table() + my_udaf = udaf(MySumUDF()) + + result = ( + ctx.table("test_table").aggregate([col("b")], [my_udaf(col("a"))]).collect() + ) + + assert len(result) == 2 + assert result[0].num_columns == 2 + + result = [r.column(1) for r in result] + expected = [ + pa.array([3], type=pa.int64()), + pa.array([3], type=pa.int64()), + ] + + assert result == expected diff --git a/examples/datafusion-ffi-example/python/tests/_test_scalar_udf.py b/examples/datafusion-ffi-example/python/tests/_test_scalar_udf.py new file mode 100644 index 000000000..0c949c34a --- /dev/null +++ b/examples/datafusion-ffi-example/python/tests/_test_scalar_udf.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import pyarrow as pa +from datafusion import SessionContext, col, udf +from datafusion_ffi_example import IsNullUDF + + +def setup_context_with_table(): + ctx = SessionContext() + + batch = pa.RecordBatch.from_arrays( + [pa.array([1, 2, 3, None])], + names=["a"], + ) + ctx.register_record_batches("test_table", [[batch]]) + return ctx + + +def test_ffi_scalar_register(): + ctx = setup_context_with_table() + my_udf = udf(IsNullUDF()) + ctx.register_udf(my_udf) + + result = ctx.sql("select my_custom_is_null(a) from test_table").collect() + + assert len(result) == 1 + assert result[0].num_columns == 1 + print(result) + + result = [r.column(0) for r in result] + expected = [ + pa.array([False, False, False, True], type=pa.bool_()), + ] + + assert result == expected + + +def test_ffi_scalar_call_directly(): + ctx = setup_context_with_table() + my_udf = udf(IsNullUDF()) + + result = ctx.table("test_table").select(my_udf(col("a"))).collect() + + assert len(result) == 1 + assert result[0].num_columns == 1 + print(result) + + result = [r.column(0) for r in result] + expected = [ + pa.array([False, False, False, True], type=pa.bool_()), + ] + + assert result == expected diff --git a/examples/datafusion-ffi-example/python/tests/_test_window_udf.py b/examples/datafusion-ffi-example/python/tests/_test_window_udf.py new file mode 100644 index 000000000..7d96994b9 --- /dev/null +++ b/examples/datafusion-ffi-example/python/tests/_test_window_udf.py @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +import pyarrow as pa +from datafusion import SessionContext, col, udwf +from datafusion_ffi_example import MyRankUDF + + +def setup_context_with_table(): + ctx = SessionContext() + + # Pick numbers here so we get the same value in both groups + # since we cannot be certain of the output order of batches + batch = pa.RecordBatch.from_arrays( + [ + pa.array([40, 10, 30, 20], type=pa.int64()), + ], + names=["a"], + ) + ctx.register_record_batches("test_table", [[batch]]) + return ctx + + +def test_ffi_window_register(): + ctx = setup_context_with_table() + my_udwf = udwf(MyRankUDF()) + ctx.register_udwf(my_udwf) + + result = ctx.sql( + "select a, my_custom_rank() over (order by a) from test_table" + ).collect() + assert len(result) == 1 + assert result[0].num_columns == 2 + + results = [ + (result[0][0][idx].as_py(), result[0][1][idx].as_py()) for idx in range(4) + ] + results.sort() + + expected = [ + (10, 1), + (20, 2), + (30, 3), + (40, 4), + ] + assert results == expected + + +def test_ffi_window_call_directly(): + ctx = setup_context_with_table() + my_udwf = udwf(MyRankUDF()) + + result = ( + ctx.table("test_table") + .select(col("a"), my_udwf().order_by(col("a")).build()) + .collect() + ) + + assert len(result) == 1 + assert result[0].num_columns == 2 + + results = [ + (result[0][0][idx].as_py(), result[0][1][idx].as_py()) for idx in range(4) + ] + results.sort() + + expected = [ + (10, 1), + (20, 2), + (30, 3), + (40, 4), + ] + assert results == expected diff --git a/examples/datafusion-ffi-example/src/aggregate_udf.rs b/examples/datafusion-ffi-example/src/aggregate_udf.rs new file mode 100644 index 000000000..9481fe9c6 --- /dev/null +++ b/examples/datafusion-ffi-example/src/aggregate_udf.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow_schema::DataType; +use datafusion::error::Result as DataFusionResult; +use datafusion::functions_aggregate::sum::Sum; +use datafusion::logical_expr::function::AccumulatorArgs; +use datafusion::logical_expr::{Accumulator, AggregateUDF, AggregateUDFImpl, Signature}; +use datafusion_ffi::udaf::FFI_AggregateUDF; +use pyo3::types::PyCapsule; +use pyo3::{pyclass, pymethods, Bound, PyResult, Python}; +use std::any::Any; +use std::sync::Arc; + +#[pyclass(name = "MySumUDF", module = "datafusion_ffi_example", subclass)] +#[derive(Debug, Clone)] +pub(crate) struct MySumUDF { + inner: Arc, +} + +#[pymethods] +impl MySumUDF { + #[new] + fn new() -> Self { + Self { + inner: Arc::new(Sum::new()), + } + } + + fn __datafusion_aggregate_udf__<'py>( + &self, + py: Python<'py>, + ) -> PyResult> { + let name = cr"datafusion_aggregate_udf".into(); + + let func = Arc::new(AggregateUDF::from(self.clone())); + let provider = FFI_AggregateUDF::from(func); + + PyCapsule::new(py, provider, Some(name)) + } +} + +impl AggregateUDFImpl for MySumUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "my_custom_sum" + } + + fn signature(&self) -> &Signature { + self.inner.signature() + } + + fn return_type(&self, arg_types: &[DataType]) -> DataFusionResult { + self.inner.return_type(arg_types) + } + + fn accumulator(&self, acc_args: AccumulatorArgs) -> DataFusionResult> { + self.inner.accumulator(acc_args) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> DataFusionResult> { + self.inner.coerce_types(arg_types) + } +} diff --git a/examples/datafusion-ffi-example/src/catalog_provider.rs b/examples/datafusion-ffi-example/src/catalog_provider.rs index 54e61cf3e..cd2616916 100644 --- a/examples/datafusion-ffi-example/src/catalog_provider.rs +++ b/examples/datafusion-ffi-example/src/catalog_provider.rs @@ -24,7 +24,6 @@ use datafusion::{ catalog::{ CatalogProvider, MemoryCatalogProvider, MemorySchemaProvider, SchemaProvider, TableProvider, }, - common::exec_err, datasource::MemTable, error::{DataFusionError, Result}, }; diff --git a/examples/datafusion-ffi-example/src/lib.rs b/examples/datafusion-ffi-example/src/lib.rs index 3a4cf2247..79af276fd 100644 --- a/examples/datafusion-ffi-example/src/lib.rs +++ b/examples/datafusion-ffi-example/src/lib.rs @@ -16,18 +16,27 @@ // under the License. use crate::catalog_provider::MyCatalogProvider; +use crate::aggregate_udf::MySumUDF; +use crate::scalar_udf::IsNullUDF; use crate::table_function::MyTableFunction; use crate::table_provider::MyTableProvider; +use crate::window_udf::MyRankUDF; use pyo3::prelude::*; pub(crate) mod catalog_provider; +pub(crate) mod aggregate_udf; +pub(crate) mod scalar_udf; pub(crate) mod table_function; pub(crate) mod table_provider; +pub(crate) mod window_udf; #[pymodule] fn datafusion_ffi_example(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; Ok(()) } diff --git a/examples/datafusion-ffi-example/src/scalar_udf.rs b/examples/datafusion-ffi-example/src/scalar_udf.rs new file mode 100644 index 000000000..727666638 --- /dev/null +++ b/examples/datafusion-ffi-example/src/scalar_udf.rs @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow_array::{Array, BooleanArray}; +use arrow_schema::DataType; +use datafusion::common::ScalarValue; +use datafusion::error::Result as DataFusionResult; +use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, TypeSignature, + Volatility, +}; +use datafusion_ffi::udf::FFI_ScalarUDF; +use pyo3::types::PyCapsule; +use pyo3::{pyclass, pymethods, Bound, PyResult, Python}; +use std::any::Any; +use std::sync::Arc; + +#[pyclass(name = "IsNullUDF", module = "datafusion_ffi_example", subclass)] +#[derive(Debug, Clone)] +pub(crate) struct IsNullUDF { + signature: Signature, +} + +#[pymethods] +impl IsNullUDF { + #[new] + fn new() -> Self { + Self { + signature: Signature::new(TypeSignature::Any(1), Volatility::Immutable), + } + } + + fn __datafusion_scalar_udf__<'py>(&self, py: Python<'py>) -> PyResult> { + let name = cr"datafusion_scalar_udf".into(); + + let func = Arc::new(ScalarUDF::from(self.clone())); + let provider = FFI_ScalarUDF::from(func); + + PyCapsule::new(py, provider, Some(name)) + } +} + +impl ScalarUDFImpl for IsNullUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "my_custom_is_null" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> DataFusionResult { + Ok(DataType::Boolean) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DataFusionResult { + let input = &args.args[0]; + + Ok(match input { + ColumnarValue::Array(arr) => match arr.is_nullable() { + true => { + let nulls = arr.nulls().unwrap(); + let nulls = BooleanArray::from_iter(nulls.iter().map(|x| Some(!x))); + ColumnarValue::Array(Arc::new(nulls)) + } + false => ColumnarValue::Scalar(ScalarValue::Boolean(Some(false))), + }, + ColumnarValue::Scalar(sv) => { + ColumnarValue::Scalar(ScalarValue::Boolean(Some(sv == &ScalarValue::Null))) + } + }) + } +} diff --git a/examples/datafusion-ffi-example/src/window_udf.rs b/examples/datafusion-ffi-example/src/window_udf.rs new file mode 100644 index 000000000..e0d397956 --- /dev/null +++ b/examples/datafusion-ffi-example/src/window_udf.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow_schema::{DataType, FieldRef}; +use datafusion::error::Result as DataFusionResult; +use datafusion::functions_window::rank::rank_udwf; +use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; +use datafusion::logical_expr::{PartitionEvaluator, Signature, WindowUDF, WindowUDFImpl}; +use datafusion_ffi::udwf::FFI_WindowUDF; +use pyo3::types::PyCapsule; +use pyo3::{pyclass, pymethods, Bound, PyResult, Python}; +use std::any::Any; +use std::sync::Arc; + +#[pyclass(name = "MyRankUDF", module = "datafusion_ffi_example", subclass)] +#[derive(Debug, Clone)] +pub(crate) struct MyRankUDF { + inner: Arc, +} + +#[pymethods] +impl MyRankUDF { + #[new] + fn new() -> Self { + Self { inner: rank_udwf() } + } + + fn __datafusion_window_udf__<'py>(&self, py: Python<'py>) -> PyResult> { + let name = cr"datafusion_window_udf".into(); + + let func = Arc::new(WindowUDF::from(self.clone())); + let provider = FFI_WindowUDF::from(func); + + PyCapsule::new(py, provider, Some(name)) + } +} + +impl WindowUDFImpl for MyRankUDF { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "my_custom_rank" + } + + fn signature(&self) -> &Signature { + self.inner.signature() + } + + fn partition_evaluator( + &self, + partition_evaluator_args: PartitionEvaluatorArgs, + ) -> DataFusionResult> { + self.inner + .inner() + .partition_evaluator(partition_evaluator_args) + } + + fn field(&self, field_args: WindowUDFFieldArgs) -> DataFusionResult { + self.inner.inner().field(field_args) + } + + fn coerce_types(&self, arg_types: &[DataType]) -> DataFusionResult> { + self.inner.coerce_types(arg_types) + } +} diff --git a/python/datafusion/user_defined.py b/python/datafusion/user_defined.py index dd634c7fb..bd686acbb 100644 --- a/python/datafusion/user_defined.py +++ b/python/datafusion/user_defined.py @@ -22,7 +22,7 @@ import functools from abc import ABCMeta, abstractmethod from enum import Enum -from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, overload +from typing import TYPE_CHECKING, Any, Callable, Optional, Protocol, TypeVar, overload import pyarrow as pa @@ -77,6 +77,12 @@ def __str__(self) -> str: return self.name.lower() +class ScalarUDFExportable(Protocol): + """Type hint for object that has __datafusion_scalar_udf__ PyCapsule.""" + + def __datafusion_scalar_udf__(self) -> object: ... # noqa: D105 + + class ScalarUDF: """Class for performing scalar user-defined functions (UDF). @@ -96,6 +102,9 @@ def __init__( See helper method :py:func:`udf` for argument details. """ + if hasattr(func, "__datafusion_scalar_udf__"): + self._udf = df_internal.ScalarUDF.from_pycapsule(func) + return if isinstance(input_types, pa.DataType): input_types = [input_types] self._udf = df_internal.ScalarUDF( @@ -134,6 +143,10 @@ def udf( name: Optional[str] = None, ) -> ScalarUDF: ... + @overload + @staticmethod + def udf(func: ScalarUDFExportable) -> ScalarUDF: ... + @staticmethod def udf(*args: Any, **kwargs: Any): # noqa: D417 """Create a new User-Defined Function (UDF). @@ -147,7 +160,10 @@ def udf(*args: Any, **kwargs: Any): # noqa: D417 Args: func (Callable, optional): Only needed when calling as a function. - Skip this argument when using ``udf`` as a decorator. + Skip this argument when using `udf` as a decorator. If you have a Rust + backed ScalarUDF within a PyCapsule, you can pass this parameter + and ignore the rest. They will be determined directly from the + underlying function. See the online documentation for more information. input_types (list[pa.DataType]): The data types of the arguments to ``func``. This list must be of the same length as the number of arguments. @@ -215,12 +231,31 @@ def wrapper(*args: Any, **kwargs: Any): return decorator + if hasattr(args[0], "__datafusion_scalar_udf__"): + return ScalarUDF.from_pycapsule(args[0]) + if args and callable(args[0]): # Case 1: Used as a function, require the first parameter to be callable return _function(*args, **kwargs) # Case 2: Used as a decorator with parameters return _decorator(*args, **kwargs) + @staticmethod + def from_pycapsule(func: ScalarUDFExportable) -> ScalarUDF: + """Create a Scalar UDF from ScalarUDF PyCapsule object. + + This function will instantiate a Scalar UDF that uses a DataFusion + ScalarUDF that is exported via the FFI bindings. + """ + name = str(func.__class__) + return ScalarUDF( + name=name, + func=func, + input_types=None, + return_type=None, + volatility=None, + ) + class Accumulator(metaclass=ABCMeta): """Defines how an :py:class:`AggregateUDF` accumulates values.""" @@ -242,6 +277,12 @@ def evaluate(self) -> pa.Scalar: """Return the resultant value.""" +class AggregateUDFExportable(Protocol): + """Type hint for object that has __datafusion_aggregate_udf__ PyCapsule.""" + + def __datafusion_aggregate_udf__(self) -> object: ... # noqa: D105 + + class AggregateUDF: """Class for performing scalar user-defined functions (UDF). @@ -263,6 +304,9 @@ def __init__( See :py:func:`udaf` for a convenience function and argument descriptions. """ + if hasattr(accumulator, "__datafusion_aggregate_udf__"): + self._udaf = df_internal.AggregateUDF.from_pycapsule(accumulator) + return self._udaf = df_internal.AggregateUDF( name, accumulator, @@ -307,7 +351,7 @@ def udaf( ) -> AggregateUDF: ... @staticmethod - def udaf(*args: Any, **kwargs: Any): # noqa: D417 + def udaf(*args: Any, **kwargs: Any): # noqa: D417, C901 """Create a new User-Defined Aggregate Function (UDAF). This class allows you to define an aggregate function that can be used in @@ -364,6 +408,10 @@ def udf4() -> Summarize: Args: accum: The accumulator python function. Only needed when calling as a function. Skip this argument when using ``udaf`` as a decorator. + If you have a Rust backed AggregateUDF within a PyCapsule, you can + pass this parameter and ignore the rest. They will be determined + directly from the underlying function. See the online documentation + for more information. input_types: The data types of the arguments to ``accum``. return_type: The data type of the return value. state_type: The data types of the intermediate accumulation. @@ -422,12 +470,32 @@ def wrapper(*args: Any, **kwargs: Any) -> Expr: return decorator + if hasattr(args[0], "__datafusion_aggregate_udf__"): + return AggregateUDF.from_pycapsule(args[0]) + if args and callable(args[0]): # Case 1: Used as a function, require the first parameter to be callable return _function(*args, **kwargs) # Case 2: Used as a decorator with parameters return _decorator(*args, **kwargs) + @staticmethod + def from_pycapsule(func: AggregateUDFExportable) -> AggregateUDF: + """Create an Aggregate UDF from AggregateUDF PyCapsule object. + + This function will instantiate a Aggregate UDF that uses a DataFusion + AggregateUDF that is exported via the FFI bindings. + """ + name = str(func.__class__) + return AggregateUDF( + name=name, + accumulator=func, + input_types=None, + return_type=None, + state_type=None, + volatility=None, + ) + class WindowEvaluator: """Evaluator class for user-defined window functions (UDWF). @@ -588,6 +656,12 @@ def include_rank(self) -> bool: return False +class WindowUDFExportable(Protocol): + """Type hint for object that has __datafusion_window_udf__ PyCapsule.""" + + def __datafusion_window_udf__(self) -> object: ... # noqa: D105 + + class WindowUDF: """Class for performing window user-defined functions (UDF). @@ -608,6 +682,9 @@ def __init__( See :py:func:`udwf` for a convenience function and argument descriptions. """ + if hasattr(func, "__datafusion_window_udf__"): + self._udwf = df_internal.WindowUDF.from_pycapsule(func) + return self._udwf = df_internal.WindowUDF( name, func, input_types, return_type, str(volatility) ) @@ -683,7 +760,10 @@ def biased_numbers() -> BiasedNumbers: Args: func: Only needed when calling as a function. Skip this argument when - using ``udwf`` as a decorator. + using ``udwf`` as a decorator. If you have a Rust backed WindowUDF + within a PyCapsule, you can pass this parameter and ignore the rest. + They will be determined directly from the underlying function. See + the online documentation for more information. input_types: The data types of the arguments. return_type: The data type of the return value. volatility: See :py:class:`Volatility` for allowed values. @@ -692,6 +772,9 @@ def biased_numbers() -> BiasedNumbers: Returns: A user-defined window function that can be used in window function calls. """ + if hasattr(args[0], "__datafusion_window_udf__"): + return WindowUDF.from_pycapsule(args[0]) + if args and callable(args[0]): # Case 1: Used as a function, require the first parameter to be callable return WindowUDF._create_window_udf(*args, **kwargs) @@ -759,6 +842,22 @@ def wrapper(*args: Any, **kwargs: Any) -> Expr: return decorator + @staticmethod + def from_pycapsule(func: WindowUDFExportable) -> WindowUDF: + """Create a Window UDF from WindowUDF PyCapsule object. + + This function will instantiate a Window UDF that uses a DataFusion + WindowUDF that is exported via the FFI bindings. + """ + name = str(func.__class__) + return WindowUDF( + name=name, + func=func, + input_types=None, + return_type=None, + volatility=None, + ) + class TableFunction: """Class for performing user-defined table functions (UDTF). diff --git a/src/functions.rs b/src/functions.rs index b40500b8b..eeef48385 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -682,7 +682,7 @@ pub fn approx_percentile_cont_with_weight( add_builder_fns_to_aggregate(agg_fn, None, filter, None, None) } -// We handle first_value explicitly because the signature expects an order_by +// We handle last_value explicitly because the signature expects an order_by // https://github.com/apache/datafusion/issues/12376 #[pyfunction] #[pyo3(signature = (expr, distinct=None, filter=None, order_by=None, null_treatment=None))] diff --git a/src/udaf.rs b/src/udaf.rs index 34a9cd51d..78f4e2b0c 100644 --- a/src/udaf.rs +++ b/src/udaf.rs @@ -19,6 +19,10 @@ use std::sync::Arc; use pyo3::{prelude::*, types::PyTuple}; +use crate::common::data_type::PyScalarValue; +use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionResult}; +use crate::expr::PyExpr; +use crate::utils::{parse_volatility, validate_pycapsule}; use datafusion::arrow::array::{Array, ArrayRef}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; @@ -27,11 +31,8 @@ use datafusion::error::{DataFusionError, Result}; use datafusion::logical_expr::{ create_udaf, Accumulator, AccumulatorFactoryFunction, AggregateUDF, }; - -use crate::common::data_type::PyScalarValue; -use crate::errors::to_datafusion_err; -use crate::expr::PyExpr; -use crate::utils::parse_volatility; +use datafusion_ffi::udaf::{FFI_AggregateUDF, ForeignAggregateUDF}; +use pyo3::types::PyCapsule; #[derive(Debug)] struct RustAccumulator { @@ -183,6 +184,26 @@ impl PyAggregateUDF { Ok(Self { function }) } + #[staticmethod] + pub fn from_pycapsule(func: Bound<'_, PyAny>) -> PyDataFusionResult { + if func.hasattr("__datafusion_aggregate_udf__")? { + let capsule = func.getattr("__datafusion_aggregate_udf__")?.call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_aggregate_udf")?; + + let udaf = unsafe { capsule.reference::() }; + let udaf: ForeignAggregateUDF = udaf.try_into()?; + + Ok(Self { + function: udaf.into(), + }) + } else { + Err(crate::errors::PyDataFusionError::Common( + "__datafusion_aggregate_udf__ does not exist on AggregateUDF object.".to_string(), + )) + } + } + /// creates a new PyExpr with the call of the udf #[pyo3(signature = (*args))] fn __call__(&self, args: Vec) -> PyResult { diff --git a/src/udf.rs b/src/udf.rs index 574c9d7b5..de1e3f18c 100644 --- a/src/udf.rs +++ b/src/udf.rs @@ -17,6 +17,8 @@ use std::sync::Arc; +use datafusion_ffi::udf::{FFI_ScalarUDF, ForeignScalarUDF}; +use pyo3::types::PyCapsule; use pyo3::{prelude::*, types::PyTuple}; use datafusion::arrow::array::{make_array, Array, ArrayData, ArrayRef}; @@ -29,8 +31,9 @@ use datafusion::logical_expr::ScalarUDF; use datafusion::logical_expr::{create_udf, ColumnarValue}; use crate::errors::to_datafusion_err; +use crate::errors::{py_datafusion_err, PyDataFusionResult}; use crate::expr::PyExpr; -use crate::utils::parse_volatility; +use crate::utils::{parse_volatility, validate_pycapsule}; /// Create a Rust callable function from a python function that expects pyarrow arrays fn pyarrow_function_to_rust( @@ -105,6 +108,26 @@ impl PyScalarUDF { Ok(Self { function }) } + #[staticmethod] + pub fn from_pycapsule(func: Bound<'_, PyAny>) -> PyDataFusionResult { + if func.hasattr("__datafusion_scalar_udf__")? { + let capsule = func.getattr("__datafusion_scalar_udf__")?.call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_scalar_udf")?; + + let udf = unsafe { capsule.reference::() }; + let udf: ForeignScalarUDF = udf.try_into()?; + + Ok(Self { + function: udf.into(), + }) + } else { + Err(crate::errors::PyDataFusionError::Common( + "__datafusion_scalar_udf__ does not exist on ScalarUDF object.".to_string(), + )) + } + } + /// creates a new PyExpr with the call of the udf #[pyo3(signature = (*args))] fn __call__(&self, args: Vec) -> PyResult { diff --git a/src/udwf.rs b/src/udwf.rs index a0c8cc59a..4fb98916b 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -27,16 +27,17 @@ use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use crate::common::data_type::PyScalarValue; -use crate::errors::to_datafusion_err; +use crate::errors::{py_datafusion_err, to_datafusion_err, PyDataFusionResult}; use crate::expr::PyExpr; -use crate::utils::parse_volatility; +use crate::utils::{parse_volatility, validate_pycapsule}; use datafusion::arrow::datatypes::DataType; use datafusion::arrow::pyarrow::{FromPyArrow, PyArrowType, ToPyArrow}; use datafusion::error::{DataFusionError, Result}; use datafusion::logical_expr::{ PartitionEvaluator, PartitionEvaluatorFactory, Signature, Volatility, WindowUDF, WindowUDFImpl, }; -use pyo3::types::{PyList, PyTuple}; +use datafusion_ffi::udwf::{FFI_WindowUDF, ForeignWindowUDF}; +use pyo3::types::{PyCapsule, PyList, PyTuple}; #[derive(Debug)] struct RustPartitionEvaluator { @@ -245,6 +246,26 @@ impl PyWindowUDF { Ok(self.function.call(args).into()) } + #[staticmethod] + pub fn from_pycapsule(func: Bound<'_, PyAny>) -> PyDataFusionResult { + if func.hasattr("__datafusion_window_udf__")? { + let capsule = func.getattr("__datafusion_window_udf__")?.call0()?; + let capsule = capsule.downcast::().map_err(py_datafusion_err)?; + validate_pycapsule(capsule, "datafusion_window_udf")?; + + let udwf = unsafe { capsule.reference::() }; + let udwf: ForeignWindowUDF = udwf.try_into()?; + + Ok(Self { + function: udwf.into(), + }) + } else { + Err(crate::errors::PyDataFusionError::Common( + "__datafusion_window_udf__ does not exist on WindowUDF object.".to_string(), + )) + } + } + fn __repr__(&self) -> PyResult { Ok(format!("WindowUDF({})", self.function.name())) } From 2e1b71369eefc97c22b82be84bbabb414f748fb9 Mon Sep 17 00:00:00 2001 From: kosiew Date: Fri, 4 Jul 2025 20:36:05 +0800 Subject: [PATCH 315/413] refactor: style loading logic in DataFrameHtmlFormatter (#1177) --- python/datafusion/dataframe_formatter.py | 117 ++++++++--------------- python/tests/test_dataframe.py | 60 +++++------- 2 files changed, 63 insertions(+), 114 deletions(-) diff --git a/python/datafusion/dataframe_formatter.py b/python/datafusion/dataframe_formatter.py index 27f00f9c3..2323224b8 100644 --- a/python/datafusion/dataframe_formatter.py +++ b/python/datafusion/dataframe_formatter.py @@ -135,9 +135,6 @@ class DataFrameHtmlFormatter: session """ - # Class variable to track if styles have been loaded in the notebook - _styles_loaded = False - def __init__( self, max_cell_length: int = 25, @@ -260,23 +257,6 @@ def set_custom_header_builder(self, builder: Callable[[Any], str]) -> None: """ self._custom_header_builder = builder - @classmethod - def is_styles_loaded(cls) -> bool: - """Check if HTML styles have been loaded in the current session. - - This method is primarily intended for debugging UI rendering issues - related to style loading. - - Returns: - True if styles have been loaded, False otherwise - - Example: - >>> from datafusion.dataframe_formatter import DataFrameHtmlFormatter - >>> DataFrameHtmlFormatter.is_styles_loaded() - False - """ - return cls._styles_loaded - def format_html( self, batches: list, @@ -315,18 +295,7 @@ def format_html( # Build HTML components html = [] - # Only include styles and scripts if: - # 1. Not using shared styles, OR - # 2. Using shared styles but they haven't been loaded yet - include_styles = ( - not self.use_shared_styles or not DataFrameHtmlFormatter._styles_loaded - ) - - if include_styles: - html.extend(self._build_html_header()) - # If we're using shared styles, mark them as loaded - if self.use_shared_styles: - DataFrameHtmlFormatter._styles_loaded = True + html.extend(self._build_html_header()) html.extend(self._build_table_container_start()) @@ -338,7 +307,7 @@ def format_html( html.append("") # Add footer (JavaScript and messages) - if include_styles and self.enable_cell_expansion: + if self.enable_cell_expansion: html.append(self._get_javascript()) # Always add truncation message if needed (independent of styles) @@ -375,14 +344,20 @@ def format_str( def _build_html_header(self) -> list[str]: """Build the HTML header with CSS styles.""" - html = [] - html.append("") + html.append(f"") return html def _build_table_container_start(self) -> list[str]: @@ -570,28 +545,31 @@ def _get_default_css(self) -> str: def _get_javascript(self) -> str: """Get JavaScript code for interactive elements.""" return """ - - """ + +""" class FormatterManager: @@ -712,24 +690,9 @@ def reset_formatter() -> None: >>> reset_formatter() # Reset formatter to default settings """ formatter = DataFrameHtmlFormatter() - # Reset the styles_loaded flag to ensure styles will be reloaded - DataFrameHtmlFormatter._styles_loaded = False set_formatter(formatter) -def reset_styles_loaded_state() -> None: - """Reset the styles loaded state to force reloading of styles. - - This can be useful when switching between notebook sessions or - when styles need to be refreshed. - - Example: - >>> from datafusion.html_formatter import reset_styles_loaded_state - >>> reset_styles_loaded_state() # Force styles to reload in next render - """ - DataFrameHtmlFormatter._styles_loaded = False - - def _refresh_formatter_reference() -> None: """Refresh formatter reference in any modules using it. diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index c9ae38d8e..a3870ead8 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -42,7 +42,6 @@ configure_formatter, get_formatter, reset_formatter, - reset_styles_loaded_state, ) from datafusion.expr import Window from pyarrow.csv import write_csv @@ -2177,27 +2176,15 @@ def test_html_formatter_shared_styles(df, clean_formatter_state): # First, ensure we're using shared styles configure_formatter(use_shared_styles=True) - # Get HTML output for first table - should include styles html_first = df._repr_html_() - - # Verify styles are included in first render - assert "
" - f"{field.name}
" - f"
" - "" - "" - f"{formatted_value}" - f"" - f"
" - f"		{formatted_value}