From 8de0149f1b9516aa8732560ea9a41c6326cccdea Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Fri, 7 Jun 2024 10:08:11 -0700 Subject: [PATCH] Use 39.0.0-rc1 --- core/Cargo.lock | 153 ++++++++++-------- core/Cargo.toml | 25 +-- .../datafusion/expressions/negative.rs | 5 +- core/src/execution/kernels/hash.rs | 16 ++ 4 files changed, 121 insertions(+), 78 deletions(-) diff --git a/core/Cargo.lock b/core/Cargo.lock index 0adac56e8..59c46c92b 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -114,9 +114,9 @@ checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" [[package]] name = "arrow" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219d05930b81663fd3b32e3bde8ce5bff3c4d23052a99f11a8fa50a3b47b2658" +checksum = "7ae9728f104939be6d8d9b368a354b4929b0569160ea1641f0721b55a861ce38" dependencies = [ "arrow-arith", "arrow-array", @@ -135,9 +135,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0272150200c07a86a390be651abdd320a2d12e84535f0837566ca87ecd8f95e0" +checksum = "a7029a5b3efbeafbf4a12d12dc16b8f9e9bff20a410b8c25c5d28acc089e1043" dependencies = [ "arrow-array", "arrow-buffer", @@ -150,16 +150,16 @@ dependencies = [ [[package]] name = "arrow-array" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8010572cf8c745e242d1b632bd97bd6d4f40fefed5ed1290a8f433abaa686fea" +checksum = "d33238427c60271710695f17742f45b1a5dc5bcfc5c15331c25ddfe7abf70d97" dependencies = [ "ahash", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "chrono-tz", + "chrono-tz 0.9.0", "half 2.4.1", "hashbrown", "num", @@ -167,9 +167,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0a2432f0cba5692bf4cb757469c66791394bac9ec7ce63c1afe74744c37b27" +checksum = "fe9b95e825ae838efaf77e366c00d3fc8cca78134c9db497d6bda425f2e7b7c1" dependencies = [ "bytes", "half 2.4.1", @@ -178,9 +178,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9abc10cd7995e83505cc290df9384d6e5412b207b79ce6bdff89a10505ed2cba" +checksum = "87cf8385a9d5b5fcde771661dd07652b79b9139fea66193eda6a88664400ccab" dependencies = [ "arrow-array", "arrow-buffer", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95cbcba196b862270bf2a5edb75927380a7f3a163622c61d40cbba416a6305f2" +checksum = "cea5068bef430a86690059665e40034625ec323ffa4dd21972048eebb0127adc" dependencies = [ "arrow-array", "arrow-buffer", @@ -218,9 +218,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2742ac1f6650696ab08c88f6dd3f0eb68ce10f8c253958a18c943a68cd04aec5" +checksum = "cb29be98f987bcf217b070512bb7afba2f65180858bca462edf4a39d84a23e10" dependencies = [ "arrow-buffer", "arrow-schema", @@ -230,9 +230,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a42ea853130f7e78b9b9d178cb4cd01dee0f78e64d96c2949dc0a915d6d9e19d" +checksum = "ffc68f6523970aa6f7ce1dc9a33a7d9284cfb9af77d4ad3e617dbe5d79cc6ec8" dependencies = [ "arrow-array", "arrow-buffer", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaafb5714d4e59feae964714d724f880511500e3569cc2a94d02456b403a2a49" +checksum = "2041380f94bd6437ab648e6c2085a045e45a0c44f91a1b9a4fe3fed3d379bfb1" dependencies = [ "arrow-array", "arrow-buffer", @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3e6b61e3dc468f503181dccc2fc705bdcc5f2f146755fa5b56d0a6c5943f412" +checksum = "fcb56ed1547004e12203652f12fe12e824161ff9d1e5cf2a7dc4ff02ba94f413" dependencies = [ "arrow-array", "arrow-buffer", @@ -280,9 +280,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "848ee52bb92eb459b811fb471175ea3afcf620157674c8794f539838920f9228" +checksum = "575b42f1fc588f2da6977b94a5ca565459f5ab07b60545e17243fb9a7ed6d43e" dependencies = [ "ahash", "arrow-array", @@ -295,18 +295,18 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d9483aaabe910c4781153ae1b6ae0393f72d9ef757d38d09d450070cf2e528" +checksum = "32aae6a60458a2389c0da89c9de0b7932427776127da1a738e2efc21d32f3393" dependencies = [ "bitflags 2.5.0", ] [[package]] name = "arrow-select" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "849524fa70e0e3c5ab58394c770cb8f514d0122d20de08475f7b472ed8075830" +checksum = "de36abaef8767b4220d7b4a8c2fe5ffc78b47db81b03d77e2136091c3ba39102" dependencies = [ "ahash", "arrow-array", @@ -318,9 +318,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9373cb5a021aee58863498c37eb484998ef13377f69989c6c5ccfbd258236cdb" +checksum = "e435ada8409bcafc910bc3e0077f532a4daa20e99060a496685c0e3e53cc2597" dependencies = [ "arrow-array", "arrow-buffer", @@ -522,7 +522,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ae0466b83e838b81a54256c39d5d7c20b9d7daa10510a242d9b75abd5936e" dependencies = [ "chrono", - "chrono-tz-build", + "chrono-tz-build 0.2.1", + "phf", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build 0.3.0", "phf", ] @@ -537,6 +548,17 @@ dependencies = [ "phf_codegen", ] +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + [[package]] name = "ciborium" version = "0.2.1" @@ -606,6 +628,7 @@ dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-buffer", "arrow-data", "arrow-schema", "arrow-string", @@ -615,7 +638,7 @@ dependencies = [ "byteorder", "bytes", "chrono", - "chrono-tz", + "chrono-tz 0.8.6", "crc32fast", "criterion", "datafusion", @@ -842,8 +865,8 @@ dependencies = [ [[package]] name = "datafusion" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "ahash", "arrow", @@ -875,6 +898,7 @@ dependencies = [ "num_cpus", "object_store", "parking_lot", + "paste", "pin-project-lite", "rand", "sqlparser", @@ -886,8 +910,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "ahash", "arrow", @@ -906,16 +930,16 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "tokio", ] [[package]] name = "datafusion-execution" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "arrow", "chrono", @@ -934,12 +958,13 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "ahash", "arrow", "arrow-array", + "arrow-buffer", "chrono", "datafusion-common", "paste", @@ -951,8 +976,8 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "arrow", "base64", @@ -977,9 +1002,10 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ + "ahash", "arrow", "arrow-schema", "datafusion-common", @@ -993,15 +1019,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "arrow", "async-trait", "chrono", "datafusion-common", "datafusion-expr", - "datafusion-functions-aggregate", "datafusion-physical-expr", "hashbrown", "indexmap", @@ -1012,8 +1037,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "ahash", "arrow", @@ -1042,8 +1067,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "arrow", "datafusion-common", @@ -1053,8 +1078,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "ahash", "arrow", @@ -1086,8 +1111,8 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "38.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=ad2b1dc#ad2b1dcac8168906e4444527320d3139a1a2ea5b" +version = "39.0.0" +source = "git+https://github.com/apache/arrow-datafusion.git?tag=39.0.0-rc1#6a4a280e3cf70fe5f1a1cfe7c2de13e4c39f89bb" dependencies = [ "arrow", "arrow-array", @@ -1191,9 +1216,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "23.5.26" +version = "24.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dac53e22462d78c16d64a1cd22371b54cc3fe94aa15e7886a2fa6e5d1ab8640" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1957,9 +1982,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" +checksum = "fbebfd32c213ba1907fa7a9c9138015a8de2b43e30c5aa45b18f7deb46786ad6" dependencies = [ "async-trait", "bytes", @@ -2031,9 +2056,9 @@ dependencies = [ [[package]] name = "parquet" -version = "51.0.0" +version = "52.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "096795d4f47f65fd3ee1ec5a98b77ab26d602f2cc785b0e4be5443add17ecc32" +checksum = "29c3b5322cc1bbf67f11c079c42be41a55949099b78732f7dba9e15edde40eab" dependencies = [ "ahash", "bytes", @@ -2588,9 +2613,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "sqlparser" -version = "0.45.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7bbffee862a796d67959a89859d6b1046bb5016d63e23835ad0da182777bbe0" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", "sqlparser_derive", diff --git a/core/Cargo.toml b/core/Cargo.toml index 42e24dab2..564c450cb 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -29,12 +29,13 @@ include = [ [dependencies] parquet-format = "4.0.0" # This must be kept in sync with that from parquet crate -arrow = { version = "51.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } -arrow-array = { version = "51.0.0" } -arrow-data = { version = "51.0.0" } -arrow-schema = { version = "51.0.0" } -arrow-string = { version = "51.0.0" } -parquet = { version = "51.0.0", default-features = false, features = ["experimental"] } +arrow = { version = "52.0.0", features = ["prettyprint", "ffi", "chrono-tz"] } +arrow-array = { version = "52.0.0" } +arrow-buffer = { version = "52.0.0" } +arrow-data = { version = "52.0.0" } +arrow-schema = { version = "52.0.0" } +arrow-string = { version = "52.0.0" } +parquet = { version = "52.0.0", default-features = false, features = ["experimental"] } half = { version = "2.4.1", default-features = false } futures = "0.3.28" mimalloc = { version = "*", default-features = false, optional = true } @@ -66,12 +67,12 @@ itertools = "0.11.0" chrono = { version = "0.4", default-features = false, features = ["clock"] } chrono-tz = { version = "0.8" } paste = "1.0.14" -datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "ad2b1dc" } -datafusion = { default-features = false, git = "https://github.com/apache/arrow-datafusion.git", rev = "ad2b1dc", features = ["unicode_expressions", "crypto_expressions"] } -datafusion-functions = { git = "https://github.com/apache/arrow-datafusion.git", rev = "ad2b1dc", features = ["crypto_expressions"] } -datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "ad2b1dc", default-features = false } -datafusion-physical-expr-common = { git = "https://github.com/apache/arrow-datafusion.git", rev = "ad2b1dc", default-features = false } -datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "ad2b1dc", default-features = false } +datafusion-common = { git = "https://github.com/apache/arrow-datafusion.git", tag = "39.0.0-rc1" } +datafusion = { default-features = false, git = "https://github.com/apache/arrow-datafusion.git", tag = "39.0.0-rc1", features = ["unicode_expressions", "crypto_expressions"] } +datafusion-functions = { git = "https://github.com/apache/arrow-datafusion.git", tag = "39.0.0-rc1", features = ["crypto_expressions"] } +datafusion-expr = { git = "https://github.com/apache/arrow-datafusion.git", tag = "39.0.0-rc1", default-features = false } +datafusion-physical-expr-common = { git = "https://github.com/apache/arrow-datafusion.git", tag = "39.0.0-rc1", default-features = false } +datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", tag = "39.0.0-rc1", default-features = false } unicode-segmentation = "^1.10.1" once_cell = "1.18.0" regex = "1.9.6" diff --git a/core/src/execution/datafusion/expressions/negative.rs b/core/src/execution/datafusion/expressions/negative.rs index b369eb717..a85cde89e 100644 --- a/core/src/execution/datafusion/expressions/negative.rs +++ b/core/src/execution/datafusion/expressions/negative.rs @@ -18,6 +18,7 @@ use crate::errors::CometError; use arrow::{compute::kernels::numeric::neg_wrapping, datatypes::IntervalDayTimeType}; use arrow_array::RecordBatch; +use arrow_buffer::IntervalDayTime; use arrow_schema::{DataType, Schema}; use datafusion::{ logical_expr::{interval_arithmetic::Interval, ColumnarValue}, @@ -62,7 +63,7 @@ macro_rules! check_overflow { for i in 0..typed_array.len() { if typed_array.value(i) == $min_val { if $type_name == "byte" || $type_name == "short" { - let value = typed_array.value(i).to_string() + " caused"; + let value = format!("{:?} caused", typed_array.value(i)); return Err(arithmetic_overflow_error(value.as_str()).into()); } return Err(arithmetic_overflow_error($type_name).into()); @@ -134,7 +135,7 @@ impl PhysicalExpr for NegativeExpr { arrow::datatypes::IntervalUnit::DayTime => check_overflow!( array, arrow::array::IntervalDayTimeArray, - i64::MIN, + IntervalDayTime::MIN, "interval" ), arrow::datatypes::IntervalUnit::MonthDayNano => { diff --git a/core/src/execution/kernels/hash.rs b/core/src/execution/kernels/hash.rs index de30f74cd..b39fd6224 100644 --- a/core/src/execution/kernels/hash.rs +++ b/core/src/execution/kernels/hash.rs @@ -22,6 +22,7 @@ use arrow_array::{ downcast_dictionary_array, downcast_primitive_array, Array, ArrayAccessor, ArrayRef, ArrowPrimitiveType, PrimitiveArray, }; +use arrow_buffer::{IntervalDayTime, IntervalMonthDayNano}; use std::fmt::Debug; pub fn hash(src: &[ArrayRef], dst: &mut [u64]) { @@ -169,3 +170,18 @@ impl Hashable for f64 { state.hash_one(u64::from_ne_bytes(self.to_ne_bytes())) } } + +impl Hashable for IntervalDayTime { + fn create_hash(&self, state: &RandomState) -> u64 { + state.hash_one(self.days); + state.hash_one(self.milliseconds) + } +} + +impl Hashable for IntervalMonthDayNano { + fn create_hash(&self, state: &RandomState) -> u64 { + state.hash_one(self.months); + state.hash_one(self.days); + state.hash_one(self.nanoseconds) + } +}