From b37ab483b521a10735e85b34f4a1f58c137eca86 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Fri, 16 Aug 2024 09:23:43 +0800 Subject: [PATCH 01/11] fix: add custom (de)serialization for infinite float values --- src/query/expression/src/types/number.rs | 54 +++++++++++++++++++ .../suites/base/issues/issue_16213.test | 11 ++++ 2 files changed, 65 insertions(+) create mode 100644 tests/sqllogictests/suites/base/issues/issue_16213.test diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 2e1b47487140..524d872565ec 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -22,10 +22,13 @@ use databend_common_arrow::arrow::buffer::Buffer; use enum_as_inner::EnumAsInner; use itertools::Itertools; use lexical_core::ToLexicalWithOptions; +use num_traits::float::FloatCore; use num_traits::NumCast; use ordered_float::OrderedFloat; use serde::Deserialize; +use serde::Deserializer; use serde::Serialize; +use serde::Serializer; use super::decimal::DecimalSize; use crate::property::Domain; @@ -314,7 +317,9 @@ pub enum NumberScalar { Int16(i16), Int32(i32), Int64(i64), + #[serde(serialize_with = "serialize_f32", deserialize_with = "deserialize_f32")] Float32(F32), + #[serde(serialize_with = "serialize_f64", deserialize_with = "deserialize_f64")] Float64(F64), } @@ -1419,3 +1424,52 @@ impl Number for F64 { } } } +fn serialize_f32(value: &F32, serializer: S) -> Result +where S: Serializer { + if value.is_infinite() { + if value.is_sign_positive() { + serializer.serialize_str("Infinity") + } else { + serializer.serialize_str("-Infinity") + } + } else { + Serialize::serialize(&value, serializer) + } +} + +fn deserialize_f32<'de, D>(deserializer: D) -> Result +where D: Deserializer<'de> { + let value = serde_json::Value::deserialize(deserializer)?; + match value { + serde_json::Value::String(s) if s == "Infinity" => Ok(F32::infinity()), + serde_json::Value::String(s) if s == "-Infinity" => Ok(F32::neg_infinity()), + _ => ::deserialize(value) + .map( as From>::from) + .map_err(|e| serde::de::Error::custom(e.to_string())), + } +} + +fn serialize_f64(value: &F64, serializer: S) -> Result +where S: Serializer { + if value.is_infinite() { + if value.is_sign_positive() { + serializer.serialize_str("Infinity") + } else { + serializer.serialize_str("-Infinity") + } + } else { + Serialize::serialize(&value, serializer) + } +} + +fn deserialize_f64<'de, D>(deserializer: D) -> Result +where D: Deserializer<'de> { + let value = serde_json::Value::deserialize(deserializer)?; + match value { + serde_json::Value::String(s) if s == "Infinity" => Ok(F64::infinity()), + serde_json::Value::String(s) if s == "-Infinity" => Ok(F64::neg_infinity()), + _ => ::deserialize(value) + .map( as From>::from) + .map_err(|e| serde::de::Error::custom(e.to_string())), + } +} diff --git a/tests/sqllogictests/suites/base/issues/issue_16213.test b/tests/sqllogictests/suites/base/issues/issue_16213.test new file mode 100644 index 000000000000..fba585d80063 --- /dev/null +++ b/tests/sqllogictests/suites/base/issues/issue_16213.test @@ -0,0 +1,11 @@ +statement ok +drop table if exists test_table_double + +statement ok +CREATE TABLE IF NOT EXISTS test_table_double (ID INT, Name VARCHAR(50), Age INT, City VARCHAR(50), Score DOUBLE) + +statement ok +INSERT INTO test_table_double (ID, Name, Age, City, Score) VALUES (1, 'Alice', 25, 'Toroto', 'inf') + +statement ok +SELECT * FROM test_table_double From a65e4f65ca852935783455e9da9e9bc6b303aab5 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Fri, 16 Aug 2024 13:27:14 +0800 Subject: [PATCH 02/11] fix: fix serialize and deserialize error for infinity float case --- src/query/expression/src/types/number.rs | 71 ++++++++++++++++-------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 524d872565ec..994f33c8953a 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -29,6 +29,7 @@ use serde::Deserialize; use serde::Deserializer; use serde::Serialize; use serde::Serializer; +use serde_json::Value; use super::decimal::DecimalSize; use crate::property::Domain; @@ -1426,50 +1427,72 @@ impl Number for F64 { } fn serialize_f32(value: &F32, serializer: S) -> Result where S: Serializer { - if value.is_infinite() { - if value.is_sign_positive() { - serializer.serialize_str("Infinity") + if serializer.is_human_readable() { + if value.is_infinite() { + if value.is_sign_positive() { + serializer.serialize_str("Infinity") + } else { + serializer.serialize_str("-Infinity") + } + } else if value.is_nan() { + serializer.serialize_str("Nan") } else { - serializer.serialize_str("-Infinity") + serde::Serialize::serialize(&value, serializer) } } else { - Serialize::serialize(&value, serializer) + serde::Serialize::serialize(&value, serializer) } } fn deserialize_f32<'de, D>(deserializer: D) -> Result where D: Deserializer<'de> { - let value = serde_json::Value::deserialize(deserializer)?; - match value { - serde_json::Value::String(s) if s == "Infinity" => Ok(F32::infinity()), - serde_json::Value::String(s) if s == "-Infinity" => Ok(F32::neg_infinity()), - _ => ::deserialize(value) - .map( as From>::from) - .map_err(|e| serde::de::Error::custom(e.to_string())), + if deserializer.is_human_readable() { + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) if s == "Infinity" => Ok(F32::infinity()), + Value::String(s) if s == "-Infinity" => Ok(F32::neg_infinity()), + Value::String(s) if s == "Nan" => Ok(F32::nan()), + _ => ::deserialize(value) + .map( as From>::from) + .map_err(|e| serde::de::Error::custom(e.to_string())), + } + } else { + serde::Deserialize::deserialize(deserializer) } } fn serialize_f64(value: &F64, serializer: S) -> Result where S: Serializer { - if value.is_infinite() { - if value.is_sign_positive() { - serializer.serialize_str("Infinity") + if serializer.is_human_readable() { + if value.is_infinite() { + if value.is_sign_positive() { + serializer.serialize_str("Infinity") + } else { + serializer.serialize_str("-Infinity") + } + } else if value.is_nan() { + serializer.serialize_str("Nan") } else { - serializer.serialize_str("-Infinity") + serde::Serialize::serialize(&value, serializer) } } else { - Serialize::serialize(&value, serializer) + serde::Serialize::serialize(&value, serializer) } } fn deserialize_f64<'de, D>(deserializer: D) -> Result where D: Deserializer<'de> { - let value = serde_json::Value::deserialize(deserializer)?; - match value { - serde_json::Value::String(s) if s == "Infinity" => Ok(F64::infinity()), - serde_json::Value::String(s) if s == "-Infinity" => Ok(F64::neg_infinity()), - _ => ::deserialize(value) - .map( as From>::from) - .map_err(|e| serde::de::Error::custom(e.to_string())), + if deserializer.is_human_readable() { + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) if s == "Infinity" => Ok(F64::infinity()), + Value::String(s) if s == "-Infinity" => Ok(F64::neg_infinity()), + Value::String(s) if s == "Nan" => Ok(F64::nan()), + _ => ::deserialize(value) + .map( as From>::from) + .map_err(|e| serde::de::Error::custom(e.to_string())), + } + } else { + serde::Deserialize::deserialize(deserializer) } } From b3dce3733d491d7d2a4dfec9c30c6d312c83694a Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 19 Aug 2024 10:19:25 +0800 Subject: [PATCH 03/11] test: add more tests for inf and nan --- .../suites/base/issues/issue_16213.test | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/sqllogictests/suites/base/issues/issue_16213.test b/tests/sqllogictests/suites/base/issues/issue_16213.test index fba585d80063..77ddaa4dcc26 100644 --- a/tests/sqllogictests/suites/base/issues/issue_16213.test +++ b/tests/sqllogictests/suites/base/issues/issue_16213.test @@ -7,5 +7,22 @@ CREATE TABLE IF NOT EXISTS test_table_double (ID INT, Name VARCHAR(50), Age INT, statement ok INSERT INTO test_table_double (ID, Name, Age, City, Score) VALUES (1, 'Alice', 25, 'Toroto', 'inf') -statement ok +query SELECT * FROM test_table_double +1 Alice 25 Toroto Infinity + +statement ok +create table if not exists test_inf_nan (a float) + +statement ok +insert into test_inf_nan select 'inf' from numbers(100); + +statement ok +insert into test_inf_nan select '-inf' from numbers(100); + +statement ok +insert into test_inf_nan select 'nan' from numbers(100); + +query +select max(a) from test_inf_nan; +inf From 6cf9dedd959c1e773cdf882db78dfd85c60fef96 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 19 Aug 2024 10:23:59 +0800 Subject: [PATCH 04/11] fix --- tests/sqllogictests/suites/base/issues/issue_16213.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/sqllogictests/suites/base/issues/issue_16213.test b/tests/sqllogictests/suites/base/issues/issue_16213.test index 77ddaa4dcc26..dcc628e06dd0 100644 --- a/tests/sqllogictests/suites/base/issues/issue_16213.test +++ b/tests/sqllogictests/suites/base/issues/issue_16213.test @@ -15,14 +15,14 @@ statement ok create table if not exists test_inf_nan (a float) statement ok -insert into test_inf_nan select 'inf' from numbers(100); +insert into test_inf_nan select 'inf' from numbers(100) statement ok -insert into test_inf_nan select '-inf' from numbers(100); +insert into test_inf_nan select '-inf' from numbers(100) statement ok -insert into test_inf_nan select 'nan' from numbers(100); +insert into test_inf_nan select 'nan' from numbers(100) query -select max(a) from test_inf_nan; +select max(a) from test_inf_nan inf From 73c14ad9e97a5496976e12a8e7a62bf25ed61287 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 19 Aug 2024 13:39:24 +0800 Subject: [PATCH 05/11] fix: add borsh f32 and f64 replaced method for nan --- src/query/expression/src/types/number.rs | 185 ++++++++++++------ .../suites/base/issues/issue_16213.test | 7 +- 2 files changed, 133 insertions(+), 59 deletions(-) diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 994f33c8953a..5b1f1f701ffe 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -26,10 +26,7 @@ use num_traits::float::FloatCore; use num_traits::NumCast; use ordered_float::OrderedFloat; use serde::Deserialize; -use serde::Deserializer; use serde::Serialize; -use serde::Serializer; -use serde_json::Value; use super::decimal::DecimalSize; use crate::property::Domain; @@ -318,10 +315,28 @@ pub enum NumberScalar { Int16(i16), Int32(i32), Int64(i64), - #[serde(serialize_with = "serialize_f32", deserialize_with = "deserialize_f32")] - Float32(F32), - #[serde(serialize_with = "serialize_f64", deserialize_with = "deserialize_f64")] - Float64(F64), + #[serde( + serialize_with = "json_serde_impl::serialize_f32", + deserialize_with = "json_serde_impl::deserialize_f32" + )] + Float32( + #[borsh( + serialize_with = "borsh_serde_impl::serialize_f32", + deserialize_with = "borsh_serde_impl::deserialize_f32" + )] + F32, + ), + #[serde( + serialize_with = "json_serde_impl::serialize_f64", + deserialize_with = "json_serde_impl::deserialize_f64" + )] + Float64( + #[borsh( + serialize_with = "borsh_serde_impl::serialize_f64", + deserialize_with = "borsh_serde_impl::deserialize_f64" + )] + F64, + ), } #[derive(Clone, PartialEq, EnumAsInner)] @@ -1425,74 +1440,128 @@ impl Number for F64 { } } } -fn serialize_f32(value: &F32, serializer: S) -> Result -where S: Serializer { - if serializer.is_human_readable() { - if value.is_infinite() { - if value.is_sign_positive() { - serializer.serialize_str("Infinity") + +/// Json serialize and deserialize implementation for `F32` and `F64`. It specially handles the +/// cases of `Infinity`, `-Infinity` and `Nan`. +mod json_serde_impl { + + use num_traits::float::FloatCore; + use ordered_float::OrderedFloat; + use serde::Deserialize; + use serde::Deserializer; + use serde::Serializer; + use serde_json::Value; + + use crate::types::F32; + use crate::types::F64; + + pub fn serialize_f32(value: &F32, serializer: S) -> Result + where S: Serializer { + if serializer.is_human_readable() { + if value.is_infinite() { + if value.is_sign_positive() { + serializer.serialize_str("Infinity") + } else { + serializer.serialize_str("-Infinity") + } + } else if value.is_nan() { + serializer.serialize_str("Nan") } else { - serializer.serialize_str("-Infinity") + serde::Serialize::serialize(&value, serializer) } - } else if value.is_nan() { - serializer.serialize_str("Nan") } else { serde::Serialize::serialize(&value, serializer) } - } else { - serde::Serialize::serialize(&value, serializer) } -} -fn deserialize_f32<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { - if deserializer.is_human_readable() { - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) if s == "Infinity" => Ok(F32::infinity()), - Value::String(s) if s == "-Infinity" => Ok(F32::neg_infinity()), - Value::String(s) if s == "Nan" => Ok(F32::nan()), - _ => ::deserialize(value) - .map( as From>::from) - .map_err(|e| serde::de::Error::custom(e.to_string())), + pub fn deserialize_f32<'de, D>(deserializer: D) -> Result + where D: Deserializer<'de> { + if deserializer.is_human_readable() { + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) if s == "Infinity" => Ok(F32::infinity()), + Value::String(s) if s == "-Infinity" => Ok(F32::neg_infinity()), + Value::String(s) if s == "Nan" => Ok(F32::nan()), + _ => ::deserialize(value) + .map( as From>::from) + .map_err(|e| serde::de::Error::custom(e.to_string())), + } + } else { + serde::Deserialize::deserialize(deserializer) } - } else { - serde::Deserialize::deserialize(deserializer) } -} -fn serialize_f64(value: &F64, serializer: S) -> Result -where S: Serializer { - if serializer.is_human_readable() { - if value.is_infinite() { - if value.is_sign_positive() { - serializer.serialize_str("Infinity") + pub fn serialize_f64(value: &F64, serializer: S) -> Result + where S: Serializer { + if serializer.is_human_readable() { + if value.is_infinite() { + if value.is_sign_positive() { + serializer.serialize_str("Infinity") + } else { + serializer.serialize_str("-Infinity") + } + } else if value.is_nan() { + serializer.serialize_str("Nan") } else { - serializer.serialize_str("-Infinity") + serde::Serialize::serialize(&value, serializer) } - } else if value.is_nan() { - serializer.serialize_str("Nan") } else { serde::Serialize::serialize(&value, serializer) } - } else { - serde::Serialize::serialize(&value, serializer) } -} -fn deserialize_f64<'de, D>(deserializer: D) -> Result -where D: Deserializer<'de> { - if deserializer.is_human_readable() { - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) if s == "Infinity" => Ok(F64::infinity()), - Value::String(s) if s == "-Infinity" => Ok(F64::neg_infinity()), - Value::String(s) if s == "Nan" => Ok(F64::nan()), - _ => ::deserialize(value) - .map( as From>::from) - .map_err(|e| serde::de::Error::custom(e.to_string())), + pub fn deserialize_f64<'de, D>(deserializer: D) -> Result + where D: Deserializer<'de> { + if deserializer.is_human_readable() { + let value = Value::deserialize(deserializer)?; + match value { + Value::String(s) if s == "Infinity" => Ok(F64::infinity()), + Value::String(s) if s == "-Infinity" => Ok(F64::neg_infinity()), + Value::String(s) if s == "Nan" => Ok(F64::nan()), + _ => ::deserialize(value) + .map( as From>::from) + .map_err(|e| serde::de::Error::custom(e.to_string())), + } + } else { + serde::Deserialize::deserialize(deserializer) } - } else { - serde::Deserialize::deserialize(deserializer) + } +} + +/// Borsh serialize and deserialize implementation for `F32` and `F64`. +mod borsh_serde_impl { + use ordered_float::OrderedFloat; + + use crate::types::F32; + use crate::types::F64; + + pub fn serialize_f32( + obj: &F32, + writer: &mut W, + ) -> Result<(), borsh::io::Error> { + writer.write_all(&obj.to_bits().to_le_bytes())?; + Ok(()) + } + + pub fn deserialize_f32(reader: &mut R) -> Result { + let mut buf = [0u8; size_of::()]; + reader.read_exact(&mut buf)?; + let res = OrderedFloat::from(f32::from_bits(u32::from_le_bytes(buf))); + Ok(res) + } + + pub fn serialize_f64( + obj: &F64, + writer: &mut W, + ) -> Result<(), borsh::io::Error> { + writer.write_all(&obj.to_bits().to_le_bytes())?; + Ok(()) + } + + pub fn deserialize_f64(reader: &mut R) -> Result { + let mut buf = [0u8; size_of::()]; + reader.read_exact(&mut buf)?; + let res = OrderedFloat::from(f64::from_bits(u64::from_le_bytes(buf))); + Ok(res) } } diff --git a/tests/sqllogictests/suites/base/issues/issue_16213.test b/tests/sqllogictests/suites/base/issues/issue_16213.test index dcc628e06dd0..99bb4b7718b3 100644 --- a/tests/sqllogictests/suites/base/issues/issue_16213.test +++ b/tests/sqllogictests/suites/base/issues/issue_16213.test @@ -9,7 +9,11 @@ INSERT INTO test_table_double (ID, Name, Age, City, Score) VALUES (1, 'Alice', 2 query SELECT * FROM test_table_double -1 Alice 25 Toroto Infinity +---- +1 Alice 25 Toroto inf + +statement ok +drop table if exists test_inf_nan statement ok create table if not exists test_inf_nan (a float) @@ -25,4 +29,5 @@ insert into test_inf_nan select 'nan' from numbers(100) query select max(a) from test_inf_nan +---- inf From 9893150c7f0e2084a685e9deaad63d5895fc5c47 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 2 Sep 2024 18:53:10 +0800 Subject: [PATCH 06/11] chore: patch borsh-rs to bypass NaN check --- Cargo.lock | 8 ++- Cargo.toml | 1 + src/query/expression/src/types/number.rs | 54 +------------------ .../suites/base/issues/issue_16213.test | 4 +- 4 files changed, 8 insertions(+), 59 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a13c497e2a9..c605a8543c9f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1676,8 +1676,7 @@ dependencies = [ [[package]] name = "borsh" version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" +source = "git+https://github.com/dqhl76/borsh-rs.git?rev=6609a44#6609a44c01ab8d6307b579a9a82b5233286e2f41" dependencies = [ "borsh-derive", "cfg_aliases 0.2.1", @@ -1686,8 +1685,7 @@ dependencies = [ [[package]] name = "borsh-derive" version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" +source = "git+https://github.com/dqhl76/borsh-rs.git?rev=6609a44#6609a44c01ab8d6307b579a9a82b5233286e2f41" dependencies = [ "once_cell", "proc-macro-crate", @@ -5067,7 +5065,7 @@ dependencies = [ "futures", "log", "rand 0.8.5", - "reqwest 0.12.4", + "reqwest 0.12.5", "serde", "serde_json", "tokio", diff --git a/Cargo.toml b/Cargo.toml index b30cdecf8be1..70caa946551a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -408,3 +408,4 @@ orc-rust = { git = "https://github.com/youngsofun/datafusion-orc", rev = "174537 recursive = { git = "https://github.com/zhang2014/recursive.git", rev = "6af35a1" } sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" } xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" } +borsh = { git = "https://github.com/dqhl76/borsh-rs.git", rev = "6609a44"} diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 5b1f1f701ffe..d49673258298 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -319,24 +319,12 @@ pub enum NumberScalar { serialize_with = "json_serde_impl::serialize_f32", deserialize_with = "json_serde_impl::deserialize_f32" )] - Float32( - #[borsh( - serialize_with = "borsh_serde_impl::serialize_f32", - deserialize_with = "borsh_serde_impl::deserialize_f32" - )] - F32, - ), + Float32(F32), #[serde( serialize_with = "json_serde_impl::serialize_f64", deserialize_with = "json_serde_impl::deserialize_f64" )] - Float64( - #[borsh( - serialize_with = "borsh_serde_impl::serialize_f64", - deserialize_with = "borsh_serde_impl::deserialize_f64" - )] - F64, - ), + Float64(F64), } #[derive(Clone, PartialEq, EnumAsInner)] @@ -1527,41 +1515,3 @@ mod json_serde_impl { } } } - -/// Borsh serialize and deserialize implementation for `F32` and `F64`. -mod borsh_serde_impl { - use ordered_float::OrderedFloat; - - use crate::types::F32; - use crate::types::F64; - - pub fn serialize_f32( - obj: &F32, - writer: &mut W, - ) -> Result<(), borsh::io::Error> { - writer.write_all(&obj.to_bits().to_le_bytes())?; - Ok(()) - } - - pub fn deserialize_f32(reader: &mut R) -> Result { - let mut buf = [0u8; size_of::()]; - reader.read_exact(&mut buf)?; - let res = OrderedFloat::from(f32::from_bits(u32::from_le_bytes(buf))); - Ok(res) - } - - pub fn serialize_f64( - obj: &F64, - writer: &mut W, - ) -> Result<(), borsh::io::Error> { - writer.write_all(&obj.to_bits().to_le_bytes())?; - Ok(()) - } - - pub fn deserialize_f64(reader: &mut R) -> Result { - let mut buf = [0u8; size_of::()]; - reader.read_exact(&mut buf)?; - let res = OrderedFloat::from(f64::from_bits(u64::from_le_bytes(buf))); - Ok(res) - } -} diff --git a/tests/sqllogictests/suites/base/issues/issue_16213.test b/tests/sqllogictests/suites/base/issues/issue_16213.test index 99bb4b7718b3..f457de9c5b88 100644 --- a/tests/sqllogictests/suites/base/issues/issue_16213.test +++ b/tests/sqllogictests/suites/base/issues/issue_16213.test @@ -10,7 +10,7 @@ INSERT INTO test_table_double (ID, Name, Age, City, Score) VALUES (1, 'Alice', 2 query SELECT * FROM test_table_double ---- -1 Alice 25 Toroto inf +1 Alice 25 Toroto Infinity statement ok drop table if exists test_inf_nan @@ -30,4 +30,4 @@ insert into test_inf_nan select 'nan' from numbers(100) query select max(a) from test_inf_nan ---- -inf +NaN From 43868ddbf3cb9bed84d7989cff15318e59376317 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Tue, 3 Sep 2024 14:19:03 +0800 Subject: [PATCH 07/11] make taplo happy --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 70caa946551a..4b4dcadb7851 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -400,6 +400,7 @@ arrow-udf-wasm = { git = "https://github.com/datafuse-extras/arrow-udf", rev = " async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" } async-recursion = { git = "https://github.com/zhang2014/async-recursion.git", rev = "a353334" } backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be" } +borsh = { git = "https://github.com/dqhl76/borsh-rs.git", rev = "6609a44" } color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" } ethnum = { git = "https://github.com/ariesdevil/ethnum-rs", rev = "4cb05f1" } @@ -408,4 +409,3 @@ orc-rust = { git = "https://github.com/youngsofun/datafusion-orc", rev = "174537 recursive = { git = "https://github.com/zhang2014/recursive.git", rev = "6af35a1" } sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" } xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" } -borsh = { git = "https://github.com/dqhl76/borsh-rs.git", rev = "6609a44"} From 52aa4d9076334679615f73eebf9d2aa59bac7e31 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Sun, 8 Sep 2024 22:53:00 +0800 Subject: [PATCH 08/11] refactor: put ordered float file in base --- Cargo.lock | 32 +- Cargo.toml | 1 - src/common/base/Cargo.toml | 5 + src/common/base/src/base/mod.rs | 2 + src/common/base/src/base/ordered_float.rs | 2155 +++++++++++++++++ src/common/hashtable/Cargo.toml | 1 - src/common/hashtable/src/traits.rs | 2 +- src/common/io/Cargo.toml | 2 +- src/common/io/src/stat_buffer.rs | 2 +- src/common/storage/Cargo.toml | 1 - src/common/storage/src/statistics.rs | 2 +- src/query/expression/Cargo.toml | 1 - .../expression/src/aggregate/group_hash.rs | 2 +- .../expression/src/converts/datavalues/to.rs | 2 +- src/query/expression/src/types/number.rs | 97 +- src/query/expression/src/values.rs | 2 +- .../tests/it/fill_field_default_value.rs | 8 +- src/query/expression/tests/it/row.rs | 2 +- src/query/formats/Cargo.toml | 2 +- src/query/formats/src/field_encoder/values.rs | 2 +- src/query/functions/Cargo.toml | 4 - .../src/aggregates/aggregate_quantile_cont.rs | 2 +- src/query/functions/src/scalars/boolean.rs | 2 +- .../functions/src/scalars/decimal/cast.rs | 2 +- src/query/functions/src/scalars/math.rs | 2 +- src/query/functions/src/scalars/other.rs | 2 +- .../functions/tests/it/scalars/parser.rs | 2 +- .../storages/fuse/operations/alter_table.rs | 2 +- src/query/sql/Cargo.toml | 1 - .../planner/optimizer/filter/infer_filter.rs | 2 +- .../planner/optimizer/rule/utils/constant.rs | 2 +- src/query/storages/delta/Cargo.toml | 1 - .../src/partition_columns/values_serde.rs | 2 +- src/query/storages/hive/hive/Cargo.toml | 1 - src/query/storages/hive/hive/src/utils.rs | 2 +- src/query/storages/iceberg/Cargo.toml | 1 + src/query/storages/iceberg/src/stats.rs | 9 +- 37 files changed, 2218 insertions(+), 144 deletions(-) create mode 100644 src/common/base/src/base/ordered_float.rs diff --git a/Cargo.lock b/Cargo.lock index c605a8543c9f..15c7b8b8739b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1676,7 +1676,8 @@ dependencies = [ [[package]] name = "borsh" version = "1.5.1" -source = "git+https://github.com/dqhl76/borsh-rs.git?rev=6609a44#6609a44c01ab8d6307b579a9a82b5233286e2f41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6362ed55def622cddc70a4746a68554d7b687713770de539e59a739b249f8ed" dependencies = [ "borsh-derive", "cfg_aliases 0.2.1", @@ -1685,7 +1686,8 @@ dependencies = [ [[package]] name = "borsh-derive" version = "1.5.1" -source = "git+https://github.com/dqhl76/borsh-rs.git?rev=6609a44#6609a44c01ab8d6307b579a9a82b5233286e2f41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3ef8005764f53cd4dca619f5bf64cafd4664dada50ece25e4d81de54c80cc0b" dependencies = [ "once_cell", "proc-macro-crate", @@ -3174,6 +3176,7 @@ dependencies = [ "anyhow", "async-backtrace", "async-trait", + "borsh", "bytesize", "chrono", "ctrlc", @@ -3185,6 +3188,8 @@ dependencies = [ "libc", "log", "logcall", + "micromarshal 0.5.0", + "num-traits", "num_cpus", "once_cell", "parking_lot 0.12.3", @@ -3198,6 +3203,7 @@ dependencies = [ "replace_with", "semver", "serde", + "serde_test", "state", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -3400,7 +3406,6 @@ dependencies = [ "micromarshal 0.5.0", "num-bigint", "num-traits", - "ordered-float 4.2.2", "pretty_assertions", "rand 0.8.5", "recursive", @@ -3427,6 +3432,7 @@ dependencies = [ "bstr", "chrono-tz 0.8.6", "databend-common-arrow", + "databend-common-base", "databend-common-exception", "databend-common-expression", "databend-common-io", @@ -3443,7 +3449,6 @@ dependencies = [ "micromarshal 0.7.0", "num", "num-traits", - "ordered-float 4.2.2", "pretty_assertions", "roaring", "serde_json", @@ -3494,7 +3499,6 @@ dependencies = [ "naive-cityhash", "num-traits", "once_cell", - "ordered-float 4.2.2", "proj4rs", "rand 0.8.5", "regex", @@ -3537,7 +3541,6 @@ dependencies = [ "databend-common-arrow", "databend-common-base", "ethnum", - "ordered-float 4.2.2", "rand 0.8.5", ] @@ -3568,6 +3571,7 @@ dependencies = [ "bytes", "chrono", "chrono-tz 0.8.6", + "databend-common-base", "databend-common-exception", "ethnum", "geo", @@ -3575,7 +3579,6 @@ dependencies = [ "geozero 0.13.0", "lexical-core", "micromarshal 0.5.0", - "ordered-float 4.2.2", "rand 0.8.5", "rmp-serde", "roaring", @@ -4157,7 +4160,6 @@ dependencies = [ "num-derive", "num-traits", "opendal", - "ordered-float 4.2.2", "parking_lot 0.12.3", "percent-encoding", "prqlc", @@ -4190,7 +4192,6 @@ dependencies = [ "futures", "log", "opendal", - "ordered-float 4.2.2", "parquet", "prometheus-client", "regex", @@ -4221,7 +4222,6 @@ dependencies = [ "maplit", "match-template", "object_store_opendal", - "ordered-float 4.2.2", "parquet", "serde", "serde_json", @@ -4351,7 +4351,6 @@ dependencies = [ "hive_metastore", "log", "opendal", - "ordered-float 4.2.2", "recursive", "serde", "typetag", @@ -4385,6 +4384,7 @@ dependencies = [ "iceberg-catalog-hms", "iceberg-catalog-rest", "match-template", + "ordered-float 4.2.2", "parquet", "serde", "tokio", @@ -10970,7 +10970,6 @@ version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ - "borsh", "num-traits", "rand 0.8.5", "serde", @@ -13600,6 +13599,15 @@ dependencies = [ "stacker", ] +[[package]] +name = "serde_test" +version = "1.0.177" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f901ee573cab6b3060453d2d5f0bae4e6d628c23c0a962ff9b5f1d7c8d4f1ed" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 4b4dcadb7851..b30cdecf8be1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -400,7 +400,6 @@ arrow-udf-wasm = { git = "https://github.com/datafuse-extras/arrow-udf", rev = " async-backtrace = { git = "https://github.com/zhang2014/async-backtrace.git", rev = "dea4553" } async-recursion = { git = "https://github.com/zhang2014/async-recursion.git", rev = "a353334" } backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "72265be" } -borsh = { git = "https://github.com/dqhl76/borsh-rs.git", rev = "6609a44" } color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" } ethnum = { git = "https://github.com/ariesdevil/ethnum-rs", rev = "4cb05f1" } diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index e80138cc9a56..f89e4eecb8a4 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -54,6 +54,10 @@ tikv-jemalloc-sys = "0.5.2" tokio = { workspace = true } unicode-segmentation = "1.10.1" uuid = { workspace = true } +num-traits = "0.2.19" +borsh = { workspace = true } +rand = { workspace = true, features = ["serde1"]} +micromarshal = "0.5.0" [target.'cfg(target_os = "linux")'.dependencies] procfs = { version = "^0.16" } @@ -62,6 +66,7 @@ procfs = { version = "^0.16" } anyerror = { workspace = true } anyhow = { workspace = true } rand = { workspace = true } +serde_test = "1.0" [build-dependencies] databend-common-building = { workspace = true } diff --git a/src/common/base/src/base/mod.rs b/src/common/base/src/base/mod.rs index d4e8faefa89b..175730845145 100644 --- a/src/common/base/src/base/mod.rs +++ b/src/common/base/src/base/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. mod net; +mod ordered_float; mod profiling; mod progress; mod select; @@ -28,6 +29,7 @@ mod watch_notify; pub use net::get_free_tcp_port; pub use net::get_free_udp_port; +pub use ordered_float::OrderedFloat; pub use profiling::Profiling; pub use progress::Progress; pub use progress::ProgressValues; diff --git a/src/common/base/src/base/ordered_float.rs b/src/common/base/src/base/ordered_float.rs new file mode 100644 index 000000000000..e29cba02039b --- /dev/null +++ b/src/common/base/src/base/ordered_float.rs @@ -0,0 +1,2155 @@ +// Copyright 2015 Jonathan Reem +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Wrappers for total order on Floats. See the [`OrderedFloat`] and [`NotNan`] docs for details. + +extern crate std; + +use core::borrow::Borrow; +use core::cmp::Ordering; +use core::convert::TryFrom; +use core::fmt; +use core::hash::Hash; +use core::hash::Hasher; +use core::iter::Product; +use core::iter::Sum; +use core::num::FpCategory; +use core::ops::Add; +use core::ops::AddAssign; +use core::ops::Deref; +use core::ops::DerefMut; +use core::ops::Div; +use core::ops::DivAssign; +use core::ops::Mul; +use core::ops::MulAssign; +use core::ops::Neg; +use core::ops::Rem; +use core::ops::RemAssign; +use core::ops::Sub; +use core::ops::SubAssign; +use core::str::FromStr; +use std::error::Error; + +use micromarshal::Unmarshal; +use num_traits::float::FloatCore; +use num_traits::AsPrimitive; +use num_traits::Bounded; +use num_traits::Float; +use num_traits::FloatConst; +use num_traits::FromPrimitive; +use num_traits::Num; +use num_traits::NumCast; +use num_traits::One; +use num_traits::Pow; +use num_traits::Signed; +use num_traits::ToPrimitive; +use num_traits::Zero; + +// masks for the parts of the IEEE 754 float +const SIGN_MASK: u64 = 0x8000000000000000u64; +const EXP_MASK: u64 = 0x7ff0000000000000u64; +const MAN_MASK: u64 = 0x000fffffffffffffu64; + +// canonical raw bit patterns (for hashing) +const CANONICAL_NAN_BITS: u64 = 0x7ff8000000000000u64; + +#[inline(always)] +fn canonicalize_signed_zero(x: T) -> T { + // -0.0 + 0.0 == +0.0 under IEEE754 roundTiesToEven rounding mode, + // which Rust guarantees. Thus by adding a positive zero we + // canonicalize signed zero without any branches in one instruction. + x + T::zero() +} + +/// A wrapper around floats providing implementations of `Eq`, `Ord`, and `Hash`. +/// +/// NaN is sorted as *greater* than all other values and *equal* +/// to itself, in contradiction with the IEEE standard. +/// +/// ``` +/// use std::f32::NAN; +/// +/// use ordered_float::OrderedFloat; +/// +/// let mut v = [OrderedFloat(NAN), OrderedFloat(2.0), OrderedFloat(1.0)]; +/// v.sort(); +/// assert_eq!(v, [OrderedFloat(1.0), OrderedFloat(2.0), OrderedFloat(NAN)]); +/// ``` +/// +/// Because `OrderedFloat` implements `Ord` and `Eq`, it can be used as a key in a `HashSet`, +/// `HashMap`, `BTreeMap`, or `BTreeSet` (unlike the primitive `f32` or `f64` types): +/// +/// ``` +/// # use ordered_float::OrderedFloat; +/// # use std::collections::HashSet; +/// # use std::f32::NAN; +/// +/// let mut s: HashSet> = HashSet::new(); +/// s.insert(OrderedFloat(NAN)); +/// assert!(s.contains(&OrderedFloat(NAN))); +/// ``` +#[derive(Default, Clone, Copy)] +#[repr(transparent)] +pub struct OrderedFloat(pub T); + +impl OrderedFloat { + /// Get the value out. + #[inline] + pub fn into_inner(self) -> T { + self.0 + } +} + +impl AsRef for OrderedFloat { + #[inline] + fn as_ref(&self) -> &T { + &self.0 + } +} + +impl AsMut for OrderedFloat { + #[inline] + fn as_mut(&mut self) -> &mut T { + &mut self.0 + } +} + +impl<'a, T: FloatCore> From<&'a T> for &'a OrderedFloat { + #[inline] + fn from(t: &'a T) -> &'a OrderedFloat { + // Safety: OrderedFloat is #[repr(transparent)] and has no invalid values. + unsafe { &*(t as *const T as *const OrderedFloat) } + } +} + +impl<'a, T: FloatCore> From<&'a mut T> for &'a mut OrderedFloat { + #[inline] + fn from(t: &'a mut T) -> &'a mut OrderedFloat { + // Safety: OrderedFloat is #[repr(transparent)] and has no invalid values. + unsafe { &mut *(t as *mut T as *mut OrderedFloat) } + } +} + +impl PartialOrd for OrderedFloat { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + + #[inline] + fn lt(&self, other: &Self) -> bool { + !self.ge(other) + } + + #[inline] + fn le(&self, other: &Self) -> bool { + other.ge(self) + } + + #[inline] + fn gt(&self, other: &Self) -> bool { + !other.ge(self) + } + + #[inline] + fn ge(&self, other: &Self) -> bool { + // We consider all NaNs equal, and NaN is the largest possible + // value. Thus if self is NaN we always return true. Otherwise + // self >= other is correct. If other is also not NaN it is trivially + // correct, and if it is we note that nothing can be greater or + // equal to NaN except NaN itself, which we already handled earlier. + self.0.is_nan() | (self.0 >= other.0) + } +} + +impl Ord for OrderedFloat { + #[inline] + fn cmp(&self, other: &Self) -> Ordering { + #[allow(clippy::comparison_chain)] + if self < other { + Ordering::Less + } else if self > other { + Ordering::Greater + } else { + Ordering::Equal + } + } +} + +impl PartialEq for OrderedFloat { + #[inline] + fn eq(&self, other: &OrderedFloat) -> bool { + if self.0.is_nan() { + other.0.is_nan() + } else { + self.0 == other.0 + } + } +} + +impl PartialEq for OrderedFloat { + #[inline] + fn eq(&self, other: &T) -> bool { + self.0 == *other + } +} + +impl Hash for OrderedFloat { + fn hash(&self, state: &mut H) { + let bits = if self.is_nan() { + CANONICAL_NAN_BITS + } else { + raw_double_bits(&canonicalize_signed_zero(self.0)) + }; + + bits.hash(state) + } +} + +impl fmt::Debug for OrderedFloat { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Display for OrderedFloat { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::LowerExp for OrderedFloat { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::UpperExp for OrderedFloat { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +impl From> for f32 { + #[inline] + fn from(f: OrderedFloat) -> f32 { + f.0 + } +} + +impl From> for f64 { + #[inline] + fn from(f: OrderedFloat) -> f64 { + f.0 + } +} + +impl From for OrderedFloat { + #[inline] + fn from(val: T) -> Self { + OrderedFloat(val) + } +} + +impl From for OrderedFloat { + fn from(val: bool) -> Self { + OrderedFloat(val as u8 as f32) + } +} + +impl From for OrderedFloat { + fn from(val: bool) -> Self { + OrderedFloat(val as u8 as f64) + } +} + +macro_rules! impl_ordered_float_from { + ($dst:ty, $src:ty) => { + impl From<$src> for OrderedFloat<$dst> { + fn from(val: $src) -> Self { + OrderedFloat(val.into()) + } + } + }; +} +impl_ordered_float_from! {f64, i8} +impl_ordered_float_from! {f64, i16} +impl_ordered_float_from! {f64, i32} +impl_ordered_float_from! {f64, u8} +impl_ordered_float_from! {f64, u16} +impl_ordered_float_from! {f64, u32} +impl_ordered_float_from! {f32, i8} +impl_ordered_float_from! {f32, i16} +impl_ordered_float_from! {f32, u8} +impl_ordered_float_from! {f32, u16} + +impl Deref for OrderedFloat { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for OrderedFloat { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl Eq for OrderedFloat {} + +macro_rules! impl_ordered_float_binop { + ($imp:ident, $method:ident, $assign_imp:ident, $assign_method:ident) => { + impl $imp for OrderedFloat { + type Output = OrderedFloat; + + #[inline] + fn $method(self, other: Self) -> Self::Output { + OrderedFloat((self.0).$method(other.0)) + } + } + + impl $imp for OrderedFloat { + type Output = OrderedFloat; + + #[inline] + fn $method(self, other: T) -> Self::Output { + OrderedFloat((self.0).$method(other)) + } + } + + impl<'a, T> $imp<&'a T> for OrderedFloat + where T: $imp<&'a T> + { + type Output = OrderedFloat<>::Output>; + + #[inline] + fn $method(self, other: &'a T) -> Self::Output { + OrderedFloat((self.0).$method(other)) + } + } + + impl<'a, T> $imp<&'a Self> for OrderedFloat + where T: $imp<&'a T> + { + type Output = OrderedFloat<>::Output>; + + #[inline] + fn $method(self, other: &'a Self) -> Self::Output { + OrderedFloat((self.0).$method(&other.0)) + } + } + + impl<'a, T> $imp> for &'a OrderedFloat + where &'a T: $imp + { + type Output = OrderedFloat<<&'a T as $imp>::Output>; + + #[inline] + fn $method(self, other: OrderedFloat) -> Self::Output { + OrderedFloat((self.0).$method(other.0)) + } + } + + impl<'a, T> $imp for &'a OrderedFloat + where &'a T: $imp + { + type Output = OrderedFloat<<&'a T as $imp>::Output>; + + #[inline] + fn $method(self, other: T) -> Self::Output { + OrderedFloat((self.0).$method(other)) + } + } + + impl<'a, T> $imp<&'a T> for &'a OrderedFloat + where &'a T: $imp + { + type Output = OrderedFloat<<&'a T as $imp>::Output>; + + #[inline] + fn $method(self, other: &'a T) -> Self::Output { + OrderedFloat((self.0).$method(other)) + } + } + + impl $assign_imp for OrderedFloat { + #[inline] + fn $assign_method(&mut self, other: T) { + (self.0).$assign_method(other); + } + } + + impl<'a, T: $assign_imp<&'a T>> $assign_imp<&'a T> for OrderedFloat { + #[inline] + fn $assign_method(&mut self, other: &'a T) { + (self.0).$assign_method(other); + } + } + + impl $assign_imp for OrderedFloat { + #[inline] + fn $assign_method(&mut self, other: Self) { + (self.0).$assign_method(other.0); + } + } + + impl<'a, T: $assign_imp<&'a T>> $assign_imp<&'a Self> for OrderedFloat { + #[inline] + fn $assign_method(&mut self, other: &'a Self) { + (self.0).$assign_method(&other.0); + } + } + }; +} + +impl_ordered_float_binop! {Add, add, AddAssign, add_assign} +impl_ordered_float_binop! {Sub, sub, SubAssign, sub_assign} +impl_ordered_float_binop! {Mul, mul, MulAssign, mul_assign} +impl_ordered_float_binop! {Div, div, DivAssign, div_assign} +impl_ordered_float_binop! {Rem, rem, RemAssign, rem_assign} + +macro_rules! impl_ordered_float_pow { + ($inner:ty, $rhs:ty) => { + impl Pow<$rhs> for OrderedFloat<$inner> { + type Output = OrderedFloat<$inner>; + #[inline] + fn pow(self, rhs: $rhs) -> OrderedFloat<$inner> { + OrderedFloat(<$inner>::pow(self.0, rhs)) + } + } + + impl<'a> Pow<&'a $rhs> for OrderedFloat<$inner> { + type Output = OrderedFloat<$inner>; + #[inline] + fn pow(self, rhs: &'a $rhs) -> OrderedFloat<$inner> { + OrderedFloat(<$inner>::pow(self.0, *rhs)) + } + } + + impl<'a> Pow<$rhs> for &'a OrderedFloat<$inner> { + type Output = OrderedFloat<$inner>; + #[inline] + fn pow(self, rhs: $rhs) -> OrderedFloat<$inner> { + OrderedFloat(<$inner>::pow(self.0, rhs)) + } + } + + impl<'a, 'b> Pow<&'a $rhs> for &'b OrderedFloat<$inner> { + type Output = OrderedFloat<$inner>; + #[inline] + fn pow(self, rhs: &'a $rhs) -> OrderedFloat<$inner> { + OrderedFloat(<$inner>::pow(self.0, *rhs)) + } + } + }; +} + +impl_ordered_float_pow! {f32, i8} +impl_ordered_float_pow! {f32, i16} +impl_ordered_float_pow! {f32, u8} +impl_ordered_float_pow! {f32, u16} +impl_ordered_float_pow! {f32, i32} +impl_ordered_float_pow! {f64, i8} +impl_ordered_float_pow! {f64, i16} +impl_ordered_float_pow! {f64, u8} +impl_ordered_float_pow! {f64, u16} +impl_ordered_float_pow! {f64, i32} +impl_ordered_float_pow! {f32, f32} +impl_ordered_float_pow! {f64, f32} +impl_ordered_float_pow! {f64, f64} + +macro_rules! impl_ordered_float_self_pow { + ($base:ty, $exp:ty) => { + impl Pow> for OrderedFloat<$base> { + type Output = OrderedFloat<$base>; + #[inline] + fn pow(self, rhs: OrderedFloat<$exp>) -> OrderedFloat<$base> { + OrderedFloat(<$base>::pow(self.0, rhs.0)) + } + } + + impl<'a> Pow<&'a OrderedFloat<$exp>> for OrderedFloat<$base> { + type Output = OrderedFloat<$base>; + #[inline] + fn pow(self, rhs: &'a OrderedFloat<$exp>) -> OrderedFloat<$base> { + OrderedFloat(<$base>::pow(self.0, rhs.0)) + } + } + + impl<'a> Pow> for &'a OrderedFloat<$base> { + type Output = OrderedFloat<$base>; + #[inline] + fn pow(self, rhs: OrderedFloat<$exp>) -> OrderedFloat<$base> { + OrderedFloat(<$base>::pow(self.0, rhs.0)) + } + } + + impl<'a, 'b> Pow<&'a OrderedFloat<$exp>> for &'b OrderedFloat<$base> { + type Output = OrderedFloat<$base>; + #[inline] + fn pow(self, rhs: &'a OrderedFloat<$exp>) -> OrderedFloat<$base> { + OrderedFloat(<$base>::pow(self.0, rhs.0)) + } + } + }; +} + +impl_ordered_float_self_pow! {f32, f32} +impl_ordered_float_self_pow! {f64, f32} +impl_ordered_float_self_pow! {f64, f64} + +/// Adds a float directly. +impl Sum for OrderedFloat { + fn sum>>(iter: I) -> Self { + OrderedFloat(iter.map(|v| v.0).sum()) + } +} + +impl<'a, T: FloatCore + Sum + 'a> Sum<&'a OrderedFloat> for OrderedFloat { + #[inline] + fn sum>>(iter: I) -> Self { + iter.cloned().sum() + } +} + +impl Product for OrderedFloat { + fn product>>(iter: I) -> Self { + OrderedFloat(iter.map(|v| v.0).product()) + } +} + +impl<'a, T: FloatCore + Product + 'a> Product<&'a OrderedFloat> for OrderedFloat { + #[inline] + fn product>>(iter: I) -> Self { + iter.cloned().product() + } +} + +impl Signed for OrderedFloat { + #[inline] + fn abs(&self) -> Self { + OrderedFloat(self.0.abs()) + } + + fn abs_sub(&self, other: &Self) -> Self { + OrderedFloat(Signed::abs_sub(&self.0, &other.0)) + } + + #[inline] + fn signum(&self) -> Self { + OrderedFloat(self.0.signum()) + } + #[inline] + fn is_positive(&self) -> bool { + #[expect(clippy::disallowed_methods)] + self.0.is_positive() + } + #[inline] + fn is_negative(&self) -> bool { + #[expect(clippy::disallowed_methods)] + self.0.is_negative() + } +} + +impl Bounded for OrderedFloat { + #[inline] + fn min_value() -> Self { + OrderedFloat(T::min_value()) + } + + #[inline] + fn max_value() -> Self { + OrderedFloat(T::max_value()) + } +} + +impl FromStr for OrderedFloat { + type Err = T::Err; + + /// Convert a &str to `OrderedFloat`. Returns an error if the string fails to parse. + /// + /// ``` + /// use ordered_float::OrderedFloat; + /// + /// assert!("-10".parse::>().is_ok()); + /// assert!("abc".parse::>().is_err()); + /// assert!("NaN".parse::>().is_ok()); + /// ``` + fn from_str(s: &str) -> Result { + T::from_str(s).map(OrderedFloat) + } +} + +impl Neg for OrderedFloat { + type Output = OrderedFloat; + + #[inline] + fn neg(self) -> Self::Output { + OrderedFloat(-self.0) + } +} + +impl<'a, T> Neg for &'a OrderedFloat +where &'a T: Neg +{ + type Output = OrderedFloat<<&'a T as Neg>::Output>; + + #[inline] + fn neg(self) -> Self::Output { + OrderedFloat(-(&self.0)) + } +} + +impl Zero for OrderedFloat { + #[inline] + fn zero() -> Self { + OrderedFloat(T::zero()) + } + + #[inline] + fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +impl One for OrderedFloat { + #[inline] + fn one() -> Self { + OrderedFloat(T::one()) + } +} + +impl NumCast for OrderedFloat { + #[inline] + fn from(n: F) -> Option { + T::from(n).map(OrderedFloat) + } +} + +macro_rules! impl_as_primitive { + (@ (NotNan<$T: ty>) => $(#[$cfg:meta])* impl (NotNan<$U: ty>) ) => { + $(#[$cfg])* + impl AsPrimitive> for NotNan<$T> { + #[inline] fn as_(self) -> NotNan<$U> { + // Safety: `NotNan` guarantees that the value is not NaN. + unsafe {NotNan::new_unchecked(self.0 as $U) } + } + } + }; + (@ ($T: ty) => $(#[$cfg:meta])* impl (NotNan<$U: ty>) ) => { + $(#[$cfg])* + impl AsPrimitive> for $T { + #[inline] fn as_(self) -> NotNan<$U> { NotNan(self as $U) } + } + }; + (@ (NotNan<$T: ty>) => $(#[$cfg:meta])* impl ($U: ty) ) => { + $(#[$cfg])* + impl AsPrimitive<$U> for NotNan<$T> { + #[inline] fn as_(self) -> $U { self.0 as $U } + } + }; + (@ (OrderedFloat<$T: ty>) => $(#[$cfg:meta])* impl (OrderedFloat<$U: ty>) ) => { + $(#[$cfg])* + impl AsPrimitive> for OrderedFloat<$T> { + #[inline] fn as_(self) -> OrderedFloat<$U> { OrderedFloat(self.0 as $U) } + } + }; + (@ ($T: ty) => $(#[$cfg:meta])* impl (OrderedFloat<$U: ty>) ) => { + $(#[$cfg])* + impl AsPrimitive> for $T { + #[inline] fn as_(self) -> OrderedFloat<$U> { OrderedFloat(self as $U) } + } + }; + (@ (OrderedFloat<$T: ty>) => $(#[$cfg:meta])* impl ($U: ty) ) => { + $(#[$cfg])* + impl AsPrimitive<$U> for OrderedFloat<$T> { + #[inline] fn as_(self) -> $U { self.0 as $U } + } + }; + ($T: tt => { $( $U: tt ),* } ) => {$( + impl_as_primitive!(@ $T => impl $U); + )*}; +} + +impl_as_primitive!((OrderedFloat) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((OrderedFloat) => { (OrderedFloat), (OrderedFloat) }); + +impl_as_primitive!((NotNan) => { (NotNan), (NotNan) }); +impl_as_primitive!((NotNan) => { (NotNan), (NotNan) }); + +impl_as_primitive!((u8) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((i8) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((u16) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((i16) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((u32) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((i32) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((u64) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((i64) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((usize) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((isize) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((f32) => { (OrderedFloat), (OrderedFloat) }); +impl_as_primitive!((f64) => { (OrderedFloat), (OrderedFloat) }); + +impl_as_primitive!((u8) => { (NotNan), (NotNan) }); +impl_as_primitive!((i8) => { (NotNan), (NotNan) }); +impl_as_primitive!((u16) => { (NotNan), (NotNan) }); +impl_as_primitive!((i16) => { (NotNan), (NotNan) }); +impl_as_primitive!((u32) => { (NotNan), (NotNan) }); +impl_as_primitive!((i32) => { (NotNan), (NotNan) }); +impl_as_primitive!((u64) => { (NotNan), (NotNan) }); +impl_as_primitive!((i64) => { (NotNan), (NotNan) }); +impl_as_primitive!((usize) => { (NotNan), (NotNan) }); +impl_as_primitive!((isize) => { (NotNan), (NotNan) }); + +impl_as_primitive!((OrderedFloat) => { (u8), (u16), (u32), (u64), (usize), (i8), (i16), (i32), (i64), (isize), (f32), (f64) }); +impl_as_primitive!((OrderedFloat) => { (u8), (u16), (u32), (u64), (usize), (i8), (i16), (i32), (i64), (isize), (f32), (f64) }); + +impl_as_primitive!((NotNan) => { (u8), (u16), (u32), (u64), (usize), (i8), (i16), (i32), (i64), (isize), (f32), (f64) }); +impl_as_primitive!((NotNan) => { (u8), (u16), (u32), (u64), (usize), (i8), (i16), (i32), (i64), (isize), (f32), (f64) }); + +impl FromPrimitive for OrderedFloat { + fn from_i64(n: i64) -> Option { + T::from_i64(n).map(OrderedFloat) + } + fn from_u64(n: u64) -> Option { + T::from_u64(n).map(OrderedFloat) + } + fn from_isize(n: isize) -> Option { + T::from_isize(n).map(OrderedFloat) + } + fn from_i8(n: i8) -> Option { + T::from_i8(n).map(OrderedFloat) + } + fn from_i16(n: i16) -> Option { + T::from_i16(n).map(OrderedFloat) + } + fn from_i32(n: i32) -> Option { + T::from_i32(n).map(OrderedFloat) + } + fn from_usize(n: usize) -> Option { + T::from_usize(n).map(OrderedFloat) + } + fn from_u8(n: u8) -> Option { + T::from_u8(n).map(OrderedFloat) + } + fn from_u16(n: u16) -> Option { + T::from_u16(n).map(OrderedFloat) + } + fn from_u32(n: u32) -> Option { + T::from_u32(n).map(OrderedFloat) + } + fn from_f32(n: f32) -> Option { + T::from_f32(n).map(OrderedFloat) + } + fn from_f64(n: f64) -> Option { + T::from_f64(n).map(OrderedFloat) + } +} + +impl ToPrimitive for OrderedFloat { + fn to_i64(&self) -> Option { + self.0.to_i64() + } + fn to_u64(&self) -> Option { + self.0.to_u64() + } + fn to_isize(&self) -> Option { + self.0.to_isize() + } + fn to_i8(&self) -> Option { + self.0.to_i8() + } + fn to_i16(&self) -> Option { + self.0.to_i16() + } + fn to_i32(&self) -> Option { + self.0.to_i32() + } + fn to_usize(&self) -> Option { + self.0.to_usize() + } + fn to_u8(&self) -> Option { + self.0.to_u8() + } + fn to_u16(&self) -> Option { + self.0.to_u16() + } + fn to_u32(&self) -> Option { + self.0.to_u32() + } + fn to_f32(&self) -> Option { + self.0.to_f32() + } + fn to_f64(&self) -> Option { + self.0.to_f64() + } +} + +impl FloatCore for OrderedFloat { + fn nan() -> Self { + OrderedFloat(T::nan()) + } + fn infinity() -> Self { + OrderedFloat(T::infinity()) + } + fn neg_infinity() -> Self { + OrderedFloat(T::neg_infinity()) + } + fn neg_zero() -> Self { + OrderedFloat(T::neg_zero()) + } + fn min_value() -> Self { + OrderedFloat(T::min_value()) + } + fn min_positive_value() -> Self { + OrderedFloat(T::min_positive_value()) + } + fn max_value() -> Self { + OrderedFloat(T::max_value()) + } + fn is_nan(self) -> bool { + self.0.is_nan() + } + fn is_infinite(self) -> bool { + self.0.is_infinite() + } + fn is_finite(self) -> bool { + self.0.is_finite() + } + fn is_normal(self) -> bool { + self.0.is_normal() + } + fn classify(self) -> FpCategory { + self.0.classify() + } + fn floor(self) -> Self { + OrderedFloat(self.0.floor()) + } + fn ceil(self) -> Self { + OrderedFloat(self.0.ceil()) + } + fn round(self) -> Self { + OrderedFloat(self.0.round()) + } + fn trunc(self) -> Self { + OrderedFloat(self.0.trunc()) + } + fn fract(self) -> Self { + OrderedFloat(self.0.fract()) + } + fn abs(self) -> Self { + OrderedFloat(self.0.abs()) + } + fn signum(self) -> Self { + OrderedFloat(self.0.signum()) + } + fn is_sign_positive(self) -> bool { + self.0.is_sign_positive() + } + fn is_sign_negative(self) -> bool { + self.0.is_sign_negative() + } + fn recip(self) -> Self { + OrderedFloat(self.0.recip()) + } + fn powi(self, n: i32) -> Self { + OrderedFloat(self.0.powi(n)) + } + fn integer_decode(self) -> (u64, i16, i8) { + self.0.integer_decode() + } + fn epsilon() -> Self { + OrderedFloat(T::epsilon()) + } + fn to_degrees(self) -> Self { + OrderedFloat(self.0.to_degrees()) + } + fn to_radians(self) -> Self { + OrderedFloat(self.0.to_radians()) + } +} + +impl Float for OrderedFloat { + fn nan() -> Self { + OrderedFloat(::nan()) + } + fn infinity() -> Self { + OrderedFloat(::infinity()) + } + fn neg_infinity() -> Self { + OrderedFloat(::neg_infinity()) + } + fn neg_zero() -> Self { + OrderedFloat(::neg_zero()) + } + fn min_value() -> Self { + OrderedFloat(::min_value()) + } + fn min_positive_value() -> Self { + OrderedFloat(::min_positive_value()) + } + fn max_value() -> Self { + OrderedFloat(::max_value()) + } + fn is_nan(self) -> bool { + Float::is_nan(self.0) + } + fn is_infinite(self) -> bool { + Float::is_infinite(self.0) + } + fn is_finite(self) -> bool { + Float::is_finite(self.0) + } + fn is_normal(self) -> bool { + Float::is_normal(self.0) + } + fn classify(self) -> FpCategory { + Float::classify(self.0) + } + fn floor(self) -> Self { + OrderedFloat(Float::floor(self.0)) + } + fn ceil(self) -> Self { + OrderedFloat(Float::ceil(self.0)) + } + fn round(self) -> Self { + OrderedFloat(Float::round(self.0)) + } + fn trunc(self) -> Self { + OrderedFloat(Float::trunc(self.0)) + } + fn fract(self) -> Self { + OrderedFloat(Float::fract(self.0)) + } + fn abs(self) -> Self { + OrderedFloat(Float::abs(self.0)) + } + fn signum(self) -> Self { + OrderedFloat(Float::signum(self.0)) + } + fn is_sign_positive(self) -> bool { + Float::is_sign_positive(self.0) + } + fn is_sign_negative(self) -> bool { + Float::is_sign_negative(self.0) + } + fn mul_add(self, a: Self, b: Self) -> Self { + OrderedFloat(self.0.mul_add(a.0, b.0)) + } + fn recip(self) -> Self { + OrderedFloat(Float::recip(self.0)) + } + fn powi(self, n: i32) -> Self { + OrderedFloat(Float::powi(self.0, n)) + } + fn powf(self, n: Self) -> Self { + OrderedFloat(self.0.powf(n.0)) + } + fn sqrt(self) -> Self { + OrderedFloat(self.0.sqrt()) + } + fn exp(self) -> Self { + OrderedFloat(self.0.exp()) + } + fn exp2(self) -> Self { + OrderedFloat(self.0.exp2()) + } + fn ln(self) -> Self { + OrderedFloat(self.0.ln()) + } + fn log(self, base: Self) -> Self { + OrderedFloat(self.0.log(base.0)) + } + fn log2(self) -> Self { + OrderedFloat(self.0.log2()) + } + fn log10(self) -> Self { + OrderedFloat(self.0.log10()) + } + fn max(self, other: Self) -> Self { + OrderedFloat(Float::max(self.0, other.0)) + } + fn min(self, other: Self) -> Self { + OrderedFloat(Float::min(self.0, other.0)) + } + fn abs_sub(self, other: Self) -> Self { + OrderedFloat(self.0.abs_sub(other.0)) + } + fn cbrt(self) -> Self { + OrderedFloat(self.0.cbrt()) + } + fn hypot(self, other: Self) -> Self { + OrderedFloat(self.0.hypot(other.0)) + } + fn sin(self) -> Self { + OrderedFloat(self.0.sin()) + } + fn cos(self) -> Self { + OrderedFloat(self.0.cos()) + } + fn tan(self) -> Self { + OrderedFloat(self.0.tan()) + } + fn asin(self) -> Self { + OrderedFloat(self.0.asin()) + } + fn acos(self) -> Self { + OrderedFloat(self.0.acos()) + } + fn atan(self) -> Self { + OrderedFloat(self.0.atan()) + } + fn atan2(self, other: Self) -> Self { + OrderedFloat(self.0.atan2(other.0)) + } + fn sin_cos(self) -> (Self, Self) { + let (a, b) = self.0.sin_cos(); + (OrderedFloat(a), OrderedFloat(b)) + } + fn exp_m1(self) -> Self { + OrderedFloat(self.0.exp_m1()) + } + fn ln_1p(self) -> Self { + OrderedFloat(self.0.ln_1p()) + } + fn sinh(self) -> Self { + OrderedFloat(self.0.sinh()) + } + fn cosh(self) -> Self { + OrderedFloat(self.0.cosh()) + } + fn tanh(self) -> Self { + OrderedFloat(self.0.tanh()) + } + fn asinh(self) -> Self { + OrderedFloat(self.0.asinh()) + } + fn acosh(self) -> Self { + OrderedFloat(self.0.acosh()) + } + fn atanh(self) -> Self { + OrderedFloat(self.0.atanh()) + } + fn integer_decode(self) -> (u64, i16, i8) { + Float::integer_decode(self.0) + } + fn epsilon() -> Self { + OrderedFloat(::epsilon()) + } + fn to_degrees(self) -> Self { + OrderedFloat(Float::to_degrees(self.0)) + } + fn to_radians(self) -> Self { + OrderedFloat(Float::to_radians(self.0)) + } +} + +impl Num for OrderedFloat { + type FromStrRadixErr = T::FromStrRadixErr; + fn from_str_radix(str: &str, radix: u32) -> Result { + T::from_str_radix(str, radix).map(OrderedFloat) + } +} + +/// A wrapper around floats providing an implementation of `Eq`, `Ord` and `Hash`. +/// +/// A NaN value cannot be stored in this type. +/// +/// ``` +/// use ordered_float::NotNan; +/// +/// let mut v = [NotNan::new(2.0).unwrap(), NotNan::new(1.0).unwrap()]; +/// v.sort(); +/// assert_eq!(v, [1.0, 2.0]); +/// ``` +/// +/// Because `NotNan` implements `Ord` and `Eq`, it can be used as a key in a `HashSet`, +/// `HashMap`, `BTreeMap`, or `BTreeSet` (unlike the primitive `f32` or `f64` types): +/// +/// ``` +/// # use ordered_float::NotNan; +/// # use std::collections::HashSet; +/// +/// let mut s: HashSet> = HashSet::new(); +/// let key = NotNan::new(1.0).unwrap(); +/// s.insert(key); +/// assert!(s.contains(&key)); +/// ``` +/// +/// Arithmetic on NotNan values will panic if it produces a NaN value: +/// +/// ```should_panic +/// # use ordered_float::NotNan; +/// let a = NotNan::new(std::f32::INFINITY).unwrap(); +/// let b = NotNan::new(std::f32::NEG_INFINITY).unwrap(); +/// +/// // This will panic: +/// let c = a + b; +/// ``` +#[derive(PartialOrd, PartialEq, Default, Clone, Copy)] +#[repr(transparent)] +pub struct NotNan(T); + +impl NotNan { + /// Create a `NotNan` value. + /// + /// Returns `Err` if `val` is NaN + pub fn new(val: T) -> Result { + match val { + ref val if val.is_nan() => Err(FloatIsNan), + val => Ok(NotNan(val)), + } + } +} + +impl NotNan { + /// Get the value out. + #[inline] + pub fn into_inner(self) -> T { + self.0 + } + + /// Create a `NotNan` value from a value that is guaranteed to not be NaN + /// + /// # Safety + /// + /// Behaviour is undefined if `val` is NaN + #[inline] + pub const unsafe fn new_unchecked(val: T) -> Self { + NotNan(val) + } + + /// Create a `NotNan` value from a value that is guaranteed to not be NaN + /// + /// # Safety + /// + /// Behaviour is undefined if `val` is NaN + #[deprecated( + since = "2.5.0", + note = "Please use the new_unchecked function instead." + )] + #[inline] + pub const unsafe fn unchecked_new(val: T) -> Self { + Self::new_unchecked(val) + } +} + +impl AsRef for NotNan { + #[inline] + fn as_ref(&self) -> &T { + &self.0 + } +} + +impl Borrow for NotNan { + #[inline] + fn borrow(&self) -> &f32 { + &self.0 + } +} + +impl Borrow for NotNan { + #[inline] + fn borrow(&self) -> &f64 { + &self.0 + } +} + +#[allow(clippy::derive_ord_xor_partial_ord)] +impl Ord for NotNan { + fn cmp(&self, other: &NotNan) -> Ordering { + // Can't use unreachable_unchecked because unsafe code can't depend on FloatCore impl. + // https://github.com/reem/rust-ordered-float/issues/150 + self.partial_cmp(other) + .expect("partial_cmp failed for non-NaN value") + } +} + +impl Hash for NotNan { + #[inline] + fn hash(&self, state: &mut H) { + let bits = raw_double_bits(&canonicalize_signed_zero(self.0)); + bits.hash(state) + } +} + +impl fmt::Debug for NotNan { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl fmt::Display for NotNan { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + self.0.fmt(f) + } +} + +impl NotNan { + /// Converts this [`NotNan`]`<`[`f64`]`>` to a [`NotNan`]`<`[`f32`]`>` while giving up on + /// precision, [using `roundTiesToEven` as rounding mode, yielding `Infinity` on + /// overflow](https://doc.rust-lang.org/reference/expressions/operator-expr.html#semantics). + /// + /// Note: For the reverse conversion (from `NotNan` to `NotNan`), you can use + /// `.into()`. + pub fn as_f32(self) -> NotNan { + // This is not destroying invariants, as it is a pure rounding operation. The only two special + // cases are where f32 would be overflowing, then the operation yields Infinity, or where + // the input is already NaN, in which case the invariant is already broken elsewhere. + NotNan(self.0 as f32) + } +} + +impl From> for f32 { + #[inline] + fn from(value: NotNan) -> Self { + value.0 + } +} + +impl From> for f64 { + #[inline] + fn from(value: NotNan) -> Self { + value.0 + } +} + +impl TryFrom for NotNan { + type Error = FloatIsNan; + #[inline] + fn try_from(v: f32) -> Result { + NotNan::new(v) + } +} + +impl TryFrom for NotNan { + type Error = FloatIsNan; + #[inline] + fn try_from(v: f64) -> Result { + NotNan::new(v) + } +} + +macro_rules! impl_from_int_primitive { + ($primitive:ty, $inner:ty) => { + impl From<$primitive> for NotNan<$inner> { + fn from(source: $primitive) -> Self { + // the primitives with which this macro will be called cannot hold a value that + // f64::from would convert to NaN, so this does not hurt invariants + NotNan(<$inner as From<$primitive>>::from(source)) + } + } + }; +} + +impl_from_int_primitive!(i8, f64); +impl_from_int_primitive!(i16, f64); +impl_from_int_primitive!(i32, f64); +impl_from_int_primitive!(u8, f64); +impl_from_int_primitive!(u16, f64); +impl_from_int_primitive!(u32, f64); + +impl_from_int_primitive!(i8, f32); +impl_from_int_primitive!(i16, f32); +impl_from_int_primitive!(u8, f32); +impl_from_int_primitive!(u16, f32); + +impl From> for NotNan { + #[inline] + fn from(v: NotNan) -> NotNan { + unsafe { NotNan::new_unchecked(v.0 as f64) } + } +} + +impl Deref for NotNan { + type Target = T; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl Eq for NotNan {} + +impl PartialEq for NotNan { + #[inline] + fn eq(&self, other: &T) -> bool { + self.0 == *other + } +} + +/// Adds a float directly. +/// +/// Panics if the provided value is NaN or the computation results in NaN +impl Add for NotNan { + type Output = Self; + + #[inline] + fn add(self, other: T) -> Self { + NotNan::new(self.0 + other).expect("Addition resulted in NaN") + } +} + +/// Adds a float directly. +/// +/// Panics if the provided value is NaN. +impl Sum for NotNan { + fn sum>>(iter: I) -> Self { + NotNan::new(iter.map(|v| v.0).sum()).expect("Sum resulted in NaN") + } +} + +impl<'a, T: FloatCore + Sum + 'a> Sum<&'a NotNan> for NotNan { + #[inline] + fn sum>>(iter: I) -> Self { + iter.cloned().sum() + } +} + +/// Subtracts a float directly. +/// +/// Panics if the provided value is NaN or the computation results in NaN +impl Sub for NotNan { + type Output = Self; + + #[inline] + fn sub(self, other: T) -> Self { + NotNan::new(self.0 - other).expect("Subtraction resulted in NaN") + } +} + +/// Multiplies a float directly. +/// +/// Panics if the provided value is NaN or the computation results in NaN +impl Mul for NotNan { + type Output = Self; + + #[inline] + fn mul(self, other: T) -> Self { + NotNan::new(self.0 * other).expect("Multiplication resulted in NaN") + } +} + +impl Product for NotNan { + fn product>>(iter: I) -> Self { + NotNan::new(iter.map(|v| v.0).product()).expect("Product resulted in NaN") + } +} + +impl<'a, T: FloatCore + Product + 'a> Product<&'a NotNan> for NotNan { + #[inline] + fn product>>(iter: I) -> Self { + iter.cloned().product() + } +} + +/// Divides a float directly. +/// +/// Panics if the provided value is NaN or the computation results in NaN +impl Div for NotNan { + type Output = Self; + + #[inline] + fn div(self, other: T) -> Self { + NotNan::new(self.0 / other).expect("Division resulted in NaN") + } +} + +/// Calculates `%` with a float directly. +/// +/// Panics if the provided value is NaN or the computation results in NaN +impl Rem for NotNan { + type Output = Self; + + #[inline] + fn rem(self, other: T) -> Self { + NotNan::new(self.0 % other).expect("Rem resulted in NaN") + } +} + +macro_rules! impl_not_nan_binop { + ($imp:ident, $method:ident, $assign_imp:ident, $assign_method:ident) => { + impl $imp for NotNan { + type Output = Self; + + #[inline] + fn $method(self, other: Self) -> Self { + self.$method(other.0) + } + } + + impl $imp<&T> for NotNan { + type Output = NotNan; + + #[inline] + fn $method(self, other: &T) -> Self::Output { + self.$method(*other) + } + } + + impl $imp<&Self> for NotNan { + type Output = NotNan; + + #[inline] + fn $method(self, other: &Self) -> Self::Output { + self.$method(other.0) + } + } + + impl $imp for &NotNan { + type Output = NotNan; + + #[inline] + fn $method(self, other: Self) -> Self::Output { + (*self).$method(other.0) + } + } + + impl $imp> for &NotNan { + type Output = NotNan; + + #[inline] + fn $method(self, other: NotNan) -> Self::Output { + (*self).$method(other.0) + } + } + + impl $imp for &NotNan { + type Output = NotNan; + + #[inline] + fn $method(self, other: T) -> Self::Output { + (*self).$method(other) + } + } + + impl $imp<&T> for &NotNan { + type Output = NotNan; + + #[inline] + fn $method(self, other: &T) -> Self::Output { + (*self).$method(*other) + } + } + + impl $assign_imp for NotNan { + #[inline] + fn $assign_method(&mut self, other: T) { + *self = (*self).$method(other); + } + } + + impl $assign_imp<&T> for NotNan { + #[inline] + fn $assign_method(&mut self, other: &T) { + *self = (*self).$method(*other); + } + } + + impl $assign_imp for NotNan { + #[inline] + fn $assign_method(&mut self, other: Self) { + (*self).$assign_method(other.0); + } + } + + impl $assign_imp<&Self> for NotNan { + #[inline] + fn $assign_method(&mut self, other: &Self) { + (*self).$assign_method(other.0); + } + } + }; +} + +impl_not_nan_binop! {Add, add, AddAssign, add_assign} +impl_not_nan_binop! {Sub, sub, SubAssign, sub_assign} +impl_not_nan_binop! {Mul, mul, MulAssign, mul_assign} +impl_not_nan_binop! {Div, div, DivAssign, div_assign} +impl_not_nan_binop! {Rem, rem, RemAssign, rem_assign} + +// Will panic if NaN value is return from the operation +macro_rules! impl_not_nan_pow { + ($inner:ty, $rhs:ty) => { + impl Pow<$rhs> for NotNan<$inner> { + type Output = NotNan<$inner>; + #[inline] + fn pow(self, rhs: $rhs) -> NotNan<$inner> { + NotNan::new(<$inner>::pow(self.0, rhs)).expect("Pow resulted in NaN") + } + } + + impl<'a> Pow<&'a $rhs> for NotNan<$inner> { + type Output = NotNan<$inner>; + #[inline] + fn pow(self, rhs: &'a $rhs) -> NotNan<$inner> { + NotNan::new(<$inner>::pow(self.0, *rhs)).expect("Pow resulted in NaN") + } + } + + impl<'a> Pow<$rhs> for &'a NotNan<$inner> { + type Output = NotNan<$inner>; + #[inline] + fn pow(self, rhs: $rhs) -> NotNan<$inner> { + NotNan::new(<$inner>::pow(self.0, rhs)).expect("Pow resulted in NaN") + } + } + + impl<'a, 'b> Pow<&'a $rhs> for &'b NotNan<$inner> { + type Output = NotNan<$inner>; + #[inline] + fn pow(self, rhs: &'a $rhs) -> NotNan<$inner> { + NotNan::new(<$inner>::pow(self.0, *rhs)).expect("Pow resulted in NaN") + } + } + }; +} + +impl_not_nan_pow! {f32, i8} +impl_not_nan_pow! {f32, i16} +impl_not_nan_pow! {f32, u8} +impl_not_nan_pow! {f32, u16} +impl_not_nan_pow! {f32, i32} +impl_not_nan_pow! {f64, i8} +impl_not_nan_pow! {f64, i16} +impl_not_nan_pow! {f64, u8} +impl_not_nan_pow! {f64, u16} +impl_not_nan_pow! {f64, i32} +impl_not_nan_pow! {f32, f32} +impl_not_nan_pow! {f64, f32} +impl_not_nan_pow! {f64, f64} + +// This also should panic on NaN +macro_rules! impl_not_nan_self_pow { + ($base:ty, $exp:ty) => { + impl Pow> for NotNan<$base> { + type Output = NotNan<$base>; + #[inline] + fn pow(self, rhs: NotNan<$exp>) -> NotNan<$base> { + NotNan::new(self.0.pow(rhs.0)).expect("Pow resulted in NaN") + } + } + + impl<'a> Pow<&'a NotNan<$exp>> for NotNan<$base> { + type Output = NotNan<$base>; + #[inline] + fn pow(self, rhs: &'a NotNan<$exp>) -> NotNan<$base> { + NotNan::new(self.0.pow(rhs.0)).expect("Pow resulted in NaN") + } + } + + impl<'a> Pow> for &'a NotNan<$base> { + type Output = NotNan<$base>; + #[inline] + fn pow(self, rhs: NotNan<$exp>) -> NotNan<$base> { + NotNan::new(self.0.pow(rhs.0)).expect("Pow resulted in NaN") + } + } + + impl<'a, 'b> Pow<&'a NotNan<$exp>> for &'b NotNan<$base> { + type Output = NotNan<$base>; + #[inline] + fn pow(self, rhs: &'a NotNan<$exp>) -> NotNan<$base> { + NotNan::new(self.0.pow(rhs.0)).expect("Pow resulted in NaN") + } + } + }; +} + +impl_not_nan_self_pow! {f32, f32} +impl_not_nan_self_pow! {f64, f32} +impl_not_nan_self_pow! {f64, f64} + +impl Neg for NotNan { + type Output = Self; + + #[inline] + fn neg(self) -> Self { + NotNan(-self.0) + } +} + +impl Neg for &NotNan { + type Output = NotNan; + + #[inline] + fn neg(self) -> Self::Output { + NotNan(-self.0) + } +} + +/// An error indicating an attempt to construct NotNan from a NaN +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct FloatIsNan; + +impl Error for FloatIsNan { + fn description(&self) -> &str { + "NotNan constructed with NaN" + } +} + +impl fmt::Display for FloatIsNan { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "NotNan constructed with NaN") + } +} + +impl From for std::io::Error { + #[inline] + fn from(e: FloatIsNan) -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidInput, e) + } +} + +#[inline] +/// Used for hashing. Input must not be zero or NaN. +fn raw_double_bits(f: &F) -> u64 { + let (man, exp, sign) = f.integer_decode(); + let exp_u64 = exp as u16 as u64; + let sign_u64 = (sign > 0) as u64; + (man & MAN_MASK) | ((exp_u64 << 52) & EXP_MASK) | ((sign_u64 << 63) & SIGN_MASK) +} + +impl Zero for NotNan { + #[inline] + fn zero() -> Self { + NotNan(T::zero()) + } + + #[inline] + fn is_zero(&self) -> bool { + self.0.is_zero() + } +} + +impl One for NotNan { + #[inline] + fn one() -> Self { + NotNan(T::one()) + } +} + +impl Bounded for NotNan { + #[inline] + fn min_value() -> Self { + NotNan(T::min_value()) + } + + #[inline] + fn max_value() -> Self { + NotNan(T::max_value()) + } +} + +impl FromStr for NotNan { + type Err = ParseNotNanError; + + /// Convert a &str to `NotNan`. Returns an error if the string fails to parse, + /// or if the resulting value is NaN + /// + /// ``` + /// use ordered_float::NotNan; + /// + /// assert!("-10".parse::>().is_ok()); + /// assert!("abc".parse::>().is_err()); + /// assert!("NaN".parse::>().is_err()); + /// ``` + fn from_str(src: &str) -> Result { + src.parse() + .map_err(ParseNotNanError::ParseFloatError) + .and_then(|f| NotNan::new(f).map_err(|_| ParseNotNanError::IsNaN)) + } +} + +impl FromPrimitive for NotNan { + fn from_i64(n: i64) -> Option { + T::from_i64(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_u64(n: u64) -> Option { + T::from_u64(n).and_then(|n| NotNan::new(n).ok()) + } + + fn from_isize(n: isize) -> Option { + T::from_isize(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_i8(n: i8) -> Option { + T::from_i8(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_i16(n: i16) -> Option { + T::from_i16(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_i32(n: i32) -> Option { + T::from_i32(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_usize(n: usize) -> Option { + T::from_usize(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_u8(n: u8) -> Option { + T::from_u8(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_u16(n: u16) -> Option { + T::from_u16(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_u32(n: u32) -> Option { + T::from_u32(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_f32(n: f32) -> Option { + T::from_f32(n).and_then(|n| NotNan::new(n).ok()) + } + fn from_f64(n: f64) -> Option { + T::from_f64(n).and_then(|n| NotNan::new(n).ok()) + } +} + +impl ToPrimitive for NotNan { + fn to_i64(&self) -> Option { + self.0.to_i64() + } + fn to_u64(&self) -> Option { + self.0.to_u64() + } + + fn to_isize(&self) -> Option { + self.0.to_isize() + } + fn to_i8(&self) -> Option { + self.0.to_i8() + } + fn to_i16(&self) -> Option { + self.0.to_i16() + } + fn to_i32(&self) -> Option { + self.0.to_i32() + } + fn to_usize(&self) -> Option { + self.0.to_usize() + } + fn to_u8(&self) -> Option { + self.0.to_u8() + } + fn to_u16(&self) -> Option { + self.0.to_u16() + } + fn to_u32(&self) -> Option { + self.0.to_u32() + } + fn to_f32(&self) -> Option { + self.0.to_f32() + } + fn to_f64(&self) -> Option { + self.0.to_f64() + } +} + +/// An error indicating a parse error from a string for `NotNan`. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum ParseNotNanError { + /// A plain parse error from the underlying float type. + ParseFloatError(E), + /// The parsed float value resulted in a NaN. + IsNaN, +} + +impl Error for ParseNotNanError { + fn description(&self) -> &str { + "Error parsing a not-NaN floating point value" + } + + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + ParseNotNanError::ParseFloatError(e) => Some(e), + ParseNotNanError::IsNaN => None, + } + } +} + +impl fmt::Display for ParseNotNanError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ParseNotNanError::ParseFloatError(e) => write!(f, "Parse error: {}", e), + ParseNotNanError::IsNaN => write!(f, "NotNan parser encounter a NaN"), + } + } +} + +impl Num for NotNan { + type FromStrRadixErr = ParseNotNanError; + + fn from_str_radix(src: &str, radix: u32) -> Result { + T::from_str_radix(src, radix) + .map_err(ParseNotNanError::ParseFloatError) + .and_then(|n| NotNan::new(n).map_err(|_| ParseNotNanError::IsNaN)) + } +} + +impl Signed for NotNan { + #[inline] + fn abs(&self) -> Self { + NotNan(self.0.abs()) + } + + fn abs_sub(&self, other: &Self) -> Self { + NotNan::new(Signed::abs_sub(&self.0, &other.0)).expect("Subtraction resulted in NaN") + } + + #[inline] + fn signum(&self) -> Self { + NotNan(self.0.signum()) + } + #[inline] + fn is_positive(&self) -> bool { + #[expect(clippy::disallowed_methods)] + self.0.is_positive() + } + #[inline] + fn is_negative(&self) -> bool { + #[expect(clippy::disallowed_methods)] + self.0.is_negative() + } +} + +impl NumCast for NotNan { + fn from(n: F) -> Option { + T::from(n).and_then(|n| NotNan::new(n).ok()) + } +} + +macro_rules! impl_float_const_method { + ($wrapper:expr, $method:ident) => { + #[allow(non_snake_case)] + #[allow(clippy::redundant_closure_call)] + fn $method() -> Self { + $wrapper(T::$method()) + } + }; +} + +macro_rules! impl_float_const { + ($type:ident, $wrapper:expr) => { + impl FloatConst for $type { + impl_float_const_method!($wrapper, E); + impl_float_const_method!($wrapper, FRAC_1_PI); + impl_float_const_method!($wrapper, FRAC_1_SQRT_2); + impl_float_const_method!($wrapper, FRAC_2_PI); + impl_float_const_method!($wrapper, FRAC_2_SQRT_PI); + impl_float_const_method!($wrapper, FRAC_PI_2); + impl_float_const_method!($wrapper, FRAC_PI_3); + impl_float_const_method!($wrapper, FRAC_PI_4); + impl_float_const_method!($wrapper, FRAC_PI_6); + impl_float_const_method!($wrapper, FRAC_PI_8); + impl_float_const_method!($wrapper, LN_10); + impl_float_const_method!($wrapper, LN_2); + impl_float_const_method!($wrapper, LOG10_E); + impl_float_const_method!($wrapper, LOG2_E); + impl_float_const_method!($wrapper, PI); + impl_float_const_method!($wrapper, SQRT_2); + } + }; +} + +impl_float_const!(OrderedFloat, OrderedFloat); +// Float constants are not NaN. +impl_float_const!(NotNan, |x| unsafe { NotNan::new_unchecked(x) }); + +mod impl_serde { + extern crate serde; + use core::f64; + + use num_traits::float::FloatCore; + + use self::serde::de::Error; + use self::serde::de::Unexpected; + use self::serde::Deserialize; + use self::serde::Deserializer; + use self::serde::Serialize; + use self::serde::Serializer; + use super::NotNan; + use super::OrderedFloat; + + #[cfg(test)] + extern crate serde_test; + #[cfg(test)] + use self::serde_test::assert_de_tokens_error; + #[cfg(test)] + use self::serde_test::assert_tokens; + #[cfg(test)] + use self::serde_test::Token; + + impl Serialize for OrderedFloat { + #[inline] + fn serialize(&self, s: S) -> Result { + self.0.serialize(s) + } + } + + impl<'de, T: FloatCore + Deserialize<'de>> Deserialize<'de> for OrderedFloat { + #[inline] + fn deserialize>(d: D) -> Result { + T::deserialize(d).map(OrderedFloat) + } + } + + impl Serialize for NotNan { + #[inline] + fn serialize(&self, s: S) -> Result { + self.0.serialize(s) + } + } + + impl<'de, T: FloatCore + Deserialize<'de>> Deserialize<'de> for NotNan { + fn deserialize>(d: D) -> Result { + let float = T::deserialize(d)?; + NotNan::new(float).map_err(|_| { + Error::invalid_value(Unexpected::Float(f64::NAN), &"float (but not NaN)") + }) + } + } + + #[test] + fn test_ordered_float() { + let float = OrderedFloat(1.0f64); + assert_tokens(&float, &[Token::F64(1.0)]); + } + + #[test] + fn test_not_nan() { + let float = NotNan(1.0f64); + assert_tokens(&float, &[Token::F64(1.0)]); + } + + #[test] + fn test_fail_on_nan() { + assert_de_tokens_error::>( + &[Token::F64(f64::NAN)], + "invalid value: floating point `NaN`, expected float (but not NaN)", + ); + } +} + +mod impl_borsh { + extern crate borsh; + use num_traits::float::FloatCore; + + use super::NotNan; + use super::OrderedFloat; + + impl borsh::BorshSerialize for OrderedFloat + where T: borsh::BorshSerialize + { + #[inline] + fn serialize(&self, writer: &mut W) -> borsh::io::Result<()> { + ::serialize(&self.0, writer) + } + } + + impl borsh::BorshDeserialize for OrderedFloat + where T: borsh::BorshDeserialize + { + #[inline] + fn deserialize_reader(reader: &mut R) -> borsh::io::Result { + ::deserialize_reader(reader).map(Self) + } + } + + impl borsh::BorshSerialize for NotNan + where T: borsh::BorshSerialize + { + #[inline] + fn serialize(&self, writer: &mut W) -> borsh::io::Result<()> { + ::serialize(&self.0, writer) + } + } + + impl borsh::BorshDeserialize for NotNan + where T: FloatCore + borsh::BorshDeserialize + { + #[inline] + fn deserialize_reader(reader: &mut R) -> borsh::io::Result { + let float = ::deserialize_reader(reader)?; + NotNan::new(float).map_err(|_| { + borsh::io::Error::new( + borsh::io::ErrorKind::InvalidData, + "expected a non-NaN float", + ) + }) + } + } + + #[test] + fn test_ordered_float() { + let float = OrderedFloat(1.0f64); + let buffer = borsh::to_vec(&float).expect("failed to serialize value"); + let deser_float: OrderedFloat = + borsh::from_slice(&buffer).expect("failed to deserialize value"); + assert_eq!(deser_float, float); + } + + #[test] + fn test_not_nan() { + let float = NotNan(1.0f64); + let buffer = borsh::to_vec(&float).expect("failed to serialize value"); + let deser_float: NotNan = + borsh::from_slice(&buffer).expect("failed to deserialize value"); + assert_eq!(deser_float, float); + } +} + +mod impl_rand { + use rand::distributions::uniform::*; + use rand::distributions::Distribution; + use rand::distributions::Open01; + use rand::distributions::OpenClosed01; + use rand::distributions::Standard; + use rand::Rng; + use serde::Deserialize; + use serde::Serialize; + + use super::NotNan; + use super::OrderedFloat; + + macro_rules! impl_distribution { + ($dist:ident, $($f:ty),+) => { + $( + impl Distribution> for $dist { + fn sample(&self, rng: &mut R) -> NotNan<$f> { + // 'rand' never generates NaN values in the Standard, Open01, or + // OpenClosed01 distributions. Using 'new_unchecked' is therefore + // safe. + unsafe { NotNan::new_unchecked(self.sample(rng)) } + } + } + + impl Distribution> for $dist { + fn sample(&self, rng: &mut R) -> OrderedFloat<$f> { + OrderedFloat(self.sample(rng)) + } + } + )* + } + } + + impl_distribution! { Standard, f32, f64 } + impl_distribution! { Open01, f32, f64 } + impl_distribution! { OpenClosed01, f32, f64 } + + /// A sampler for a uniform distribution + #[derive(Clone, Copy, Debug, Serialize, Deserialize)] + pub struct UniformNotNan(UniformFloat); + impl SampleUniform for NotNan { + type Sampler = UniformNotNan; + } + impl SampleUniform for NotNan { + type Sampler = UniformNotNan; + } + impl PartialEq for UniformNotNan + where UniformFloat: PartialEq + { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + /// A sampler for a uniform distribution + #[derive(Clone, Copy, Debug, Serialize, Deserialize)] + pub struct UniformOrdered(UniformFloat); + impl SampleUniform for OrderedFloat { + type Sampler = UniformOrdered; + } + impl SampleUniform for OrderedFloat { + type Sampler = UniformOrdered; + } + impl PartialEq for UniformOrdered + where UniformFloat: PartialEq + { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } + } + + macro_rules! impl_uniform_sampler { + ($f:ty) => { + impl UniformSampler for UniformNotNan<$f> { + type X = NotNan<$f>; + fn new(low: B1, high: B2) -> Self + where + B1: SampleBorrow + Sized, + B2: SampleBorrow + Sized, + { + UniformNotNan(UniformFloat::<$f>::new(low.borrow().0, high.borrow().0)) + } + fn new_inclusive(low: B1, high: B2) -> Self + where + B1: SampleBorrow + Sized, + B2: SampleBorrow + Sized, + { + UniformSampler::new(low, high) + } + fn sample(&self, rng: &mut R) -> Self::X { + // UniformFloat.sample() will never return NaN. + unsafe { NotNan::new_unchecked(self.0.sample(rng)) } + } + } + + impl UniformSampler for UniformOrdered<$f> { + type X = OrderedFloat<$f>; + fn new(low: B1, high: B2) -> Self + where + B1: SampleBorrow + Sized, + B2: SampleBorrow + Sized, + { + UniformOrdered(UniformFloat::<$f>::new(low.borrow().0, high.borrow().0)) + } + fn new_inclusive(low: B1, high: B2) -> Self + where + B1: SampleBorrow + Sized, + B2: SampleBorrow + Sized, + { + UniformSampler::new(low, high) + } + fn sample(&self, rng: &mut R) -> Self::X { + OrderedFloat(self.0.sample(rng)) + } + } + }; + } + + impl_uniform_sampler! { f32 } + impl_uniform_sampler! { f64 } +} + +impl Unmarshal> for OrderedFloat { + fn unmarshal(scratch: &[u8]) -> Self { + let bits = u32::from_le_bytes(scratch.try_into().unwrap()); + f32::from_bits(bits).into() + } +} + +impl Unmarshal> for OrderedFloat { + fn unmarshal(scratch: &[u8]) -> Self { + let bits = u64::from_le_bytes(scratch.try_into().unwrap()); + f64::from_bits(bits).into() + } +} diff --git a/src/common/hashtable/Cargo.toml b/src/common/hashtable/Cargo.toml index 72403ad715cc..22c153fcef18 100644 --- a/src/common/hashtable/Cargo.toml +++ b/src/common/hashtable/Cargo.toml @@ -18,7 +18,6 @@ ahash = { version = "0.8.2", features = ["no-rng"] } bumpalo = { workspace = true } cfg-if = "1.0.0" ethnum = { workspace = true } -ordered-float = { workspace = true, features = ["serde"] } [dev-dependencies] rand = { workspace = true } diff --git a/src/common/hashtable/src/traits.rs b/src/common/hashtable/src/traits.rs index ab95b3f3874a..3515bbdbd177 100644 --- a/src/common/hashtable/src/traits.rs +++ b/src/common/hashtable/src/traits.rs @@ -19,9 +19,9 @@ use std::mem::MaybeUninit; use std::num::NonZeroU64; use databend_common_arrow::arrow::bitmap::Bitmap; +use databend_common_base::base::OrderedFloat; use ethnum::i256; use ethnum::U256; -use ordered_float::OrderedFloat; use crate::RowPtr; diff --git a/src/common/io/Cargo.toml b/src/common/io/Cargo.toml index 0a7711f92a6f..e1f2b03461c2 100644 --- a/src/common/io/Cargo.toml +++ b/src/common/io/Cargo.toml @@ -17,13 +17,13 @@ bytes = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } databend-common-exception = { workspace = true } +databend-common-base = { workspace = true } ethnum = { workspace = true } geo = { workspace = true } geos = { workspace = true } geozero = { workspace = true } lexical-core = "0.8.5" micromarshal = "0.5.0" -ordered-float = { workspace = true } roaring = { version = "0.10.1", features = ["serde"] } serde = { workspace = true } wkt = "0.10.3" diff --git a/src/common/io/src/stat_buffer.rs b/src/common/io/src/stat_buffer.rs index 75cd3860005d..5fc37be3f2e7 100644 --- a/src/common/io/src/stat_buffer.rs +++ b/src/common/io/src/stat_buffer.rs @@ -14,7 +14,7 @@ use std::fmt::Debug; -use ordered_float::OrderedFloat; +use databend_common_base::base::OrderedFloat; pub trait StatBuffer { type Buffer: AsMut<[u8]> + AsRef<[u8]> + Copy + Sync + Debug; diff --git a/src/common/storage/Cargo.toml b/src/common/storage/Cargo.toml index 58351d393fba..bacc5c9ecfd6 100644 --- a/src/common/storage/Cargo.toml +++ b/src/common/storage/Cargo.toml @@ -26,7 +26,6 @@ flagset = "0.4" futures = { workspace = true } log = { workspace = true } opendal = { workspace = true } -ordered-float = { workspace = true } parquet = { workspace = true } prometheus-client = { workspace = true } regex = { workspace = true } diff --git a/src/common/storage/src/statistics.rs b/src/common/storage/src/statistics.rs index 307c71f67c4a..917d84f46d51 100644 --- a/src/common/storage/src/statistics.rs +++ b/src/common/storage/src/statistics.rs @@ -15,11 +15,11 @@ use std::fmt::Display; use std::fmt::Formatter; +use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::number::NumberScalar; use databend_common_expression::Scalar; -use ordered_float::OrderedFloat; pub type F64 = OrderedFloat; diff --git a/src/query/expression/Cargo.toml b/src/query/expression/Cargo.toml index a8d9a55c35ff..0b2701b18095 100644 --- a/src/query/expression/Cargo.toml +++ b/src/query/expression/Cargo.toml @@ -46,7 +46,6 @@ memchr = { version = "2", default-features = false } micromarshal = "0.5.0" num-bigint = "0.4.6" num-traits = "0.2.15" -ordered-float = { workspace = true, features = ["serde", "rand", "borsh"] } rand = { workspace = true } recursive = "0.1.1" roaring = { version = "0.10.1", features = ["serde"] } diff --git a/src/query/expression/src/aggregate/group_hash.rs b/src/query/expression/src/aggregate/group_hash.rs index 4172e93ca783..660fc11ace3b 100644 --- a/src/query/expression/src/aggregate/group_hash.rs +++ b/src/query/expression/src/aggregate/group_hash.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_base::base::OrderedFloat; use ethnum::i256; -use ordered_float::OrderedFloat; use crate::types::decimal::DecimalType; use crate::types::geometry::GeometryType; diff --git a/src/query/expression/src/converts/datavalues/to.rs b/src/query/expression/src/converts/datavalues/to.rs index 139a82483325..11d409e5d0e6 100644 --- a/src/query/expression/src/converts/datavalues/to.rs +++ b/src/query/expression/src/converts/datavalues/to.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_base::base::OrderedFloat; use databend_common_datavalues::DataValue; -use ordered_float::OrderedFloat; use crate::Scalar; diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index d49673258298..0afe13311c63 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -19,12 +19,12 @@ use std::ops::Range; use borsh::BorshDeserialize; use borsh::BorshSerialize; use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_base::base::OrderedFloat; use enum_as_inner::EnumAsInner; use itertools::Itertools; use lexical_core::ToLexicalWithOptions; use num_traits::float::FloatCore; use num_traits::NumCast; -use ordered_float::OrderedFloat; use serde::Deserialize; use serde::Serialize; @@ -315,15 +315,7 @@ pub enum NumberScalar { Int16(i16), Int32(i32), Int64(i64), - #[serde( - serialize_with = "json_serde_impl::serialize_f32", - deserialize_with = "json_serde_impl::deserialize_f32" - )] Float32(F32), - #[serde( - serialize_with = "json_serde_impl::serialize_f64", - deserialize_with = "json_serde_impl::deserialize_f64" - )] Float64(F64), } @@ -1428,90 +1420,3 @@ impl Number for F64 { } } } - -/// Json serialize and deserialize implementation for `F32` and `F64`. It specially handles the -/// cases of `Infinity`, `-Infinity` and `Nan`. -mod json_serde_impl { - - use num_traits::float::FloatCore; - use ordered_float::OrderedFloat; - use serde::Deserialize; - use serde::Deserializer; - use serde::Serializer; - use serde_json::Value; - - use crate::types::F32; - use crate::types::F64; - - pub fn serialize_f32(value: &F32, serializer: S) -> Result - where S: Serializer { - if serializer.is_human_readable() { - if value.is_infinite() { - if value.is_sign_positive() { - serializer.serialize_str("Infinity") - } else { - serializer.serialize_str("-Infinity") - } - } else if value.is_nan() { - serializer.serialize_str("Nan") - } else { - serde::Serialize::serialize(&value, serializer) - } - } else { - serde::Serialize::serialize(&value, serializer) - } - } - - pub fn deserialize_f32<'de, D>(deserializer: D) -> Result - where D: Deserializer<'de> { - if deserializer.is_human_readable() { - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) if s == "Infinity" => Ok(F32::infinity()), - Value::String(s) if s == "-Infinity" => Ok(F32::neg_infinity()), - Value::String(s) if s == "Nan" => Ok(F32::nan()), - _ => ::deserialize(value) - .map( as From>::from) - .map_err(|e| serde::de::Error::custom(e.to_string())), - } - } else { - serde::Deserialize::deserialize(deserializer) - } - } - - pub fn serialize_f64(value: &F64, serializer: S) -> Result - where S: Serializer { - if serializer.is_human_readable() { - if value.is_infinite() { - if value.is_sign_positive() { - serializer.serialize_str("Infinity") - } else { - serializer.serialize_str("-Infinity") - } - } else if value.is_nan() { - serializer.serialize_str("Nan") - } else { - serde::Serialize::serialize(&value, serializer) - } - } else { - serde::Serialize::serialize(&value, serializer) - } - } - - pub fn deserialize_f64<'de, D>(deserializer: D) -> Result - where D: Deserializer<'de> { - if deserializer.is_human_readable() { - let value = Value::deserialize(deserializer)?; - match value { - Value::String(s) if s == "Infinity" => Ok(F64::infinity()), - Value::String(s) if s == "-Infinity" => Ok(F64::neg_infinity()), - Value::String(s) if s == "Nan" => Ok(F64::nan()), - _ => ::deserialize(value) - .map( as From>::from) - .map_err(|e| serde::de::Error::custom(e.to_string())), - } - } else { - serde::Deserialize::deserialize(deserializer) - } - } -} diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 9023958a775a..dad81a912e2b 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -26,6 +26,7 @@ use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_arrow::arrow::buffer::Buffer; use databend_common_arrow::arrow::trusted_len::TrustedLen; +use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_io::prelude::BinaryRead; @@ -36,7 +37,6 @@ use geo::Point; use geozero::CoordDimensions; use geozero::ToWkb; use itertools::Itertools; -use ordered_float::OrderedFloat; use roaring::RoaringTreemap; use serde::de::Visitor; use serde::Deserialize; diff --git a/src/query/expression/tests/it/fill_field_default_value.rs b/src/query/expression/tests/it/fill_field_default_value.rs index aa97039257c6..9d4db03eef20 100644 --- a/src/query/expression/tests/it/fill_field_default_value.rs +++ b/src/query/expression/tests/it/fill_field_default_value.rs @@ -84,7 +84,9 @@ fn test_data_block_create_with_default_value_functions() -> Result<()> { let default_vals = vec![ Scalar::Number(NumberScalar::Int32(1)), Scalar::Number(NumberScalar::Int32(0)), - Scalar::Number(NumberScalar::Float32(ordered_float::OrderedFloat(10.0))), + Scalar::Number(NumberScalar::Float32( + databend_common_base::base::OrderedFloat(10.0), + )), Scalar::Number(NumberScalar::Int32(0)), Scalar::String("ab".into()), ]; @@ -112,7 +114,9 @@ fn test_data_block_create_with_default_value_functions() -> Result<()> { let default_vals = vec![ Scalar::Number(NumberScalar::Int32(1)), Scalar::Number(NumberScalar::Int32(2)), - Scalar::Number(NumberScalar::Float32(ordered_float::OrderedFloat(10.0))), + Scalar::Number(NumberScalar::Float32( + databend_common_base::base::OrderedFloat(10.0), + )), Scalar::Number(NumberScalar::Int32(3)), Scalar::String("ab".into()), ]; diff --git a/src/query/expression/tests/it/row.rs b/src/query/expression/tests/it/row.rs index 1e8106395c28..8f9fc31a0436 100644 --- a/src/query/expression/tests/it/row.rs +++ b/src/query/expression/tests/it/row.rs @@ -19,6 +19,7 @@ use arrow_ord::sort::SortColumn; use arrow_schema::SortOptions; use databend_common_arrow::arrow::bitmap::MutableBitmap; use databend_common_arrow::arrow::offset::OffsetsBuffer; +use databend_common_base::base::OrderedFloat; use databend_common_expression::converts::arrow2::set_validities; use databend_common_expression::types::binary::BinaryColumnBuilder; use databend_common_expression::types::decimal::*; @@ -32,7 +33,6 @@ use ethnum::i256; use itertools::Itertools; use jsonb::convert_to_comparable; use jsonb::parse_value; -use ordered_float::OrderedFloat; use rand::distributions::Alphanumeric; use rand::distributions::Standard; use rand::prelude::Distribution; diff --git a/src/query/formats/Cargo.toml b/src/query/formats/Cargo.toml index 31d5311a67c6..4e5335bfda9b 100644 --- a/src/query/formats/Cargo.toml +++ b/src/query/formats/Cargo.toml @@ -16,6 +16,7 @@ async-trait = { workspace = true } base64 = "0.21.0" bstr = "1.0.1" chrono-tz = { workspace = true } +databend-common-base = { workspace = true } databend-common-arrow = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } @@ -33,7 +34,6 @@ match-template = { workspace = true } micromarshal = "0.7.0" num = "0.4.0" num-traits = "0.2.15" -ordered-float = { workspace = true } roaring = { version = "0.10.1", features = ["serde"] } serde_json = { workspace = true } diff --git a/src/query/formats/src/field_encoder/values.rs b/src/query/formats/src/field_encoder/values.rs index 044f055082d8..ce8144eb0376 100644 --- a/src/query/formats/src/field_encoder/values.rs +++ b/src/query/formats/src/field_encoder/values.rs @@ -16,6 +16,7 @@ use bstr::ByteSlice; use chrono_tz::Tz; use databend_common_arrow::arrow::bitmap::Bitmap; use databend_common_arrow::arrow::buffer::Buffer; +use databend_common_base::base::OrderedFloat; use databend_common_expression::types::array::ArrayColumn; use databend_common_expression::types::binary::BinaryColumn; use databend_common_expression::types::date::date_to_string; @@ -44,7 +45,6 @@ use geozero::ToWkt; use lexical_core::ToLexical; use micromarshal::Marshal; use micromarshal::Unmarshal; -use ordered_float::OrderedFloat; use crate::field_encoder::helpers::write_quoted_string; use crate::field_encoder::helpers::PrimitiveWithFormat; diff --git a/src/query/functions/Cargo.toml b/src/query/functions/Cargo.toml index 5195a0f29a11..9b5013aabd32 100644 --- a/src/query/functions/Cargo.toml +++ b/src/query/functions/Cargo.toml @@ -47,10 +47,6 @@ memchr = { version = "2", default-features = false } naive-cityhash = "0.2.0" num-traits = "0.2.15" once_cell = { workspace = true } -ordered-float = { workspace = true, features = [ - "serde", - "rand", -] } proj4rs = { workspace = true } rand = { workspace = true } regex = { workspace = true } diff --git a/src/query/functions/src/aggregates/aggregate_quantile_cont.rs b/src/query/functions/src/aggregates/aggregate_quantile_cont.rs index 790cf23760fa..5c9c26d8361b 100644 --- a/src/query/functions/src/aggregates/aggregate_quantile_cont.rs +++ b/src/query/functions/src/aggregates/aggregate_quantile_cont.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use borsh::BorshDeserialize; use borsh::BorshSerialize; +use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::type_check::check_number; @@ -33,7 +34,6 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use ethnum::i256; use num_traits::AsPrimitive; -use ordered_float::OrderedFloat; use super::AggregateUnaryFunction; use super::FunctionData; diff --git a/src/query/functions/src/scalars/boolean.rs b/src/query/functions/src/scalars/boolean.rs index 8f28f5d8b5fc..4235e6fa47e0 100644 --- a/src/query/functions/src/scalars/boolean.rs +++ b/src/query/functions/src/scalars/boolean.rs @@ -17,6 +17,7 @@ use std::sync::Arc; +use databend_common_base::base::OrderedFloat; use databend_common_expression::error_to_null; use databend_common_expression::types::boolean::BooleanDomain; use databend_common_expression::types::nullable::NullableColumn; @@ -42,7 +43,6 @@ use databend_common_expression::FunctionRegistry; use databend_common_expression::FunctionSignature; use databend_common_expression::Value; use databend_common_expression::ValueRef; -use ordered_float::OrderedFloat; pub fn register(registry: &mut FunctionRegistry) { registry.register_function_factory("and_filters", |_, args_type| { diff --git a/src/query/functions/src/scalars/decimal/cast.rs b/src/query/functions/src/scalars/decimal/cast.rs index 1a06634cda9b..dd4494bc0165 100644 --- a/src/query/functions/src/scalars/decimal/cast.rs +++ b/src/query/functions/src/scalars/decimal/cast.rs @@ -15,6 +15,7 @@ use std::ops::Mul; use std::sync::Arc; +use databend_common_base::base::OrderedFloat; use databend_common_expression::serialize::read_decimal_with_size; use databend_common_expression::types::decimal::*; use databend_common_expression::types::string::StringColumnBuilder; @@ -39,7 +40,6 @@ use databend_common_expression::Value; use databend_common_expression::ValueRef; use ethnum::i256; use num_traits::AsPrimitive; -use ordered_float::OrderedFloat; // int float to decimal pub fn register_to_decimal(registry: &mut FunctionRegistry) { diff --git a/src/query/functions/src/scalars/math.rs b/src/query/functions/src/scalars/math.rs index a81c592307ef..b07d25459638 100644 --- a/src/query/functions/src/scalars/math.rs +++ b/src/query/functions/src/scalars/math.rs @@ -17,6 +17,7 @@ use std::f64::consts::E; use std::f64::consts::PI; use std::marker::PhantomData; +use databend_common_base::base::OrderedFloat; use databend_common_expression::types::number::SimpleDomain; use databend_common_expression::types::number::F64; use databend_common_expression::types::NumberDataType; @@ -33,7 +34,6 @@ use databend_common_expression::Value; use num_traits::AsPrimitive; use num_traits::Float; use num_traits::Pow; -use ordered_float::OrderedFloat; use crate::scalars::decimal::register_decimal_math; diff --git a/src/query/functions/src/scalars/other.rs b/src/query/functions/src/scalars/other.rs index fbd8cb36ada8..98eaffec0995 100644 --- a/src/query/functions/src/scalars/other.rs +++ b/src/query/functions/src/scalars/other.rs @@ -20,6 +20,7 @@ use std::time::Duration; use databend_common_base::base::convert_byte_size; use databend_common_base::base::convert_number_size; use databend_common_base::base::uuid::Uuid; +use databend_common_base::base::OrderedFloat; use databend_common_expression::error_to_null; use databend_common_expression::types::boolean::BooleanDomain; use databend_common_expression::types::nullable::NullableColumn; @@ -57,7 +58,6 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::Value; use databend_common_expression::ValueRef; -use ordered_float::OrderedFloat; use rand::Rng; use rand::SeedableRng; diff --git a/src/query/functions/tests/it/scalars/parser.rs b/src/query/functions/tests/it/scalars/parser.rs index ec35b8c4747f..e9a3501bf1f1 100644 --- a/src/query/functions/tests/it/scalars/parser.rs +++ b/src/query/functions/tests/it/scalars/parser.rs @@ -23,6 +23,7 @@ use databend_common_ast::ast::UnaryOperator; use databend_common_ast::parser::parse_expr; use databend_common_ast::parser::tokenize_sql; use databend_common_ast::parser::Dialect; +use databend_common_base::base::OrderedFloat; use databend_common_expression::shrink_scalar; use databend_common_expression::type_check; use databend_common_expression::types::decimal::DecimalDataType; @@ -36,7 +37,6 @@ use databend_common_expression::FunctionContext; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use ordered_float::OrderedFloat; pub fn parse_raw_expr(text: &str, columns: &[(&str, DataType)]) -> RawExpr { let tokens = tokenize_sql(text).unwrap(); diff --git a/src/query/service/tests/it/storages/fuse/operations/alter_table.rs b/src/query/service/tests/it/storages/fuse/operations/alter_table.rs index 54fea6709015..8c4646879ba7 100644 --- a/src/query/service/tests/it/storages/fuse/operations/alter_table.rs +++ b/src/query/service/tests/it/storages/fuse/operations/alter_table.rs @@ -15,6 +15,7 @@ use std::collections::HashSet; use databend_common_base::base::tokio; +use databend_common_base::base::OrderedFloat; use databend_common_exception::Result; use databend_common_expression::types::Float64Type; use databend_common_expression::types::Int32Type; @@ -45,7 +46,6 @@ use databend_storages_common_table_meta::meta::TableSnapshot; use databend_storages_common_table_meta::meta::Versioned; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use futures_util::TryStreamExt; -use ordered_float::OrderedFloat; async fn check_segment_column_ids( fixture: &TestFixture, diff --git a/src/query/sql/Cargo.toml b/src/query/sql/Cargo.toml index 3e340dd8163e..e258b744ff45 100644 --- a/src/query/sql/Cargo.toml +++ b/src/query/sql/Cargo.toml @@ -61,7 +61,6 @@ log = { workspace = true } num-derive = "0.3.3" num-traits = "0.2.15" opendal = { workspace = true } -ordered-float = { workspace = true } parking_lot = { workspace = true } percent-encoding = "2" prqlc = "0.11.3" diff --git a/src/query/sql/src/planner/optimizer/filter/infer_filter.rs b/src/query/sql/src/planner/optimizer/filter/infer_filter.rs index 0d6bcf048cef..04333e81efb6 100644 --- a/src/query/sql/src/planner/optimizer/filter/infer_filter.rs +++ b/src/query/sql/src/planner/optimizer/filter/infer_filter.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use databend_common_base::base::OrderedFloat; use databend_common_exception::Result; use databend_common_expression::type_check::common_super_type; use databend_common_expression::types::DataType; @@ -22,7 +23,6 @@ use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; use databend_common_functions::BUILTIN_FUNCTIONS; -use ordered_float::OrderedFloat; use crate::optimizer::rule::constant::check_float_range; use crate::optimizer::rule::constant::check_int_range; diff --git a/src/query/sql/src/planner/optimizer/rule/utils/constant.rs b/src/query/sql/src/planner/optimizer/rule/utils/constant.rs index 16d123e04b8b..6564c6a294d3 100644 --- a/src/query/sql/src/planner/optimizer/rule/utils/constant.rs +++ b/src/query/sql/src/planner/optimizer/rule/utils/constant.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use databend_common_base::base::OrderedFloat; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; -use ordered_float::OrderedFloat; use crate::plans::BoundColumnRef; use crate::plans::CastExpr; diff --git a/src/query/storages/delta/Cargo.toml b/src/query/storages/delta/Cargo.toml index fa460e861bfc..428816eac2af 100644 --- a/src/query/storages/delta/Cargo.toml +++ b/src/query/storages/delta/Cargo.toml @@ -23,7 +23,6 @@ deltalake = { workspace = true } fastrace = { workspace = true } match-template = "0.0.1" object_store_opendal = { workspace = true } -ordered-float = { workspace = true } parquet = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/src/query/storages/delta/src/partition_columns/values_serde.rs b/src/query/storages/delta/src/partition_columns/values_serde.rs index eab44babf951..e283b0f4338e 100644 --- a/src/query/storages/delta/src/partition_columns/values_serde.rs +++ b/src/query/storages/delta/src/partition_columns/values_serde.rs @@ -14,6 +14,7 @@ // TODO: support other data types +use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::DataType; @@ -22,7 +23,6 @@ use databend_common_expression::types::NumberScalar; use databend_common_expression::Scalar; use databend_common_expression::TableField; use deltalake::kernel::Add; -use ordered_float::OrderedFloat; pub fn str_to_scalar(value: &str, data_type: &DataType) -> Result { if value.is_empty() { diff --git a/src/query/storages/hive/hive/Cargo.toml b/src/query/storages/hive/hive/Cargo.toml index dec13b8be965..2bc99a9aff48 100644 --- a/src/query/storages/hive/hive/Cargo.toml +++ b/src/query/storages/hive/hive/Cargo.toml @@ -39,7 +39,6 @@ futures = { workspace = true } hive_metastore = "0.1.0" log = { workspace = true } opendal = { workspace = true } -ordered-float = { workspace = true } recursive = "0.1.1" serde = { workspace = true } typetag = { workspace = true } diff --git a/src/query/storages/hive/hive/src/utils.rs b/src/query/storages/hive/hive/src/utils.rs index bb616f1e7b2f..f833109d4a9a 100644 --- a/src/query/storages/hive/hive/src/utils.rs +++ b/src/query/storages/hive/hive/src/utils.rs @@ -14,13 +14,13 @@ use std::fmt::Debug; +use databend_common_base::base::OrderedFloat; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::types::number::NumberScalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; use databend_common_expression::Scalar; -use ordered_float::OrderedFloat; use volo_thrift::MaybeException; use crate::hive_table::HIVE_DEFAULT_PARTITION; diff --git a/src/query/storages/iceberg/Cargo.toml b/src/query/storages/iceberg/Cargo.toml index b6ce09873b46..61a066b26a1a 100644 --- a/src/query/storages/iceberg/Cargo.toml +++ b/src/query/storages/iceberg/Cargo.toml @@ -34,6 +34,7 @@ parquet = { workspace = true } serde = { workspace = true } tokio = { workspace = true } typetag = { workspace = true } +ordered-float = { workspace = true } [lints] workspace = true diff --git a/src/query/storages/iceberg/src/stats.rs b/src/query/storages/iceberg/src/stats.rs index bd6455068ecf..95c9dc65bf4f 100644 --- a/src/query/storages/iceberg/src/stats.rs +++ b/src/query/storages/iceberg/src/stats.rs @@ -25,6 +25,7 @@ use databend_storages_common_table_meta::meta::StatisticsOfColumns; use iceberg::spec::DataFile; use iceberg::spec::Datum; use iceberg::spec::PrimitiveLiteral; +use ordered_float::OrderedFloat; /// Try to convert statistics in [`DataFile`] to [`StatisticsOfColumns`]. pub fn get_stats_of_data_file(schema: &TableSchema, df: &DataFile) -> Option { @@ -74,8 +75,12 @@ fn parse_datum(data: &Datum) -> Option { PrimitiveLiteral::Boolean(v) => Some(Scalar::Boolean(*v)), PrimitiveLiteral::Int(v) => Some(Scalar::Number(i32::upcast_scalar(*v))), PrimitiveLiteral::Long(v) => Some(Scalar::Number(i64::upcast_scalar(*v))), - PrimitiveLiteral::Float(v) => Some(Scalar::Number(F32::upcast_scalar(F32::from(*v)))), - PrimitiveLiteral::Double(v) => Some(Scalar::Number(F64::upcast_scalar(F64::from(*v)))), + PrimitiveLiteral::Float(OrderedFloat(v)) => { + Some(Scalar::Number(F32::upcast_scalar(F32::from(*v)))) + } + PrimitiveLiteral::Double(OrderedFloat(v)) => { + Some(Scalar::Number(F64::upcast_scalar(F64::from(*v)))) + } PrimitiveLiteral::String(v) => Some(Scalar::String(v.clone())), _ => None, } From 99c6fc3b2b27a94901093b0cc6c0b5fe08566853 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 9 Sep 2024 00:01:33 +0800 Subject: [PATCH 09/11] fix: override ordered_float's json and borsh (de)serde methods to allow use NaN and inf --- src/common/base/src/base/ordered_float.rs | 68 +++++++++++++++++++---- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/src/common/base/src/base/ordered_float.rs b/src/common/base/src/base/ordered_float.rs index e29cba02039b..fa2bc65b21b1 100644 --- a/src/common/base/src/base/ordered_float.rs +++ b/src/common/base/src/base/ordered_float.rs @@ -1879,6 +1879,7 @@ mod impl_serde { use core::f64; use num_traits::float::FloatCore; + use serde::de::IntoDeserializer; use self::serde::de::Error; use self::serde::de::Unexpected; @@ -1901,14 +1902,40 @@ mod impl_serde { impl Serialize for OrderedFloat { #[inline] fn serialize(&self, s: S) -> Result { - self.0.serialize(s) + if s.is_human_readable() { + if self.0.is_infinite() { + if self.0.is_sign_positive() { + s.serialize_str("Infinity") + } else { + s.serialize_str("-Infinity") + } + } else if self.0.is_nan() { + s.serialize_str("Nan") + } else { + self.0.serialize(s) + } + } else { + self.0.serialize(s) + } } } impl<'de, T: FloatCore + Deserialize<'de>> Deserialize<'de> for OrderedFloat { #[inline] fn deserialize>(d: D) -> Result { - T::deserialize(d).map(OrderedFloat) + if d.is_human_readable() { + let value = serde_json::Value::deserialize(d)?; + match value { + serde_json::Value::String(s) if s == "Infinity" => Ok(Self::infinity()), + serde_json::Value::String(s) if s == "-Infinity" => Ok(Self::neg_infinity()), + serde_json::Value::String(s) if s == "Nan" => Ok(Self::nan()), + _ => T::deserialize(serde_json::Value::into_deserializer(value)) + .map(OrderedFloat) + .map_err(Error::custom), + } + } else { + T::deserialize(d).map(OrderedFloat) + } } } @@ -1951,26 +1978,45 @@ mod impl_serde { mod impl_borsh { extern crate borsh; + + use std::io::Read; + use num_traits::float::FloatCore; use super::NotNan; use super::OrderedFloat; - impl borsh::BorshSerialize for OrderedFloat - where T: borsh::BorshSerialize - { + impl borsh::BorshSerialize for OrderedFloat { #[inline] fn serialize(&self, writer: &mut W) -> borsh::io::Result<()> { - ::serialize(&self.0, writer) + writer.write_all(&self.0.to_bits().to_le_bytes())?; + Ok(()) } } - impl borsh::BorshDeserialize for OrderedFloat - where T: borsh::BorshDeserialize - { + impl borsh::BorshSerialize for OrderedFloat { #[inline] - fn deserialize_reader(reader: &mut R) -> borsh::io::Result { - ::deserialize_reader(reader).map(Self) + fn serialize(&self, writer: &mut W) -> borsh::io::Result<()> { + writer.write_all(&self.0.to_bits().to_le_bytes())?; + Ok(()) + } + } + + impl borsh::BorshDeserialize for OrderedFloat { + fn deserialize_reader(reader: &mut R) -> std::io::Result { + let mut buf = [0u8; size_of::>()]; + reader.read_exact(&mut buf)?; + let res = OrderedFloat::from(f32::from_bits(u32::from_le_bytes(buf))); + Ok(res) + } + } + + impl borsh::BorshDeserialize for OrderedFloat { + fn deserialize_reader(reader: &mut R) -> std::io::Result { + let mut buf = [0u8; size_of::>()]; + reader.read_exact(&mut buf)?; + let res = OrderedFloat::from(f64::from_bits(u64::from_le_bytes(buf))); + Ok(res) } } From 22e9524de5181c1f034d223f02f033e120a53757 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 9 Sep 2024 00:06:28 +0800 Subject: [PATCH 10/11] taplo fmt --- Cargo.lock | 1 + src/common/base/Cargo.toml | 9 +++++---- src/common/io/Cargo.toml | 2 +- src/query/formats/Cargo.toml | 2 +- src/query/storages/iceberg/Cargo.toml | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 15c7b8b8739b..41a8f3027e36 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3203,6 +3203,7 @@ dependencies = [ "replace_with", "semver", "serde", + "serde_json", "serde_test", "state", "tikv-jemalloc-ctl", diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml index f89e4eecb8a4..62a88b3efce1 100644 --- a/src/common/base/Cargo.toml +++ b/src/common/base/Cargo.toml @@ -23,6 +23,7 @@ memory-profiling = [ [dependencies] async-backtrace = { workspace = true } async-trait = { workspace = true } +borsh = { workspace = true } bytesize = "1.1.0" chrono = { workspace = true } ctrlc = { version = "3.2.3", features = ["termination"] } @@ -33,6 +34,8 @@ futures = { workspace = true } libc = "0.2.153" log = { workspace = true } logcall = { workspace = true } +micromarshal = "0.5.0" +num-traits = "0.2.19" num_cpus = "1.13.1" once_cell = { workspace = true } parking_lot = { workspace = true } @@ -44,20 +47,18 @@ pprof = { version = "0.11.1", features = [ ] } prometheus-client = { workspace = true } prometheus-parse = "0.2.3" +rand = { workspace = true, features = ["serde1"] } regex = { workspace = true } replace_with = "0.1.7" semver = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } state = "0.5" tikv-jemalloc-ctl = { workspace = true } tikv-jemalloc-sys = "0.5.2" tokio = { workspace = true } unicode-segmentation = "1.10.1" uuid = { workspace = true } -num-traits = "0.2.19" -borsh = { workspace = true } -rand = { workspace = true, features = ["serde1"]} -micromarshal = "0.5.0" [target.'cfg(target_os = "linux")'.dependencies] procfs = { version = "^0.16" } diff --git a/src/common/io/Cargo.toml b/src/common/io/Cargo.toml index e1f2b03461c2..23addb6c9c12 100644 --- a/src/common/io/Cargo.toml +++ b/src/common/io/Cargo.toml @@ -16,8 +16,8 @@ borsh = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } chrono-tz = { workspace = true } -databend-common-exception = { workspace = true } databend-common-base = { workspace = true } +databend-common-exception = { workspace = true } ethnum = { workspace = true } geo = { workspace = true } geos = { workspace = true } diff --git a/src/query/formats/Cargo.toml b/src/query/formats/Cargo.toml index 4e5335bfda9b..7a6b86420214 100644 --- a/src/query/formats/Cargo.toml +++ b/src/query/formats/Cargo.toml @@ -16,8 +16,8 @@ async-trait = { workspace = true } base64 = "0.21.0" bstr = "1.0.1" chrono-tz = { workspace = true } -databend-common-base = { workspace = true } databend-common-arrow = { workspace = true } +databend-common-base = { workspace = true } databend-common-exception = { workspace = true } databend-common-expression = { workspace = true } databend-common-io = { workspace = true } diff --git a/src/query/storages/iceberg/Cargo.toml b/src/query/storages/iceberg/Cargo.toml index 61a066b26a1a..244cf457d596 100644 --- a/src/query/storages/iceberg/Cargo.toml +++ b/src/query/storages/iceberg/Cargo.toml @@ -30,11 +30,11 @@ iceberg = { git = "https://github.com/apache/iceberg-rust", package = "iceberg", iceberg-catalog-hms = { git = "https://github.com/apache/iceberg-rust", package = "iceberg-catalog-hms", rev = "v0.3.0-rc.1" } iceberg-catalog-rest = { git = "https://github.com/apache/iceberg-rust", package = "iceberg-catalog-rest", rev = "v0.3.0-rc.1" } match-template = { workspace = true } +ordered-float = { workspace = true } parquet = { workspace = true } serde = { workspace = true } tokio = { workspace = true } typetag = { workspace = true } -ordered-float = { workspace = true } [lints] workspace = true From 1fc42a68a84ea43460ebee0f61c8c42e78845b36 Mon Sep 17 00:00:00 2001 From: Liuqing Yue Date: Mon, 9 Sep 2024 15:41:15 +0800 Subject: [PATCH 11/11] test: add more test --- src/common/base/src/base/ordered_float.rs | 30 ++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/common/base/src/base/ordered_float.rs b/src/common/base/src/base/ordered_float.rs index fa2bc65b21b1..23fd5946c592 100644 --- a/src/common/base/src/base/ordered_float.rs +++ b/src/common/base/src/base/ordered_float.rs @@ -1897,6 +1897,8 @@ mod impl_serde { #[cfg(test)] use self::serde_test::assert_tokens; #[cfg(test)] + use self::serde_test::Configure; + #[cfg(test)] use self::serde_test::Token; impl Serialize for OrderedFloat { @@ -1958,7 +1960,33 @@ mod impl_serde { #[test] fn test_ordered_float() { let float = OrderedFloat(1.0f64); - assert_tokens(&float, &[Token::F64(1.0)]); + assert_tokens(&float.readable(), &[Token::F64(1.0)]); + } + + #[test] + fn test_ordered_float_with_nan() { + let float = OrderedFloat(f64::nan()); + assert_tokens(&float.readable(), &[Token::Str("Nan")]); + } + + #[test] + fn test_ordered_float_with_inf() { + let float = OrderedFloat(f64::infinity()); + assert_tokens(&float.readable(), &[Token::Str("Infinity")]); + let float = OrderedFloat(f64::neg_infinity()); + assert_tokens(&float.readable(), &[Token::Str("-Infinity")]); + } + + #[test] + fn test_non_readable_ordered_float() { + let float = OrderedFloat(1.0f64); + assert_tokens(&float.compact(), &[Token::F64(1.0)]); + + let float = OrderedFloat(f64::infinity()); + assert_tokens(&float.compact(), &[Token::F64(f64::infinity())]); + + let float = OrderedFloat(f64::neg_infinity()); + assert_tokens(&float.compact(), &[Token::F64(f64::neg_infinity())]); } #[test]