Merge remote-tracking branch 'up/main' into spill-parquet

Signed-off-by: coldWater <[email protected]>

.github/actions/test_unit/action.yml

@@ -16,7 +16,7 @@ runs:
         RUST_TEST_THREADS: "8"
         RUST_LOG: ERROR
         RUST_MIN_STACK: 104857600
-        # RUST_BACKTRACE: full
+        # RUST_BACKTRACE: 1
 
     - name: Upload failure
       if: failure()

Cargo.toml

@@ -351,18 +351,26 @@ useless_format = "allow"
 mutable_key_type = "allow"
 result_large_err = "allow"
 
+## DONT'T DELETE THIS: If we want best performance, we should use this profile but it will take longer time to compile.
+## Test SQL: 
+## select sum(number) from numbers_mt(10000000000);   ~ 3x performance 
+## select max(number) from numbers_mt(10000000000);   ~ 3x performance
+# [profile.release]
+# debug = 1
+# lto = "thin"
+# overflow-checks = false
+# incremental = false
+# codegen-units = 1
+
 [profile.release]
 debug = 1
 lto = "thin"
 overflow-checks = false
+opt-level = "s"         ## defaults to be 3
 incremental = false
-opt-level = "s"
-
-# codegen-units = 1 # Reduce number of codegen units to increase optimizations.
 
 # [profile.release.package]
-# arrow2 = { codegen-units = 4 }
-# common-functions = { codegen-units = 16 }
+# databend-common-arrow = { codegen-units = 16 }
 # databend-query = { codegen-units = 4 }
 # databend-binaries = { codegen-units = 4 }
 

src/common/base/src/base/dma.rs

@@ -182,7 +182,8 @@ struct DmaFile {
 }
 
 impl DmaFile {
-    async fn open_raw(path: impl AsRef<Path>, dio: bool) -> io::Result<File> {
+    async fn open_raw(path: impl AsRef<Path>, #[allow(unused)] dio: bool) -> io::Result<File> {
+        #[allow(unused_mut)]
         let mut flags = 0;
         #[cfg(target_os = "linux")]
         if dio {
@@ -196,7 +197,8 @@ impl DmaFile {
             .await
     }
 
-    async fn create_raw(path: impl AsRef<Path>, dio: bool) -> io::Result<File> {
+    async fn create_raw(path: impl AsRef<Path>, #[allow(unused)] dio: bool) -> io::Result<File> {
+        #[allow(unused_mut)]
         let mut flags = OFlags::EXCL;
         #[cfg(target_os = "linux")]
         if dio {

src/meta/types/src/cluster.rs

@@ -78,6 +78,7 @@ pub struct NodeInfo {
     pub secret: String,
     pub cpu_nums: u64,
     pub version: u32,
+    pub http_address: String,
     pub flight_address: String,
     pub discovery_address: String,
     pub binary_version: String,
@@ -88,6 +89,7 @@ impl NodeInfo {
         id: String,
         secret: String,
         cpu_nums: u64,
+        http_address: String,
         flight_address: String,
         discovery_address: String,
         binary_version: String,
@@ -97,6 +99,7 @@ impl NodeInfo {
             secret,
             cpu_nums,
             version: 0,
+            http_address,
             flight_address,
             discovery_address,
             binary_version,

src/meta/types/tests/it/cluster.rs

@@ -21,6 +21,7 @@ fn test_node_info_ip_port() -> anyhow::Result<()> {
         secret: "".to_string(),
         cpu_nums: 1,
         version: 1,
+        http_address: "7.8.9.10:987".to_string(),
         flight_address: "1.2.3.4:123".to_string(),
         discovery_address: "4.5.6.7:456".to_string(),
         binary_version: "v0.8-binary-version".to_string(),

src/query/ast/src/ast/format/syntax/dml.rs

@@ -236,7 +236,7 @@ pub(crate) fn pretty_copy_into_location(copy_stmt: CopyIntoLocationStmt) -> RcDo
         .append(
             RcDoc::line()
                 .append(RcDoc::text("SINGLE = "))
-                .append(RcDoc::text(copy_stmt.single.to_string())),
+                .append(RcDoc::text(copy_stmt.options.single.to_string())),
         )
 }
 

src/query/ast/src/ast/query.rs

@@ -29,6 +29,8 @@ use crate::ast::Identifier;
 use crate::ast::Lambda;
 use crate::ast::SelectStageOptions;
 use crate::ast::WindowDefinition;
+use crate::ParseError;
+use crate::Result;
 use crate::Span;
 
 /// Root node of a query tree
@@ -623,56 +625,82 @@ impl Display for TemporalClause {
     }
 }
 
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, Drive, DriveMut)]
-pub enum SampleLevel {
-    ROW,
-    BLOCK,
-}
-
 #[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut)]
-pub enum SampleConfig {
-    Probability(f64),
+pub enum SampleRowLevel {
     RowsNum(f64),
+    Probability(f64),
 }
 
-impl Eq for SampleConfig {}
-
-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Eq, Drive, DriveMut)]
-pub struct Sample {
-    pub sample_level: SampleLevel,
-    pub sample_conf: SampleConfig,
-}
-
-impl Sample {
-    pub fn sample_probability(&self, stats_rows: Option<u64>) -> Option<f64> {
-        let rand = match &self.sample_conf {
-            SampleConfig::Probability(probability) => probability / 100.0,
-            SampleConfig::RowsNum(rows) => {
+impl SampleRowLevel {
+    pub fn sample_probability(&self, stats_rows: Option<u64>) -> Result<Option<f64>> {
+        let rand = match &self {
+            SampleRowLevel::Probability(probability) => probability / 100.0,
+            SampleRowLevel::RowsNum(rows) => {
                 if let Some(row_num) = stats_rows {
                     if row_num > 0 {
                         rows / row_num as f64
                     } else {
-                        return None;
+                        return Ok(None);
                     }
                 } else {
-                    return None;
+                    return Ok(None);
                 }
             }
         };
-        Some(rand)
+        if rand > 1.0 {
+            return Err(ParseError(
+                None,
+                format!(
+                    "Sample value should be less than or equal to 100, but got {}",
+                    rand * 100.0
+                ),
+            ));
+        }
+        Ok(Some(rand))
+    }
+}
+
+impl Eq for SampleRowLevel {}
+
+#[derive(
+    serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq, Drive, DriveMut, Default,
+)]
+pub struct SampleConfig {
+    pub row_level: Option<SampleRowLevel>,
+    pub block_level: Option<f64>,
+}
+
+impl SampleConfig {
+    pub fn set_row_level_sample(&mut self, value: f64, rows: bool) {
+        if rows {
+            self.row_level = Some(SampleRowLevel::RowsNum(value));
+        } else {
+            self.row_level = Some(SampleRowLevel::Probability(value));
+        }
+    }
+
+    pub fn set_block_level_sample(&mut self, probability: f64) {
+        self.block_level = Some(probability);
     }
 }
 
-impl Display for Sample {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+impl Eq for SampleConfig {}
+
+impl Display for SampleConfig {
+    fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         write!(f, "SAMPLE ")?;
-        match self.sample_level {
-            SampleLevel::ROW => write!(f, "ROW ")?,
-            SampleLevel::BLOCK => write!(f, "BLOCK ")?,
+        if let Some(block_level) = self.block_level {
+            write!(f, "BLOCK ({}) ", block_level)?;
         }
-        match &self.sample_conf {
-            SampleConfig::Probability(prob) => write!(f, "({})", prob)?,
-            SampleConfig::RowsNum(rows) => write!(f, "({} ROWS)", rows)?,
+        if let Some(row_level) = &self.row_level {
+            match row_level {
+                SampleRowLevel::RowsNum(rows) => {
+                    write!(f, "ROW ({} ROWS)", rows)?;
+                }
+                SampleRowLevel::Probability(probability) => {
+                    write!(f, "ROW ({})", probability)?;
+                }
+            }
         }
         Ok(())
     }
@@ -692,7 +720,7 @@ pub enum TableReference {
         with_options: Option<WithOptions>,
         pivot: Option<Box<Pivot>>,
         unpivot: Option<Box<Unpivot>>,
-        sample: Option<Sample>,
+        sample: Option<SampleConfig>,
     },
     // `TABLE(expr)[ AS alias ]`
     TableFunction {
@@ -703,7 +731,7 @@ pub enum TableReference {
         params: Vec<Expr>,
         named_params: Vec<(Identifier, Expr)>,
         alias: Option<TableAlias>,
-        sample: Option<Sample>,
+        sample: Option<SampleConfig>,
     },
     // Derived table, which can be a subquery or joined tables or combination of them
     Subquery {