feat: Bump OpenDAL to 0.50.1

Cargo.toml

@@ -229,7 +229,7 @@ criterion = "0.5"
 ctor = "0.2"
 dashmap = "5.4.0"
 deepsize = { version = "0.2.0" }
-deltalake = "0.18"
+deltalake = "0.20"
 derive-visitor = { version = "0.4.0", features = ["std-types-drive"] }
 derive_more = "0.99.17"
 enumflags2 = { version = "0.7.7", features = ["serde"] }
@@ -256,7 +256,7 @@ mysql_async = { version = "0.34", default-features = false, features = ["native-
 object_store_opendal = "0.46"
 once_cell = "1.15.0"
 openai_api_rust = "0.1"
-opendal = { version = "0.49.0", features = [
+opendal = { version = "0.50.1", features = [
     "layers-fastrace",
     "layers-prometheus-client",
     "layers-async-backtrace",
@@ -275,6 +275,7 @@ opendal = { version = "0.49.0", features = [
     "services-huggingface",
     "services-redis",
 ] }
+opendal_compat = { version = "1" }
 openraft = { git = "https://github.com/drmingdrmer/openraft", tag = "v0.10.0-alpha.6", features = [
     "serde",
     "tracing-log",
@@ -413,14 +414,18 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226
     "serialize-serde",
 ] }
 color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" }
-deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "57795da" }
 ethnum = { git = "https://github.com/datafuse-extras/ethnum-rs", rev = "4cb05f1" }
 jsonb = { git = "https://github.com/databendlabs/jsonb", rev = "ada713c" }
 openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" }
+opendal_compat = { git = "https://github.com/apache/opendal", rev = "f6e60f6" }
 orc-rust = { git = "https://github.com/datafuse-extras/datafusion-orc", rev = "03372b97" }
 recursive = { git = "https://github.com/datafuse-extras/recursive.git", rev = "6af35a1" }
 sled = { git = "https://github.com/datafuse-extras/sled", tag = "v0.34.7-datafuse.1" }
 tantivy = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370" }
 tantivy-common = { git = "https://github.com/datafuse-extras/tantivy", rev = "7502370", package = "tantivy-common" }
 tantivy-jieba = { git = "https://github.com/datafuse-extras/tantivy-jieba", rev = "0e300e9" }
 xorfilter-rs = { git = "https://github.com/datafuse-extras/xorfilter", tag = "databend-alpha.4" }
+
+# This patch makes sure opendal_compat also used the same version of opendal instead of git one.
+[patch.'https://github.com/apache/opendal']
+opendal = { version = "0.50.1" }

src/common/base/Cargo.toml

@@ -50,6 +50,8 @@ prometheus-parse = "0.2.3"
 rand = { workspace = true, features = ["serde1"] }
 regex = { workspace = true }
 replace_with = "0.1.7"
+reqwest = { workspace = true }
+reqwest-hickory-resolver = { workspace = true }
 rustix = "0.38.37"
 semver = { workspace = true }
 serde = { workspace = true }

src/common/base/src/http_client.rs

@@ -0,0 +1,88 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+use std::sync::Arc;
+use std::sync::LazyLock;
+use std::time::Duration;
+
+use reqwest_hickory_resolver::HickoryResolver;
+
+/// Global shared hickory resolver.
+static GLOBAL_HICKORY_RESOLVER: LazyLock<Arc<HickoryResolver>> =
+    LazyLock::new(|| Arc::new(HickoryResolver::default()));
+
+/// Global shared http client.
+///
+/// Please create your own http client if you want dedicated http connection pool.
+pub static GLOBAL_HTTP_CLIENT: LazyLock<HttpClient> = LazyLock::new(HttpClient::new);
+
+/// HttpClient that used by databend.
+pub struct HttpClient {
+    client: reqwest::Client,
+}
+
+impl Default for HttpClient {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl HttpClient {
+    /// Create a new http client.
+    ///
+    /// # Notes
+    ///
+    /// This client is optimized for interact with storage services.
+    /// Please tune the settings if you want to use it for other purposes.
+    pub fn new() -> Self {
+        let mut builder = reqwest::ClientBuilder::new();
+
+        // Disable http2 for better performance.
+        builder = builder.http1_only();
+
+        // Set dns resolver.
+        builder = builder.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone());
+
+        // Pool max idle per host controls connection pool size.
+        // Default to no limit, set to `0` for disable it.
+        let pool_max_idle_per_host = env::var("_DATABEND_INTERNAL_POOL_MAX_IDLE_PER_HOST")
+            .ok()
+            .and_then(|v| v.parse::<usize>().ok())
+            .unwrap_or(usize::MAX);
+        builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+
+        // Connect timeout default to 30s.
+        let connect_timeout = env::var("_DATABEND_INTERNAL_CONNECT_TIMEOUT")
+            .ok()
+            .and_then(|v| v.parse::<u64>().ok())
+            .unwrap_or(30);
+        builder = builder.connect_timeout(Duration::from_secs(connect_timeout));
+
+        // Enable TCP keepalive if set.
+        if let Ok(v) = env::var("_DATABEND_INTERNAL_TCP_KEEPALIVE") {
+            if let Ok(v) = v.parse::<u64>() {
+                builder = builder.tcp_keepalive(Duration::from_secs(v));
+            }
+        }
+
+        let client = builder.build().expect("http client must be created");
+        HttpClient { client }
+    }
+
+    /// Get the inner reqwest client.
+    pub fn inner(&self) -> reqwest::Client {
+        self.client.clone()
+    }
+}

src/common/base/src/lib.rs

@@ -31,6 +31,7 @@ pub mod containers;
 pub mod display;
 pub mod future;
 pub mod headers;
+pub mod http_client;
 pub mod mem_allocator;
 pub mod rangemap;
 pub mod runtime;

src/common/storage/Cargo.toml

@@ -29,8 +29,6 @@ opendal = { workspace = true }
 parquet = { workspace = true }
 prometheus-client = { workspace = true }
 regex = { workspace = true }
-reqwest = { workspace = true }
-reqwest-hickory-resolver = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }