Merge branch 'main' into refactor/optimize_shuffle

databendlabs · Sep 12, 2024 · 7a5a3ee · 7a5a3ee
2 parents 03c5b42 + a620190
commit 7a5a3ee
Show file tree

Hide file tree

Showing 41 changed files with 647 additions and 811 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -324,19 +324,19 @@ walkdir = "2.3.2"
 xorfilter-rs = "0.5"
 
 # Observability
-fastrace = { version = "0.6", features = ["enable"] }
-fastrace-opentelemetry = "0.6"
+fastrace = { version = "0.7", features = ["enable"] }
+fastrace-opentelemetry = "0.7"
 log = { version = "0.4.21", features = ["serde", "kv_unstable_std"] }
 logcall = "0.1.9"
-opentelemetry = { version = "0.23", features = ["trace", "logs"] }
-opentelemetry-otlp = { version = "0.16", features = [
+opentelemetry = { version = "0.24", features = ["trace", "logs"] }
+opentelemetry-otlp = { version = "0.17", features = [
     "trace",
     "logs",
     "grpc-tonic",
     "http-proto",
     "reqwest-client",
 ] }
-opentelemetry_sdk = { version = "0.23", features = ["trace", "logs", "rt-tokio"] }
+opentelemetry_sdk = { version = "0.24", features = ["trace", "logs", "rt-tokio"] }
 tracing = "0.1.40"
 tracing-appender = "0.2.3"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter", "json", "valuable"] }

diff --git a/scripts/ci/deploy/config/databend-query-node-otlp-logs.toml b/scripts/ci/deploy/config/databend-query-node-otlp-logs.toml
@@ -52,14 +52,14 @@ dir = "./.databend/logs_1"
 
 [log.query]
 on = true
-otlp_endpoint = "http://127.0.0.1:4317"
+otlp_endpoint = "http://127.0.0.1:4317/v1/logs"
 [log.query.otlp_labels]
 qkey1 = "qvalue1"
 qkey2 = "qvalue2"
 
 [log.profile]
 on = true
-otlp_endpoint = "http://127.0.0.1:4318"
+otlp_endpoint = "http://127.0.0.1:4318/v1/logs"
 otlp_protocol = "http"
 [log.profile.otlp_labels]
 pkey1 = "pvalue1"

diff --git a/src/common/tracing/Cargo.toml b/src/common/tracing/Cargo.toml
@@ -19,18 +19,21 @@ databend-common-exception = { workspace = true }
 defer = "0.2"
 fastrace = { workspace = true }
 fastrace-opentelemetry = { workspace = true }
-fern = "0.6.2"
 itertools = { workspace = true }
 libc = "0.2.153"
 log = { workspace = true }
+logforth = { version = "0.11", git = "http://github.com/andylokandy/logforth", rev = "0ca61ca", features = [
+    'json',
+    'rolling_file',
+    'opentelemetry',
+    'fastrace',
+] }
 opentelemetry = { workspace = true }
 opentelemetry-otlp = { workspace = true }
 opentelemetry_sdk = { workspace = true }
 serde = { workspace = true }
-serde_json = { workspace = true }
 strip-ansi-escapes = "0.2"
 tonic = { workspace = true }
-tracing-appender = { workspace = true }
 
 [dev-dependencies]
 

diff --git a/src/common/tracing/src/config.rs b/src/common/tracing/src/config.rs
@@ -246,7 +246,7 @@ impl Default for TracingConfig {
     }
 }
 
-#[derive(Clone, Debug, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum OTLPProtocol {
     Http,
     Grpc,
@@ -292,6 +292,17 @@ impl<'de> serde::Deserialize<'de> for OTLPProtocol {
     }
 }
 
+impl From<OTLPProtocol> for logforth::append::opentelemetry::OpentelemetryWireProtocol {
+    fn from(protocol: OTLPProtocol) -> Self {
+        match protocol {
+            OTLPProtocol::Http => {
+                logforth::append::opentelemetry::OpentelemetryWireProtocol::HttpBinary
+            }
+            OTLPProtocol::Grpc => logforth::append::opentelemetry::OpentelemetryWireProtocol::Grpc,
+        }
+    }
+}
+
 #[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
 pub struct OTLPEndpointConfig {
     pub endpoint: String,