feat: create physical table based on the engine config

GreptimeTeam · Jan 23, 2025 · 22445c4 · 22445c4
1 parent 6291ac1
commit 22445c4
Show file tree

Hide file tree

Showing 8 changed files with 57 additions and 13 deletions.
diff --git a/config/config.md b/config/config.md
@@ -172,6 +172,7 @@
 | `region_engine.file` | -- | -- | Enable the file engine. |
 | `region_engine.metric` | -- | -- | Metric engine options. |
 | `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. |
+| `region_engine.metric.experimental_bloom_filter_index` | Bool | `false` | Whether to use the bloom filter index as default index for primary key columns. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |
@@ -510,6 +511,7 @@
 | `region_engine.file` | -- | -- | Enable the file engine. |
 | `region_engine.metric` | -- | -- | Metric engine options. |
 | `region_engine.metric.experimental_sparse_primary_key_encoding` | Bool | `false` | Whether to enable the experimental sparse primary key encoding. |
+| `region_engine.metric.experimental_bloom_filter_index` | Bool | `false` | Whether to use the bloom filter index as default index for primary key columns. |
 | `logging` | -- | -- | The logging options. |
 | `logging.dir` | String | `/tmp/greptimedb/logs` | The directory to store the log files. If set to empty, logs will not be written to files. |
 | `logging.level` | String | Unset | The log level. Can be `info`/`debug`/`warn`/`error`. |

diff --git a/config/datanode.example.toml b/config/datanode.example.toml
@@ -628,6 +628,9 @@ fork_dictionary_bytes = "1GiB"
 ## Whether to enable the experimental sparse primary key encoding.
 experimental_sparse_primary_key_encoding = false
 
+## Whether to use the bloom filter index as default index for primary key columns.
+experimental_bloom_filter_index = false
+
 ## The logging options.
 [logging]
 ## The directory to store the log files. If set to empty, logs will not be written to files.

diff --git a/config/standalone.example.toml b/config/standalone.example.toml
@@ -677,6 +677,9 @@ fork_dictionary_bytes = "1GiB"
 ## Whether to enable the experimental sparse primary key encoding.
 experimental_sparse_primary_key_encoding = false
 
+## Whether to use the bloom filter index as default index for primary key columns.
+experimental_bloom_filter_index = false
+
 ## The logging options.
 [logging]
 ## The directory to store the log files. If set to empty, logs will not be written to files.

diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
@@ -75,6 +75,7 @@ fn test_load_datanode_example_config() {
                 }),
                 RegionEngineConfig::File(FileEngineConfig {}),
                 RegionEngineConfig::Metric(MetricEngineConfig {
+                    experimental_bloom_filter_index: false,
                     experimental_sparse_primary_key_encoding: false,
                 }),
             ],
@@ -217,6 +218,7 @@ fn test_load_standalone_example_config() {
                 RegionEngineConfig::File(FileEngineConfig {}),
                 RegionEngineConfig::Metric(MetricEngineConfig {
                     experimental_sparse_primary_key_encoding: false,
+                    experimental_bloom_filter_index: false,
                 }),
             ],
             storage: StorageConfig {

diff --git a/src/metric-engine/src/config.rs b/src/metric-engine/src/config.rs
@@ -17,4 +17,5 @@ use serde::{Deserialize, Serialize};
 #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
 pub struct EngineConfig {
     pub experimental_sparse_primary_key_encoding: bool,
+    pub experimental_bloom_filter_index: bool,
 }
diff --git a/src/metric-engine/src/engine/create.rs b/src/metric-engine/src/engine/create.rs
@@ -93,11 +93,13 @@ impl MetricEngineInner {
         region_id: RegionId,
         request: RegionCreateRequest,
     ) -> Result<()> {
-        let physical_region_options = PhysicalRegionOptions::try_from(&request.options)?;
         let (data_region_id, metadata_region_id) = Self::transform_region_id(region_id);
 
         // create metadata region
         let create_metadata_region_request = self.create_request_for_metadata_region(&request);
+        let create_data_region_request = self.create_request_for_data_region(&request);
+        let physical_region_options =
+            PhysicalRegionOptions::try_from(&create_data_region_request.options)?;
         self.mito
             .handle_request(
                 metadata_region_id,
@@ -109,7 +111,7 @@ impl MetricEngineInner {
             })?;
 
         // create data region
-        let create_data_region_request = self.create_request_for_data_region(&request);
+
         let physical_columns = create_data_region_request
             .column_metadatas
             .iter()
@@ -130,7 +132,7 @@ impl MetricEngineInner {
             },
         )?;
 
-        info!("Created physical metric region {region_id}, primary key encoding={primary_key_encoding}");
+        info!("Created physical metric region {region_id}, physical_region_options={physical_region_options:?}");
         PHYSICAL_REGION_COUNT.inc();
 
         // remember this table
@@ -524,7 +526,11 @@ impl MetricEngineInner {
         data_region_request.primary_key = primary_key;
 
         // set data region options
-        set_data_region_options(&mut data_region_request.options);
+        set_data_region_options(
+            &mut data_region_request.options,
+            self.config.experimental_bloom_filter_index,
+            self.config.experimental_sparse_primary_key_encoding,
+        );
 
         data_region_request
     }

diff --git a/src/metric-engine/src/engine/open.rs b/src/metric-engine/src/engine/open.rs
@@ -48,9 +48,7 @@ impl MetricEngineInner {
     ) -> Result<AffectedRows> {
         if request.is_physical_table() {
             // open physical region and recover states
-            let physical_region_options: PhysicalRegionOptions =
-                PhysicalRegionOptions::try_from(&request.options)?;
-            self.open_physical_region(region_id, request).await?;
+            let physical_region_options = self.open_physical_region(region_id, request).await?;
             let data_region_id = utils::to_data_region_id(region_id);
             let primary_key_encoding = self.mito.get_primary_key_encoding(data_region_id).context(
                 PhysicalRegionNotFoundSnafu {
@@ -75,7 +73,7 @@ impl MetricEngineInner {
         &self,
         region_id: RegionId,
         request: RegionOpenRequest,
-    ) -> Result<AffectedRows> {
+    ) -> Result<PhysicalRegionOptions> {
         let metadata_region_dir = join_dir(&request.region_dir, METADATA_REGION_SUBDIR);
         let data_region_dir = join_dir(&request.region_dir, DATA_REGION_SUBDIR);
 
@@ -88,7 +86,13 @@ impl MetricEngineInner {
         };
 
         let mut data_region_options = request.options;
-        set_data_region_options(&mut data_region_options);
+        set_data_region_options(
+            &mut data_region_options,
+            self.config.experimental_bloom_filter_index,
+            self.config.experimental_sparse_primary_key_encoding,
+        );
+        let physical_region_options: PhysicalRegionOptions =
+            PhysicalRegionOptions::try_from(&data_region_options)?;
         let open_data_region_request = RegionOpenRequest {
             region_dir: data_region_dir,
             options: data_region_options,
@@ -118,10 +122,10 @@ impl MetricEngineInner {
                 region_type: "data",
             })?;
 
-        info!("Opened physical metric region {region_id}");
+        info!("Opened physical metric region {region_id}, options={physical_region_options:?}");
         PHYSICAL_REGION_COUNT.inc();
 
-        Ok(0)
+        Ok(physical_region_options)
     }
 
     /// Recovers [MetricEngineState](crate::engine::state::MetricEngineState) from

diff --git a/src/metric-engine/src/engine/options.rs b/src/metric-engine/src/engine/options.rs
@@ -20,6 +20,7 @@ use store_api::metric_engine_consts::{
     METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION,
     METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION_DEFAULT, METRIC_ENGINE_INDEX_TYPE_OPTION,
 };
+use store_api::mito_engine_options::MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING;
 
 use crate::error::{Error, ParseRegionOptionsSnafu, Result};
 
@@ -46,7 +47,11 @@ pub enum IndexOptions {
 }
 
 /// Sets data region specific options.
-pub fn set_data_region_options(options: &mut HashMap<String, String>) {
+pub fn set_data_region_options(
+    options: &mut HashMap<String, String>,
+    bloom_filter_index: bool,
+    sparse_primary_key_encoding: bool,
+) {
     options.remove(METRIC_ENGINE_INDEX_TYPE_OPTION);
     options.remove(METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION);
     options.insert(
@@ -55,6 +60,24 @@ pub fn set_data_region_options(options: &mut HashMap<String, String>) {
     );
     // Set memtable options for the data region.
     options.insert("memtable.type".to_string(), "partition_tree".to_string());
+    if sparse_primary_key_encoding
+        && !options.contains_key(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING)
+    {
+        options.insert(
+            MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING.to_string(),
+            "sparse".to_string(),
+        );
+    }
+    if bloom_filter_index && !options.contains_key(METRIC_ENGINE_INDEX_TYPE_OPTION) {
+        options.insert(
+            METRIC_ENGINE_INDEX_TYPE_OPTION.to_string(),
+            "skipping".to_string(),
+        );
+        options.insert(
+            METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION.to_string(),
+            "102400".to_string(),
+        );
+    }
 }
 
 impl TryFrom<&HashMap<String, String>> for PhysicalRegionOptions {
@@ -108,7 +131,7 @@ mod tests {
             METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION.to_string(),
             "102400".to_string(),
         );
-        set_data_region_options(&mut options);
+        set_data_region_options(&mut options, false, false);
 
         for key in [
             METRIC_ENGINE_INDEX_TYPE_OPTION,