refactor: add DatabaseMeta.gc_in_progress

If `gc_in_progress` is set, no un-drop can be done on this database.
Because the related data may already have been deleted.

This commit does not use this flag yet, and will be used in next commit.

- Part of #16433

src/meta/api/src/name_id_value_api.rs

@@ -449,6 +449,7 @@ mod tests {
             updated_on: Default::default(),
             comment: "".to_string(),
             drop_on: None,
+            gc_in_progress: false,
         };
 
         let v = db_meta(1).to_pb()?.encode_to_vec();

src/meta/app/src/schema/database.rs

@@ -69,8 +69,29 @@ pub struct DatabaseMeta {
     pub updated_on: DateTime<Utc>,
     pub comment: String,
 
-    // if used in CreateDatabaseReq, this field MUST set to None.
+    /// if used in CreateDatabaseReq, this field MUST set to None.
     pub drop_on: Option<DateTime<Utc>>,
+
+    /// Indicates whether garbage collection is currently in progress for this dropped database.
+    ///
+    /// If it is in progress, the database should not be un-dropped, because the data may be incomplete.
+    ///
+    /// ```text
+    /// normal <----.
+    ///   |         |
+    ///   | drop()  | undrop()
+    ///   v         |
+    /// dropped ----'
+    ///   |
+    ///   | gc()
+    ///   v
+    /// gc_in_progress=True
+    ///   |
+    ///   | purge data from meta-service
+    ///   v
+    /// completed removed
+    /// ```
+    pub gc_in_progress: bool,
 }
 
 impl Default for DatabaseMeta {
@@ -83,6 +104,7 @@ impl Default for DatabaseMeta {
             updated_on: Utc::now(),
             comment: "".to_string(),
             drop_on: None,
+            gc_in_progress: false,
         }
     }
 }

src/meta/proto-conv/src/database_from_to_protobuf_impl.rs

@@ -44,6 +44,7 @@ impl FromToProto for mt::DatabaseMeta {
                 Some(drop_on) => Some(DateTime::<Utc>::from_pb(drop_on)?),
                 None => None,
             },
+            gc_in_progress: p.gc_in_progress,
             comment: p.comment,
         };
         Ok(v)
@@ -62,6 +63,7 @@ impl FromToProto for mt::DatabaseMeta {
                 Some(drop_on) => Some(drop_on.to_pb()?),
                 None => None,
             },
+            gc_in_progress: self.gc_in_progress,
             comment: self.comment.clone(),
             shared_by: vec![],
             from_share: None,

src/meta/proto-conv/src/util.rs

@@ -139,6 +139,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (107, "2024-08-09: Add: datatype.proto/DataType Geography type"),
     (108, "2024-08-29: Add: procedure.proto: ProcedureMeta and ProcedureIdentity"),
     (109, "2024-08-29: Refactor: ProcedureMeta add arg_names"),
+    (110, "2024-09-18: Add: database.proto: DatabaseMeta.gc_in_progress"),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -107,3 +107,4 @@ mod v106_query_token;
 mod v107_geography_datatype;
 mod v108_procedure;
 mod v109_procedure_with_args;
+mod v110_database_meta_gc_in_progress;

src/meta/proto-conv/tests/it/proto_conv.rs

@@ -52,6 +52,7 @@ fn new_db_meta_share() -> mt::DatabaseMeta {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     }
 }
 
@@ -64,6 +65,7 @@ fn new_db_meta() -> mt::DatabaseMeta {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     }
 }
 

src/meta/proto-conv/tests/it/v002_database_meta.rs

@@ -48,6 +48,7 @@ fn test_decode_v2_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v005_database_meta.rs

@@ -49,6 +49,7 @@ fn test_decode_v5_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v055_table_meta.rs

@@ -119,6 +119,7 @@ fn test_decode_v51_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v074_table_db_meta.rs

@@ -116,6 +116,7 @@ fn test_decode_v74_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v096_database_meta.rs

@@ -39,6 +39,7 @@ fn test_decode_v96_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v101_database_meta.rs

@@ -40,6 +40,7 @@ fn v101_database_meta() -> anyhow::Result<()> {
         updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
         comment: "foo bar".to_string(),
         drop_on: None,
+        gc_in_progress: false,
     };
 
     common::test_pb_from_to(func_name!(), want())?;

src/meta/proto-conv/tests/it/v110_database_meta_gc_in_progress.rs

@@ -0,0 +1,60 @@
+// Copyright 2023 Datafuse Labs.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use chrono::TimeZone;
+use chrono::Utc;
+use databend_common_meta_app::schema as mt;
+use fastrace::func_name;
+use maplit::btreemap;
+
+use crate::common;
+
+// These bytes are built when a new version in introduced,
+// and are kept for backward compatibility test.
+//
+// *************************************************************
+// * These messages should never be updated,                   *
+// * only be added when a new version is added,                *
+// * or be removed when an old version is no longer supported. *
+// *************************************************************
+//
+// The message bytes are built from the output of `test_pb_from_to()`
+#[test]
+fn test_decode_v109_database_meta() -> anyhow::Result<()> {
+    let database_meta_v109 = vec![
+        34, 10, 10, 3, 120, 121, 122, 18, 3, 102, 111, 111, 42, 2, 52, 52, 50, 10, 10, 3, 97, 98,
+        99, 18, 3, 100, 101, 102, 162, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 56, 32, 49, 50,
+        58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 170, 1, 23, 50, 48, 49, 52, 45, 49, 49, 45, 50, 57,
+        32, 49, 50, 58, 48, 48, 58, 48, 57, 32, 85, 84, 67, 178, 1, 7, 102, 111, 111, 32, 98, 97,
+        114, 232, 1, 1, 160, 6, 109, 168, 6, 24,
+    ];
+
+    let want = || mt::DatabaseMeta {
+        engine: "44".to_string(),
+        engine_options: btreemap! {s("abc") => s("def")},
+        options: btreemap! {s("xyz") => s("foo")},
+        created_on: Utc.with_ymd_and_hms(2014, 11, 28, 12, 0, 9).unwrap(),
+        updated_on: Utc.with_ymd_and_hms(2014, 11, 29, 12, 0, 9).unwrap(),
+        comment: "foo bar".to_string(),
+        drop_on: None,
+        gc_in_progress: true,
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), database_meta_v109.as_slice(), 109, want())
+}
+
+fn s(ss: impl ToString) -> String {
+    ss.to_string()
+}

src/meta/protos/proto/database.proto

@@ -70,6 +70,11 @@ message DatabaseMeta {
   // The time table dropped.
   optional string drop_on = 23;
 
+  // Indicates whether garbage collection is currently in progress for this dropped database.
+  //
+  // If it is in progress, the database should not be un-dropped, because the data may be incomplete.
+  bool gc_in_progress = 29;
+
   repeated uint64 shared_by = 24;
 
   optional TIdent from_share = 25;