Merge branch 'main' into refactor/cluster

src/common/base/src/runtime/metrics/family.rs

@@ -107,6 +107,10 @@ impl<S: FamilyLabels, M: FamilyMetric> Family<S, M> {
         }
     }
 
+    pub fn get(&self, label_set: &S) -> Option<Arc<M>> {
+        self.metrics.read().get(label_set).cloned()
+    }
+
     pub fn remove(&self, label_set: &S) -> bool {
         ScopedRegistry::op(self.index, |m: &Self| {
             m.metrics.write().remove(label_set);

src/common/exception/src/exception_code.rs

@@ -116,6 +116,7 @@ build_exceptions! {
     PermissionDenied(1063),
     UnmarshalError(1064),
     SemanticError(1065),
+    NeedChangePasswordDenied(1066),
     UnknownException(1067),
     TokioError(1068),
     HttpNotFound(1072),

src/common/metrics/src/metrics/cache.rs

@@ -40,6 +40,30 @@ static CACHE_POPULATION_PENDING_COUNT: LazyLock<FamilyCounter<CacheLabels>> =
 static CACHE_POPULATION_OVERFLOW_COUNT: LazyLock<FamilyCounter<CacheLabels>> =
     LazyLock::new(|| register_counter_family("cache_population_overflow_count"));
 
+pub fn get_cache_access_count(cache_name: &str) -> u64 {
+    get_metric_count_by_name(&CACHE_ACCESS_COUNT, cache_name)
+}
+
+pub fn get_cache_hit_count(cache_name: &str) -> u64 {
+    get_metric_count_by_name(&CACHE_HIT_COUNT, cache_name)
+}
+
+pub fn get_cache_miss_count(cache_name: &str) -> u64 {
+    get_metric_count_by_name(&CACHE_MISS_COUNT, cache_name)
+}
+
+fn get_metric_count_by_name(
+    metric: &LazyLock<FamilyCounter<CacheLabels>>,
+    cache_name: &str,
+) -> u64 {
+    metric
+        .get(&CacheLabels {
+            cache_name: cache_name.to_string(),
+        })
+        .map(|v| v.get())
+        .unwrap_or_default()
+}
+
 pub fn metrics_inc_cache_access_count(c: u64, cache_name: &str) {
     CACHE_ACCESS_COUNT
         .get_or_create(&CacheLabels {

src/meta/app/src/principal/user_auth.rs

@@ -100,6 +100,7 @@ pub enum AuthInfo {
     Password {
         hash_value: Vec<u8>,
         hash_method: PasswordHashMethod,
+        need_change: bool,
     },
     JWT,
 }
@@ -115,7 +116,11 @@ fn double_sha1(v: &[u8]) -> [u8; 20] {
 }
 
 impl AuthInfo {
-    pub fn new(auth_type: AuthType, auth_string: &Option<String>) -> Result<AuthInfo> {
+    pub fn new(
+        auth_type: AuthType,
+        auth_string: &Option<String>,
+        need_change: bool,
+    ) -> Result<AuthInfo> {
         match auth_type {
             AuthType::NoPassword => Ok(AuthInfo::None),
             AuthType::JWT => Ok(AuthInfo::JWT),
@@ -125,6 +130,7 @@ impl AuthInfo {
                     Ok(AuthInfo::Password {
                         hash_value: method.hash(p.as_bytes()),
                         hash_method: method,
+                        need_change,
                     })
                 }
                 None => Err(ErrorCode::InvalidAuthInfo("need password".to_string())),
@@ -139,13 +145,17 @@ impl AuthInfo {
             .map(|s| AuthType::from_str(&s))
             .transpose()?
             .unwrap_or(default);
-        AuthInfo::new(auth_type, auth_string)
+        AuthInfo::new(auth_type, auth_string, false)
     }
 
-    pub fn create2(auth_type: &Option<AuthType>, auth_string: &Option<String>) -> Result<AuthInfo> {
+    pub fn create2(
+        auth_type: &Option<AuthType>,
+        auth_string: &Option<String>,
+        need_change: bool,
+    ) -> Result<AuthInfo> {
         let default = AuthType::DoubleSha1Password;
         let auth_type = auth_type.clone().unwrap_or(default);
-        AuthInfo::new(auth_type, auth_string)
+        AuthInfo::new(auth_type, auth_string, need_change)
     }
 
     pub fn alter(
@@ -159,39 +169,46 @@ impl AuthInfo {
             .map(|s| AuthType::from_str(&s))
             .transpose()?
             .unwrap_or(old_auth_type);
-        AuthInfo::new(new_auth_type, auth_string)
+        AuthInfo::new(new_auth_type, auth_string, false)
     }
 
     pub fn alter2(
         &self,
         auth_type: &Option<AuthType>,
         auth_string: &Option<String>,
+        need_change: bool,
     ) -> Result<AuthInfo> {
         let old_auth_type = self.get_type();
         let new_auth_type = auth_type.clone().unwrap_or(old_auth_type);
 
-        AuthInfo::new(new_auth_type, auth_string)
+        AuthInfo::new(new_auth_type, auth_string, need_change)
     }
 
     pub fn get_type(&self) -> AuthType {
         match self {
             AuthInfo::None => AuthType::NoPassword,
             AuthInfo::JWT => AuthType::JWT,
-            AuthInfo::Password {
-                hash_value: _,
-                hash_method: t,
-            } => match t {
+            AuthInfo::Password { hash_method: t, .. } => match t {
                 PasswordHashMethod::Sha256 => AuthType::Sha256Password,
                 PasswordHashMethod::DoubleSha1 => AuthType::DoubleSha1Password,
             },
         }
     }
 
+    pub fn get_need_change(&self) -> bool {
+        match self {
+            AuthInfo::None => false,
+            AuthInfo::JWT => false,
+            AuthInfo::Password { need_change, .. } => *need_change,
+        }
+    }
+
     pub fn get_auth_string(&self) -> String {
         match self {
             AuthInfo::Password {
                 hash_value: p,
                 hash_method: t,
+                ..
             } => t.to_string(p),
             AuthInfo::None | AuthInfo::JWT => "".to_string(),
         }
@@ -202,6 +219,7 @@ impl AuthInfo {
             AuthInfo::Password {
                 hash_value: p,
                 hash_method: _,
+                ..
             } => Some(p.to_vec()),
             _ => None,
         }
@@ -212,6 +230,7 @@ impl AuthInfo {
             AuthInfo::Password {
                 hash_value: _,
                 hash_method: t,
+                ..
             } => Some(*t),
             _ => None,
         }
@@ -240,6 +259,7 @@ impl AuthInfo {
             AuthInfo::Password {
                 hash_value: p,
                 hash_method: t,
+                ..
             } => match t {
                 PasswordHashMethod::DoubleSha1 => {
                     let password_sha1 = AuthInfo::restore_sha1_mysql(salt, password_input, p)?;

src/meta/app/src/principal/user_info.rs

@@ -129,6 +129,21 @@ impl UserInfo {
         }
     }
 
+    pub fn update_auth_need_change_password(&mut self) {
+        if let AuthInfo::Password {
+            hash_value,
+            hash_method,
+            ..
+        } = self.auth_info.clone()
+        {
+            self.auth_info = AuthInfo::Password {
+                hash_value,
+                hash_method,
+                need_change: true,
+            };
+        }
+    }
+
     pub fn update_user_time(&mut self) {
         self.update_on = Utc::now();
     }
@@ -173,6 +188,7 @@ pub struct UserOption {
     network_policy: Option<String>,
     password_policy: Option<String>,
     disabled: Option<bool>,
+    must_change_password: Option<bool>,
 }
 
 impl UserOption {
@@ -183,6 +199,7 @@ impl UserOption {
             network_policy: None,
             password_policy: None,
             disabled: None,
+            must_change_password: None,
         }
     }
 
@@ -215,6 +232,11 @@ impl UserOption {
         self
     }
 
+    pub fn with_must_change_password(mut self, must_change_password: Option<bool>) -> Self {
+        self.must_change_password = must_change_password;
+        self
+    }
+
     pub fn with_set_flag(mut self, flag: UserOptionFlag) -> Self {
         self.flags.insert(flag);
         self
@@ -240,6 +262,10 @@ impl UserOption {
         self.disabled.as_ref()
     }
 
+    pub fn must_change_password(&self) -> Option<&bool> {
+        self.must_change_password.as_ref()
+    }
+
     pub fn set_default_role(&mut self, default_role: Option<String>) {
         self.default_role = default_role;
     }
@@ -256,6 +282,10 @@ impl UserOption {
         self.disabled = disabled;
     }
 
+    pub fn set_must_change_password(&mut self, must_change_password: Option<bool>) {
+        self.must_change_password = must_change_password;
+    }
+
     pub fn set_all_flag(&mut self) {
         self.flags = BitFlags::all();
     }
@@ -295,6 +325,7 @@ impl UserOption {
             UserOptionItem::SetPasswordPolicy(v) => self.password_policy = Some(v.clone()),
             UserOptionItem::UnsetPasswordPolicy => self.password_policy = None,
             UserOptionItem::Disabled(v) => self.disabled = Some(*v),
+            UserOptionItem::MustChangePassword(v) => self.must_change_password = Some(*v),
         }
     }
 }

src/meta/app/tests/it/user_info.rs

@@ -37,6 +37,7 @@ fn test_user_info() -> Result<()> {
         auth_info: AuthInfo::Password {
             hash_value: Vec::from("pwd"),
             hash_method: PasswordHashMethod::Sha256,
+            need_change: false,
         },
     };
 
@@ -46,6 +47,7 @@ fn test_user_info() -> Result<()> {
     let mut expect = UserInfo::new("old-name", "old-host", AuthInfo::Password {
         hash_value: Vec::from("pwd"),
         hash_method: PasswordHashMethod::Sha256,
+        need_change: false,
     });
     expect.created_on = DateTime::<Utc>::default();
     expect.update_on = DateTime::<Utc>::default();

src/meta/proto-conv/src/user_from_to_protobuf_impl.rs

@@ -50,11 +50,13 @@ impl FromToProto for mt::principal::AuthInfo {
             Some(pb::auth_info::Info::Password(pb::auth_info::Password {
                 hash_value,
                 hash_method,
+                need_change,
             })) => Ok(mt::principal::AuthInfo::Password {
                 hash_value,
                 hash_method: FromPrimitive::from_i32(hash_method).ok_or_else(|| Incompatible {
                     reason: format!("invalid PasswordHashMethod: {}", hash_method),
                 })?,
+                need_change: need_change.unwrap_or_default(),
             }),
             None => Err(Incompatible {
                 reason: "AuthInfo cannot be None".to_string(),
@@ -71,9 +73,11 @@ impl FromToProto for mt::principal::AuthInfo {
             mt::principal::AuthInfo::Password {
                 hash_value,
                 hash_method,
+                need_change,
             } => Some(pb::auth_info::Info::Password(pb::auth_info::Password {
                 hash_value: hash_value.clone(),
                 hash_method: *hash_method as i32,
+                need_change: Some(*need_change),
             })),
         };
         Ok(pb::AuthInfo {
@@ -101,7 +105,8 @@ impl FromToProto for mt::principal::UserOption {
             .with_default_role(p.default_role)
             .with_network_policy(p.network_policy)
             .with_password_policy(p.password_policy)
-            .with_disabled(p.disabled))
+            .with_disabled(p.disabled)
+            .with_must_change_password(p.must_change_password))
     }
 
     fn to_pb(&self) -> Result<pb::UserOption, Incompatible> {
@@ -113,6 +118,7 @@ impl FromToProto for mt::principal::UserOption {
             network_policy: self.network_policy().cloned(),
             password_policy: self.password_policy().cloned(),
             disabled: self.disabled().cloned(),
+            must_change_password: self.must_change_password().cloned(),
         })
     }
 }

src/meta/proto-conv/src/util.rs

@@ -131,6 +131,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (99, "2024-07-08: Add: missing_field_as in user.proto/ParquetFileFormatParams"),
     (100, "2024-06-21: Add: tenant.proto/TenantQuota"),
     (101, "2024-07-06: Add: add from_share_db_id field into DatabaseMeta"),
+    (102, "2024-07-11: Add: UserOption add must_change_password, AuthInfo.Password add need_change"),
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

src/meta/proto-conv/tests/it/main.rs

@@ -105,3 +105,4 @@ mod v098_catalog_option;
 mod v099_parquet_format_params;
 mod v100_tenant_quota;
 mod v101_database_meta;
+mod v102_user_must_change_password;

src/meta/proto-conv/tests/it/user_proto_conv.rs

@@ -55,6 +55,7 @@ fn test_user_info() -> mt::principal::UserInfo {
             ]
             .to_vec(),
             hash_method: mt::principal::PasswordHashMethod::DoubleSha1,
+            need_change: false,
         },
         grants: mt::principal::UserGrantSet::new(
             vec![mt::principal::GrantEntry::new(

src/meta/proto-conv/tests/it/v050_user_info.rs

@@ -52,6 +52,7 @@ fn test_decode_v50_user_info() -> anyhow::Result<()> {
             ]
             .to_vec(),
             hash_method: databend_common_meta_app::principal::PasswordHashMethod::DoubleSha1,
+            need_change: false,
         },
         grants: databend_common_meta_app::principal::UserGrantSet::new(
             vec![databend_common_meta_app::principal::GrantEntry::new(

src/meta/proto-conv/tests/it/v067_password_policy.rs

@@ -86,6 +86,7 @@ fn test_decode_v67_password_policy() -> anyhow::Result<()> {
             ]
             .to_vec(),
             hash_method: databend_common_meta_app::principal::PasswordHashMethod::DoubleSha1,
+            need_change: false,
         },
         grants: databend_common_meta_app::principal::UserGrantSet::new(
             vec![databend_common_meta_app::principal::GrantEntry::new(

src/meta/proto-conv/tests/it/v071_user_password.rs

@@ -73,8 +73,8 @@ fn test_decode_v71_user() -> anyhow::Result<()> {
         quota: Default::default(),
         option: Default::default(),
         history_auth_infos: vec![
-            AuthInfo::create2(&None, &Some("1234".to_string())).unwrap(),
-            AuthInfo::create2(&None, &Some("abcd".to_string())).unwrap(),
+            AuthInfo::create2(&None, &Some("1234".to_string()), false).unwrap(),
+            AuthInfo::create2(&None, &Some("abcd".to_string()), false).unwrap(),
         ],
         password_fails: vec![
             Utc.with_ymd_and_hms(2023, 12, 25, 1, 0, 0).unwrap(),

src/meta/proto-conv/tests/it/v091_role_user_create_time_info.rs

@@ -96,8 +96,8 @@ fn test_decode_v91_user() -> anyhow::Result<()> {
         quota: Default::default(),
         option: Default::default(),
         history_auth_infos: vec![
-            AuthInfo::create2(&None, &Some("1234".to_string())).unwrap(),
-            AuthInfo::create2(&None, &Some("abcd".to_string())).unwrap(),
+            AuthInfo::create2(&None, &Some("1234".to_string()), false).unwrap(),
+            AuthInfo::create2(&None, &Some("abcd".to_string()), false).unwrap(),
         ],
         password_fails: vec![
             Utc.with_ymd_and_hms(2023, 12, 25, 1, 0, 0).unwrap(),