Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(query): Support show dictionaries DQL. #16602

Merged
merged 18 commits into from
Oct 15, 2024
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions src/query/ast/src/ast/statements/dictionary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ use std::fmt::Formatter;
use derive_visitor::Drive;
use derive_visitor::DriveMut;

use super::ShowLimit;
use crate::ast::write_comma_separated_list;
use crate::ast::write_dot_separated_list;
use crate::ast::write_space_separated_string_map;
Expand Down Expand Up @@ -123,3 +124,22 @@ impl Display for ShowCreateDictionaryStmt {
)
}
}

#[derive(Debug, Clone, PartialEq, Drive, DriveMut)]
pub struct ShowDictionariesStmt {
pub database: Option<Identifier>,
pub limit: Option<ShowLimit>,
}

impl Display for ShowDictionariesStmt {
fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
write!(f, "SHOW DICTIONARIES")?;
if let Some(database) = &self.database {
write!(f, " FROM {database}")?;
}
if let Some(limit) = &self.limit {
write!(f, " {limit}")?;
}
Ok(())
}
}
11 changes: 2 additions & 9 deletions src/query/ast/src/ast/statements/statement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -158,9 +158,7 @@ pub enum Statement {
CreateDictionary(CreateDictionaryStmt),
DropDictionary(DropDictionaryStmt),
ShowCreateDictionary(ShowCreateDictionaryStmt),
ShowDictionaries {
show_options: Option<ShowOptions>,
},
ShowDictionaries(ShowDictionariesStmt),

// Columns
ShowColumns(ShowColumnsStmt),
Expand Down Expand Up @@ -613,12 +611,7 @@ impl Display for Statement {
Statement::CreateDictionary(stmt) => write!(f, "{stmt}")?,
Statement::DropDictionary(stmt) => write!(f, "{stmt}")?,
Statement::ShowCreateDictionary(stmt) => write!(f, "{stmt}")?,
Statement::ShowDictionaries { show_options } => {
write!(f, "SHOW DICTIONARIES")?;
if let Some(show_options) = show_options {
write!(f, " {show_options}")?;
}
}
Statement::ShowDictionaries(stmt) => write!(f, "{stmt}")?,
Statement::CreateView(stmt) => write!(f, "{stmt}")?,
Statement::AlterView(stmt) => write!(f, "{stmt}")?,
Statement::DropView(stmt) => write!(f, "{stmt}")?,
Expand Down
10 changes: 8 additions & 2 deletions src/query/ast/src/parser/statement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -957,9 +957,15 @@ pub fn statement_body(i: Input) -> IResult<Statement> {
);
let show_dictionaries = map(
rule! {
SHOW ~ DICTIONARIES ~ #show_options?
SHOW ~ DICTIONARIES ~ ((FROM|IN) ~ #ident)? ~ #show_limit?
},
|(_, _, db, limit)| {
let database = match db {
Some((_, d)) => Some(d),
_ => None,
};
Statement::ShowDictionaries(ShowDictionariesStmt { database, limit })
},
|(_, _, show_options)| Statement::ShowDictionaries { show_options },
);
let show_create_dictionary = map(
rule! {
Expand Down
2 changes: 2 additions & 0 deletions src/query/service/src/databases/system/system_database.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ use databend_common_storages_system::ConfigsTable;
use databend_common_storages_system::ContributorsTable;
use databend_common_storages_system::CreditsTable;
use databend_common_storages_system::DatabasesTable;
use databend_common_storages_system::DictionariesTable;
use databend_common_storages_system::EnginesTable;
use databend_common_storages_system::FullStreamsTable;
use databend_common_storages_system::FunctionsTable;
Expand Down Expand Up @@ -144,6 +145,7 @@ impl SystemDatabase {
ViewsTableWithoutHistory::create(sys_db_meta.next_table_id()),
TemporaryTablesTable::create(sys_db_meta.next_table_id()),
ProceduresTable::create(sys_db_meta.next_table_id()),
DictionariesTable::create(sys_db_meta.next_table_id()),
];

let disable_tables = Self::disable_system_tables();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -709,7 +709,8 @@ impl AccessChecker for PrivilegeAccess {
Some(RewriteKind::ShowDatabases)
| Some(RewriteKind::ShowEngines)
| Some(RewriteKind::ShowFunctions)
| Some(RewriteKind::ShowUserFunctions) => {
| Some(RewriteKind::ShowUserFunctions)
| Some(RewriteKind::ShowDictionaries(_)) => {
return Ok(());
}
| Some(RewriteKind::ShowTableFunctions) => {
Expand Down
2 changes: 1 addition & 1 deletion src/query/sql/src/planner/binder/binder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ impl<'a> Binder {
Statement::CreateDictionary(stmt) => self.bind_create_dictionary(stmt).await?,
Statement::DropDictionary(stmt) => self.bind_drop_dictionary(stmt).await?,
Statement::ShowCreateDictionary(stmt) => self.bind_show_create_dictionary(stmt).await?,
Statement::ShowDictionaries { show_options: _ } => todo!(),
Statement::ShowDictionaries(stmt) => self.bind_show_dictionaries(bind_context, stmt).await?,
// Views
Statement::CreateView(stmt) => self.bind_create_view(stmt).await?,
Statement::AlterView(stmt) => self.bind_alter_view(stmt).await?,
Expand Down
52 changes: 52 additions & 0 deletions src/query/sql/src/planner/binder/ddl/dictionary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ use std::sync::LazyLock;
use databend_common_ast::ast::CreateDictionaryStmt;
use databend_common_ast::ast::DropDictionaryStmt;
use databend_common_ast::ast::ShowCreateDictionaryStmt;
use databend_common_ast::ast::ShowDictionariesStmt;
use databend_common_ast::ast::ShowLimit;
use databend_common_exception::ErrorCode;
use databend_common_exception::Result;
use databend_common_expression::types::DataType;
Expand All @@ -28,12 +30,16 @@ use databend_common_expression::TableDataType;
use databend_common_expression::TableSchema;
use databend_common_meta_app::schema::DictionaryMeta;
use itertools::Itertools;
use log::debug;

use crate::plans::CreateDictionaryPlan;
use crate::plans::DropDictionaryPlan;
use crate::plans::Plan;
use crate::plans::RewriteKind;
use crate::plans::ShowCreateDictionaryPlan;
use crate::BindContext;
use crate::Binder;
use crate::SelectBuilder;

pub const DICT_OPT_KEY_SQL_HOST: &str = "host";
pub const DICT_OPT_KEY_SQL_PORT: &str = "port";
Expand Down Expand Up @@ -383,4 +389,50 @@ impl Binder {
},
)))
}

#[async_backtrace::framed]
pub(in crate::planner::binder) async fn bind_show_dictionaries(
&mut self,
bind_context: &mut BindContext,
stmt: &ShowDictionariesStmt,
) -> Result<Plan> {
let ShowDictionariesStmt { database, limit } = stmt;

let mut select_builder = SelectBuilder::from("system.dictionaries");

select_builder
.with_column("database AS Database")
.with_column("name AS Dictionary")
.with_column("key_names AS Key_Names")
.with_column("key_types AS key_Types")
.with_column("attribute_names AS Attribute_Names")
.with_column("attribute_types AS Attribute_Types")
.with_column("source AS Source")
.with_column("comment AS Comment");

select_builder
.with_order_by("database")
.with_order_by("name");

let database = self.check_database_exist(&None, database).await?;
select_builder.with_filter(format!("database = '{}'", database.clone()));

match limit {
None => (),
Some(ShowLimit::Like { pattern }) => {
select_builder.with_filter(format!("name LIKE '{pattern}'"));
}
Some(ShowLimit::Where { selection }) => {
select_builder.with_filter(format!("({selection})"));
}
};
let query = select_builder.build();
debug!("show dictionaries rewrite to: {:?}", query);
self.bind_rewrite_to_query(
bind_context,
query.as_str(),
RewriteKind::ShowDictionaries(database.clone()),
)
.await
}
}
1 change: 1 addition & 0 deletions src/query/sql/src/planner/plans/plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,7 @@ pub enum RewriteKind {
ShowColumns(String, String, String),
ShowTablesStatus,
ShowVirtualColumns,
ShowDictionaries(String),

ShowStreams(String),

Expand Down
200 changes: 200 additions & 0 deletions src/query/storages/system/src/dictionaries_table.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
// Copyright 2021 Datafuse Labs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::sync::Arc;

use databend_common_catalog::plan::PushDownInfo;
use databend_common_catalog::table::Table;
use databend_common_catalog::table_context::TableContext;
use databend_common_exception::Result;
use databend_common_expression::types::DataType;
use databend_common_expression::types::StringType;
use databend_common_expression::types::TimestampType;
use databend_common_expression::ColumnBuilder;
use databend_common_expression::DataBlock;
use databend_common_expression::FromData;
use databend_common_expression::ScalarRef;
use databend_common_expression::TableDataType;
use databend_common_expression::TableField;
use databend_common_expression::TableSchemaRefExt;
use databend_common_meta_app::schema::ListDictionaryReq;
use databend_common_meta_app::schema::TableIdent;
use databend_common_meta_app::schema::TableInfo;
use databend_common_meta_app::schema::TableMeta;

use crate::table::AsyncOneBlockSystemTable;
use crate::table::AsyncSystemTable;

pub struct DictionariesTable {
table_info: TableInfo,
}

#[async_trait::async_trait]
impl AsyncSystemTable for DictionariesTable {
const NAME: &'static str = "system.dictionaries";

fn get_table_info(&self) -> &TableInfo {
&self.table_info
}

#[async_backtrace::framed]
async fn get_full_data(
&self,
ctx: Arc<dyn TableContext>,
_push_downs: Option<PushDownInfo>,
) -> Result<DataBlock> {
let tenant = ctx.get_tenant();

let mut db_names = vec![];
let mut names = vec![];

let mut key_names_builder =
ColumnBuilder::with_capacity(&DataType::Array(Box::new(DataType::String)), 0);
let mut attribute_names_builder =
ColumnBuilder::with_capacity(&DataType::Array(Box::new(DataType::String)), 0);
let mut key_types_builder =
ColumnBuilder::with_capacity(&DataType::Array(Box::new(DataType::String)), 0);
let mut attribute_types_builder =
ColumnBuilder::with_capacity(&DataType::Array(Box::new(DataType::String)), 0);

let mut sources = vec![];
let mut comments = vec![];
let mut created_ons = vec![];
let mut updated_ons = vec![];

let catalog = ctx.get_default_catalog().unwrap();
let databases = catalog.list_databases(&tenant).await?;
for database in databases {
let db_id = database.get_db_info().database_id.db_id;
let req = ListDictionaryReq {
tenant: tenant.clone(),
db_id,
};
let dictionaries = catalog.list_dictionaries(req).await?;
for (dict_name, dict_meta) in dictionaries {
db_names.push(database.get_db_name().to_string());

names.push(dict_name.clone());

let comment = dict_meta.comment;
comments.push(comment);

let created_on = dict_meta.created_on.timestamp_micros();
created_ons.push(created_on);
let updated_on = match dict_meta.updated_on {
Some(updated_on) => updated_on.timestamp_micros(),
None => created_on,
};
updated_ons.push(updated_on);

let schema = dict_meta.schema;
let fields = &schema.fields;
let primary_column_ids = dict_meta.primary_column_ids;

let mut key_names = vec![];
let mut attribute_names = vec![];
let mut key_types = vec![];
let mut attribute_types = vec![];

for field in fields {
if primary_column_ids.contains(&field.column_id) {
key_names.push(field.name.clone());
key_types.push(field.data_type.sql_name());
} else {
attribute_names.push(field.name.clone());
attribute_types.push(field.data_type.sql_name());
}
}
let key_names_column = ScalarRef::Array(StringType::from_data(key_names));
key_names_builder.push(key_names_column);
let attribute_names_column =
ScalarRef::Array(StringType::from_data(attribute_names));
attribute_names_builder.push(attribute_names_column);
let key_types_column = ScalarRef::Array(StringType::from_data(key_types));
key_types_builder.push(key_types_column);
let attribute_types_column =
ScalarRef::Array(StringType::from_data(attribute_types));
attribute_types_builder.push(attribute_types_column);

let dict_source = dict_meta.source;
let mut options = dict_meta.options;
if let Some(password) = options.get_mut("password") {
*password = "[hidden]".to_string();
}
let options_str: Vec<String> = options
.iter()
.map(|(k, v)| format!("{}={}", k, v))
.collect();
let options_joined = options_str.join(" ");
let source = format!("{}({})", dict_source, options_joined);
sources.push(source);
}
}
return Ok(DataBlock::new_from_columns(vec![
StringType::from_data(db_names),
StringType::from_data(names),
key_names_builder.build(),
key_types_builder.build(),
attribute_names_builder.build(),
attribute_types_builder.build(),
StringType::from_data(sources),
StringType::from_data(comments),
TimestampType::from_data(created_ons),
TimestampType::from_data(updated_ons),
]));
}
}

impl DictionariesTable {
pub fn create(table_id: u64) -> Arc<dyn Table> {
let schema = TableSchemaRefExt::create(vec![
TableField::new("database", TableDataType::String),
TableField::new("name", TableDataType::String),
TableField::new(
"key_names",
TableDataType::Array(Box::new(TableDataType::String)),
),
TableField::new(
"key_types",
TableDataType::Array(Box::new(TableDataType::String)),
),
TableField::new(
"attribute_names",
TableDataType::Array(Box::new(TableDataType::String)),
),
TableField::new(
"attribute_types",
TableDataType::Array(Box::new(TableDataType::String)),
),
TableField::new("source", TableDataType::String),
TableField::new("comment", TableDataType::String),
TableField::new("created_on", TableDataType::Timestamp),
TableField::new("updated_on", TableDataType::Timestamp),
]);

let table_info = TableInfo {
desc: "'system'.'dictionaries'".to_string(),
name: "dictionaries".to_string(),
ident: TableIdent::new(table_id, 0),
meta: TableMeta {
schema,
engine: "SystemDictionaries".to_string(),
..Default::default()
},
..Default::default()
};

AsyncOneBlockSystemTable::create(DictionariesTable { table_info })
}
}
Loading
Loading