feat: add tables and table data

cmd/api-server/service/parser_excel.go

@@ -0,0 +1,157 @@
+/*
+ * Tencent is pleased to support the open source community by making Blueking Container Service available.
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ * http://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package service
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/TencentBlueKing/bk-bscp/pkg/dal/table"
+	"github.com/TencentBlueKing/bk-bscp/pkg/i18n"
+	"github.com/TencentBlueKing/bk-bscp/pkg/kit"
+	"github.com/xuri/excelize/v2"
+)
+
+type excelImport struct {
+}
+
+func NewExcelImport() *excelImport {
+	return &excelImport{}
+}
+
+// 解析excel
+func (e *excelImport) Import(kit *kit.Kit, r io.Reader) (string, []table.Columns_, []map[string]interface{}, error) {
+	f, err := excelize.OpenReader(r)
+	if err != nil {
+		return "", nil, nil, errors.New(i18n.T(kit, "open Excel file failed %v", err))
+	}
+	defer f.Close()
+
+	tableName := ""
+	// 字段和行
+	columns := make([]table.Columns_, 0)
+	rowsData := make([]map[string]interface{}, 0)
+
+	sheetList := f.GetSheetList()
+
+	for _, sheetName := range sheetList {
+		tableName = sheetName
+		rows, err := f.Rows(sheetName)
+		if err != nil {
+			return "", nil, nil, err
+		}
+		// 解析表头
+		var headers []string
+		if rows.Next() {
+			headers, err = rows.Columns()
+			if err != nil {
+				return "", nil, nil, err
+			}
+		}
+		// 初始化列信息
+		columns = initializeColumns(headers)
+		// 逐行解析数据
+		rowIndex := 0
+		uniqueCheck := make([]map[string]bool, len(headers))
+		for i := range uniqueCheck {
+			uniqueCheck[i] = make(map[string]bool)
+		}
+
+		for rows.Next() {
+			rowIndex++
+			cells, err := rows.Columns()
+			if err != nil {
+				return "", nil, nil, err
+			}
+
+			// 解析行数据
+			rowData := make(map[string]interface{})
+			for i, cell := range cells {
+				if i < len(headers) {
+					rowData[headers[i]] = cell
+					// 更新列信息
+					updateColumnInfo(&columns[i], cell, uniqueCheck[i])
+				}
+			}
+			rowsData = append(rowsData, rowData)
+		}
+	}
+
+	return tableName, columns, rowsData, nil
+}
+
+func initializeColumns(headers []string) []table.Columns_ {
+	columns := make([]table.Columns_, len(headers))
+	for i, header := range headers {
+		columns[i] = table.Columns_{
+			Name:       header,
+			Alias:      header,
+			Length:     0,
+			ColumnType: table.StringColumn,
+			Primary:    false,
+			NotNull:    false,
+			Unique:     false,
+		}
+	}
+	return columns
+}
+
+func updateColumnInfo(column *table.Columns_, value interface{}, uniqueMap map[string]bool) {
+	// 更新 NotNull
+	if value == "" {
+		column.NotNull = false
+		return
+	}
+
+	// 检查唯一性
+	strValue := fmt.Sprintf("%v", value) // 转成字符串方便比较
+	if uniqueMap[strValue] {
+		column.Unique = false // 一旦重复，不再唯一
+	} else {
+		uniqueMap[strValue] = true
+	}
+
+	// 检测数据类型
+	column.ColumnType = detectColumnType(value)
+}
+
+func detectColumnType(value interface{}) table.ColumnType {
+
+	// 检查是否为 nil 或空值
+	if value == nil {
+		return "string" // 默认为字符串
+	}
+
+	// 转为字符串以便进一步解析
+	strValue, ok := value.(string)
+	if !ok {
+		strValue = fmt.Sprintf("%v", value)
+	}
+
+	// 尝试检测数据类型
+	if _, err := strconv.Atoi(strValue); err == nil {
+		return table.NumberColumn // 整数类型
+	}
+	if _, err := strconv.ParseFloat(strValue, 64); err == nil {
+		return table.StringColumn // 浮点类型
+	}
+	if strings.ToLower(strValue) == "true" || strings.ToLower(strValue) == "false" {
+		return table.StringColumn // 布尔类型
+	}
+
+	// 默认返回字符串类型
+	return table.StringColumn
+}

cmd/api-server/service/parser_sql.go

@@ -0,0 +1,192 @@
+/*
+ * Tencent is pleased to support the open source community by making Blueking Container Service available.
+ * Copyright (C) 2019 THL A29 Limited, a Tencent company. All rights reserved.
+ * Licensed under the MIT License (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ * http://opensource.org/licenses/MIT
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+ * either express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package service
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/TencentBlueKing/bk-bscp/pkg/dal/table"
+	"github.com/TencentBlueKing/bk-bscp/pkg/i18n"
+	"github.com/TencentBlueKing/bk-bscp/pkg/kit"
+	"github.com/pingcap/tidb/pkg/parser"
+	"github.com/pingcap/tidb/pkg/parser/ast"
+	"github.com/pingcap/tidb/pkg/parser/mysql"
+	"github.com/pingcap/tidb/pkg/parser/test_driver"
+	"github.com/pingcap/tidb/pkg/parser/types"
+)
+
+type sqlImport struct {
+}
+
+func NewSqlImport() *sqlImport {
+	return &sqlImport{}
+}
+
+// 解析mysql
+func (s *sqlImport) Import(kit *kit.Kit, r io.Reader) (string, []table.Columns_, []map[string]interface{}, error) {
+	sql, err := io.ReadAll(r)
+	if err != nil {
+		return "", nil, nil, errors.New(i18n.T(kit, "read file failed, err: %v", err))
+	}
+
+	p := parser.New()
+	stmtNode, _, err := p.ParseSQL(string(sql))
+	if err != nil {
+		return "", nil, nil, errors.New(i18n.T(kit, "parses a query string to raw ast.StmtNode failed, err: %v", err))
+	}
+
+	tableName := ""
+	// 字段和行
+	columns := make([]table.Columns_, 0)
+	rows := make([]map[string]interface{}, 0)
+	colNames := []string{}
+	for _, stmt := range stmtNode {
+		switch stmt := stmt.(type) {
+		// 解析列名、数据类型等
+		case *ast.CreateTableStmt:
+			tableName = stmt.Table.Name.String()
+			for _, col := range stmt.Cols {
+				var unique, primaryKey bool
+				primaryKey = isPrimaryKey(col, stmt)
+				unique = isUnique(col.Name.String(), stmt)
+				if primaryKey {
+					unique = primaryKey
+				}
+				columns = append(columns, table.Columns_{
+					Name:          col.Name.String(),
+					Length:        col.Tp.GetFlen(),
+					Primary:       primaryKey,
+					ColumnType:    parseColumnType(col.Tp),
+					NotNull:       isNotNull(col),
+					DefaultValue:  getDefaultValue(col),
+					Unique:        unique,
+					AutoIncrement: isAutoIncrement(col),
+					EnumValue:     getEnumValues(col),
+				})
+				colNames = append(colNames, col.Name.String())
+			}
+		// 解析 Insert 语法
+		case *ast.InsertStmt:
+			for _, value := range stmt.Lists {
+				row := make(map[string]interface{})
+				for i, val := range value {
+					switch v := val.(type) {
+					case *test_driver.ValueExpr:
+						row[colNames[i]] = v.GetValue()
+					}
+				}
+				rows = append(rows, row)
+			}
+		}
+	}
+
+	return tableName, columns, rows, nil
+}
+
+// isPrimaryKey checks if the column is a primary key.
+func isPrimaryKey(col *ast.ColumnDef, stmt *ast.CreateTableStmt) bool {
+	for _, constraint := range stmt.Constraints {
+		if constraint.Tp == ast.ConstraintPrimaryKey {
+			for _, column := range constraint.Keys {
+				if column.Column.Name.L == col.Name.String() {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// isAutoIncrement checks if the column is auto-increment.
+func isAutoIncrement(col *ast.ColumnDef) bool {
+	for _, opt := range col.Options {
+		if opt.Tp == ast.ColumnOptionAutoIncrement {
+			return true
+		}
+	}
+	return false
+}
+
+// getDefaultValue returns the default value of the column.
+func getDefaultValue(col *ast.ColumnDef) string {
+	for _, opt := range col.Options {
+		if opt.Tp == ast.ColumnOptionDefaultValue {
+			return toString(opt.Expr.(*test_driver.ValueExpr).GetValue())
+		}
+	}
+
+	return ""
+}
+
+// isNotNull checks if the column is not null.
+func isNotNull(col *ast.ColumnDef) bool {
+	for _, opt := range col.Options {
+		if opt.Tp == ast.ColumnOptionNotNull {
+			return true
+		}
+	}
+
+	return false
+}
+
+// isUnique checks if the column has a unique constraint.
+func isUnique(colName string, stmt *ast.CreateTableStmt) bool {
+	for _, constraint := range stmt.Constraints {
+		if constraint.Tp == ast.ConstraintUniq {
+			for _, column := range constraint.Keys {
+				if column.Column.Name.L == colName {
+					return true
+				}
+			}
+		}
+	}
+
+	return false
+}
+
+// getEnumValues extracts the enum values for a column if it's an enum type.
+func getEnumValues(col *ast.ColumnDef) string {
+	if col.Tp.GetElems() == nil {
+		return ""
+	}
+	enumValues, err := json.Marshal(col.Tp.GetElems())
+	if err != nil {
+		return ""
+	}
+
+	return string(enumValues)
+}
+
+// parseColumnType maps the column type based on the expression.
+func parseColumnType(colType *types.FieldType) table.ColumnType {
+	switch colType.GetType() {
+	case mysql.TypeBit, mysql.TypeTiny, mysql.TypeShort, mysql.TypeLong, mysql.TypeFloat,
+		mysql.TypeDouble, mysql.TypeLonglong, mysql.TypeInt24:
+		return table.NumberColumn
+	case mysql.TypeEnum:
+		return table.EnumColumn
+	default:
+		return table.StringColumn
+	}
+}
+
+func toString(value interface{}) string {
+	if value == nil {
+		return ""
+	}
+
+	return fmt.Sprintf("%v", value)
+}