diff --git a/cmd/gemini/root.go b/cmd/gemini/root.go index f51ca0cf..768b2f78 100644 --- a/cmd/gemini/root.go +++ b/cmd/gemini/root.go @@ -125,7 +125,8 @@ func readSchema(confFile string) (*typedef.Schema, error) { schemaBuilder := builders.NewSchemaBuilder() schemaBuilder.Keyspace(shm.Keyspace) - for _, tbl := range shm.Tables { + for t, tbl := range shm.Tables { + shm.Tables[t].LinkIndexAndColumns() schemaBuilder.Table(tbl) } return schemaBuilder.Build(), nil diff --git a/cmd/gemini/schema.json b/cmd/gemini/schema.json index 2bb3c6f3..71b0f797 100644 --- a/cmd/gemini/schema.json +++ b/cmd/gemini/schema.json @@ -1,6 +1,14 @@ { "keyspace": { - "name": "ks1" + "name": "ks1", + "replication": { + "class": "SimpleStrategy", + "replication_factor": 1 + }, + "oracle_replication": { + "class": "SimpleStrategy", + "replication_factor": 1 + } }, "tables": [ { @@ -29,9 +37,10 @@ { "name": "col0", "type": { + "complex_type": "udt", "type_name": "udt_672245080", "frozen": true, - "coltypes": { + "value_types": { "udt_672245080_0": "ascii", "udt_672245080_1": "boolean", "udt_672245080_2": "bigint", @@ -41,28 +50,18 @@ }, { "name": "col1", - "type": "timestamp" - }, - { - "name": "col2", - "type": "decimal" - }, - { - "name": "col3", - "type": "uuid" - }, - { - "name": "col4", "type": { + "complex_type": "map", "key_type": "boolean", "value_type": "duration", "frozen": false } }, { - "name": "col5", + "name": "col2", "type": { - "types": [ + "complex_type": "tuple", + "value_types": [ "varchar", "smallint" ], @@ -70,30 +69,114 @@ } }, { - "name": "col6", + "name": "col3", "type": { - "kind": "list", - "type": "int", + "complex_type": "list", + "value_type": "int", "frozen": true } + }, + { + "name": "col4", + "type": { + "complex_type": "set", + "value_type": "int", + "frozen": true + } + }, + { + "name": "col5", + "type": "ascii" + }, + { + "name": "col6", + "type": "bigint" + }, + { + "name": "col7", + "type": "blob" + }, + { + "name": "col8", + "type": "boolean" + }, + { + "name": "col9", + "type": "date" + }, + { + "name": "col10", + "type": "decimal" + }, + { + "name": "col11", + "type": "double" + }, + { + "name": "col12", + "type": "duration" + }, + { + "name": "col13", + "type": "float" + }, + { + "name": "col14", + "type": "inet" + }, + { + "name": "col15", + "type": "int" + }, + { + "name": "col16", + "type": "smallint" + }, + { + "name": "col17", + "type": "text" + }, + { + "name": "col19", + "type": "timestamp" + }, + { + "name": "col20", + "type": "timeuuid" + }, + { + "name": "col21", + "type": "tinyint" + }, + { + "name": "col22", + "type": "uuid" + }, + { + "name": "col23", + "type": "varchar" + }, + { + "name": "col24", + "type": "varint" } ], "indexes": [ { - "name": "col0_idx", - "column": "col0" + "index_name": "col5_idx", + "column_name": "col5" }, { - "name": "col1_idx", - "column": "col1" + "index_name": "col6_idx", + "column_name": "col6" }, { - "name": "col2_idx", - "column": "col2" + "index_name": "col7_idx", + "column_name": "col7" }, { - "name": "col3_idx", - "column": "col3" + "index_name": "col8_idx", + "column_name": "col8" } ], "known_issues": { diff --git a/cmd/gemini/strategies_test.go b/cmd/gemini/strategies_test.go index 59450c85..d328110c 100644 --- a/cmd/gemini/strategies_test.go +++ b/cmd/gemini/strategies_test.go @@ -15,13 +15,15 @@ package main import ( + "encoding/json" "testing" - "github.com/scylladb/gemini/pkg/replication" - "github.com/google/go-cmp/cmp" - + "github.com/google/go-cmp/cmp/cmpopts" "go.uber.org/zap" + + "github.com/scylladb/gemini/pkg/replication" + "github.com/scylladb/gemini/pkg/typedef" ) func TestGetReplicationStrategy(t *testing.T) { @@ -57,3 +59,30 @@ func TestGetReplicationStrategy(t *testing.T) { }) } } + +func TestReadSchema(t *testing.T) { + filePath := "schema.json" + + testSchema, err := readSchema(filePath) + if err != nil { + t.Fatalf("failed to open schema example json file %s, error:%s", filePath, err) + } + + opts := cmp.Options{ + cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}), + cmpopts.IgnoreUnexported(typedef.Table{}, typedef.MaterializedView{}), + } + + testSchemaMarshaled, err := json.MarshalIndent(testSchema, " ", " ") + if err != nil { + t.Fatalf("unable to marshal schema example json, error=%s\n", err) + } + testSchemaUnMarshaled := typedef.Schema{} + if err = json.Unmarshal(testSchemaMarshaled, &testSchemaUnMarshaled); err != nil { + t.Fatalf("unable to unmarshal json, error=%s\n", err) + } + + if diff := cmp.Diff(*testSchema, testSchemaUnMarshaled, opts); diff != "" { + t.Errorf("schema not the same after marshal/unmarshal, diff=%s", diff) + } +} diff --git a/pkg/generators/column.go b/pkg/generators/column.go index 7e5fdd4a..d8fe009c 100644 --- a/pkg/generators/column.go +++ b/pkg/generators/column.go @@ -23,7 +23,11 @@ func CreateIndexesForColumn(table *typedef.Table, maxIndexes int) []typedef.Inde indexes := make([]typedef.IndexDef, 0, maxIndexes) for i, col := range table.Columns { if col.Type.Indexable() && typedef.TypesForIndex.Contains(col.Type) { - indexes = append(indexes, typedef.IndexDef{Name: GenIndexName(table.Name+"_col", i), Column: table.Columns[i]}) + indexes = append(indexes, typedef.IndexDef{ + IndexName: GenIndexName(table.Name+"_col", i), + ColumnName: table.Columns[i].Name, + Column: table.Columns[i], + }) createdCount++ } if createdCount == maxIndexes { diff --git a/pkg/generators/column_generator.go b/pkg/generators/column_generator.go index fb5308f8..bcd00faa 100644 --- a/pkg/generators/column_generator.go +++ b/pkg/generators/column_generator.go @@ -58,8 +58,9 @@ func GenTupleType(sc *typedef.SchemaConfig) typedef.Type { typeList[i] = GenSimpleType(sc) } return &typedef.TupleType{ - Types: typeList, - Frozen: rand.Uint32()%2 == 0, + ComplexType: typedef.TYPE_TUPLE, + ValueTypes: typeList, + Frozen: rand.Uint32()%2 == 0, } } @@ -73,18 +74,19 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType { } return &typedef.UDTType{ - Types: ts, - TypeName: typeName, - Frozen: true, + ComplexType: typedef.TYPE_UDT, + ValueTypes: ts, + TypeName: typeName, + Frozen: true, } } func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType { - return genBagType("set", sc) + return genBagType(typedef.TYPE_SET, sc) } func GenListType(sc *typedef.SchemaConfig) *typedef.BagType { - return genBagType("list", sc) + return genBagType(typedef.TYPE_LIST, sc) } func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType { @@ -96,9 +98,9 @@ func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType { } } return &typedef.BagType{ - Kind: kind, - Type: t, - Frozen: rand.Uint32()%2 == 0, + ComplexType: kind, + ValueType: t, + Frozen: rand.Uint32()%2 == 0, } } @@ -111,9 +113,10 @@ func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType { t = GenSimpleType(sc) } return &typedef.MapType{ - KeyType: t, - ValueType: GenSimpleType(sc), - Frozen: rand.Uint32()%2 == 0, + ComplexType: typedef.TYPE_MAP, + KeyType: t, + ValueType: GenSimpleType(sc), + Frozen: rand.Uint32()%2 == 0, } } diff --git a/pkg/generators/statement_generator.go b/pkg/generators/statement_generator.go index 4bda53e3..30e358af 100644 --- a/pkg/generators/statement_generator.go +++ b/pkg/generators/statement_generator.go @@ -106,7 +106,7 @@ func GetCreateSchema(s *typedef.Schema) []string { createTable := GetCreateTable(t, s.Keyspace) stmts = append(stmts, createTable) for _, idef := range t.Indexes { - stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.Name, s.Keyspace.Name, t.Name, idef.Column.Name)) + stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.IndexName, s.Keyspace.Name, t.Name, idef.ColumnName)) } for _, mv := range t.MaterializedViews { var ( diff --git a/pkg/generators/table.go b/pkg/generators/table.go index 206e8001..3f9da286 100644 --- a/pkg/generators/table.go +++ b/pkg/generators/table.go @@ -68,7 +68,7 @@ func GetCreateTypes(t *typedef.Table, keyspace typedef.Keyspace) []string { } createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s)" var typs []string - for name, typ := range c.Types { + for name, typ := range c.ValueTypes { typs = append(typs, name+" "+typ.CQLDef()) } stmts = append(stmts, fmt.Sprintf(createType, keyspace.Name, c.TypeName, strings.Join(typs, ","))) diff --git a/pkg/jobs/gen_check_stmt.go b/pkg/jobs/gen_check_stmt.go index 41565ced..9a5d9dfb 100644 --- a/pkg/jobs/gen_check_stmt.go +++ b/pkg/jobs/gen_check_stmt.go @@ -521,7 +521,7 @@ func genSingleIndexQuery( builder := qb.Select(s.Keyspace.Name + "." + t.Name) builder.AllowFiltering() for i := 0; i < idxCount; i++ { - builder = builder.Where(qb.Eq(t.Indexes[i].Column.Name)) + builder = builder.Where(qb.Eq(t.Indexes[i].ColumnName)) values = append(values, t.Indexes[i].Column.Type.GenValue(r, p)...) typs = append(typs, t.Indexes[i].Column.Type) } diff --git a/pkg/jobs/gen_ddl_stmt.go b/pkg/jobs/gen_ddl_stmt.go index 2d2837e4..4d3fed50 100644 --- a/pkg/jobs/gen_ddl_stmt.go +++ b/pkg/jobs/gen_ddl_stmt.go @@ -52,7 +52,7 @@ func genAddColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnD if c, ok := column.Type.(*typedef.UDTType); ok { createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s);" var typs []string - for name, typ := range c.Types { + for name, typ := range c.ValueTypes { typs = append(typs, name+" "+typ.CQLDef()) } stmt := fmt.Sprintf(createType, keyspace, c.TypeName, strings.Join(typs, ",")) diff --git a/pkg/jobs/gen_mutate_stmt.go b/pkg/jobs/gen_mutate_stmt.go index 36f6363b..2af4eb2b 100644 --- a/pkg/jobs/gen_mutate_stmt.go +++ b/pkg/jobs/gen_mutate_stmt.go @@ -144,9 +144,9 @@ func genInsertJSONStmt( values[pk.Name] = "0x" + v } case *typedef.TupleType: - tupVals := make([]interface{}, len(t.Types)) - for j := 0; j < len(t.Types); j++ { - if t.Types[j] == typedef.TYPE_BLOB { + tupVals := make([]interface{}, len(t.ValueTypes)) + for j := 0; j < len(t.ValueTypes); j++ { + if t.ValueTypes[j] == typedef.TYPE_BLOB { v, ok = vs[i+j].(string) if ok { v = "0x" + v diff --git a/pkg/jobs/gen_utils_test.go b/pkg/jobs/gen_utils_test.go index 85b5a5bf..877e52c5 100644 --- a/pkg/jobs/gen_utils_test.go +++ b/pkg/jobs/gen_utils_test.go @@ -476,15 +476,19 @@ func createIdxFromColumns(t testInterface, table *typedef.Table, all bool) (inde switch all { case true: for i := range table.Columns { - var index typedef.IndexDef - index.Name = table.Columns[i].Name + "_idx" - index.Column = table.Columns[i] + index := typedef.IndexDef{ + IndexName: table.Columns[i].Name + "_idx", + ColumnName: table.Columns[i].Name, + Column: table.Columns[i], + } indexes = append(indexes, index) } default: - var index typedef.IndexDef - index.Name = table.Columns[0].Name + "_idx" - index.Column = table.Columns[0] + index := typedef.IndexDef{ + IndexName: table.Columns[0].Name + "_idx", + ColumnName: table.Columns[0].Name, + Column: table.Columns[0], + } indexes = append(indexes, index) } diff --git a/pkg/jobs/jobs.go b/pkg/jobs/jobs.go index ee0543a5..c065b5cf 100644 --- a/pkg/jobs/jobs.go +++ b/pkg/jobs/jobs.go @@ -323,7 +323,7 @@ func ddl( defer table.Unlock() ddlStmts, err := GenDDLStmt(schema, table, r, p, sc) if err != nil { - logger.Error("Failed! Mutation statement generation failed", zap.Error(err)) + logger.Error("Failed! DDL Mutation statement generation failed", zap.Error(err)) globalStatus.WriteErrors.Add(1) return err } diff --git a/pkg/querycache/querycache.go b/pkg/querycache/querycache.go index b69d29c7..f76a6312 100644 --- a/pkg/querycache/querycache.go +++ b/pkg/querycache/querycache.go @@ -114,7 +114,7 @@ func genInsertStmtCache( for _, col := range t.Columns { switch colType := col.Type.(type) { case *typedef.TupleType: - builder = builder.TupleColumn(col.Name, len(colType.Types)) + builder = builder.TupleColumn(col.Name, len(colType.ValueTypes)) default: builder = builder.Columns(col.Name) } @@ -143,7 +143,7 @@ func genUpdateStmtCache(s *typedef.Schema, t *typedef.Table) *typedef.StmtCache for _, cdef := range t.Columns { switch t := cdef.Type.(type) { case *typedef.TupleType: - builder = builder.SetTuple(cdef.Name, len(t.Types)) + builder = builder.SetTuple(cdef.Name, len(t.ValueTypes)) case *typedef.CounterType: builder = builder.SetLit(cdef.Name, cdef.Name+"+1") continue diff --git a/pkg/typedef/bag.go b/pkg/typedef/bag.go index bc596254..3a05d525 100644 --- a/pkg/typedef/bag.go +++ b/pkg/typedef/bag.go @@ -24,14 +24,14 @@ import ( ) type BagType struct { - Kind string `json:"kind"` // We need to differentiate between sets and lists - Type SimpleType `json:"type"` - Frozen bool `json:"frozen"` + ComplexType string `json:"complex_type"` // We need to differentiate between sets and lists + ValueType SimpleType `json:"value_type"` + Frozen bool `json:"frozen"` } func (ct *BagType) CQLType() gocql.TypeInfo { - switch ct.Kind { - case "set": + switch ct.ComplexType { + case TYPE_SET: return goCQLTypeMap[gocql.TypeSet] default: return goCQLTypeMap[gocql.TypeList] @@ -40,16 +40,16 @@ func (ct *BagType) CQLType() gocql.TypeInfo { func (ct *BagType) Name() string { if ct.Frozen { - return "frozen<" + ct.Kind + "<" + ct.Type.Name() + ">>" + return "frozen<" + ct.ComplexType + "<" + ct.ValueType.Name() + ">>" } - return ct.Kind + "<" + ct.Type.Name() + ">" + return ct.ComplexType + "<" + ct.ValueType.Name() + ">" } func (ct *BagType) CQLDef() string { if ct.Frozen { - return "frozen<" + ct.Kind + "<" + ct.Type.Name() + ">>" + return "frozen<" + ct.ComplexType + "<" + ct.ValueType.Name() + ">>" } - return ct.Kind + "<" + ct.Type.Name() + ">" + return ct.ComplexType + "<" + ct.ValueType.Name() + ">" } func (ct *BagType) CQLHolder() string { @@ -65,7 +65,7 @@ func (ct *BagType) CQLPretty(query string, value []interface{}) (string, int) { } s := reflect.ValueOf(value[0]) op, cl := "[", "]" - if ct.Kind == "set" { + if ct.ComplexType == TYPE_SET { op, cl = "{", "}" } vv := op @@ -73,7 +73,7 @@ func (ct *BagType) CQLPretty(query string, value []interface{}) (string, int) { vv = strings.TrimRight(vv, ",") vv += cl for i := 0; i < s.Len(); i++ { - vv, _ = ct.Type.CQLPretty(vv, []interface{}{s.Index(i).Interface()}) + vv, _ = ct.ValueType.CQLPretty(vv, []interface{}{s.Index(i).Interface()}) } return strings.Replace(query, "?", vv, 1), 1 } @@ -82,7 +82,7 @@ func (ct *BagType) GenValue(r *rand.Rand, p *PartitionRangeConfig) []interface{} count := r.Intn(9) + 1 out := make([]interface{}, count) for i := 0; i < count; i++ { - out[i] = ct.Type.GenValue(r, p)[0] + out[i] = ct.ValueType.GenValue(r, p)[0] } return []interface{}{out} } @@ -91,7 +91,7 @@ func (ct *BagType) GenJSONValue(r *rand.Rand, p *PartitionRangeConfig) interface count := r.Intn(9) + 1 out := make([]interface{}, count) for i := 0; i < count; i++ { - out[i] = ct.Type.GenJSONValue(r, p) + out[i] = ct.ValueType.GenJSONValue(r, p) } return out } diff --git a/pkg/typedef/columns.go b/pkg/typedef/columns.go index fc1525df..dbf267e6 100644 --- a/pkg/typedef/columns.go +++ b/pkg/typedef/columns.go @@ -19,7 +19,6 @@ import ( "github.com/mitchellh/mapstructure" "github.com/pkg/errors" - "golang.org/x/exp/rand" ) @@ -28,6 +27,8 @@ type ColumnDef struct { Name string `json:"name"` } +var ErrSchemaValidation = errors.New("validation failed") + func (cd *ColumnDef) IsValidForPrimaryKey() bool { for _, pkType := range PkTypes { if cd.Type.Name() == pkType.Name() { @@ -42,21 +43,34 @@ func (cd *ColumnDef) UnmarshalJSON(data []byte) error { if err := json.Unmarshal(data, &dataMap); err != nil { return err } - t, err := GetSimpleTypeColumn(dataMap) if err != nil { - t, err = GetUDTTypeColumn(dataMap) - if err != nil { + typeMap, typeOk := dataMap["type"] + if !typeOk { + return errors.Wrapf(ErrSchemaValidation, "missing definition of column 'type': [%T]%+[1]v", dataMap) + } + complexTypeMap, typeMapOk := typeMap.(map[string]interface{}) + if !typeMapOk { + return errors.Wrapf(ErrSchemaValidation, "unknown definition column 'type': [%T]%+[1]v", typeMap) + } + complexType, complexTypeOk := complexTypeMap["complex_type"] + if !complexTypeOk { + return errors.Wrapf(ErrSchemaValidation, "missing definition of column 'complex_type': [%T]%+[1]v", complexTypeMap) + } + switch complexType { + case TYPE_LIST, TYPE_SET: + t, err = GetBagTypeColumn(dataMap) + case TYPE_MAP: + t, err = GetMapTypeColumn(dataMap) + case TYPE_TUPLE: t, err = GetTupleTypeColumn(dataMap) - if err != nil { - t, err = GetMapTypeColumn(dataMap) - if err != nil { - t, err = GetBagTypeColumn(dataMap) - if err != nil { - return err - } - } - } + case TYPE_UDT: + t, err = GetUDTTypeColumn(dataMap) + default: + return errors.Wrapf(ErrSchemaValidation, "unknown 'complex_type': [%T]%+[1]v", complexType) + } + if err != nil { + return err } } *cd = ColumnDef{ @@ -169,9 +183,10 @@ func GetMapTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) { return &ColumnDef{ Name: st.Name, Type: &MapType{ - Frozen: frozen, - ValueType: valueType, - KeyType: keyType, + ComplexType: TYPE_MAP, + Frozen: frozen, + ValueType: valueType, + KeyType: keyType, }, }, err } @@ -185,9 +200,8 @@ func GetBagTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) { if err = mapstructure.Decode(data, &st); err != nil { return nil, errors.Wrapf(err, "can't decode string value for BagType, value=%+v", data) } - - var kind string - if err = mapstructure.Decode(st.Type["kind"], &kind); err != nil { + var complexType string + if err = mapstructure.Decode(st.Type["complex_type"], &complexType); err != nil { return nil, errors.Wrapf(err, "can't decode string value for BagType::Frozen, value=%v", st) } var frozen bool @@ -195,15 +209,15 @@ func GetBagTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) { return nil, errors.Wrapf(err, "can't decode bool value for BagType::Frozen, value=%v", st) } var typ SimpleType - if err = mapstructure.Decode(st.Type["type"], &typ); err != nil { + if err = mapstructure.Decode(st.Type["value_type"], &typ); err != nil { return nil, errors.Wrapf(err, "can't decode SimpleType value for BagType::ValueType, value=%v", st) } return &ColumnDef{ Name: st.Name, Type: &BagType{ - Kind: kind, - Frozen: frozen, - Type: typ, + ComplexType: complexType, + Frozen: frozen, + ValueType: typ, }, }, err } @@ -218,23 +232,24 @@ func GetTupleTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) return nil, errors.Wrapf(err, "can't decode []SimpleType value, value=%+v", data) } - if _, ok := st.Type["coltypes"]; !ok { + if _, ok := st.Type["value_types"]; !ok { return nil, errors.Errorf("not a tuple type, value=%v", st) } var dbTypes []SimpleType - if err = mapstructure.Decode(st.Type["coltypes"], &dbTypes); err != nil { - return nil, errors.Wrapf(err, "can't decode []SimpleType value for TupleType::Types, value=%v", st) + if err = mapstructure.Decode(st.Type["value_types"], &dbTypes); err != nil { + return nil, errors.Wrapf(err, "can't decode []SimpleType value for TupleType::ValueTypes, value=%v", st) } var frozen bool if err = mapstructure.Decode(st.Type["frozen"], &frozen); err != nil { - return nil, errors.Wrapf(err, "can't decode bool value for TupleType::Types, value=%v", st) + return nil, errors.Wrapf(err, "can't decode bool value for TupleType::ValueTypes, value=%v", st) } return &ColumnDef{ Name: st.Name, Type: &TupleType{ - Types: dbTypes, - Frozen: frozen, + ComplexType: TYPE_TUPLE, + ValueTypes: dbTypes, + Frozen: frozen, }, }, nil } @@ -249,7 +264,7 @@ func GetUDTTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) { return nil, errors.Wrapf(err, "can't decode []SimpleType , value=%+v", data) } - if _, ok := st.Type["coltypes"]; !ok { + if _, ok := st.Type["value_types"]; !ok { return nil, errors.Errorf("not a UDT type, value=%v", st) } if _, ok := st.Type["type_name"]; !ok { @@ -257,8 +272,8 @@ func GetUDTTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) { } var dbTypes map[string]SimpleType - if err = mapstructure.Decode(st.Type["coltypes"], &dbTypes); err != nil { - return nil, errors.Wrapf(err, "can't decode []SimpleType value for UDTType::Types, value=%v", st) + if err = mapstructure.Decode(st.Type["value_types"], &dbTypes); err != nil { + return nil, errors.Wrapf(err, "can't decode []SimpleType value for UDTType::ValueTypes, value=%v", st) } var frozen bool if err = mapstructure.Decode(st.Type["frozen"], &frozen); err != nil { @@ -271,9 +286,10 @@ func GetUDTTypeColumn(data map[string]interface{}) (out *ColumnDef, err error) { return &ColumnDef{ Name: st.Name, Type: &UDTType{ - Types: dbTypes, - TypeName: typeName, - Frozen: frozen, + ComplexType: TYPE_UDT, + ValueTypes: dbTypes, + TypeName: typeName, + Frozen: frozen, }, }, nil } @@ -287,6 +303,22 @@ func GetSimpleTypeColumn(data map[string]interface{}) (*ColumnDef, error) { if err != nil { return nil, err } + if st.Name == "" { + return nil, errors.Wrapf(ErrSchemaValidation, "wrong definition of column 'name' [%T]%+[1]v", data) + } + if st.Type == "" { + return nil, errors.Wrapf(ErrSchemaValidation, "empty definition of column 'type' [%T]%+[1]v", data) + } + + knownType := false + for _, sType := range AllTypes { + if sType == st.Type { + knownType = true + } + } + if !knownType { + return nil, errors.Wrapf(ErrSchemaValidation, "not simple type in column 'type' [%T]%+[1]v", data) + } return &ColumnDef{ Name: st.Name, Type: st.Type, diff --git a/pkg/typedef/columns_test.go b/pkg/typedef/columns_test.go index d7020d76..dd0e17cb 100644 --- a/pkg/typedef/columns_test.go +++ b/pkg/typedef/columns_test.go @@ -41,7 +41,8 @@ var allSimpleTypes = []typedef.SimpleType{ typedef.TYPE_INT, typedef.TYPE_SMALLINT, typedef.TYPE_TEXT, - typedef.TYPE_TIME, + // TODO: Add support for time when gocql bug is fixed. + // typedef.TYPE_TIME, typedef.TYPE_TIMESTAMP, typedef.TYPE_TIMEUUID, typedef.TYPE_TINYINT, @@ -72,13 +73,14 @@ func TestColumnMarshalUnmarshal(t *testing.T) { testCases = append(testCases, testCase{ def: typedef.ColumnDef{ Type: &typedef.UDTType{ - TypeName: "udt1", - Types: udtTypes, + ComplexType: typedef.TYPE_UDT, + TypeName: "udt1", + ValueTypes: udtTypes, }, Name: "udt1", }, //nolint:lll - expected: "{\"type\":{\"coltypes\":{\"col_ascii\":\"ascii\",\"col_bigint\":\"bigint\",\"col_blob\":\"blob\",\"col_boolean\":\"boolean\",\"col_date\":\"date\",\"col_decimal\":\"decimal\",\"col_double\":\"double\",\"col_duration\":\"duration\",\"col_float\":\"float\",\"col_inet\":\"inet\",\"col_int\":\"int\",\"col_smallint\":\"smallint\",\"col_text\":\"text\",\"col_time\":\"time\",\"col_timestamp\":\"timestamp\",\"col_timeuuid\":\"timeuuid\",\"col_tinyint\":\"tinyint\",\"col_uuid\":\"uuid\",\"col_varchar\":\"varchar\",\"col_varint\":\"varint\"},\"type_name\":\"udt1\",\"frozen\":false},\"name\":\"udt1\"}", + expected: "{\"type\":{\"complex_type\":\"udt\",\"value_types\":{\"col_ascii\":\"ascii\",\"col_bigint\":\"bigint\",\"col_blob\":\"blob\",\"col_boolean\":\"boolean\",\"col_date\":\"date\",\"col_decimal\":\"decimal\",\"col_double\":\"double\",\"col_duration\":\"duration\",\"col_float\":\"float\",\"col_inet\":\"inet\",\"col_int\":\"int\",\"col_smallint\":\"smallint\",\"col_text\":\"text\",\"col_timestamp\":\"timestamp\",\"col_timeuuid\":\"timeuuid\",\"col_tinyint\":\"tinyint\",\"col_uuid\":\"uuid\",\"col_varchar\":\"varchar\",\"col_varint\":\"varint\"},\"type_name\":\"udt1\",\"frozen\":false},\"name\":\"udt1\"}", }) for id := range testCases { @@ -95,7 +97,6 @@ func TestColumnMarshalUnmarshal(t *testing.T) { t.Errorf(diff) } var unmarshaledDef typedef.ColumnDef - err = json.Unmarshal(marshaledData, &unmarshaledDef) if err != nil { t.Fatal(err.Error()) @@ -291,12 +292,14 @@ func getTestSchema() *typedef.Schema { } sch.Tables[0].Indexes = []typedef.IndexDef{ { - Name: generators.GenIndexName(sch.Tables[0].Name+"_col", 0), - Column: columns[0], + IndexName: generators.GenIndexName(sch.Tables[0].Name+"_col", 0), + ColumnName: columns[0].Name, + Column: columns[0], }, { - Name: generators.GenIndexName(sch.Tables[0].Name+"_col", 1), - Column: columns[1], + IndexName: generators.GenIndexName(sch.Tables[0].Name+"_col", 1), + ColumnName: columns[1].Name, + Column: columns[1], }, } diff --git a/pkg/typedef/table.go b/pkg/typedef/table.go index 48bf8129..1af82947 100644 --- a/pkg/typedef/table.go +++ b/pkg/typedef/table.go @@ -98,7 +98,7 @@ func (t *Table) ValidColumnsForDelete() Columns { if len(t.Indexes) != 0 { for _, idx := range t.Indexes { for j := range validCols { - if validCols[j].Name == idx.Column.Name { + if validCols[j].Name == idx.ColumnName { validCols = append(validCols[:j], validCols[j+1:]...) break } @@ -119,3 +119,14 @@ func (t *Table) ValidColumnsForDelete() Columns { } return validCols } + +func (t *Table) LinkIndexAndColumns() { + for i, index := range t.Indexes { + for c, column := range t.Columns { + if index.ColumnName == column.Name { + t.Indexes[i].Column = t.Columns[c] + break + } + } + } +} diff --git a/pkg/typedef/tuple.go b/pkg/typedef/tuple.go index cc05a4b1..75e2cd7f 100644 --- a/pkg/typedef/tuple.go +++ b/pkg/typedef/tuple.go @@ -22,8 +22,9 @@ import ( ) type TupleType struct { - Types []SimpleType `json:"coltypes"` - Frozen bool `json:"frozen"` + ComplexType string `json:"complex_type"` + ValueTypes []SimpleType `json:"value_types"` + Frozen bool `json:"frozen"` } func (t *TupleType) CQLType() gocql.TypeInfo { @@ -31,16 +32,16 @@ func (t *TupleType) CQLType() gocql.TypeInfo { } func (t *TupleType) Name() string { - names := make([]string, len(t.Types)) - for i, tp := range t.Types { + names := make([]string, len(t.ValueTypes)) + for i, tp := range t.ValueTypes { names[i] = tp.Name() } return "Type: " + strings.Join(names, ",") } func (t *TupleType) CQLDef() string { - names := make([]string, len(t.Types)) - for i, tp := range t.Types { + names := make([]string, len(t.ValueTypes)) + for i, tp := range t.ValueTypes { names[i] = tp.CQLDef() } if t.Frozen { @@ -50,7 +51,7 @@ func (t *TupleType) CQLDef() string { } func (t *TupleType) CQLHolder() string { - return "(" + strings.TrimRight(strings.Repeat("?,", len(t.Types)), ",") + ")" + return "(" + strings.TrimRight(strings.Repeat("?,", len(t.ValueTypes)), ",") + ")" } func (t *TupleType) CQLPretty(query string, value []interface{}) (string, int) { @@ -58,7 +59,7 @@ func (t *TupleType) CQLPretty(query string, value []interface{}) (string, int) { return query, 0 } var cnt, tmp int - for i, tp := range t.Types { + for i, tp := range t.ValueTypes { query, tmp = tp.CQLPretty(query, value[i:]) cnt += tmp } @@ -66,7 +67,7 @@ func (t *TupleType) CQLPretty(query string, value []interface{}) (string, int) { } func (t *TupleType) Indexable() bool { - for _, t := range t.Types { + for _, t := range t.ValueTypes { if t == TYPE_DURATION { return false } @@ -75,16 +76,16 @@ func (t *TupleType) Indexable() bool { } func (t *TupleType) GenJSONValue(r *rand.Rand, p *PartitionRangeConfig) interface{} { - out := make([]interface{}, 0, len(t.Types)) - for _, tp := range t.Types { + out := make([]interface{}, 0, len(t.ValueTypes)) + for _, tp := range t.ValueTypes { out = append(out, tp.GenJSONValue(r, p)) } return out } func (t *TupleType) GenValue(r *rand.Rand, p *PartitionRangeConfig) []interface{} { - out := make([]interface{}, 0, len(t.Types)) - for _, tp := range t.Types { + out := make([]interface{}, 0, len(t.ValueTypes)) + for _, tp := range t.ValueTypes { out = append(out, tp.GenValue(r, p)...) } return out @@ -92,7 +93,7 @@ func (t *TupleType) GenValue(r *rand.Rand, p *PartitionRangeConfig) []interface{ func (t *TupleType) LenValue() int { out := 0 - for _, tp := range t.Types { + for _, tp := range t.ValueTypes { out += tp.LenValue() } return out diff --git a/pkg/typedef/typedef.go b/pkg/typedef/typedef.go index 6ecf5c8d..f559d0d8 100644 --- a/pkg/typedef/typedef.go +++ b/pkg/typedef/typedef.go @@ -34,8 +34,9 @@ type ( } IndexDef struct { - Column *ColumnDef - Name string `json:"name"` + Column *ColumnDef + IndexName string `json:"index_name"` + ColumnName string `json:"column_name"` } PartitionRangeConfig struct { diff --git a/pkg/typedef/types.go b/pkg/typedef/types.go index 864566ec..41d9b7fc 100644 --- a/pkg/typedef/types.go +++ b/pkg/typedef/types.go @@ -26,6 +26,15 @@ import ( "github.com/scylladb/gemini/pkg/utils" ) +// nolint:revive +const ( + TYPE_UDT = "udt" + TYPE_MAP = "map" + TYPE_LIST = "list" + TYPE_SET = "set" + TYPE_TUPLE = "tuple" +) + // nolint:revive const ( TYPE_ASCII = SimpleType("ascii") @@ -100,9 +109,10 @@ var goCQLTypeMap = map[gocql.Type]gocql.TypeInfo{ } type MapType struct { - KeyType SimpleType `json:"key_type"` - ValueType SimpleType `json:"value_type"` - Frozen bool `json:"frozen"` + ComplexType string `json:"complex_type"` + KeyType SimpleType `json:"key_type"` + ValueType SimpleType `json:"value_type"` + Frozen bool `json:"frozen"` } func (mt *MapType) CQLType() gocql.TypeInfo { diff --git a/pkg/typedef/types_test.go b/pkg/typedef/types_test.go index 72096a83..0a098dd6 100644 --- a/pkg/typedef/types_test.go +++ b/pkg/typedef/types_test.go @@ -155,9 +155,9 @@ var prettytests = []struct { }, { typ: &typedef.BagType{ - Kind: "set", - Type: typedef.TYPE_ASCII, - Frozen: false, + ComplexType: typedef.TYPE_SET, + ValueType: typedef.TYPE_ASCII, + Frozen: false, }, query: "SELECT * FROM tbl WHERE pk0=?", values: []interface{}{[]string{"a", "b"}}, @@ -165,9 +165,9 @@ var prettytests = []struct { }, { typ: &typedef.BagType{ - Kind: "list", - Type: typedef.TYPE_ASCII, - Frozen: false, + ComplexType: typedef.TYPE_LIST, + ValueType: typedef.TYPE_ASCII, + Frozen: false, }, query: "SELECT * FROM tbl WHERE pk0=?", values: []interface{}{[]string{"a", "b"}}, @@ -195,8 +195,8 @@ var prettytests = []struct { }, { typ: &typedef.TupleType{ - Types: []typedef.SimpleType{typedef.TYPE_ASCII}, - Frozen: false, + ValueTypes: []typedef.SimpleType{typedef.TYPE_ASCII}, + Frozen: false, }, query: "SELECT * FROM tbl WHERE pk0=?", values: []interface{}{"a"}, @@ -204,8 +204,8 @@ var prettytests = []struct { }, { typ: &typedef.TupleType{ - Types: []typedef.SimpleType{typedef.TYPE_ASCII, typedef.TYPE_ASCII}, - Frozen: false, + ValueTypes: []typedef.SimpleType{typedef.TYPE_ASCII, typedef.TYPE_ASCII}, + Frozen: false, }, query: "SELECT * FROM tbl WHERE pk0={?,?}", values: []interface{}{"a", "b"}, diff --git a/pkg/typedef/udt.go b/pkg/typedef/udt.go index 8d0749e4..387641c9 100644 --- a/pkg/typedef/udt.go +++ b/pkg/typedef/udt.go @@ -23,9 +23,10 @@ import ( ) type UDTType struct { - Types map[string]SimpleType `json:"coltypes"` - TypeName string `json:"type_name"` - Frozen bool `json:"frozen"` + ComplexType string `json:"complex_type"` + ValueTypes map[string]SimpleType `json:"value_types"` + TypeName string `json:"type_name"` + Frozen bool `json:"frozen"` } func (t *UDTType) CQLType() gocql.TypeInfo { @@ -53,7 +54,7 @@ func (t *UDTType) CQLPretty(query string, value []interface{}) (string, int) { } if s, ok := value[0].(map[string]interface{}); ok { vv := "{" - for k, v := range t.Types { + for k, v := range t.ValueTypes { vv += fmt.Sprintf("%s:?,", k) vv, _ = v.CQLPretty(vv, []interface{}{s[k]}) } @@ -65,7 +66,7 @@ func (t *UDTType) CQLPretty(query string, value []interface{}) (string, int) { } func (t *UDTType) Indexable() bool { - for _, t := range t.Types { + for _, t := range t.ValueTypes { if t == TYPE_DURATION { return false } @@ -75,7 +76,7 @@ func (t *UDTType) Indexable() bool { func (t *UDTType) GenJSONValue(r *rand.Rand, p *PartitionRangeConfig) interface{} { vals := make(map[string]interface{}) - for name, typ := range t.Types { + for name, typ := range t.ValueTypes { vals[name] = typ.GenJSONValue(r, p) } return vals @@ -83,7 +84,7 @@ func (t *UDTType) GenJSONValue(r *rand.Rand, p *PartitionRangeConfig) interface{ func (t *UDTType) GenValue(r *rand.Rand, p *PartitionRangeConfig) []interface{} { vals := make(map[string]interface{}) - for name, typ := range t.Types { + for name, typ := range t.ValueTypes { vals[name] = typ.GenValue(r, p)[0] } return []interface{}{vals}