Redo columnDef.UnmarshalJSON

cmd/gemini/root.go

@@ -125,8 +125,9 @@ func readSchema(confFile string) (*typedef.Schema, error) {
 
 	schemaBuilder := builders.NewSchemaBuilder()
 	schemaBuilder.Keyspace(shm.Keyspace)
-	for _, tbl := range shm.Tables {
+	for t, tbl := range shm.Tables {
 		schemaBuilder.Table(tbl)
+		generators.AddReferencesForIndexes(shm.Tables[t])
 	}
 	return schemaBuilder.Build(), nil
 }

cmd/gemini/schema.json

@@ -1,6 +1,14 @@
 {
   "keyspace": {
-    "name": "ks1"
+    "name": "ks1",
+    "replication": {
+      "class": "SimpleStrategy",
+      "replication_factor": 1
+    },
+    "oracle_replication": {
+      "class": "SimpleStrategy",
+      "replication_factor": 1
+    }
   },
   "tables": [
     {
@@ -29,9 +37,10 @@
         {
           "name": "col0",
           "type": {
+            "complex_type": "udt",
             "type_name": "udt_672245080",
             "frozen": true,
-            "coltypes": {
+            "value_types": {
               "udt_672245080_0": "ascii",
               "udt_672245080_1": "boolean",
               "udt_672245080_2": "bigint",
@@ -41,59 +50,133 @@
         },
         {
           "name": "col1",
-          "type": "timestamp"
-        },
-        {
-          "name": "col2",
-          "type": "decimal"
-        },
-        {
-          "name": "col3",
-          "type": "uuid"
-        },
-        {
-          "name": "col4",
           "type": {
+            "complex_type": "map",
             "key_type": "boolean",
             "value_type": "duration",
             "frozen": false
           }
         },
         {
-          "name": "col5",
+          "name": "col2",
           "type": {
-            "types": [
+            "complex_type": "tuple",
+            "value_types": [
               "varchar",
               "smallint"
             ],
             "frozen": false
           }
         },
         {
-          "name": "col6",
+          "name": "col3",
           "type": {
-            "kind": "list",
-            "type": "int",
+            "complex_type": "list",
+            "value_type": "int",
             "frozen": true
           }
+        },
+        {
+          "name": "col4",
+          "type": {
+            "complex_type": "set",
+            "value_type": "int",
+            "frozen": true
+          }
+        },
+        {
+          "name": "col5",
+          "type": "ascii"
+        },
+        {
+          "name": "col6",
+          "type": "bigint"
+        },
+        {
+          "name": "col7",
+          "type": "blob"
+        },
+        {
+          "name": "col8",
+          "type": "boolean"
+        },
+        {
+          "name": "col9",
+          "type": "date"
+        },
+        {
+          "name": "col10",
+          "type": "decimal"
+        },
+        {
+          "name": "col11",
+          "type": "double"
+        },
+        {
+          "name": "col12",
+          "type": "duration"
+        },
+        {
+          "name": "col13",
+          "type": "float"
+        },
+        {
+          "name": "col14",
+          "type": "inet"
+        },
+        {
+          "name": "col15",
+          "type": "int"
+        },
+        {
+          "name": "col16",
+          "type": "smallint"
+        },
+        {
+          "name": "col17",
+          "type": "text"
+        },
+        {
+          "name": "col19",
+          "type": "timestamp"
+        },
+        {
+          "name": "col20",
+          "type": "timeuuid"
+        },
+        {
+          "name": "col21",
+          "type": "tinyint"
+        },
+        {
+          "name": "col22",
+          "type": "uuid"
+        },
+        {
+          "name": "col23",
+          "type": "varchar"
+        },
+        {
+          "name": "col24",
+          "type": "varint"
         }
       ],
       "indexes": [
         {
-          "name": "col0_idx",
-          "column": "col0"
+          "index_name": "col0_idx",
+          "column_name": "col56"
         },
         {
-          "name": "col1_idx",
-          "column": "col1"
+          "index_name": "col1_idx",
+          "column_name": "col6"
         },
         {
-          "name": "col2_idx",
-          "column": "col2"
+          "index_name": "col2_idx",
+          "column_name": "col7"
         },
         {
-          "name": "col3_idx",
-          "column": "col3"
+          "index_name": "col3_idx",
+          "column_name": "col8"
         }
       ],
       "known_issues": {

pkg/generators/column.go

@@ -15,6 +15,8 @@
 package generators
 
 import (
+	"fmt"
+
 	"github.com/scylladb/gemini/pkg/typedef"
 )
 
@@ -23,7 +25,11 @@ func CreateIndexesForColumn(table *typedef.Table, maxIndexes int) []typedef.Inde
 	indexes := make([]typedef.IndexDef, 0, maxIndexes)
 	for i, col := range table.Columns {
 		if col.Type.Indexable() && typedef.TypesForIndex.Contains(col.Type) {
-			indexes = append(indexes, typedef.IndexDef{Name: GenIndexName(table.Name+"_col", i), Column: table.Columns[i]})
+			indexes = append(indexes, typedef.IndexDef{
+				IndexName:  GenIndexName(table.Name+"_col", i),
+				ColumnName: table.Columns[i].Name,
+				Column:     table.Columns[i],
+			})
 			createdCount++
 		}
 		if createdCount == maxIndexes {
@@ -32,3 +38,20 @@ func CreateIndexesForColumn(table *typedef.Table, maxIndexes int) []typedef.Inde
 	}
 	return indexes
 }
+
+func AddReferencesForIndexes(table *typedef.Table) {
+	wrongIndex := -1
+	for i, index := range table.Indexes {
+		for c, column := range table.Columns {
+			if index.ColumnName == column.Name {
+				table.Indexes[i].Column = table.Columns[c]
+				break
+			} else if len(table.Columns) == c {
+				wrongIndex = i
+			}
+		}
+	}
+	if wrongIndex != -1 {
+		panic(fmt.Sprintf("wrong column_name in index defenition:%+v", table.Indexes[wrongIndex]))
+	}
+}

pkg/generators/column_generator.go

@@ -58,8 +58,9 @@ func GenTupleType(sc *typedef.SchemaConfig) typedef.Type {
 		typeList[i] = GenSimpleType(sc)
 	}
 	return &typedef.TupleType{
-		Types:  typeList,
-		Frozen: rand.Uint32()%2 == 0,
+		ComplexType: typedef.TYPE_TUPLE,
+		ValueTypes:  typeList,
+		Frozen:      rand.Uint32()%2 == 0,
 	}
 }
 
@@ -73,18 +74,19 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
 	}
 
 	return &typedef.UDTType{
-		Types:    ts,
-		TypeName: typeName,
-		Frozen:   true,
+		ComplexType: typedef.TYPE_UDT,
+		ValueTypes:  ts,
+		TypeName:    typeName,
+		Frozen:      true,
 	}
 }
 
 func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType {
-	return genBagType("set", sc)
+	return genBagType(typedef.TYPE_SET, sc)
 }
 
 func GenListType(sc *typedef.SchemaConfig) *typedef.BagType {
-	return genBagType("list", sc)
+	return genBagType(typedef.TYPE_LIST, sc)
 }
 
 func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
@@ -96,9 +98,9 @@ func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
 		}
 	}
 	return &typedef.BagType{
-		Kind:   kind,
-		Type:   t,
-		Frozen: rand.Uint32()%2 == 0,
+		ComplexType: kind,
+		ValueType:   t,
+		Frozen:      rand.Uint32()%2 == 0,
 	}
 }
 
@@ -111,9 +113,10 @@ func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType {
 		t = GenSimpleType(sc)
 	}
 	return &typedef.MapType{
-		KeyType:   t,
-		ValueType: GenSimpleType(sc),
-		Frozen:    rand.Uint32()%2 == 0,
+		ComplexType: typedef.TYPE_MAP,
+		KeyType:     t,
+		ValueType:   GenSimpleType(sc),
+		Frozen:      rand.Uint32()%2 == 0,
 	}
 }
 

pkg/generators/statement_generator.go

@@ -106,7 +106,7 @@ func GetCreateSchema(s *typedef.Schema) []string {
 		createTable := GetCreateTable(t, s.Keyspace)
 		stmts = append(stmts, createTable)
 		for _, idef := range t.Indexes {
-			stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.Name, s.Keyspace.Name, t.Name, idef.Column.Name))
+			stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.IndexName, s.Keyspace.Name, t.Name, idef.ColumnName))
 		}
 		for _, mv := range t.MaterializedViews {
 			var (

pkg/generators/table.go

@@ -68,7 +68,7 @@ func GetCreateTypes(t *typedef.Table, keyspace typedef.Keyspace) []string {
 		}
 		createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s)"
 		var typs []string
-		for name, typ := range c.Types {
+		for name, typ := range c.ValueTypes {
 			typs = append(typs, name+" "+typ.CQLDef())
 		}
 		stmts = append(stmts, fmt.Sprintf(createType, keyspace.Name, c.TypeName, strings.Join(typs, ",")))

pkg/jobs/gen_check_stmt.go

@@ -521,7 +521,7 @@ func genSingleIndexQuery(
 	builder := qb.Select(s.Keyspace.Name + "." + t.Name)
 	builder.AllowFiltering()
 	for i := 0; i < idxCount; i++ {
-		builder = builder.Where(qb.Eq(t.Indexes[i].Column.Name))
+		builder = builder.Where(qb.Eq(t.Indexes[i].ColumnName))
 		values = append(values, t.Indexes[i].Column.Type.GenValue(r, p)...)
 		typs = append(typs, t.Indexes[i].Column.Type)
 	}

pkg/jobs/gen_ddl_stmt.go

@@ -52,7 +52,7 @@ func genAddColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnD
 	if c, ok := column.Type.(*typedef.UDTType); ok {
 		createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s);"
 		var typs []string
-		for name, typ := range c.Types {
+		for name, typ := range c.ValueTypes {
 			typs = append(typs, name+" "+typ.CQLDef())
 		}
 		stmt := fmt.Sprintf(createType, keyspace, c.TypeName, strings.Join(typs, ","))

pkg/jobs/gen_mutate_stmt.go

@@ -144,9 +144,9 @@ func genInsertJSONStmt(
 				values[pk.Name] = "0x" + v
 			}
 		case *typedef.TupleType:
-			tupVals := make([]interface{}, len(t.Types))
-			for j := 0; j < len(t.Types); j++ {
-				if t.Types[j] == typedef.TYPE_BLOB {
+			tupVals := make([]interface{}, len(t.ValueTypes))
+			for j := 0; j < len(t.ValueTypes); j++ {
+				if t.ValueTypes[j] == typedef.TYPE_BLOB {
 					v, ok = vs[i+j].(string)
 					if ok {
 						v = "0x" + v

pkg/jobs/gen_utils_test.go

@@ -223,7 +223,7 @@ func (r *Result) Diff(t *testing.T, received *Result) {
 	getErrorMsgIfDifferent(t, r.Query, received.Query, " error: value stmt.Query.ToCql().stmt expected and received are different:")
 	getErrorMsgIfDifferent(t, r.Names, received.Names, " error: value stmt.Query.ToCql().Names expected and received are different:")
 	getErrorMsgIfDifferent(t, r.Values, received.Values, " error: value stmt.Values expected and received are different:")
-	getErrorMsgIfDifferent(t, r.Types, received.Types, " error: value stmt.Types expected and received are different:")
+	getErrorMsgIfDifferent(t, r.Types, received.Types, " error: value stmt.ValueTypes expected and received are different:")
 	getErrorMsgIfDifferent(t, r.Values, received.Values, " error: value stmt.Values expected and received are different:")
 	getErrorMsgIfDifferent(t, r.QueryType, received.QueryType, " error: value stmt.QueryType expected and received are different:")
 }
@@ -476,15 +476,19 @@ func createIdxFromColumns(t testInterface, table *typedef.Table, all bool) (inde
 	switch all {
 	case true:
 		for i := range table.Columns {
-			var index typedef.IndexDef
-			index.Name = table.Columns[i].Name + "_idx"
-			index.Column = table.Columns[i]
+			index := typedef.IndexDef{
+				IndexName:  table.Columns[i].Name + "_idx",
+				ColumnName: table.Columns[i].Name,
+				Column:     table.Columns[i],
+			}
 			indexes = append(indexes, index)
 		}
 	default:
-		var index typedef.IndexDef
-		index.Name = table.Columns[0].Name + "_idx"
-		index.Column = table.Columns[0]
+		index := typedef.IndexDef{
+			IndexName:  table.Columns[0].Name + "_idx",
+			ColumnName: table.Columns[0].Name,
+			Column:     table.Columns[0],
+		}
 		indexes = append(indexes, index)
 
 	}

pkg/jobs/jobs.go

@@ -325,7 +325,7 @@ func ddl(
 	defer table.Unlock()
 	ddlStmts, err := GenDDLStmt(schema, table, r, p, sc)
 	if err != nil {
-		logger.Error("Failed! Mutation statement generation failed", zap.Error(err))
+		logger.Error("Failed! DDl Mutation statement generation failed", zap.Error(err))
 		globalStatus.WriteErrors.Add(1)
 		return err
 	}