Merge pull request #360 from illia-li/il/redo-UnmarshalJSON

Redo columnDef.UnmarshalJSON

cmd/gemini/root.go

@@ -125,7 +125,8 @@ func readSchema(confFile string) (*typedef.Schema, error) {
 
 	schemaBuilder := builders.NewSchemaBuilder()
 	schemaBuilder.Keyspace(shm.Keyspace)
-	for _, tbl := range shm.Tables {
+	for t, tbl := range shm.Tables {
+		shm.Tables[t].LinkIndexAndColumns()
 		schemaBuilder.Table(tbl)
 	}
 	return schemaBuilder.Build(), nil

cmd/gemini/schema.json

@@ -1,6 +1,14 @@
 {
   "keyspace": {
-    "name": "ks1"
+    "name": "ks1",
+    "replication": {
+      "class": "SimpleStrategy",
+      "replication_factor": 1
+    },
+    "oracle_replication": {
+      "class": "SimpleStrategy",
+      "replication_factor": 1
+    }
   },
   "tables": [
     {
@@ -29,9 +37,10 @@
         {
           "name": "col0",
           "type": {
+            "complex_type": "udt",
             "type_name": "udt_672245080",
             "frozen": true,
-            "coltypes": {
+            "value_types": {
               "udt_672245080_0": "ascii",
               "udt_672245080_1": "boolean",
               "udt_672245080_2": "bigint",
@@ -41,59 +50,133 @@
         },
         {
           "name": "col1",
-          "type": "timestamp"
-        },
-        {
-          "name": "col2",
-          "type": "decimal"
-        },
-        {
-          "name": "col3",
-          "type": "uuid"
-        },
-        {
-          "name": "col4",
           "type": {
+            "complex_type": "map",
             "key_type": "boolean",
             "value_type": "duration",
             "frozen": false
           }
         },
         {
-          "name": "col5",
+          "name": "col2",
           "type": {
-            "types": [
+            "complex_type": "tuple",
+            "value_types": [
               "varchar",
               "smallint"
             ],
             "frozen": false
           }
         },
         {
-          "name": "col6",
+          "name": "col3",
           "type": {
-            "kind": "list",
-            "type": "int",
+            "complex_type": "list",
+            "value_type": "int",
             "frozen": true
           }
+        },
+        {
+          "name": "col4",
+          "type": {
+            "complex_type": "set",
+            "value_type": "int",
+            "frozen": true
+          }
+        },
+        {
+          "name": "col5",
+          "type": "ascii"
+        },
+        {
+          "name": "col6",
+          "type": "bigint"
+        },
+        {
+          "name": "col7",
+          "type": "blob"
+        },
+        {
+          "name": "col8",
+          "type": "boolean"
+        },
+        {
+          "name": "col9",
+          "type": "date"
+        },
+        {
+          "name": "col10",
+          "type": "decimal"
+        },
+        {
+          "name": "col11",
+          "type": "double"
+        },
+        {
+          "name": "col12",
+          "type": "duration"
+        },
+        {
+          "name": "col13",
+          "type": "float"
+        },
+        {
+          "name": "col14",
+          "type": "inet"
+        },
+        {
+          "name": "col15",
+          "type": "int"
+        },
+        {
+          "name": "col16",
+          "type": "smallint"
+        },
+        {
+          "name": "col17",
+          "type": "text"
+        },
+        {
+          "name": "col19",
+          "type": "timestamp"
+        },
+        {
+          "name": "col20",
+          "type": "timeuuid"
+        },
+        {
+          "name": "col21",
+          "type": "tinyint"
+        },
+        {
+          "name": "col22",
+          "type": "uuid"
+        },
+        {
+          "name": "col23",
+          "type": "varchar"
+        },
+        {
+          "name": "col24",
+          "type": "varint"
         }
       ],
       "indexes": [
         {
-          "name": "col0_idx",
-          "column": "col0"
+          "index_name": "col5_idx",
+          "column_name": "col5"
         },
         {
-          "name": "col1_idx",
-          "column": "col1"
+          "index_name": "col6_idx",
+          "column_name": "col6"
         },
         {
-          "name": "col2_idx",
-          "column": "col2"
+          "index_name": "col7_idx",
+          "column_name": "col7"
         },
         {
-          "name": "col3_idx",
-          "column": "col3"
+          "index_name": "col8_idx",
+          "column_name": "col8"
         }
       ],
       "known_issues": {

cmd/gemini/strategies_test.go

@@ -15,13 +15,15 @@
 package main
 
 import (
+	"encoding/json"
 	"testing"
 
-	"github.com/scylladb/gemini/pkg/replication"
-
 	"github.com/google/go-cmp/cmp"
-
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"go.uber.org/zap"
+
+	"github.com/scylladb/gemini/pkg/replication"
+	"github.com/scylladb/gemini/pkg/typedef"
 )
 
 func TestGetReplicationStrategy(t *testing.T) {
@@ -57,3 +59,30 @@ func TestGetReplicationStrategy(t *testing.T) {
 		})
 	}
 }
+
+func TestReadSchema(t *testing.T) {
+	filePath := "schema.json"
+
+	testSchema, err := readSchema(filePath)
+	if err != nil {
+		t.Fatalf("failed to open schema example json file %s, error:%s", filePath, err)
+	}
+
+	opts := cmp.Options{
+		cmp.AllowUnexported(typedef.Table{}, typedef.MaterializedView{}),
+		cmpopts.IgnoreUnexported(typedef.Table{}, typedef.MaterializedView{}),
+	}
+
+	testSchemaMarshaled, err := json.MarshalIndent(testSchema, "  ", "  ")
+	if err != nil {
+		t.Fatalf("unable to marshal schema example json, error=%s\n", err)
+	}
+	testSchemaUnMarshaled := typedef.Schema{}
+	if err = json.Unmarshal(testSchemaMarshaled, &testSchemaUnMarshaled); err != nil {
+		t.Fatalf("unable to unmarshal json, error=%s\n", err)
+	}
+
+	if diff := cmp.Diff(*testSchema, testSchemaUnMarshaled, opts); diff != "" {
+		t.Errorf("schema not the same after marshal/unmarshal, diff=%s", diff)
+	}
+}

pkg/generators/column.go

@@ -23,7 +23,11 @@ func CreateIndexesForColumn(table *typedef.Table, maxIndexes int) []typedef.Inde
 	indexes := make([]typedef.IndexDef, 0, maxIndexes)
 	for i, col := range table.Columns {
 		if col.Type.Indexable() && typedef.TypesForIndex.Contains(col.Type) {
-			indexes = append(indexes, typedef.IndexDef{Name: GenIndexName(table.Name+"_col", i), Column: table.Columns[i]})
+			indexes = append(indexes, typedef.IndexDef{
+				IndexName:  GenIndexName(table.Name+"_col", i),
+				ColumnName: table.Columns[i].Name,
+				Column:     table.Columns[i],
+			})
 			createdCount++
 		}
 		if createdCount == maxIndexes {

pkg/generators/column_generator.go

@@ -58,8 +58,9 @@ func GenTupleType(sc *typedef.SchemaConfig) typedef.Type {
 		typeList[i] = GenSimpleType(sc)
 	}
 	return &typedef.TupleType{
-		Types:  typeList,
-		Frozen: rand.Uint32()%2 == 0,
+		ComplexType: typedef.TYPE_TUPLE,
+		ValueTypes:  typeList,
+		Frozen:      rand.Uint32()%2 == 0,
 	}
 }
 
@@ -73,18 +74,19 @@ func GenUDTType(sc *typedef.SchemaConfig) *typedef.UDTType {
 	}
 
 	return &typedef.UDTType{
-		Types:    ts,
-		TypeName: typeName,
-		Frozen:   true,
+		ComplexType: typedef.TYPE_UDT,
+		ValueTypes:  ts,
+		TypeName:    typeName,
+		Frozen:      true,
 	}
 }
 
 func GenSetType(sc *typedef.SchemaConfig) *typedef.BagType {
-	return genBagType("set", sc)
+	return genBagType(typedef.TYPE_SET, sc)
 }
 
 func GenListType(sc *typedef.SchemaConfig) *typedef.BagType {
-	return genBagType("list", sc)
+	return genBagType(typedef.TYPE_LIST, sc)
 }
 
 func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
@@ -96,9 +98,9 @@ func genBagType(kind string, sc *typedef.SchemaConfig) *typedef.BagType {
 		}
 	}
 	return &typedef.BagType{
-		Kind:   kind,
-		Type:   t,
-		Frozen: rand.Uint32()%2 == 0,
+		ComplexType: kind,
+		ValueType:   t,
+		Frozen:      rand.Uint32()%2 == 0,
 	}
 }
 
@@ -111,9 +113,10 @@ func GenMapType(sc *typedef.SchemaConfig) *typedef.MapType {
 		t = GenSimpleType(sc)
 	}
 	return &typedef.MapType{
-		KeyType:   t,
-		ValueType: GenSimpleType(sc),
-		Frozen:    rand.Uint32()%2 == 0,
+		ComplexType: typedef.TYPE_MAP,
+		KeyType:     t,
+		ValueType:   GenSimpleType(sc),
+		Frozen:      rand.Uint32()%2 == 0,
 	}
 }
 

pkg/generators/statement_generator.go

@@ -106,7 +106,7 @@ func GetCreateSchema(s *typedef.Schema) []string {
 		createTable := GetCreateTable(t, s.Keyspace)
 		stmts = append(stmts, createTable)
 		for _, idef := range t.Indexes {
-			stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.Name, s.Keyspace.Name, t.Name, idef.Column.Name))
+			stmts = append(stmts, fmt.Sprintf("CREATE INDEX IF NOT EXISTS %s ON %s.%s (%s)", idef.IndexName, s.Keyspace.Name, t.Name, idef.ColumnName))
 		}
 		for _, mv := range t.MaterializedViews {
 			var (

pkg/generators/table.go

@@ -68,7 +68,7 @@ func GetCreateTypes(t *typedef.Table, keyspace typedef.Keyspace) []string {
 		}
 		createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s)"
 		var typs []string
-		for name, typ := range c.Types {
+		for name, typ := range c.ValueTypes {
 			typs = append(typs, name+" "+typ.CQLDef())
 		}
 		stmts = append(stmts, fmt.Sprintf(createType, keyspace.Name, c.TypeName, strings.Join(typs, ",")))

pkg/jobs/gen_check_stmt.go

@@ -521,7 +521,7 @@ func genSingleIndexQuery(
 	builder := qb.Select(s.Keyspace.Name + "." + t.Name)
 	builder.AllowFiltering()
 	for i := 0; i < idxCount; i++ {
-		builder = builder.Where(qb.Eq(t.Indexes[i].Column.Name))
+		builder = builder.Where(qb.Eq(t.Indexes[i].ColumnName))
 		values = append(values, t.Indexes[i].Column.Type.GenValue(r, p)...)
 		typs = append(typs, t.Indexes[i].Column.Type)
 	}

pkg/jobs/gen_ddl_stmt.go

@@ -52,7 +52,7 @@ func genAddColumnStmt(t *typedef.Table, keyspace string, column *typedef.ColumnD
 	if c, ok := column.Type.(*typedef.UDTType); ok {
 		createType := "CREATE TYPE IF NOT EXISTS %s.%s (%s);"
 		var typs []string
-		for name, typ := range c.Types {
+		for name, typ := range c.ValueTypes {
 			typs = append(typs, name+" "+typ.CQLDef())
 		}
 		stmt := fmt.Sprintf(createType, keyspace, c.TypeName, strings.Join(typs, ","))

pkg/jobs/gen_mutate_stmt.go

@@ -144,9 +144,9 @@ func genInsertJSONStmt(
 				values[pk.Name] = "0x" + v
 			}
 		case *typedef.TupleType:
-			tupVals := make([]interface{}, len(t.Types))
-			for j := 0; j < len(t.Types); j++ {
-				if t.Types[j] == typedef.TYPE_BLOB {
+			tupVals := make([]interface{}, len(t.ValueTypes))
+			for j := 0; j < len(t.ValueTypes); j++ {
+				if t.ValueTypes[j] == typedef.TYPE_BLOB {
 					v, ok = vs[i+j].(string)
 					if ok {
 						v = "0x" + v

pkg/jobs/gen_utils_test.go

@@ -476,15 +476,19 @@ func createIdxFromColumns(t testInterface, table *typedef.Table, all bool) (inde
 	switch all {
 	case true:
 		for i := range table.Columns {
-			var index typedef.IndexDef
-			index.Name = table.Columns[i].Name + "_idx"
-			index.Column = table.Columns[i]
+			index := typedef.IndexDef{
+				IndexName:  table.Columns[i].Name + "_idx",
+				ColumnName: table.Columns[i].Name,
+				Column:     table.Columns[i],
+			}
 			indexes = append(indexes, index)
 		}
 	default:
-		var index typedef.IndexDef
-		index.Name = table.Columns[0].Name + "_idx"
-		index.Column = table.Columns[0]
+		index := typedef.IndexDef{
+			IndexName:  table.Columns[0].Name + "_idx",
+			ColumnName: table.Columns[0].Name,
+			Column:     table.Columns[0],
+		}
 		indexes = append(indexes, index)
 
 	}