fix(prettyCQL): now gemini does not crash on out-of-bounds access

.github/workflows/integration-tests.yml

@@ -32,14 +32,17 @@ jobs:
       matrix:
         gemini-features: ["basic", "normal", "all"]
         gemini-concurrency: [16]
-        oracle-scylla-version: ["6.1.1"]
-        test-scylla-version: ["6.1.1"]
+        duration: ["5m"]
+        dataset-size: [large, small]
+        oracle-scylla-version: ["6.1"]
+        test-scylla-version: ["6.1"]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
       - uses: actions/download-artifact@v4
       - name: Start ScyllaDB
         id: scylla
+        shell: bash
         run: |
           chmod +x ./bin/gemini
           make scylla-setup \
@@ -51,16 +54,19 @@ jobs:
           make integration-test \
             CONCURRENCY=${{ matrix.gemini-concurrency }} \
             CQL_FEATURES=${{ matrix.gemini-features }} \
-            DURATION=5m
+            DURATION=${{ matrix.duration }} \
+            DATASET_SIZE=${{ matrix.dataset-size }} \
       - name: Shutdown ScyllaDB
+        shell: bash
         run: |
           make scylla-shutdown \
             SCYLLA_TEST_VERSION=${{ matrix.test-scylla-version }} \
             SCYLLA_ORACLE_VERSION=${{ matrix.oracle-scylla-version }}
 
       - uses: actions/upload-artifact@v4
+        if: always()
         with:
-          name: results-${{ matrix.gemini-features }}-${{ matrix.gemini-concurrency }}
+          name: results-${{ matrix.gemini-features }}-${{ matrix.gemini-concurrency }}-${{ matrix.duration }}-${{ matrix.dataset-size }}-${{ matrix.test-scylla-version }}-${{ matrix.oracle-scylla-version }}-${{ github.run_number }}
           path: ./results
           if-no-files-found: error
           retention-days: 30

.gitignore

@@ -5,3 +5,5 @@ cmd/gemini/dist/
 bin/
 coverage.txt
 dist/
+results/
+

Makefile

@@ -35,57 +35,73 @@ fix: $(GOBIN)/golangci-lint
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 go build -ldflags="-s -w" -o bin/gemini ./cmd/gemini
+	@CGO_ENABLED=0 go build  -o bin/gemini ./cmd/gemini
 
 debug-build:
-	CGO_ENABLED=0 go build -gcflags "all=-N -l" -o bin/gemini ./cmd/gemini
+	@CGO_ENABLED=0 go build -ldflags="-asan" -gcflags "all=-N -l" -o bin/gemini ./cmd/gemini
 
 .PHONY: build-docker
 build-docker:
-	docker build --target production -t scylladb/gemini:$(DOCKER_VERSION) --compress .
+	@docker build --target production -t scylladb/gemini:$(DOCKER_VERSION) --compress .
 
 .PHONY: scylla-setup
 scylla-setup:
-	@docker compose -f scripts/docker-compose-$(DOCKER_COMPOSE_TESTING).yml up -d
+	@docker compose -f docker/docker-compose-$(DOCKER_COMPOSE_TESTING).yml up -d
 
 	until docker logs gemini-oracle 2>&1 | grep "Starting listening for CQL clients" > /dev/null; do sleep 0.2; done
 	until docker logs gemini-test 2>&1 | grep "Starting listening for CQL clients" > /dev/null; do sleep 0.2; done
 
 .PHONY: scylla-shutdown
 scylla-shutdown:
-	docker compose -f scripts/docker-compose-$(DOCKER_COMPOSE_TESTING).yml down --volumes
+	docker compose -f docker/docker-compose-$(DOCKER_COMPOSE_TESTING).yml down --volumes
 
 .PHONY: test
 test:
-	go test -covermode=atomic -race -coverprofile=coverage.txt -timeout 5m -json -v ./... 2>&1 | gotestfmt -showteststatus
+	@go test -covermode=atomic -race -coverprofile=coverage.txt -timeout 5m -json -v ./... 2>&1 | gotestfmt -showteststatus
 
-CQL_FEATURES := normal
-CONCURRENCY := 1
-DURATION := 10m
-WARMUP := 1m
+CQL_FEATURES ?= normal
+CONCURRENCY ?= 16
+DURATION ?= 1m
+WARMUP ?= 1m
+DATASET_SIZE ?= large
+SEED ?= $(shell date +%s | tee ./results/gemini_seed)
 
 .PHONY: integration-test
-integration-test:
-	mkdir -p ./results
-	touch ./results/gemini_seed
-	./bin/gemini \
+integration-test: debug-build
+	@mkdir -p $(PWD)/results
+	@touch $(PWD)/results/gemini_seed
+	@./bin/gemini \
+		--dataset-size=$(DATASET_SIZE) \
+		--seed=$(SEED) \
+		--schema-seed=$(SEED) \
+		--cql-features $(CQL_FEATURES) \
+		--duration $(DURATION) \
+		--warmup $(WARMUP) \
+		--drop-schema true \
 		--fail-fast \
-		--dataset-size=small \
-		--seed=$(shell date +%s | tee ./results/gemini_seed) \
+		--level info \
+		--non-interactive \
+		--materialized-views false \
+		--consistency LOCAL_QUORUM \
+		--outfile $(PWD)/results/gemini.log \
+		--test-statement-log-file $(PWD)/results/gemini_test_statements.log \
+		--oracle-statement-log-file $(PWD)/results/gemini_oracle_statements.log \
+		--test-host-selection-policy token-aware \
+		--oracle-host-selection-policy token-aware \
 		--test-cluster=$(shell docker inspect --format='{{ .NetworkSettings.Networks.gemini.IPAddress }}' gemini-test) \
 		--oracle-cluster=$(shell docker inspect --format='{{ .NetworkSettings.Networks.gemini.IPAddress }}' gemini-oracle) \
-		--outfile ./results/gemini_result.log \
-		--duration $(DURATION) \
-		--warmup $(WARMUP) \
-		-m mixed \
+		--outfile $(PWD)/results/gemini_result.log \
+		--mode mixed \
 		--non-interactive \
-		--cql-features $(CQL_FEATURES) \
 		--request-timeout 180s \
 		--connect-timeout 120s \
-		--async-objects-stabilization-attempts 5 \
-		--async-objects-stabilization-backoff 500ms \
+		--consistency LOCAL_QUORUM \
+		--use-server-timestamps false \
+		--async-objects-stabilization-attempts 10 \
+		--async-objects-stabilization-backoff 100ms \
 		--replication-strategy "{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}" \
 		--oracle-replication-strategy "{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}" \
-		--max-mutation-retries 10 \
-		--max-mutation-retries-backoff 500ms \
-		-c $(CONCURRENCY)
+		--max-mutation-retries 5 \
+		--max-mutation-retries-backoff 1000ms \
+		--concurrency $(CONCURRENCY) \
+		--tracing-outfile $(PWD)/results/gemini_tracing.log

cmd/gemini/spinner.go

@@ -25,7 +25,7 @@ type spinningFeedback struct {
 	s *spinner.Spinner
 }
 
-func (sf *spinningFeedback) Set(format string, args ...interface{}) {
+func (sf *spinningFeedback) Set(format string, args ...any) {
 	if sf.s != nil {
 		sf.s.Suffix = fmt.Sprintf(format, args...)
 	}

docs/architecture.md

@@ -9,7 +9,7 @@ database clusters.
 ## The different Jobs
 
 1. ___MutationJob___: This job applies mutations to the clusters. The mutations can be of several types.
-   The basic _INSERT_ and _DELETE_ with various conditions or ___DDL___ type statements such as _ALTER_ the 
+   The basic _INSERT_ and _DELETE_ with various conditions or ___DDL___ type statements such as _ALTER_ the
    structure of the table. These type of mutations happen with different frequency with normal _INSERT_
    being the most common and _ALTER_ the most infrequent.
 
@@ -39,7 +39,7 @@ The application allows the user to decide the level of concurrency that Gemini o
 The toggle `--concurrency` currently means that the application will create that number of
 ___READ___ and ___WRITE___ jobs when running _mixed_ mode. If running in ___WRITE___ or ___READ___
 mode it will correspond to the exact number of job executing goroutines. Each goroutine is only
-working on a subset of the data (from now known as bucket) when mutating and validating to avoid 
+working on a subset of the data (from now known as bucket) when mutating and validating to avoid
 concurrent modification races when validating the system under test.
 This can still happen when executing read queries that performs an index scan.
 
@@ -56,7 +56,7 @@ in use but the idea is to introduce some jitter into the execution flow.
 The application generates partition ids through a `Generator` that creates a steady flow of partition
 key components for the desired [concurrency](architecture.md#Concurrency).
 Each goroutine is connected to a `partition` that the generator controls. This partition continuously emits
-new partition ids in the form of a `[]interface{}`. These keys are created in the same way as the the
+new partition ids in the form of a `[]any`. These keys are created in the same way as the the
 driver does to ensure that each goroutine only processes partition keys from it's designated bucket.
 These partition keys These values are copied into another list that keeps the old partition ids for
 later reuse. The idea of reusing the partition keys is that probability of hitting the same partition
@@ -72,14 +72,14 @@ ___NB___:There are probably issues with this approach and we may want to refine
 There are a number of core data structures that has a more central place in Gemini's design.
 
 * Schema
-  
+
   Gemini has a top level data structure named `Schema`. This structure a loose wrapper around a keyspace
 and a list of tables. It furthermore contains exported methods for generating a schema and it's
 corresponding CQL DDL statements allowing for creating the tables in the database. It also holds the
 methods for creating queries of all kinds which are used in the main Gemini program.
 
 * Table
-  
+
   Tables are conceptually very similar to regular CQL tables. Their base elements are partition keys,
   clustering keys and columns. They also may contain materialized views and indexes depending on user
   preferences.

pkg/generators/utils.go

@@ -20,8 +20,8 @@ import (
 	"github.com/scylladb/gemini/pkg/typedef"
 )
 
-func CreatePartitionKeyValues(table *typedef.Table, r *rand.Rand, g *typedef.PartitionRangeConfig) []interface{} {
-	values := make([]interface{}, 0, table.PartitionKeysLenValues())
+func CreatePartitionKeyValues(table *typedef.Table, r *rand.Rand, g *typedef.PartitionRangeConfig) []any {
+	values := make([]any, 0, table.PartitionKeysLenValues())
 	for _, pk := range table.PartitionKeys {
 		values = append(values, pk.Type.GenValue(r, g)...)
 	}

pkg/inflight/inflight_test.go

@@ -69,10 +69,10 @@ func TestDeleteSharded(t *testing.T) {
 func TestInflight(t *testing.T) {
 	t.Parallel()
 	flight := newSyncU64set(shrinkInflightsLimit)
-	f := func(v uint64) interface{} {
+	f := func(v uint64) any {
 		return flight.AddIfNotPresent(v)
 	}
-	g := func(v uint64) interface{} {
+	g := func(v uint64) any {
 		flight.Delete(v)
 		return !flight.Has(v)
 	}
@@ -105,10 +105,10 @@ func TestAutoShrink(t *testing.T) {
 func TestInflightSharded(t *testing.T) {
 	t.Parallel()
 	flight := newShardedSyncU64set()
-	f := func(v uint64) interface{} {
+	f := func(v uint64) any {
 		return flight.AddIfNotPresent(v)
 	}
-	g := func(v uint64) interface{} {
+	g := func(v uint64) any {
 		flight.Delete(v)
 		return !flight.shards[v%256].Has(v)
 	}

pkg/joberror/joberror_test.go

@@ -33,7 +33,7 @@ func TestParallel(t *testing.T) {
 	wg := sync.WaitGroup{}
 	resultList := joberror.NewErrorList(1000)
 	expectedList := joberror.NewErrorList(1000)
-	baseDate := time.Date(2020, 02, 01, 0, 0, 0, 0, time.UTC)
+	baseDate := time.Date(2020, 0o2, 0o1, 0, 0, 0, 0, time.UTC)
 	idx := atomic.Int32{}
 	for y := 0; y < 1000; y++ {
 		expectedList.AddError(&joberror.JobError{
@@ -73,7 +73,7 @@ func TestErrorSerialization(t *testing.T) {
 	t.Parallel()
 	expected := []byte(`{"timestamp":"2020-02-01T00:00:00Z","message":"Some Message","query":"Some Query","stmt-type":"Some Type"}`)
 	result, err := json.Marshal(joberror.JobError{
-		Timestamp: time.Date(2020, 02, 01, 0, 0, 0, 0, time.UTC),
+		Timestamp: time.Date(2020, 0o2, 0o1, 0, 0, 0, 0, time.UTC),
 		Message:   "Some Message",
 		Query:     "Some Query",
 		StmtType:  "Some Type",
@@ -92,7 +92,7 @@ func TestErrorListSerialization(t *testing.T) {
 	//nolint:lll
 	expected := []byte(`[{"timestamp":"2020-02-01T00:00:00Z","message":"Some Message 0","query":"Some Query 0","stmt-type":"Some Stmt Type 0"},{"timestamp":"2020-02-02T00:00:00Z","message":"Some Message 1","query":"Some Query 1","stmt-type":"Some Stmt Type 1"},{"timestamp":"2020-02-03T00:00:00Z","message":"Some Message 2","query":"Some Query 2","stmt-type":"Some Stmt Type 2"},{"timestamp":"2020-02-04T00:00:00Z","message":"Some Message 3","query":"Some Query 3","stmt-type":"Some Stmt Type 3"},{"timestamp":"2020-02-05T00:00:00Z","message":"Some Message 4","query":"Some Query 4","stmt-type":"Some Stmt Type 4"},{"timestamp":"2020-02-06T00:00:00Z","message":"Some Message 5","query":"Some Query 5","stmt-type":"Some Stmt Type 5"},{"timestamp":"2020-02-07T00:00:00Z","message":"Some Message 6","query":"Some Query 6","stmt-type":"Some Stmt Type 6"},{"timestamp":"2020-02-08T00:00:00Z","message":"Some Message 7","query":"Some Query 7","stmt-type":"Some Stmt Type 7"},{"timestamp":"2020-02-09T00:00:00Z","message":"Some Message 8","query":"Some Query 8","stmt-type":"Some Stmt Type 8"},{"timestamp":"2020-02-10T00:00:00Z","message":"Some Message 9","query":"Some Query 9","stmt-type":"Some Stmt Type 9"}]`)
 	lst := joberror.NewErrorList(1000)
-	baseDate := time.Date(2020, 02, 01, 0, 0, 0, 0, time.UTC)
+	baseDate := time.Date(2020, 0o2, 0o1, 0, 0, 0, 0, time.UTC)
 	for y := 0; y < 10; y++ {
 		lst.AddError(&joberror.JobError{
 			Timestamp: baseDate.AddDate(0, 0, y),

pkg/jobs/gen_check_stmt.go

@@ -165,7 +165,7 @@ func genSinglePartitionQueryMv(
 
 	values := valuesWithToken.Value.Copy()
 	if mv.HaveNonPrimaryKey() {
-		var mvValues []interface{}
+		var mvValues []any
 		mvValues = append(mvValues, mv.NonPrimaryKey.Type.GenValue(r, p)...)
 		values = append(mvValues, values...)
 	}
@@ -189,7 +189,7 @@ func genMultiplePartitionQuery(
 	t.RLock()
 	defer t.RUnlock()
 	typs := make([]typedef.Type, numQueryPKs*t.PartitionKeys.Len())
-	values := make([]interface{}, numQueryPKs*t.PartitionKeys.Len())
+	values := make([]any, numQueryPKs*t.PartitionKeys.Len())
 
 	builder := qb.Select(s.Keyspace.Name + "." + t.Name)
 	tokens := make([]*typedef.ValueWithToken, 0, numQueryPKs)
@@ -233,7 +233,7 @@ func genMultiplePartitionQueryMv(
 
 	mv := t.MaterializedViews[mvNum]
 	typs := make([]typedef.Type, numQueryPKs*mv.PartitionKeys.Len())
-	values := make([]interface{}, numQueryPKs*mv.PartitionKeys.Len())
+	values := make([]any, numQueryPKs*mv.PartitionKeys.Len())
 
 	builder := qb.Select(s.Keyspace.Name + "." + t.Name)
 	tokens := make([]*typedef.ValueWithToken, 0, numQueryPKs)
@@ -245,7 +245,7 @@ func genMultiplePartitionQueryMv(
 			return nil
 		}
 		tokens = append(tokens, vs)
-		vals := make([]interface{}, mv.PartitionKeys.Len())
+		vals := make([]any, mv.PartitionKeys.Len())
 		if mv.HaveNonPrimaryKey() {
 			vals[0] = mv.NonPrimaryKey.Type.GenValue(r, p)
 			copy(vals[1:], vs.Value.Copy())
@@ -335,7 +335,7 @@ func genClusteringRangeQueryMv(
 	values := vs.Value.Copy()
 	mv := t.MaterializedViews[mvNum]
 	if mv.HaveNonPrimaryKey() {
-		mvValues := append([]interface{}{}, mv.NonPrimaryKey.Type.GenValue(r, p)...)
+		mvValues := append([]any{}, mv.NonPrimaryKey.Type.GenValue(r, p)...)
 		values = append(mvValues, values...)
 	}
 	builder := qb.Select(s.Keyspace.Name + "." + mv.Name)
@@ -525,7 +525,7 @@ func genSingleIndexQuery(
 	defer t.RUnlock()
 
 	var (
-		values []interface{}
+		values []any
 		typs   []typedef.Type
 	)
 

pkg/jobs/gen_ddl_stmt.go

@@ -45,7 +45,7 @@ func GenDDLStmt(s *typedef.Schema, t *typedef.Table, r *rand.Rand, _ *typedef.Pa
 	}
 }
 
-func appendValue(columnType typedef.Type, r *rand.Rand, p *typedef.PartitionRangeConfig, values []interface{}) []interface{} {
+func appendValue(columnType typedef.Type, r *rand.Rand, p *typedef.PartitionRangeConfig, values []any) []any {
 	return append(values, columnType.GenValue(r, p)...)
 }
 

pkg/jobs/gen_mutate_stmt.go

@@ -129,13 +129,13 @@ func genInsertJSONStmt(
 		return nil, nil
 	}
 	vs := valuesWithToken.Value.Copy()
-	values := make(map[string]interface{})
+	values := make(map[string]any)
 	for i, pk := range table.PartitionKeys {
 		switch t := pk.Type.(type) {
 		case typedef.SimpleType:
 			values[pk.Name] = convertForJSON(t, vs[i])
 		case *typedef.TupleType:
-			tupVals := make([]interface{}, len(t.ValueTypes))
+			tupVals := make([]any, len(t.ValueTypes))
 			for j := 0; j < len(t.ValueTypes); j++ {
 				tupVals[i] = convertForJSON(t, vs[i])
 				i++
@@ -161,7 +161,7 @@ func genInsertJSONStmt(
 			QueryType: typedef.InsertJSONStatementType,
 		},
 		ValuesWithToken: []*typedef.ValueWithToken{valuesWithToken},
-		Values:          []interface{}{string(jsonString)},
+		Values:          []any{string(jsonString)},
 	}, nil
 }
 
@@ -180,7 +180,7 @@ func genDeleteRows(_ *typedef.Schema, t *typedef.Table, valuesWithToken *typedef
 	}, nil
 }
 
-func convertForJSON(vType typedef.Type, value interface{}) interface{} {
+func convertForJSON(vType typedef.Type, value any) any {
 	switch vType {
 	case typedef.TYPE_BLOB:
 		val, _ := value.(string)

pkg/jobs/gen_utils_test.go

@@ -151,7 +151,7 @@ func (r results) Diff(t results) string {
 	return strings.Join(out, "\n")
 }
 
-func convertStmtsToResults(stmt interface{}) results {
+func convertStmtsToResults(stmt any) results {
 	var out results
 	switch stmts := stmt.(type) {
 	case *typedef.Stmts:
@@ -239,7 +239,7 @@ func GetCkCountFromOptions(options testutils.TestCaseOptions, allValue int) int
 	return ckCount
 }
 
-func validateStmt(t *testing.T, stmt interface{}, err error) {
+func validateStmt(t *testing.T, stmt any, err error) {
 	t.Helper()
 	if err != nil {
 		t.Fatalf("error: get an error on create test inputs:%v", err)

pkg/replication/replication.go

@@ -19,7 +19,7 @@ import (
 	"strings"
 )
 
-type Replication map[string]interface{}
+type Replication map[string]any
 
 func (r *Replication) ToCQL() string {
 	b, _ := json.Marshal(r)
@@ -41,7 +41,7 @@ func NewNetworkTopologyStrategy() *Replication {
 }
 
 func (r *Replication) UnmarshalJSON(data []byte) error {
-	dataMap := make(map[string]interface{})
+	dataMap := make(map[string]any)
 	if err := json.Unmarshal(data, &dataMap); err != nil {
 		return err
 	}