Merge pull request #240 from scylladb/issue_229

schema: simple LWT support
scylladb · May 25, 2020 · 843428e · 843428e
2 parents 407eaac + c589d97
commit 843428e
Show file tree

Hide file tree

Showing 11 changed files with 29 additions and 17 deletions.
diff --git a/cmd/gemini/generators.go b/cmd/gemini/generators.go
@@ -27,6 +27,7 @@ func createGenerators(ctx context.Context, schema *gemini.Schema, schemaConfig g
 		MinBlobLength:   schemaConfig.MinBlobLength,
 		MaxStringLength: schemaConfig.MaxStringLength,
 		MinStringLength: schemaConfig.MinStringLength,
+		UseLWT:          schemaConfig.UseLWT,
 	}
 
 	var gs []*gemini.Generator

diff --git a/cmd/gemini/jobs.go b/cmd/gemini/jobs.go
@@ -144,6 +144,7 @@ func job(ctx context.Context, f testJob, actors uint64, schema *gemini.Schema, s
 		MinBlobLength:   schemaConfig.MinBlobLength,
 		MaxStringLength: schemaConfig.MaxStringLength,
 		MinStringLength: schemaConfig.MinStringLength,
+		UseLWT:          schemaConfig.UseLWT,
 	}
 
 	for j, table := range schema.Tables {

diff --git a/cmd/gemini/root.go b/cmd/gemini/root.go
@@ -84,6 +84,7 @@ var (
 	useCounters                      bool
 	asyncObjectStabilizationAttempts int
 	asyncObjectStabilizationDelay    time.Duration
+	useLWT                           bool
 )
 
 const (
@@ -477,6 +478,7 @@ func init() {
 	rootCmd.Flags().BoolVarP(&useCounters, "use-counters", "", false, "Ensure that at least one table is a counter table")
 	rootCmd.Flags().IntVarP(&asyncObjectStabilizationAttempts, "async-objects-stabilization-attempts", "", 10, "Maximum number of attempts to validate result sets from MV and SI")
 	rootCmd.Flags().DurationVarP(&asyncObjectStabilizationDelay, "async-objects-stabilization-backoff", "", 10*time.Millisecond, "Duration between attempts to validate result sets from MV and SI for example 10ms or 1s")
+	rootCmd.Flags().BoolVarP(&useLWT, "use-lwt", "", false, "Emit LWT based updates")
 }
 
 func printSetup() error {

diff --git a/cmd/gemini/schema.go b/cmd/gemini/schema.go
@@ -42,6 +42,7 @@ func createSchemaConfig(logger *zap.Logger) gemini.SchemaConfig {
 			MaxBlobLength:                    20,
 			MaxStringLength:                  20,
 			UseCounters:                      defaultConfig.UseCounters,
+			UseLWT:                           defaultConfig.UseLWT,
 			CQLFeature:                       defaultConfig.CQLFeature,
 			AsyncObjectStabilizationAttempts: defaultConfig.AsyncObjectStabilizationAttempts,
 			AsyncObjectStabilizationDelay:    defaultConfig.AsyncObjectStabilizationDelay,
@@ -80,6 +81,7 @@ func createDefaultSchemaConfig(logger *zap.Logger) gemini.SchemaConfig {
 		MaxStringLength:                  MaxStringLength,
 		MinStringLength:                  MinStringLength,
 		UseCounters:                      useCounters,
+		UseLWT:                           useLWT,
 		CQLFeature:                       getCQLFeature(cqlFeatures),
 		AsyncObjectStabilizationAttempts: asyncObjectStabilizationAttempts,
 		AsyncObjectStabilizationDelay:    asyncObjectStabilizationDelay,

diff --git a/go.mod b/go.mod
@@ -4,14 +4,14 @@ go 1.12
 
 require (
 	github.com/briandowns/spinner v1.11.1
-	github.com/gocql/gocql v0.0.0-20200103014340-68f928edb90a
+	github.com/gocql/gocql v0.0.0-20200131111108-92af2e088537
 	github.com/google/go-cmp v0.4.1
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/pkg/errors v0.8.1
 	github.com/prometheus/client_golang v0.9.3
 	github.com/scylladb/go-set v1.0.2
-	github.com/scylladb/gocqlx v1.3.3
+	github.com/scylladb/gocqlx/v2 v2.0.2
 	github.com/segmentio/ksuid v1.0.2
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3 // indirect
@@ -27,6 +27,6 @@ require (
 )
 
 replace (
-	github.com/gocql/gocql => github.com/scylladb/gocql v1.3.1
+	github.com/gocql/gocql => github.com/scylladb/gocql v1.4.0
 	golang.org/x/sync => golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a
 )
diff --git a/go.sum b/go.sum
@@ -36,6 +36,7 @@ github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
 github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
@@ -79,13 +80,14 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzrgykg80hfFph1mryUeLRsUfoocVVmRY=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ=
 github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc=
 github.com/scylladb/go-set v1.0.2 h1:SkvlMCKhP0wyyct6j+0IHJkBkSZL+TDzZ4E7f7BCcRE=
 github.com/scylladb/go-set v1.0.2/go.mod h1:DkpGd78rljTxKAnTDPFqXSGxvETQnJyuSOQwsHycqfs=
-github.com/scylladb/gocql v1.3.1 h1:LkK3OXns6QaMTJrMlWTCGroSnrt/7u1UfBcDzEeU6u0=
-github.com/scylladb/gocql v1.3.1/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY=
-github.com/scylladb/gocqlx v1.3.3 h1:p4R6sDDPvkmIMBlomKw+WyGmQc7o9LM3JlFDtcT+PoM=
-github.com/scylladb/gocqlx v1.3.3/go.mod h1:g+GaoDa4O9T/THP3z1TdA589P7ZyxVm+/Zj897g6DUc=
+github.com/scylladb/gocql v1.4.0 h1:JVePQWv+o9ezElfVESvNAzgxG1nmiiojYOaJRxO1pqo=
+github.com/scylladb/gocql v1.4.0/go.mod h1:S154F0u6zQlF3JjuHAidQIExQf9H45yT8z68h0FQYdU=
+github.com/scylladb/gocqlx/v2 v2.0.2 h1:+4mLbgN0N3sO8A7bimXBIW8jjTJqGkJ0liACpioBgxo=
+github.com/scylladb/gocqlx/v2 v2.0.2/go.mod h1:G1Cuw/NcqM0Mnj5zTRihdx7aW3yBk9zxR+XqCV3aCAA=
 github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc=
 github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

diff --git a/schema.go b/schema.go
@@ -25,7 +25,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/scylladb/gemini/replication"
 	"github.com/scylladb/gemini/tableopts"
-	"github.com/scylladb/gocqlx/qb"
+	"github.com/scylladb/gocqlx/v2/qb"
 	"golang.org/x/exp/rand"
 )
 
@@ -59,6 +59,7 @@ type SchemaConfig struct {
 	MinBlobLength                    int
 	MinStringLength                  int
 	UseCounters                      bool
+	UseLWT                           bool
 	CQLFeature                       CQLFeature
 	AsyncObjectStabilizationAttempts int
 	AsyncObjectStabilizationDelay    time.Duration
@@ -184,10 +185,10 @@ type Table struct {
 	PartitionKeys     Columns            `json:"partition_keys"`
 	ClusteringKeys    Columns            `json:"clustering_keys"`
 	Columns           Columns            `json:"columns"`
-	Indexes           []IndexDef         `json:"indexes"`
-	MaterializedViews []MaterializedView `json:"materialized_views"`
+	Indexes           []IndexDef         `json:"indexes,omitempty"`
+	MaterializedViews []MaterializedView `json:"materialized_views,omitempty"`
 	KnownIssues       map[string]bool    `json:"known_issues"`
-	TableOptions      []string           `json:"table_options"`
+	TableOptions      []string           `json:"table_options,omitempty"`
 
 	// mu protects the table during schema changes
 	mu sync.RWMutex
@@ -404,6 +405,7 @@ type PartitionRangeConfig struct {
 	MinBlobLength   int
 	MaxStringLength int
 	MinStringLength int
+	UseLWT          bool
 }
 
 func (s *Schema) GetDropSchema() []string {
@@ -658,7 +660,6 @@ func (s *Schema) insertStmt(t *Table, g *Generator, r *rand.Rand, p PartitionRan
 		builder = builder.Columns(pk.Name)
 		typs = append(typs, pk.Type)
 	}
-
 	valuesWithToken, ok := g.Get()
 	if !ok {
 		return nil, nil
@@ -679,6 +680,9 @@ func (s *Schema) insertStmt(t *Table, g *Generator, r *rand.Rand, p PartitionRan
 		values = appendValue(cdef.Type, r, p, values)
 		typs = append(typs, cdef.Type)
 	}
+	if p.UseLWT && r.Uint32()%10 == 0 {
+		builder = builder.Unique()
+	}
 	return &Stmt{
 		Query: builder,
 		Values: func() (uint64, []interface{}) {

diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml
@@ -18,7 +18,7 @@ services:
       gemini:
 
   gemini-test:
-    image: scylladb/scylla:3.0.7
+    image: scylladb/scylla:4.0.0
     container_name: gemini-test
     command: --smp 1 --memory 128M --api-address 0.0.0.0
     networks:

diff --git a/scripts/gemini-launcher b/scripts/gemini-launcher
@@ -21,9 +21,9 @@ go mod download || quit $? "Downloading dependencies failed"
 GOBIN="$(dirname ${GEMINI_CMD})" go install ./... || quit $? "Compilation failed"
 
 echo "Waiting for ${ORACLE_NAME} to start"
-until docker logs ${ORACLE_NAME} | grep "Starting listening for CQL clients" > /dev/null; do sleep 2; done
+until docker logs ${ORACLE_NAME} 2>&1 | grep "Starting listening for CQL clients" > /dev/null; do sleep 2; done
 echo "Waiting for ${TEST_NAME} to start"
-until docker logs ${TEST_NAME} | grep "Starting listening for CQL clients" > /dev/null; do sleep 2; done
+until docker logs ${TEST_NAME} 2>&1 | grep "Starting listening for CQL clients" > /dev/null; do sleep 2; done
 
 $GEMINI_CMD \
 	--duration=10m \

diff --git a/store/cqlstore.go b/store/cqlstore.go
@@ -23,7 +23,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/scylladb/gemini"
-	"github.com/scylladb/gocqlx/qb"
+	"github.com/scylladb/gocqlx/v2/qb"
 	"go.uber.org/multierr"
 	"go.uber.org/zap"
 )

diff --git a/store/store.go b/store/store.go
@@ -32,7 +32,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/scylladb/gemini"
 	"github.com/scylladb/go-set/strset"
-	"github.com/scylladb/gocqlx/qb"
+	"github.com/scylladb/gocqlx/v2/qb"
 	"go.uber.org/multierr"
 	"gopkg.in/inf.v0"
 )