improvement(stmt): refactoring statement genration

Statement generation currently does not work as intended,
gemini generates values for primary and clustering keys,
that are completply invalid. The are not just off by value,
but also off by type, e.g. generates `text` for `timeuuid`.
This is the first step in resolving this issue, by refactoring
how, when and where statements are generated.

Signed-off-by: Dusan Malusev <[email protected]>

.run/Run Gemini.run.xml

@@ -0,0 +1,14 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Run Gemini" type="GoApplicationRunConfiguration" factoryName="Go Application">
+    <module name="gemini" />
+    <working_directory value="$PROJECT_DIR$" />
+    <go_parameters value="-ldflags=&quot;-asan&quot; -gcflags &quot;all=-N -l&quot;" />
+    <parameters value="--dataset-size=small &#9;&#9;--cql-features all &#9;&#9;--duration 2m &#9;&#9;--drop-schema true &#9;&#9;--fail-fast &#9;&#9;--level info &#9;&#9;--non-interactive &#9;&#9;--materialized-views false &#9;&#9;--outfile ./results/gemini.log &#9;&#9;--test-statement-log-file ./results/gemini_test_statements.log &#9;&#9;--oracle-statement-log-file ./results/gemini_oracle_statements.log &#9;&#9;--test-host-selection-policy token-aware &#9;&#9;--oracle-host-selection-policy token-aware &#9;&#9;--test-cluster=192.168.100.2 &#9;&#9;--oracle-cluster=192.168.100.3 &#9;&#9;--outfile ./results/gemini_result.log &#9;&#9;--mode mixed &#9;&#9;--non-interactive &#9;&#9;--request-timeout 180s &#9;&#9;--connect-timeout 120s &#9;&#9;--use-server-timestamps false &#9;&#9;--async-objects-stabilization-attempts 10 &#9;&#9;--async-objects-stabilization-backoff 100ms &#9;&#9;--replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; &#9;&#9;--oracle-replication-strategy &quot;{'class': 'NetworkTopologyStrategy', 'replication_factor': '1'}&quot; &#9;&#9;--max-mutation-retries 5 &#9;&#9;--max-mutation-retries-backoff 1000ms &#9;&#9;--concurrency 1 &#9;&#9;--tracing-outfile ./results/gemini_tracing.log" />
+    <kind value="PACKAGE" />
+    <package value="github.com/scylladb/gemini/cmd/gemini" />
+    <directory value="$PROJECT_DIR$" />
+    <filePath value="$PROJECT_DIR$/cmd/gemini/main.go" />
+    <output_directory value="bin/" />
+    <method v="2" />
+  </configuration>
+</component>

cmd/gemini/root.go

@@ -292,16 +292,16 @@ func run(_ *cobra.Command, _ []string) error {
 	}
 
 	if warmup > 0 && !stopFlag.IsHardOrSoft() {
-		jobsList := jobs.ListFromMode(jobs.WarmupMode, warmup, concurrency)
-		if err = jobsList.Run(ctx, schema, schemaConfig, st, pump, gens, globalStatus, logger, intSeed, warmupStopFlag, failFast, verbose); err != nil {
+		jobsList := jobs.New(jobs.WarmupMode, warmup, concurrency)
+		if err = jobsList.Do(ctx, schema, schemaConfig, st, pump, gens, globalStatus, logger, intSeed, warmupStopFlag, failFast, verbose); err != nil {
 			logger.Error("warmup encountered an error", zap.Error(err))
 			stopFlag.SetHard(true)
 		}
 	}
 
 	if !stopFlag.IsHardOrSoft() {
-		jobsList := jobs.ListFromMode(mode, duration, concurrency)
-		if err = jobsList.Run(ctx, schema, schemaConfig, st, pump, gens, globalStatus, logger, intSeed, stopFlag.CreateChild("workload"), failFast, verbose); err != nil {
+		jobsList := jobs.New(mode, duration, concurrency)
+		if err = jobsList.Do(ctx, schema, schemaConfig, st, pump, gens, globalStatus, logger, intSeed, stopFlag.CreateChild("workload"), failFast, verbose); err != nil {
 			logger.Debug("error detected", zap.Error(err))
 		}
 	}
@@ -473,7 +473,7 @@ func init() {
 	rootCmd.Flags().BoolVarP(&dropSchema, "drop-schema", "d", false, "Drop schema before starting tests run")
 	rootCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Verbose output during test run")
 	rootCmd.Flags().BoolVarP(&failFast, "fail-fast", "f", false, "Stop on the first failure")
-	rootCmd.Flags().BoolVarP(&nonInteractive, "non-interactive", "", false, "Run in non-interactive mode (disable progress indicator)")
+	rootCmd.Flags().BoolVarP(&nonInteractive, "non-interactive", "", false, "Statement in non-interactive mode (disable progress indicator)")
 	rootCmd.Flags().DurationVarP(&duration, "duration", "", 30*time.Second, "")
 	rootCmd.Flags().StringVarP(&outFileArg, "outfile", "", "", "Specify the name of the file where the results should go")
 	rootCmd.Flags().StringVarP(&bind, "bind", "b", ":2112", "Specify the interface and port which to bind prometheus metrics on. Default is ':2112'")
@@ -497,7 +497,7 @@ func init() {
 	rootCmd.Flags().IntVarP(&minColumns, "min-columns", "", 8, "Minimum number of generated columns")
 	rootCmd.Flags().StringVarP(&datasetSize, "dataset-size", "", "large", "Specify the type of dataset size to use, small|large")
 	rootCmd.Flags().StringVarP(&cqlFeatures, "cql-features", "", "basic", "Specify the type of cql features to use, basic|normal|all")
-	rootCmd.Flags().BoolVarP(&useMaterializedViews, "materialized-views", "", false, "Run gemini with materialized views support")
+	rootCmd.Flags().BoolVarP(&useMaterializedViews, "materialized-views", "", false, "Statement gemini with materialized views support")
 	rootCmd.Flags().StringVarP(&level, "level", "", "info", "Specify the logging level, debug|info|warn|error|dpanic|panic|fatal")
 	rootCmd.Flags().IntVarP(&maxRetriesMutate, "max-mutation-retries", "", 2, "Maximum number of attempts to apply a mutation")
 	rootCmd.Flags().DurationVarP(

pkg/generators/generator.go

@@ -37,12 +37,11 @@ type TokenIndex uint64
 
 type DistributionFunc func() TokenIndex
 
-type GeneratorInterface interface {
+type Interface interface {
 	Get() *typedef.ValueWithToken
 	GetOld() *typedef.ValueWithToken
-	GiveOld(_ *typedef.ValueWithToken)
-	GiveOlds(_ []*typedef.ValueWithToken)
-	ReleaseToken(_ uint64)
+	GiveOld(...*typedef.ValueWithToken)
+	ReleaseToken(uint64)
 }
 
 type Generator struct {
@@ -118,15 +117,10 @@ func (g *Generator) GetOld() *typedef.ValueWithToken {
 	return targetPart.getOld()
 }
 
-// GiveOld returns the supplied value for later reuse unless
-func (g *Generator) GiveOld(v *typedef.ValueWithToken) {
-	g.GetPartitionForToken(TokenIndex(v.Token)).giveOld(v)
-}
-
-// GiveOlds returns the supplied values for later reuse unless
-func (g *Generator) GiveOlds(tokens []*typedef.ValueWithToken) {
+// GiveOld returns the supplied value for later reuse
+func (g *Generator) GiveOld(tokens ...*typedef.ValueWithToken) {
 	for _, token := range tokens {
-		g.GiveOld(token)
+		g.GetPartitionForToken(TokenIndex(token.Token)).giveOld(token)
 	}
 }
 

pkg/generators/statement_generator.go

@@ -87,7 +87,7 @@ func genTable(sc typedef.SchemaConfig, tableName string, r *rand.Rand) *typedef.
 	table.Indexes = indexes
 
 	var mvs []typedef.MaterializedView
-	if sc.CQLFeature > typedef.CQL_FEATURE_BASIC && sc.UseMaterializedViews && len(clusteringKeys) > 0 && columns.ValidColumnsForPrimaryKey().Len() != 0 {
+	if sc.UseMaterializedViews && sc.CQLFeature > typedef.CQL_FEATURE_BASIC && len(clusteringKeys) > 0 && columns.ValidColumnsForPrimaryKey().Len() != 0 {
 		mvs = CreateMaterializedViews(columns, table.Name, partitionKeys, clusteringKeys, r)
 	}