From 9edf1693ca891dc37d744eaa264f8914dd226cbb Mon Sep 17 00:00:00 2001 From: Dusan Malusev Date: Tue, 29 Oct 2024 23:50:42 +0100 Subject: [PATCH] improvement(benchmarks): performance improvements to the prettyCQL Not prettyCQL uses the `strings.Builder` to generate the `CQL` statament that was previously executed on ScyllaDB. Instaface has been changed accomadate the `builder` being passed to down. Signed-off-by: Dusan Malusev --- pkg/typedef/bag.go | 21 ++++-- pkg/typedef/interfaces.go | 4 +- pkg/typedef/simple_type.go | 89 ++++++++++++++++++++---- pkg/typedef/tuple.go | 16 +++-- pkg/typedef/typedef.go | 84 ++++------------------- pkg/typedef/typedef_test.go | 132 ++++++++++++++++++++++++++++++++++++ pkg/typedef/types.go | 39 ++++++++--- pkg/typedef/types_test.go | 27 ++++++-- pkg/typedef/udt.go | 17 +++-- 9 files changed, 313 insertions(+), 116 deletions(-) diff --git a/pkg/typedef/bag.go b/pkg/typedef/bag.go index 51a9d619..473adab8 100644 --- a/pkg/typedef/bag.go +++ b/pkg/typedef/bag.go @@ -59,20 +59,27 @@ func (ct *BagType) CQLHolder() string { return "?" } -func (ct *BagType) CQLPretty(value any) string { +func (ct *BagType) CQLPretty(builder *strings.Builder, value any) { if reflect.TypeOf(value).Kind() != reflect.Slice { panic(fmt.Sprintf("set cql pretty, unknown type %v", ct)) } - s := reflect.ValueOf(value) - format := "[%s]" + if ct.ComplexType == TYPE_SET { - format = "{%s}" + builder.WriteRune('{') + defer builder.WriteRune('}') + } else { + builder.WriteRune('[') + defer builder.WriteRune(']') } - out := make([]string, s.Len()) + + s := reflect.ValueOf(value) + for i := 0; i < s.Len(); i++ { - out[i] = ct.ValueType.CQLPretty(s.Index(i).Interface()) + ct.ValueType.CQLPretty(builder, s.Index(i).Interface()) + if i < s.Len()-1 { + builder.WriteRune(',') + } } - return fmt.Sprintf(format, strings.Join(out, ",")) } func (ct *BagType) GenValue(r *rand.Rand, p *PartitionRangeConfig) []any { diff --git a/pkg/typedef/interfaces.go b/pkg/typedef/interfaces.go index 1e6e5d65..7e6ceef3 100644 --- a/pkg/typedef/interfaces.go +++ b/pkg/typedef/interfaces.go @@ -15,6 +15,8 @@ package typedef import ( + "strings" + "github.com/gocql/gocql" "golang.org/x/exp/rand" ) @@ -23,7 +25,7 @@ type Type interface { Name() string CQLDef() string CQLHolder() string - CQLPretty(any) string + CQLPretty(*strings.Builder, any) GenValue(*rand.Rand, *PartitionRangeConfig) []any GenJSONValue(*rand.Rand, *PartitionRangeConfig) any LenValue() int diff --git a/pkg/typedef/simple_type.go b/pkg/typedef/simple_type.go index f2ad0c04..36dbe85b 100644 --- a/pkg/typedef/simple_type.go +++ b/pkg/typedef/simple_type.go @@ -20,6 +20,8 @@ import ( "math" "math/big" "net" + "strconv" + "strings" "time" "github.com/gocql/gocql" @@ -66,50 +68,113 @@ func (st SimpleType) LenValue() int { return 1 } -func (st SimpleType) CQLPretty(value any) string { +func (st SimpleType) CQLPretty(builder *strings.Builder, value any) { switch st { - case TYPE_ASCII, TYPE_TEXT, TYPE_VARCHAR, TYPE_INET, TYPE_DATE: - return fmt.Sprintf("'%s'", value) + case TYPE_INET: + builder.WriteRune('\'') + builder.WriteString(value.(net.IP).String()) + builder.WriteRune('\'') + case TYPE_ASCII, TYPE_TEXT, TYPE_VARCHAR, TYPE_DATE: + builder.WriteRune('\'') + builder.WriteString(value.(string)) + builder.WriteRune('\'') case TYPE_BLOB: if v, ok := value.(string); ok { if len(v) > 100 { v = v[:100] } - return "textasblob('" + v + "')" + builder.WriteString("textasblob('") + builder.WriteString(v) + builder.WriteString("')") + return } + panic(fmt.Sprintf("unexpected blob value [%T]%+v", value, value)) case TYPE_BIGINT, TYPE_INT, TYPE_SMALLINT, TYPE_TINYINT: - return fmt.Sprintf("%d", value) + var i int64 + switch v := value.(type) { + case int8: + i = int64(v) + case int16: + i = int64(v) + case int32: + i = int64(v) + case int: + i = int64(v) + case int64: + i = v + case *big.Int: + builder.WriteString(v.Text(10)) + return + default: + panic(fmt.Sprintf("unexpected int value [%T]%+v", value, value)) + } + builder.WriteString(strconv.FormatInt(i, 10)) case TYPE_DECIMAL, TYPE_DOUBLE, TYPE_FLOAT: - return fmt.Sprintf("%.2f", value) + var f float64 + switch v := value.(type) { + case float32: + f = float64(v) + case float64: + f = v + case *inf.Dec: + builder.WriteString(v.String()) + return + default: + panic(fmt.Sprintf("unexpected float value [%T]%+v", value, value)) + } + builder.WriteString(strconv.FormatFloat(f, 'f', 2, 64)) case TYPE_BOOLEAN: if v, ok := value.(bool); ok { - return fmt.Sprintf("%t", v) + builder.WriteString(strconv.FormatBool(v)) + return } + panic(fmt.Sprintf("unexpected boolean value [%T]%+v", value, value)) case TYPE_TIME: if v, ok := value.(int64); ok { + builder.WriteRune('\'') // CQL supports only 3 digits microseconds: // '10:10:55.83275+0000': marshaling error: Milliseconds length exceeds expected (5)" - return fmt.Sprintf("'%s'", time.Time{}.Add(time.Duration(v)).Format("15:04:05.999")) + builder.WriteString(time.Time{}.Add(time.Duration(v)).Format("15:04:05.999")) + builder.WriteRune('\'') + return } + panic(fmt.Sprintf("unexpected time value [%T]%+v", value, value)) case TYPE_TIMESTAMP: if v, ok := value.(int64); ok { // CQL supports only 3 digits milliseconds: // '1976-03-25T10:10:55.83275+0000': marshaling error: Milliseconds length exceeds expected (5)" - return time.UnixMilli(v).UTC().Format("'2006-01-02T15:04:05.999-0700'") + builder.WriteString(time.UnixMilli(v).UTC().Format("'2006-01-02T15:04:05.999-0700'")) + return } + panic(fmt.Sprintf("unexpected timestamp value [%T]%+v", value, value)) case TYPE_DURATION, TYPE_TIMEUUID, TYPE_UUID: - return fmt.Sprintf("%s", value) + switch v := value.(type) { + case string: + builder.WriteString(v) + return + case time.Duration: + builder.WriteString(v.String()) + return + case gocql.UUID: + builder.WriteString(v.String()) + return + } + + panic(fmt.Sprintf("unexpected (duration|timeuuid|uuid) value [%T]%+v", value, value)) case TYPE_VARINT: if s, ok := value.(*big.Int); ok { - return fmt.Sprintf("%d", s.Int64()) + builder.WriteString(s.Text(10)) + return } + panic(fmt.Sprintf("unexpected varint value [%T]%+v", value, value)) default: - panic(fmt.Sprintf("cql pretty: not supported type %s", st)) + panic(fmt.Sprintf("cql pretty: not supported type %s [%T]%+v", st, value, value)) + } } diff --git a/pkg/typedef/tuple.go b/pkg/typedef/tuple.go index 6c720b08..e534d3dc 100644 --- a/pkg/typedef/tuple.go +++ b/pkg/typedef/tuple.go @@ -15,7 +15,6 @@ package typedef import ( - "fmt" "strings" "github.com/gocql/gocql" @@ -55,16 +54,21 @@ func (t *TupleType) CQLHolder() string { return "(" + strings.TrimRight(strings.Repeat("?,", len(t.ValueTypes)), ",") + ")" } -func (t *TupleType) CQLPretty(value any) string { +func (t *TupleType) CQLPretty(builder *strings.Builder, value any) { values, ok := value.([]any) if !ok { - return "()" + builder.WriteString("()") } - out := make([]string, len(values)) + + builder.WriteRune('(') + defer builder.WriteRune(')') + for i, tp := range t.ValueTypes { - out[i] = tp.CQLPretty(values[i]) + tp.CQLPretty(builder, values[i]) + if i < len(values)-1 { + builder.WriteRune(',') + } } - return fmt.Sprintf("(%s)", strings.Join(out, ",")) } func (t *TupleType) Indexable() bool { diff --git a/pkg/typedef/typedef.go b/pkg/typedef/typedef.go index 6176e3b4..a64ac96c 100644 --- a/pkg/typedef/typedef.go +++ b/pkg/typedef/typedef.go @@ -16,9 +16,7 @@ package typedef import ( "fmt" - "iter" "strings" - "sync" "github.com/scylladb/gocqlx/v2/qb" @@ -199,78 +197,22 @@ const ( CacheArrayLen ) -func splitString(str, delimiter string) func(func(int, string) bool) { - lastPos := 0 - delLen := len(delimiter) - return func(yield func(int, string) bool) { - for i := 0; ; i++ { - pos := strings.Index(str[lastPos:], delimiter) +func prettyCQL(query string, values Values, types []Type) string { + var ( + index int + builder strings.Builder + ) - if pos == -1 || str[lastPos:] == "" { - yield(-1, str[lastPos:]) + builder.Grow(len(query)) - break - } + for pos, i := strings.Index(query[index:], "?"), 0; pos != -1; pos, i = strings.Index(query[index:], "?"), i+1 { + actualPos := index + pos + builder.WriteString(query[index:actualPos]) + types[i].CQLPretty(&builder, values[i]) - if str[lastPos:lastPos+pos] == "" || !yield(i, str[lastPos:lastPos+pos]) { - break - } - - lastPos += pos + delLen - } - } -} - -var builderPool = &sync.Pool{ - New: func() any { - builder := &strings.Builder{} - - builder.Grow(1024) - - return builder - }, -} - -func prettyCQL(query string, values Values, types Types) string { - if len(values) == 0 { - return query - } - - out := builderPool.Get().(*strings.Builder) - defer func() { - out.Reset() - builderPool.Put(out) - }() - - next, stop := iter.Pull2(splitString(query, "?")) - - for { - i, str, more := next() - - _, _ = out.WriteString(str) - - if !more || i == -1 { - stop() - break - } - - switch ty := types[i].(type) { - case *TupleType: - for count, t := range ty.ValueTypes { - _, _ = out.WriteString(t.CQLPretty(values[count])) - - _, str, more = next() - if !more { - stop() - break - } - - _, _ = out.WriteString(str) - } - default: - _, _ = out.WriteString(ty.CQLPretty(values[i])) - } + index = actualPos + 1 } - return out.String() + builder.WriteString(query[index:]) + return builder.String() } diff --git a/pkg/typedef/typedef_test.go b/pkg/typedef/typedef_test.go index 3a67b887..32938e4b 100644 --- a/pkg/typedef/typedef_test.go +++ b/pkg/typedef/typedef_test.go @@ -15,7 +15,14 @@ package typedef import ( + "fmt" + "math/big" + "net" + "strings" "testing" + "time" + + "gopkg.in/inf.v0" "github.com/google/go-cmp/cmp" ) @@ -41,3 +48,128 @@ func TestValues(t *testing.T) { t.Error("%i != %i", tmp, expected) } } + +var stmt = &Stmt{ + StmtCache: &StmtCache{ + Query: SimpleQuery{`INSERT INTO tbl(col1, col2, col3, col4, col5, col6,col7,col8,col9,cold10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);`}, + QueryType: InsertStatementType, + Types: Types{ + TYPE_ASCII, + TYPE_BIGINT, + TYPE_BLOB, + TYPE_BOOLEAN, + TYPE_DATE, + TYPE_DECIMAL, + TYPE_DOUBLE, + TYPE_DURATION, + TYPE_FLOAT, + TYPE_INET, + TYPE_INT, + TYPE_SMALLINT, + TYPE_TEXT, + TYPE_TIME, + TYPE_TIMESTAMP, + TYPE_TIMEUUID, + TYPE_UUID, + TYPE_TINYINT, + TYPE_VARCHAR, + TYPE_VARINT, + }, + }, + Values: Values{ + "a", + big.NewInt(10), + "a", + true, + millennium.Format("2006-01-02"), + inf.NewDec(1000, 0), + 10.0, + 10 * time.Minute, + 10.0, + net.ParseIP("192.168.0.1"), + 10, + 2, + "a", + millennium.UnixNano(), + millennium.UnixMilli(), + "63176980-bfde-11d3-bc37-1c4d704231dc", + "63176980-bfde-11d3-bc37-1c4d704231dc", + 1, + "a", + big.NewInt(1001), + }, +} + +func TestPrettyCQL(t *testing.T) { + t.Parallel() + + query := stmt.PrettyCQL() + + expected := fmt.Sprintf( + `INSERT INTO tbl(col1, col2, col3, col4, col5, col6,col7,col8,col9,cold10,col11,col12,col13,col14,col15,col16,col17,col18,col19,col20) VALUES ('a',10,textasblob('a'),true,'1999-12-31',1000,10.00,10m0s,10.00,'192.168.0.1',10,2,'a','%s','%s',63176980-bfde-11d3-bc37-1c4d704231dc,63176980-bfde-11d3-bc37-1c4d704231dc,1,'a',1001);`, + millennium.Format("15:04:05.999"), + millennium.Format("2006-01-02T15:04:05.999-0700"), + ) + + if query != expected { + t.Error("expected", expected, "got", query) + } +} + +func prettyCQLOld(query string, values Values, types Types) string { + if len(values) == 0 { + return query + } + + k := 0 + out := make([]string, 0, len(values)*2) + queryChunks := strings.Split(query, "?") + out = append(out, queryChunks[0]) + qID := 1 + var builder strings.Builder + for _, typ := range types { + builder.Reset() + tupleType, ok := typ.(*TupleType) + if !ok { + typ.CQLPretty(&builder, values[k]) + out = append(out, builder.String()) + out = append(out, queryChunks[qID]) + qID++ + k++ + continue + } + for _, t := range tupleType.ValueTypes { + builder.Reset() + t.CQLPretty(&builder, values[k]) + out = append(out, builder.String()) + out = append(out, queryChunks[qID]) + qID++ + k++ + } + } + out = append(out, queryChunks[qID:]...) + return strings.Join(out, "") +} + +func BenchmarkPrettyCQLOLD(b *testing.B) { + b.Run("New", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + query, _ := stmt.Query.ToCql() + values := stmt.Values.Copy() + prettyCQL(query, values, stmt.Types) + } + }) + + b.Run("Old", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + query, _ := stmt.Query.ToCql() + values := stmt.Values.Copy() + prettyCQLOld(query, values, stmt.Types) + } + }) +} diff --git a/pkg/typedef/types.go b/pkg/typedef/types.go index 8854613b..73ce6337 100644 --- a/pkg/typedef/types.go +++ b/pkg/typedef/types.go @@ -18,6 +18,7 @@ import ( "fmt" "math" "reflect" + "strconv" "strings" "sync/atomic" @@ -140,19 +141,26 @@ func (mt *MapType) CQLHolder() string { return "?" } -func (mt *MapType) CQLPretty(value any) string { +func (mt *MapType) CQLPretty(builder *strings.Builder, value any) { if reflect.TypeOf(value).Kind() != reflect.Map { panic(fmt.Sprintf("map cql pretty, unknown type %v", mt)) } + + builder.WriteRune('{') + defer builder.WriteRune('}') + vof := reflect.ValueOf(value) s := vof.MapRange() - out := make([]string, len(vof.MapKeys())) - id := 0 - for s.Next() { - out[id] = fmt.Sprintf("%s:%s", mt.KeyType.CQLPretty(s.Key().Interface()), mt.ValueType.CQLPretty(s.Value().Interface())) - id++ + length := vof.Len() + + for id := 0; s.Next(); id++ { + mt.KeyType.CQLPretty(builder, s.Key().Interface()) + builder.WriteRune(':') + mt.ValueType.CQLPretty(builder, s.Value().Interface()) + if id < length-1 { + builder.WriteRune(',') + } } - return fmt.Sprintf("{%s}", strings.Join(out, ",")) } func (mt *MapType) GenJSONValue(r *rand.Rand, p *PartitionRangeConfig) any { @@ -209,8 +217,21 @@ func (ct *CounterType) CQLHolder() string { return "?" } -func (ct *CounterType) CQLPretty(value any) string { - return fmt.Sprintf("%d", value) +func (ct *CounterType) CQLPretty(builder *strings.Builder, value interface{}) { + switch v := value.(type) { + case int64: + builder.WriteString(strconv.FormatInt(v, 10)) + case int: + builder.WriteString(strconv.FormatInt(int64(v), 10)) + case int32: + builder.WriteString(strconv.FormatInt(int64(v), 10)) + case uint64: + builder.WriteString(strconv.FormatUint(v, 10)) + case uint32: + builder.WriteString(strconv.FormatUint(uint64(v), 10)) + case uint: + builder.WriteString(strconv.FormatUint(uint64(v), 10)) + } } func (ct *CounterType) GenJSONValue(r *rand.Rand, _ *PartitionRangeConfig) any { diff --git a/pkg/typedef/types_test.go b/pkg/typedef/types_test.go index fa028dde..330c78ee 100644 --- a/pkg/typedef/types_test.go +++ b/pkg/typedef/types_test.go @@ -196,18 +196,33 @@ var prettytests = []struct { ValueTypes: []SimpleType{TYPE_ASCII}, Frozen: false, }, - query: "SELECT * FROM tbl WHERE pk0=?", - values: []any{"a"}, - expected: "SELECT * FROM tbl WHERE pk0='a'", + query: "SELECT * FROM tbl WHERE pk0=?", + values: []interface{}{ + []any{"a"}, + }, + expected: "SELECT * FROM tbl WHERE pk0=('a')", }, { typ: &TupleType{ ValueTypes: []SimpleType{TYPE_ASCII, TYPE_ASCII}, Frozen: false, }, - query: "SELECT * FROM tbl WHERE pk0={?,?}", - values: []any{"a", "b"}, - expected: "SELECT * FROM tbl WHERE pk0={'a','b'}", + query: "SELECT * FROM tbl WHERE pk0=?", + values: []interface{}{ + []any{"a", "b"}, + }, + expected: "SELECT * FROM tbl WHERE pk0=('a','b')", + }, + { + typ: &TupleType{ + ValueTypes: []SimpleType{TYPE_ASCII, TYPE_ASCII}, + Frozen: false, + }, + query: "SELECT * FROM tbl WHERE pk0=?", + values: []interface{}{ + []any{"a", "b"}, + }, + expected: "SELECT * FROM tbl WHERE pk0=('a','b')", }, } diff --git a/pkg/typedef/udt.go b/pkg/typedef/udt.go index d8899e49..b7575917 100644 --- a/pkg/typedef/udt.go +++ b/pkg/typedef/udt.go @@ -48,21 +48,30 @@ func (t *UDTType) CQLHolder() string { return "?" } -func (t *UDTType) CQLPretty(value any) string { +func (t *UDTType) CQLPretty(builder *strings.Builder, value any) { s, ok := value.(map[string]any) if !ok { panic(fmt.Sprintf("udt pretty, unknown type %v", t)) } - out := make([]string, 0, len(t.ValueTypes)) + builder.WriteRune('{') + defer builder.WriteRune('}') + + i := 0 for k, v := range t.ValueTypes { keyVal, kexExists := s[k] if !kexExists { continue } - out = append(out, fmt.Sprintf("%s:%s", k, v.CQLPretty(keyVal))) + + builder.WriteString(k) + builder.WriteRune(':') + v.CQLPretty(builder, keyVal) + if i != len(s)-1 { + builder.WriteRune(',') + } + i++ } - return fmt.Sprintf("{%s}", strings.Join(out, ",")) } func (t *UDTType) Indexable() bool {