From 41f0abf7a0a6c1c183e420d905cc723f2622695a Mon Sep 17 00:00:00 2001 From: Lz Date: Mon, 26 Feb 2024 06:29:16 +0800 Subject: [PATCH] feat(sharding): enabled sharding feature on db (#13) --- CHANGELOG.md | 4 +- README.md | 88 ++++++++++--- context.go | 155 ++++++++++++++++++++++ stmt.go => context_stmt.go | 4 +- stmt_test.go => context_stmt_test.go | 0 db.go | 164 +++--------------------- db_test.go | 78 +++++++++-- {sharding => shardid}/README.md | 4 +- {sharding => shardid}/generator.go | 6 +- {sharding => shardid}/generator_test.go | 132 +++++++++---------- {sharding => shardid}/id.go | 26 ++-- {sharding => shardid}/id_test.go | 46 ++++--- {sharding => shardid}/nocopy.go | 2 +- {sharding => shardid}/option.go | 4 +- {sharding => shardid}/woker.go | 2 +- sqlbuilder.go | 41 +++--- sqlbuilder_test.go | 85 ++++++++++++ tx_test.go | 6 +- use.go | 24 ++++ 19 files changed, 550 insertions(+), 321 deletions(-) create mode 100644 context.go rename stmt.go => context_stmt.go (86%) rename stmt_test.go => context_stmt_test.go (100%) rename {sharding => shardid}/README.md (98%) rename {sharding => shardid}/generator.go (95%) rename {sharding => shardid}/generator_test.go (69%) rename {sharding => shardid}/id.go (80%) rename {sharding => shardid}/id_test.go (66%) rename {sharding => shardid}/nocopy.go (88%) rename {sharding => shardid}/option.go (90%) rename {sharding => shardid}/woker.go (92%) create mode 100644 use.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a1b0db..29f27a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [1.2.0] ### Added - added `BitBool` for mysql bit type (#11) - added `sharding` feature (#12) +- added `On` on `DB` to enable AutoSharding feature(#13) +- added `On` on `SQLBuilder` to enable AutoRotation feature(#13) ### Fixed - fixed parameterized placeholder for postgresql(#12) diff --git a/README.md b/README.md index 1ecb546..764c351 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,9 @@ You’ll find the SQLE package useful if you’re not a fan of full-featured ORM slices of map/structs/primitive types. - 100% compatible drop-in replacement of "database/sql". Code is really easy to migrate from `database/sql` to `SQLE`. see [examples](row_test.go) - [Migration](migrate/migrator_test.go) -- _Configurable and extendable sharding_ +- [ShardID](shardid/README.md) is a `snowflake-like` distributed sequence unique identifier with extended features for table rotation and database sharding. +- Table AutoRotation +- Database AutoSharding ## Tutorials > All examples on https://go.dev/doc/tutorial/database-access can directly work with `sqle.DB` instance. @@ -47,28 +49,29 @@ SQLE directly connects to a database by `sql.DB` instance. var db *sqle.DB switch driver { - case "sqlite": - sqldb, err := sql.Open("sqlite3", "file:"+dsn+"?cache=shared") - if err != nil { - panic(fmt.Sprintf("db: failed to open sqlite database %s", dsn)) - } + case "sqlite": + sqldb, err := sql.Open("sqlite3", "file:"+dsn+"?cache=shared") + if err != nil { + panic(fmt.Sprintf("db: failed to open sqlite database %s", dsn)) + } - db = sqle.Open(sqldb) + db = sqle.Open(sqldb) - case "mysql": - sqldb, err := sql.Open("mysql", dsn) - if err != nil { - panic(fmt.Sprintf("db: failed to open mysql database %s", dsn)) - } - db = sqle.Open(sqldb) + case "mysql": + sqldb, err := sql.Open("mysql", dsn) + if err != nil { + panic(fmt.Sprintf("db: failed to open mysql database %s", dsn)) + } - default: - panic(fmt.Sprintf("db: driver %s is not supported yet", driver)) + db = sqle.Open(sqldb) + + default: + panic(fmt.Sprintf("db: driver %s is not supported yet", driver)) } if err := db.Ping(); err == nil { - panic("db: database is unreachable") + panic("db: database is unreachable") } ``` @@ -418,4 +421,57 @@ func deleteAlbums(ids []int64) error { } }) } +``` + + +### Table Rotation +use `shardid.ID` to enable rotate feature for a table based on option (NoRotate/MonthlyRotate/WeeklyRotate/DailyRotate) + +``` +gen := shardid.New(WithTableRotate(shardid.DailyRotate)) +id := gen.Next() + +b := New().On(id) //call `On` to enable rotate feature, and setup a input variable +b.Delete("orders").Where(). + If(true).And("order_id = {order_id}"). + If(false).And("member_id"). + Param("order_id", "order_123456") + + +db.ExecBuilder(context.TODO(),b) //DELETE FROM `orders_20240220` WHERE order_id = ? +``` +see more [examples](sqlbuilder_test.go#L490) + + +### Database Sharding +use `shardid.ID` to enable sharding feature for any sql +``` +gen := shardid.New(WithDatabase(10)) // 10 database instances +id := gen.Next() + +b := New().On(id) //call `On` to setup an input variable named `rotate`, and enable rotation feature +b.Delete("orders").Where(). + If(true).And("order_id = {order_id}"). + If(false).And("member_id"). + Param("order_id", "order_123456") + + +db.On(id). //automatically select database based on `id.DatabaseID` + ExecBuilder(context.TODO(),b) //DELETE FROM `orders` WHERE order_id = ? + +``` + +see more [examples](db_test.go#L49) + + +## SQL Injection +SQLE uses the database/sql‘s argument placeholders to build parameterized SQL statement, which will automatically escape arguments to avoid SQL injection. eg if postgresql is used in your app, please call [UsePostgres](use.go#L5) on SQLBuilder or change [DefaultSQLQuote](sqlbuilder.go?L16) and [DefaultSQLParameterize](sqlbuilder.go?L17) to update parameterization options. + +``` +func UsePostgres(b *Builder) { + b.Quote = "`" + b.Parameterize = func(name string, index int) string { + return "$" + strconv.Itoa(index) + } +} ``` \ No newline at end of file diff --git a/context.go b/context.go new file mode 100644 index 0000000..574dea2 --- /dev/null +++ b/context.go @@ -0,0 +1,155 @@ +package sqle + +import ( + "context" + "database/sql" + "sync" + + "github.com/rs/zerolog/log" +) + +type Context struct { + *sql.DB + sync.Mutex + _ noCopy + + index int + stmts map[string]*cachedStmt + stmtsMutex sync.RWMutex +} + +func (db *Context) Query(query string, args ...any) (*Rows, error) { + return db.QueryContext(context.Background(), query, args...) +} + +func (db *Context) QueryBuilder(ctx context.Context, b *Builder) (*Rows, error) { + query, args, err := b.Build() + if err != nil { + return nil, err + } + + return db.QueryContext(ctx, query, args...) +} + +func (db *Context) QueryContext(ctx context.Context, query string, args ...any) (*Rows, error) { + var rows *sql.Rows + var stmt *sql.Stmt + var err error + if len(args) > 0 { + stmt, err = db.prepareStmt(ctx, query) + if err == nil { + rows, err = stmt.QueryContext(ctx, args...) + if err != nil { + return nil, err + } + } + + } else { + rows, err = db.DB.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + } + + return &Rows{Rows: rows, query: query}, nil +} + +func (db *Context) QueryRow(query string, args ...any) *Row { + return db.QueryRowContext(context.Background(), query, args...) +} + +func (db *Context) QueryRowBuilder(ctx context.Context, b *Builder) *Row { + query, args, err := b.Build() + if err != nil { + return &Row{ + err: err, + query: query, + } + } + + return db.QueryRowContext(ctx, query, args...) +} + +func (db *Context) QueryRowContext(ctx context.Context, query string, args ...any) *Row { + var rows *sql.Rows + var stmt *sql.Stmt + var err error + + if len(args) > 0 { + stmt, err = db.prepareStmt(ctx, query) + if err != nil { + return &Row{ + err: err, + query: query, + } + } + rows, err = stmt.QueryContext(ctx, args...) + return &Row{ + rows: rows, + err: err, + query: query, + } + } + + rows, err = db.DB.QueryContext(ctx, query, args...) + return &Row{ + rows: rows, + err: err, + query: query, + } +} + +func (db *Context) Exec(query string, args ...any) (sql.Result, error) { + return db.ExecContext(context.Background(), query, args...) +} + +func (db *Context) ExecBuilder(ctx context.Context, b *Builder) (sql.Result, error) { + query, args, err := b.Build() + if err != nil { + return nil, err + } + + return db.ExecContext(ctx, query, args...) +} + +func (db *Context) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { + if len(args) > 0 { + stmt, err := db.prepareStmt(ctx, query) + if err != nil { + return nil, err + } + + return stmt.ExecContext(ctx, args...) + } + return db.DB.ExecContext(context.Background(), query, args...) +} + +func (db *Context) Begin(opts *sql.TxOptions) (*Tx, error) { + return db.BeginTx(context.TODO(), opts) + +} + +func (db *Context) BeginTx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) { + tx, err := db.DB.BeginTx(ctx, opts) + if err != nil { + return nil, err + } + + return &Tx{Tx: tx, cachedStmts: make(map[string]*sql.Stmt)}, nil +} + +func (db *Context) Transaction(ctx context.Context, opts *sql.TxOptions, fn func(ctx context.Context, tx *Tx) error) error { + tx, err := db.BeginTx(ctx, opts) + if err != nil { + return err + } + + err = fn(ctx, tx) + if err != nil { + if e := tx.Rollback(); e != nil { + log.Error().Str("pkg", "sqle").Str("tag", "tx").Err(e) + } + return err + } + return tx.Commit() +} diff --git a/stmt.go b/context_stmt.go similarity index 86% rename from stmt.go rename to context_stmt.go index 17783ff..503fb4e 100644 --- a/stmt.go +++ b/context_stmt.go @@ -13,7 +13,7 @@ type cachedStmt struct { lastUsed time.Time } -func (db *DB) prepareStmt(ctx context.Context, query string) (*sql.Stmt, error) { +func (db *Context) prepareStmt(ctx context.Context, query string) (*sql.Stmt, error) { db.stmtsMutex.RLock() s, ok := db.stmts[query] db.stmtsMutex.RUnlock() @@ -39,7 +39,7 @@ func (db *DB) prepareStmt(ctx context.Context, query string) (*sql.Stmt, error) return stmt, nil } -func (db *DB) closeIdleStmt() { +func (db *Context) closeIdleStmt() { for { <-time.After(1 * time.Minute) diff --git a/stmt_test.go b/context_stmt_test.go similarity index 100% rename from stmt_test.go rename to context_stmt_test.go diff --git a/db.go b/db.go index daa0f9c..2817738 100644 --- a/db.go +++ b/db.go @@ -1,165 +1,39 @@ package sqle import ( - "context" "database/sql" - "sync" - "github.com/rs/zerolog/log" + "github.com/yaitoo/sqle/shardid" ) type DB struct { - *sql.DB - noCopy //nolint + *Context + _ noCopy //nolint: unused - sync.Mutex - stmts map[string]*cachedStmt - stmtsMutex sync.RWMutex + dbs []*Context } -func Open(db *sql.DB) *DB { +func Open(dbs ...*sql.DB) *DB { d := &DB{ - DB: db, - stmts: make(map[string]*cachedStmt), + Context: &Context{ + DB: dbs[0], + stmts: make(map[string]*cachedStmt), + }, } - go d.closeIdleStmt() - - return d -} - -func (db *DB) Query(query string, args ...any) (*Rows, error) { - return db.QueryContext(context.Background(), query, args...) -} - -func (db *DB) QueryBuilder(ctx context.Context, b *Builder) (*Rows, error) { - query, args, err := b.Build() - if err != nil { - return nil, err - } - - return db.QueryContext(ctx, query, args...) -} - -func (db *DB) QueryContext(ctx context.Context, query string, args ...any) (*Rows, error) { - var rows *sql.Rows - var stmt *sql.Stmt - var err error - if len(args) > 0 { - stmt, err = db.prepareStmt(ctx, query) - if err == nil { - rows, err = stmt.QueryContext(ctx, args...) - if err != nil { - return nil, err - } - } - - } else { - rows, err = db.DB.QueryContext(ctx, query, args...) - if err != nil { - return nil, err - } - } - - return &Rows{Rows: rows, query: query}, nil -} - -func (db *DB) QueryRow(query string, args ...any) *Row { - return db.QueryRowContext(context.Background(), query, args...) -} - -func (db *DB) QueryRowBuilder(ctx context.Context, b *Builder) *Row { - query, args, err := b.Build() - if err != nil { - return &Row{ - err: err, - query: query, - } - } - - return db.QueryRowContext(ctx, query, args...) -} - -func (db *DB) QueryRowContext(ctx context.Context, query string, args ...any) *Row { - var rows *sql.Rows - var stmt *sql.Stmt - var err error - - if len(args) > 0 { - stmt, err = db.prepareStmt(ctx, query) - if err != nil { - return &Row{ - err: err, - query: query, - } + for i, db := range dbs { + ctx := &Context{ + DB: db, + index: i, + stmts: make(map[string]*cachedStmt), } - rows, err = stmt.QueryContext(ctx, args...) - return &Row{ - rows: rows, - err: err, - query: query, - } - } - - rows, err = db.DB.QueryContext(ctx, query, args...) - return &Row{ - rows: rows, - err: err, - query: query, + d.dbs = append(d.dbs, ctx) + go ctx.closeIdleStmt() } -} -func (db *DB) Exec(query string, args ...any) (sql.Result, error) { - return db.ExecContext(context.Background(), query, args...) -} - -func (db *DB) ExecBuilder(ctx context.Context, b *Builder) (sql.Result, error) { - query, args, err := b.Build() - if err != nil { - return nil, err - } - - return db.ExecContext(ctx, query, args...) -} - -func (db *DB) ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) { - if len(args) > 0 { - stmt, err := db.prepareStmt(ctx, query) - if err != nil { - return nil, err - } - - return stmt.ExecContext(ctx, args...) - } - return db.DB.ExecContext(context.Background(), query, args...) -} - -func (db *DB) Begin(opts *sql.TxOptions) (*Tx, error) { - return db.BeginTx(context.TODO(), opts) - -} - -func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*Tx, error) { - tx, err := db.DB.BeginTx(ctx, opts) - if err != nil { - return nil, err - } - - return &Tx{Tx: tx, cachedStmts: make(map[string]*sql.Stmt)}, nil + return d } -func (db *DB) Transaction(ctx context.Context, opts *sql.TxOptions, fn func(ctx context.Context, tx *Tx) error) error { - tx, err := db.BeginTx(ctx, opts) - if err != nil { - return err - } - - err = fn(ctx, tx) - if err != nil { - if e := tx.Rollback(); e != nil { - log.Error().Str("pkg", "sqle").Str("tag", "tx").Err(e) - } - return err - } - return tx.Commit() +func (db *DB) On(id shardid.ID) *Context { + return db.dbs[int(id.DatabaseID)] } diff --git a/db_test.go b/db_test.go index 3c73e15..ab203f1 100644 --- a/db_test.go +++ b/db_test.go @@ -1,32 +1,84 @@ package sqle import ( + "context" "database/sql" - "os" + "testing" + "time" _ "github.com/mattn/go-sqlite3" + "github.com/stretchr/testify/require" + "github.com/yaitoo/sqle/shardid" ) -func createSQLite3() (*sql.DB, func(), error) { - f, err := os.CreateTemp(".", "*.db") - f.Close() +func createSQLite3() *sql.DB { + // f, err := os.CreateTemp(".", "*.db") + // f.Close() - clean := func() { - os.Remove(f.Name()) //nolint - } + // clean := func() { + // os.Remove(f.Name()) //nolint + // } - if err != nil { - return nil, clean, err - } + // if err != nil { + // return nil, clean, err + // } //db, err := sql.Open("sqlite3", "file:"+f.Name()+"?cache=shared") - db, err := sql.Open("sqlite3", "file::memory:") if err != nil { - return nil, clean, err + return nil } //https://github.com/mattn/go-sqlite3/issues/209 // db.SetMaxOpenConns(1) - return db, clean, nil + return db +} + +func TestSharding(t *testing.T) { + dbs := make([]*sql.DB, 0, 10) + + for i := 0; i < 10; i++ { + db3 := createSQLite3() + + db3.Exec("CREATE TABLE `users` (`id` bigint , `status` tinyint,`email` varchar(50),`passwd` varchar(120), `salt` varchar(45), `created` DATETIME, PRIMARY KEY (`id`))") //nolint: errcheck + + dbs = append(dbs, db3) + } + + db := Open(dbs...) + gen := shardid.New(shardid.WithDatabase(10)) + + ids := make([]shardid.ID, 10) + + for i := 0; i < 10; i++ { + id := gen.Next() + b := New().On(id). + Insert("users"). + Set("id", id.Value). + Set("status", 1). + Set("created", time.Now()). + End() + result, err := db.On(id).ExecBuilder(context.TODO(), b) + + require.NoError(t, err) + rows, err := result.RowsAffected() + require.NoError(t, err) + require.Equal(t, int64(1), rows) + + ids[i] = id + } + + for i, id := range ids { + b := New().On(id).Select("users", "id") + + ctx := db.On(id) + + require.Equal(t, i, ctx.index) + + var userID int64 + err := ctx.QueryRowBuilder(context.TODO(), b).Scan(&userID) + require.NoError(t, err) + require.Equal(t, id.Value, userID) + } + } diff --git a/sharding/README.md b/shardid/README.md similarity index 98% rename from sharding/README.md rename to shardid/README.md index 87811cf..6e43402 100644 --- a/sharding/README.md +++ b/shardid/README.md @@ -1,6 +1,6 @@ -# sharding +# shardid -## sid-64-bit +## 64-bit // +----------+-------------------+------------+----------------+----------------------+---------------+ // | signed 1 | millis (39) | worker(2) | db-sharding(10)| table-rotate(2) | sequence(10) | // +----------+-------------------+------------+----------------+----------------------+---------------+ diff --git a/sharding/generator.go b/shardid/generator.go similarity index 95% rename from sharding/generator.go rename to shardid/generator.go index 212e411..56a2d55 100644 --- a/sharding/generator.go +++ b/shardid/generator.go @@ -1,4 +1,4 @@ -package sharding +package shardid import ( "sync" @@ -23,7 +23,7 @@ func New(options ...Option) *Generator { g := &Generator{ now: time.Now, databaseTotal: 1, - tableRotate: None, + tableRotate: NoRotate, workerID: acquireWorkerID(), } for _, option := range options { @@ -32,7 +32,7 @@ func New(options ...Option) *Generator { return g } -func (g *Generator) Next() int64 { +func (g *Generator) Next() ID { g.Lock() defer func() { diff --git a/sharding/generator_test.go b/shardid/generator_test.go similarity index 69% rename from sharding/generator_test.go rename to shardid/generator_test.go index b16e512..b7ba9d2 100644 --- a/sharding/generator_test.go +++ b/shardid/generator_test.go @@ -1,4 +1,4 @@ -package sharding +package shardid import ( "testing" @@ -24,16 +24,16 @@ func TestGenerator(t *testing.T) { }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, None, 0) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, NoRotate, 0) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, None, 1) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, NoRotate, 1) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, None, 2) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, NoRotate, 2) + require.Equal(t, want.Value, id.Value) }, }, @@ -48,16 +48,16 @@ func TestGenerator(t *testing.T) { }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, None, 0) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, NoRotate, 0) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, None, 1) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, NoRotate, 1) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, None, 2) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, NoRotate, 2) + require.Equal(t, want.Value, id.Value) }, }, @@ -72,16 +72,16 @@ func TestGenerator(t *testing.T) { }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, None, 0) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, NoRotate, 0) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 1, None, 1) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 1, NoRotate, 1) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 2, None, 2) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 2, NoRotate, 2) + require.Equal(t, want.Value, id.Value) }, }, @@ -96,20 +96,20 @@ func TestGenerator(t *testing.T) { }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, None, 0) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, NoRotate, 0) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 1, None, 1) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 1, NoRotate, 1) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, None, 2) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, NoRotate, 2) + require.Equal(t, want.Value, id.Value) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 1, None, 3) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 1, NoRotate, 3) + require.Equal(t, want.Value, id.Value) }, }, @@ -118,17 +118,15 @@ func TestGenerator(t *testing.T) { new: func() *Generator { g := New(WithTimeNow(func() time.Time { return time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC) - }), WithWorkerID(1), WithDatabase(3), WithTableRotate(Monthly)) + }), WithWorkerID(1), WithDatabase(3), WithTableRotate(MonthlyRotate)) return g }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, Monthly, 0) - require.Equal(t, want, id) - - md := Parse(id) - require.Equal(t, "202402", md.RotateName()) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, MonthlyRotate, 0) + require.Equal(t, want.Value, id.Value) + require.Equal(t, "202402", id.RotateName()) }, }, { @@ -136,17 +134,16 @@ func TestGenerator(t *testing.T) { new: func() *Generator { g := New(WithTimeNow(func() time.Time { return time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC) - }), WithWorkerID(1), WithDatabase(3), WithTableRotate(Weekly)) + }), WithWorkerID(1), WithDatabase(3), WithTableRotate(WeeklyRotate)) return g }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, Weekly, 0) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, WeeklyRotate, 0) + require.Equal(t, want.Value, id.Value) - md := Parse(id) - require.Equal(t, "2024008", md.RotateName()) + require.Equal(t, "2024008", id.RotateName()) }, }, { @@ -154,17 +151,16 @@ func TestGenerator(t *testing.T) { new: func() *Generator { g := New(WithTimeNow(func() time.Time { return time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC) - }), WithWorkerID(1), WithDatabase(3), WithTableRotate(Daily)) + }), WithWorkerID(1), WithDatabase(3), WithTableRotate(DailyRotate)) return g }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, Daily, 0) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, DailyRotate, 0) + require.Equal(t, want.Value, id.Value) - md := Parse(id) - require.Equal(t, "20240220", md.RotateName()) + require.Equal(t, "20240220", id.RotateName()) }, }, { @@ -178,25 +174,23 @@ func TestGenerator(t *testing.T) { return time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Millisecond) - }), WithWorkerID(1), WithTableRotate(Daily)) + }), WithWorkerID(1), WithTableRotate(DailyRotate)) return g }, assert: func(t *testing.T, gen *Generator) { gen.nextSequence = MaxSequence id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, Daily, MaxSequence) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, DailyRotate, MaxSequence) + require.Equal(t, want.Value, id.Value) - md := Parse(id) - require.Equal(t, "20240220", md.RotateName()) + require.Equal(t, "20240220", id.RotateName()) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(1*time.Millisecond).UnixMilli(), 1, 0, Daily, 0) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(1*time.Millisecond).UnixMilli(), 1, 0, DailyRotate, 0) + require.Equal(t, want.Value, id.Value) - md = Parse(id) - require.Equal(t, "20240220", md.RotateName()) + require.Equal(t, "20240220", id.RotateName()) }, }, { @@ -214,24 +208,20 @@ func TestGenerator(t *testing.T) { return time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Millisecond) - }), WithWorkerID(1), WithTableRotate(Daily)) + }), WithWorkerID(1), WithTableRotate(DailyRotate)) return g }, assert: func(t *testing.T, gen *Generator) { id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, Daily, 0) - require.Equal(t, want, id) - - md := Parse(id) - require.Equal(t, "20240220", md.RotateName()) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, DailyRotate, 0) + require.Equal(t, want.Value, id.Value) + require.Equal(t, "20240220", id.RotateName()) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(1*time.Millisecond).UnixMilli(), 1, 0, Daily, 1) - require.Equal(t, want, id) - - md = Parse(id) - require.Equal(t, "20240220", md.RotateName()) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(1*time.Millisecond).UnixMilli(), 1, 0, DailyRotate, 1) + require.Equal(t, want.Value, id.Value) + require.Equal(t, "20240220", id.RotateName()) }, }, @@ -250,25 +240,23 @@ func TestGenerator(t *testing.T) { return time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Millisecond) - }), WithWorkerID(1), WithTableRotate(Daily)) + }), WithWorkerID(1), WithTableRotate(DailyRotate)) return g }, assert: func(t *testing.T, gen *Generator) { gen.nextSequence = MaxSequence id := gen.Next() - want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, Daily, MaxSequence) - require.Equal(t, want, id) + want := Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 1, 0, DailyRotate, MaxSequence) + require.Equal(t, want.Value, id.Value) - md := Parse(id) - require.Equal(t, "20240220", md.RotateName()) + require.Equal(t, "20240220", id.RotateName()) id = gen.Next() - want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(2*time.Millisecond).UnixMilli(), 1, 0, Daily, 0) - require.Equal(t, want, id) + want = Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).Add(2*time.Millisecond).UnixMilli(), 1, 0, DailyRotate, 0) + require.Equal(t, want.Value, id.Value) - md = Parse(id) - require.Equal(t, "20240220", md.RotateName()) + require.Equal(t, "20240220", id.RotateName()) }, }, diff --git a/sharding/id.go b/shardid/id.go similarity index 80% rename from sharding/id.go rename to shardid/id.go index 250efad..dbd163f 100644 --- a/sharding/id.go +++ b/shardid/id.go @@ -1,4 +1,4 @@ -package sharding +package shardid import ( "fmt" @@ -38,15 +38,15 @@ const ( type TableRotate int8 var ( - None TableRotate = 0 - Monthly TableRotate = 1 - Weekly TableRotate = 2 - Daily TableRotate = 3 + NoRotate TableRotate = 0 + MonthlyRotate TableRotate = 1 + WeeklyRotate TableRotate = 2 + DailyRotate TableRotate = 3 ) type ID struct { Time time.Time - ID int64 + Value int64 TimeMillis int64 Sequence int16 @@ -58,25 +58,27 @@ type ID struct { func (i *ID) RotateName() string { switch i.TableRotate { - case Daily: + case DailyRotate: return i.Time.Format("20060102") - case Weekly: + case WeeklyRotate: _, week := i.Time.ISOWeek() //1-53 week return i.Time.Format("2006") + fmt.Sprintf("%03d", week) - case Monthly: + case MonthlyRotate: return i.Time.Format("200601") default: return "" } } -func Build(timeNow int64, workerID int8, databaseID int16, tr TableRotate, sequence int16) int64 { - return int64(timeNow-TimeEpoch)<>TableShift) & MaxTableShard), DatabaseID: int16(id>>DatabaseShift) & MaxDatabaseID, diff --git a/sharding/id_test.go b/shardid/id_test.go similarity index 66% rename from sharding/id_test.go rename to shardid/id_test.go index a636023..e2824df 100644 --- a/sharding/id_test.go +++ b/shardid/id_test.go @@ -1,4 +1,4 @@ -package sharding +package shardid import ( "fmt" @@ -25,7 +25,7 @@ func TestID(t *testing.T) { timeNow: time.UnixMilli(TimeEpoch), workerID: 0, databaseID: 0, - tableRotate: None, + tableRotate: NoRotate, sequence: 0, }, { @@ -33,7 +33,7 @@ func TestID(t *testing.T) { timeNow: time.UnixMilli(TimeEnd), workerID: MaxWorkerID, databaseID: MaxDatabaseID, - tableRotate: Daily, + tableRotate: DailyRotate, sequence: MaxSequence, }, { @@ -41,7 +41,7 @@ func TestID(t *testing.T) { timeNow: time.Now(), workerID: int8(rand.Intn(4)), databaseID: int16(rand.Intn(1024)), - tableRotate: Weekly, + tableRotate: WeeklyRotate, sequence: int16(rand.Intn(1024)), }, { @@ -49,7 +49,7 @@ func TestID(t *testing.T) { timeNow: time.Now(), workerID: 0, databaseID: 0, - tableRotate: Monthly, + tableRotate: MonthlyRotate, sequence: 0, orderby: true, }, @@ -59,26 +59,24 @@ func TestID(t *testing.T) { t.Run(test.name, func(t *testing.T) { id := Build(test.timeNow.UnixMilli(), test.workerID, test.databaseID, test.tableRotate, test.sequence) - result := Parse(id) - - require.Equal(t, test.timeNow.UnixMilli(), result.Time.UnixMilli()) - require.Equal(t, test.workerID, result.WorkerID) - require.Equal(t, test.databaseID, result.DatabaseID) - require.Equal(t, test.tableRotate, result.TableRotate) - require.Equal(t, test.sequence, result.Sequence) + require.Equal(t, test.timeNow.UnixMilli(), id.Time.UnixMilli()) + require.Equal(t, test.workerID, id.WorkerID) + require.Equal(t, test.databaseID, id.DatabaseID) + require.Equal(t, test.tableRotate, id.TableRotate) + require.Equal(t, test.sequence, id.Sequence) switch test.tableRotate { - case None: - require.Equal(t, "", result.RotateName()) - case Monthly: - require.Equal(t, test.timeNow.UTC().Format("200601"), result.RotateName()) - case Weekly: + case NoRotate: + require.Equal(t, "", id.RotateName()) + case MonthlyRotate: + require.Equal(t, test.timeNow.UTC().Format("200601"), id.RotateName()) + case WeeklyRotate: _, week := test.timeNow.UTC().ISOWeek() - require.Equal(t, test.timeNow.UTC().Format("2006")+fmt.Sprintf("%03d", week), result.RotateName()) - case Daily: - require.Equal(t, test.timeNow.UTC().Format("20060102"), result.RotateName()) + require.Equal(t, test.timeNow.UTC().Format("2006")+fmt.Sprintf("%03d", week), id.RotateName()) + case DailyRotate: + require.Equal(t, test.timeNow.UTC().Format("20060102"), id.RotateName()) default: - require.Equal(t, "", result.RotateName()) + require.Equal(t, "", id.RotateName()) } if test.orderby { @@ -88,9 +86,9 @@ func TestID(t *testing.T) { id4 := Build(test.timeNow.Add(1*time.Millisecond).UnixMilli(), test.workerID, test.databaseID, test.tableRotate, test.sequence+3) - require.Greater(t, id2, id) - require.Greater(t, id3, id2) - require.Greater(t, id4, id3) + require.Greater(t, id2.Value, id.Value) + require.Greater(t, id3.Value, id2.Value) + require.Greater(t, id4.Value, id3.Value) } }) diff --git a/sharding/nocopy.go b/shardid/nocopy.go similarity index 88% rename from sharding/nocopy.go rename to shardid/nocopy.go index be9a08a..4b756f4 100644 --- a/sharding/nocopy.go +++ b/shardid/nocopy.go @@ -1,4 +1,4 @@ -package sharding +package shardid // nolint: unused type noCopy struct{} diff --git a/sharding/option.go b/shardid/option.go similarity index 90% rename from sharding/option.go rename to shardid/option.go index dcd52f8..209e85a 100644 --- a/sharding/option.go +++ b/shardid/option.go @@ -1,4 +1,4 @@ -package sharding +package shardid import ( "time" @@ -24,7 +24,7 @@ func WithDatabase(total int16) Option { func WithTableRotate(ts TableRotate) Option { return func(g *Generator) { - if ts >= None && ts <= Daily { + if ts >= NoRotate && ts <= DailyRotate { g.tableRotate = ts } } diff --git a/sharding/woker.go b/shardid/woker.go similarity index 92% rename from sharding/woker.go rename to shardid/woker.go index c1da974..2098d00 100644 --- a/sharding/woker.go +++ b/shardid/woker.go @@ -1,4 +1,4 @@ -package sharding +package shardid import ( "os" diff --git a/sqlbuilder.go b/sqlbuilder.go index 1f79fcd..4263bd7 100644 --- a/sqlbuilder.go +++ b/sqlbuilder.go @@ -2,8 +2,9 @@ package sqle import ( "errors" - "strconv" "strings" + + "github.com/yaitoo/sqle/shardid" ) var ( @@ -11,6 +12,13 @@ var ( ErrInvalidParamVariable = errors.New("sqle: invalid param variable") ) +var ( + DefaultSQLQuote = "`" + DefaultSQLParameterize = func(name string, index int) string { + return "?" + } +) + type Builder struct { stmt strings.Builder inputs map[string]string @@ -23,13 +31,12 @@ type Builder struct { func New(cmd ...string) *Builder { b := &Builder{ - inputs: make(map[string]string), - params: make(map[string]any), + inputs: make(map[string]string), + params: make(map[string]any), + Quote: DefaultSQLQuote, + Parameterize: DefaultSQLParameterize, } - // MySQL as default - UseMySQL(b) - for i, it := range cmd { if i > 0 { b.stmt.WriteString(" ") @@ -171,23 +178,11 @@ func (b *Builder) Delete(table string) *Builder { return b } -func UsePostgres(b *Builder) { - b.Quote = "`" - b.Parameterize = func(name string, index int) string { - return "$" + strconv.Itoa(index) +func (b *Builder) On(id shardid.ID) *Builder { + rn := id.RotateName() + if rn != "" { + rn = "_" + rn } -} - -func UseMySQL(b *Builder) { - b.Quote = "`" - b.Parameterize = func(name string, index int) string { - return "?" - } -} -func UseOracle(b *Builder) { - b.Quote = "`" - b.Parameterize = func(name string, index int) string { - return ":" + name - } + return b.Input("rotate", rn) } diff --git a/sqlbuilder_test.go b/sqlbuilder_test.go index e7d6924..609e95f 100644 --- a/sqlbuilder_test.go +++ b/sqlbuilder_test.go @@ -6,6 +6,7 @@ import ( "github.com/iancoleman/strcase" "github.com/stretchr/testify/require" + "github.com/yaitoo/sqle/shardid" ) func TestBuilder(t *testing.T) { @@ -443,6 +444,90 @@ func TestBuilder(t *testing.T) { }, }, + { + name: "build_none_rotate_should_work", + build: func() *Builder { + id := shardid.Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, shardid.NoRotate, 0) + b := New().On(id) + b.Delete("orders").Where(). + If(true).And("order_id = {order_id}"). + If(false).And("member_id"). + Param("order_id", "order_123456") + + return b + }, + assert: func(t *testing.T, b *Builder) { + s, vars, err := b.Build() + require.NoError(t, err) + require.Equal(t, "DELETE FROM `orders` WHERE order_id = ?", s) + require.Len(t, vars, 1) + require.Equal(t, "order_123456", vars[0]) + + }, + }, + { + name: "build_monthly_rotate_should_work", + build: func() *Builder { + id := shardid.Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, shardid.MonthlyRotate, 0) + b := New().On(id) + b.Delete("orders").Where(). + If(true).And("order_id = {order_id}"). + If(false).And("member_id"). + Param("order_id", "order_123456") + + return b + }, + assert: func(t *testing.T, b *Builder) { + s, vars, err := b.Build() + require.NoError(t, err) + require.Equal(t, "DELETE FROM `orders_202402` WHERE order_id = ?", s) + require.Len(t, vars, 1) + require.Equal(t, "order_123456", vars[0]) + + }, + }, + { + name: "build_weekly_rotate_should_work", + build: func() *Builder { + id := shardid.Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, shardid.WeeklyRotate, 0) + b := New().On(id) + b.Delete("orders").Where(). + If(true).And("order_id = {order_id}"). + If(false).And("member_id"). + Param("order_id", "order_123456") + + return b + }, + assert: func(t *testing.T, b *Builder) { + s, vars, err := b.Build() + require.NoError(t, err) + require.Equal(t, "DELETE FROM `orders_2024008` WHERE order_id = ?", s) + require.Len(t, vars, 1) + require.Equal(t, "order_123456", vars[0]) + + }, + }, + { + name: "build_daily_rotate_should_work", + build: func() *Builder { + id := shardid.Build(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli(), 0, 0, shardid.DailyRotate, 0) + b := New().On(id) + b.Delete("orders").Where(). + If(true).And("order_id = {order_id}"). + If(false).And("member_id"). + Param("order_id", "order_123456") + + return b + }, + assert: func(t *testing.T, b *Builder) { + s, vars, err := b.Build() + require.NoError(t, err) + require.Equal(t, "DELETE FROM `orders_20240220` WHERE order_id = ?", s) + require.Len(t, vars, 1) + require.Equal(t, "order_123456", vars[0]) + + }, + }, } for _, test := range tests { diff --git a/tx_test.go b/tx_test.go index dfeae70..c549716 100644 --- a/tx_test.go +++ b/tx_test.go @@ -10,11 +10,9 @@ import ( ) func TestTx(t *testing.T) { - d, teardown, err := createSQLite3() - defer teardown() - require.NoError(t, err) + d := createSQLite3() - _, err = d.Exec("CREATE TABLE `users` (`id` int , `status` tinyint,`email` varchar(50),`passwd` varchar(120), `salt` varchar(45), `created` DATETIME, PRIMARY KEY (`id`))") + _, err := d.Exec("CREATE TABLE `users` (`id` int , `status` tinyint,`email` varchar(50),`passwd` varchar(120), `salt` varchar(45), `created` DATETIME, PRIMARY KEY (`id`))") require.NoError(t, err) now := time.Now() diff --git a/use.go b/use.go new file mode 100644 index 0000000..ab62af0 --- /dev/null +++ b/use.go @@ -0,0 +1,24 @@ +package sqle + +import "strconv" + +func UsePostgres(b *Builder) { + b.Quote = "`" + b.Parameterize = func(name string, index int) string { + return "$" + strconv.Itoa(index) + } +} + +func UseMySQL(b *Builder) { + b.Quote = "`" + b.Parameterize = func(name string, index int) string { + return "?" + } +} + +func UseOracle(b *Builder) { + b.Quote = "`" + b.Parameterize = func(name string, index int) string { + return ":" + name + } +}