[CLI] Add database migration tooling

Signed-off-by: Knut Ahlers <[email protected]>

README.md

@@ -36,11 +36,12 @@ Usage of twitch-bot:
 
 # twitch-bot help
 Supported sub-commands are:
-  actor-docs                                 Generate markdown documentation for available actors
-  api-token <token-name> <scope> [...scope]  Generate an api-token to be entered into the config
-  reset-secrets                              Remove encrypted data to reset encryption passphrase
-  tpl-docs                                   Generate markdown documentation for available template functions
-  validate-config                            Try to load configuration file and report errors if any
+  actor-docs                                        Generate markdown documentation for available actors
+  api-token <token-name> <scope> [...scope]         Generate an api-token to be entered into the config
+  copy-database <target storage-type> <target DSN>  Copies database contents to a new storage DSN i.e. for migrating to a new DBMS
+  reset-secrets                                     Remove encrypted data to reset encryption passphrase
+  tpl-docs                                          Generate markdown documentation for available template functions
+  validate-config                                   Try to load configuration file and report errors if any
 ```
 
 ### Database Connection Strings

cli_migrateDatabase.go

@@ -0,0 +1,67 @@
+package main
+
+import (
+	"sync"
+
+	"github.com/Luzifer/twitch-bot/v3/pkg/database"
+	"github.com/Luzifer/twitch-bot/v3/plugins"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
+)
+
+var (
+	dbCopyFuncs     = map[string]plugins.DatabaseCopyFunc{}
+	dbCopyFuncsLock sync.Mutex
+)
+
+func init() {
+	cli.Add(cliRegistryEntry{
+		Name:        "copy-database",
+		Description: "Copies database contents to a new storage DSN i.e. for migrating to a new DBMS",
+		Params:      []string{"<target storage-type>", "<target DSN>"},
+		Run: func(args []string) error {
+			if len(args) < 3 { //nolint:gomnd // Just a count of parameters
+				return errors.New("Usage: twitch-bot copy-database <target storage-type> <target DSN>")
+			}
+
+			// Core functions cannot register themselves, we take that for them
+			registerDatabaseCopyFunc("core-values", db.CopyDatabase)
+			registerDatabaseCopyFunc("permissions", accessService.CopyDatabase)
+			registerDatabaseCopyFunc("timers", timerService.CopyDatabase)
+
+			targetDB, err := database.New(args[1], args[2], cfg.StorageEncryptionPass)
+			if err != nil {
+				return errors.Wrap(err, "connecting to target db")
+			}
+			defer func() {
+				if err := targetDB.Close(); err != nil {
+					logrus.WithError(err).Error("closing connection to target db")
+				}
+			}()
+
+			return errors.Wrap(
+				targetDB.DB().Transaction(func(tx *gorm.DB) (err error) {
+					for name, dbcf := range dbCopyFuncs {
+						logrus.WithField("name", name).Info("running migration")
+						if err = dbcf(db.DB(), tx); err != nil {
+							return errors.Wrapf(err, "running DatabaseCopyFunc %q", name)
+						}
+					}
+
+					logrus.Info("database has been copied successfully")
+
+					return nil
+				}),
+				"copying database to target",
+			)
+		},
+	})
+}
+
+func registerDatabaseCopyFunc(name string, fn plugins.DatabaseCopyFunc) {
+	dbCopyFuncsLock.Lock()
+	defer dbCopyFuncsLock.Unlock()
+
+	dbCopyFuncs[name] = fn
+}

internal/actors/counter/actor.go

@@ -9,6 +9,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"gopkg.in/irc.v4"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/plugins"
@@ -28,6 +29,10 @@ func Register(args plugins.RegistrationArguments) error {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("counter", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &Counter{})
+	})
+
 	formatMessage = args.FormatMessage
 
 	args.RegisterActor("counter", func() plugins.Actor { return &ActorCounter{} })

internal/actors/punish/actor.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pkg/errors"
 	"gopkg.in/irc.v4"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/pkg/twitch"
@@ -34,6 +35,10 @@ func Register(args plugins.RegistrationArguments) error {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("punish", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &punishLevel{})
+	})
+
 	botTwitchClient = args.GetTwitchClient()
 	formatMessage = args.FormatMessage
 

internal/actors/quotedb/actor.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/pkg/errors"
 	"gopkg.in/irc.v4"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/plugins"
@@ -30,6 +31,10 @@ func Register(args plugins.RegistrationArguments) error {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("quote", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &quote{})
+	})
+
 	formatMessage = args.FormatMessage
 	send = args.SendMessage
 

internal/actors/variables/actor.go

@@ -7,6 +7,7 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"gopkg.in/irc.v4"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/plugins"
@@ -27,6 +28,10 @@ func Register(args plugins.RegistrationArguments) error {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("variable", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &variable{})
+	})
+
 	formatMessage = args.FormatMessage
 
 	args.RegisterActor("setvariable", func() plugins.Actor { return &ActorSetVariable{} })

internal/apimodules/customevent/customevent.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/plugins"
@@ -32,6 +33,10 @@ func Register(args plugins.RegistrationArguments) error {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("custom_event", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &storedCustomEvent{})
+	})
+
 	mc = &memoryCache{dbc: db}
 
 	eventCreatorFunc = args.CreateEvent

internal/apimodules/overlays/database.go

@@ -14,6 +14,7 @@ import (
 
 type (
 	overlaysEvent struct {
+		ID        uint64    `gorm:"primaryKey"`
 		Channel   string    `gorm:"not null;index:overlays_events_sort_idx"`
 		CreatedAt time.Time `gorm:"index:overlays_events_sort_idx"`
 		EventType string
@@ -28,7 +29,7 @@ func AddChannelEvent(db database.Connector, channel string, evt SocketMessage) e
 	}
 
 	return errors.Wrap(
-		db.DB().Create(overlaysEvent{
+		db.DB().Create(&overlaysEvent{
 			Channel:   channel,
 			CreatedAt: evt.Time.UTC(),
 			EventType: evt.Type,

internal/apimodules/overlays/overlays.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gorilla/websocket"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/go_helpers/v2/str"
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
@@ -69,6 +70,10 @@ func Register(args plugins.RegistrationArguments) error {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("overlay_events", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &overlaysEvent{})
+	})
+
 	validateToken = args.ValidateToken
 
 	args.RegisterAPIRoute(plugins.HTTPRouteRegistrationArgs{

internal/apimodules/raffle/raffle.go

@@ -5,6 +5,7 @@ package raffle
 import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/pkg/twitch"
@@ -28,6 +29,10 @@ func Register(args plugins.RegistrationArguments) (err error) {
 		return errors.Wrap(err, "applying schema migration")
 	}
 
+	args.RegisterCopyDatabaseFunc("raffle", func(src, target *gorm.DB) error {
+		return database.CopyObjects(src, target, &raffle{}, &raffleEntry{})
+	})
+
 	dbc = newDBClient(db)
 	if err = dbc.RefreshActiveRaffles(); err != nil {
 		return errors.Wrap(err, "refreshing active raffle cache")

internal/service/access/access.go

@@ -47,6 +47,10 @@ func New(db database.Connector) (*Service, error) {
 	)
 }
 
+func (s *Service) CopyDatabase(src, target *gorm.DB) error {
+	return database.CopyObjects(src, target, &extendedPermission{})
+}
+
 func (s Service) GetBotUsername() (string, error) {
 	var botUsername string
 	err := s.db.ReadCoreMeta(coreMetaKeyBotUsername, &botUsername)

internal/service/timer/timer.go

@@ -44,6 +44,10 @@ func New(db database.Connector, cronService *cron.Cron) (*Service, error) {
 	return s, errors.Wrap(s.db.DB().AutoMigrate(&timer{}), "applying migrations")
 }
 
+func (s *Service) CopyDatabase(src, target *gorm.DB) error {
+	return database.CopyObjects(src, target, &timer{})
+}
+
 func (s *Service) UpdatePermitTimeout(d time.Duration) {
 	s.permitTimeout = d
 }

pkg/database/connector.go

@@ -42,7 +42,7 @@ func New(driverName, connString, encryptionSecret string) (Connector, error) {
 
 	switch driverName {
 	case "mysql":
-		mysqlDriver.SetLogger(newLogrusLogWriterWithLevel(logrus.ErrorLevel, driverName))
+		mysqlDriver.SetLogger(NewLogrusLogWriterWithLevel(logrus.StandardLogger(), logrus.ErrorLevel, driverName))
 		innerDB = mysql.Open(connString)
 		dbTuner = tuneMySQLDatabase
 
@@ -63,7 +63,13 @@ func New(driverName, connString, encryptionSecret string) (Connector, error) {
 
 	db, err := gorm.Open(innerDB, &gorm.Config{
 		DisableForeignKeyConstraintWhenMigrating: true,
-		Logger:                                   logger.New(newLogrusLogWriterWithLevel(logrus.TraceLevel, driverName), logger.Config{}),
+		Logger: logger.New(NewLogrusLogWriterWithLevel(logrus.StandardLogger(), logrus.TraceLevel, driverName), logger.Config{
+			SlowThreshold:             time.Second,
+			Colorful:                  false,
+			IgnoreRecordNotFoundError: false,
+			ParameterizedQueries:      false,
+			LogLevel:                  logger.Info,
+		}),
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "connecting database")
@@ -87,6 +93,10 @@ func (c connector) Close() error {
 	return nil
 }
 
+func (c connector) CopyDatabase(src, target *gorm.DB) error {
+	return CopyObjects(src, target, &coreKV{})
+}
+
 func (c connector) DB() *gorm.DB {
 	return c.db
 }

pkg/database/copyhelper.go

@@ -0,0 +1,42 @@
+package database
+
+import (
+	"reflect"
+
+	"github.com/pkg/errors"
+	"gorm.io/gorm"
+)
+
+const copyBatchSize = 100
+
+// CopyObjects is a helper to copy elements of a given type from the
+// src to the target GORM database interface
+func CopyObjects(src, target *gorm.DB, objects ...any) (err error) {
+	for _, obj := range objects {
+		copySlice := reflect.New(reflect.SliceOf(reflect.TypeOf(obj))).Elem().Addr().Interface()
+
+		if err = target.AutoMigrate(obj); err != nil {
+			return errors.Wrap(err, "applying migration to target")
+		}
+
+		if err = target.Where("1 = 1").Delete(obj).Error; err != nil {
+			return errors.Wrap(err, "cleaning target table")
+		}
+
+		if err = src.FindInBatches(copySlice, copyBatchSize, func(tx *gorm.DB, _ int) error {
+			if err = target.Save(copySlice).Error; err != nil {
+				if errors.Is(err, gorm.ErrEmptySlice) {
+					// That's fine and no reason to exit here
+					return nil
+				}
+				return errors.Wrap(err, "inserting collected elements")
+			}
+
+			return nil
+		}).Error; err != nil {
+			return errors.Wrap(err, "batch-copying data")
+		}
+	}
+
+	return nil
+}

pkg/database/database.go

@@ -11,6 +11,7 @@ type (
 	// convenience methods
 	Connector interface {
 		Close() error
+		CopyDatabase(src, target *gorm.DB) error
 		DB() *gorm.DB
 		DeleteCoreMeta(key string) error
 		ReadCoreMeta(key string, value any) error

pkg/database/logger.go

@@ -8,18 +8,18 @@ import (
 )
 
 type (
-	logWriter struct{ io.Writer }
+	LogWriter struct{ io.Writer }
 )
 
-func newLogrusLogWriterWithLevel(level logrus.Level, dbDriver string) logWriter {
-	writer := logrus.WithField("database", dbDriver).WriterLevel(level)
-	return logWriter{writer}
+func NewLogrusLogWriterWithLevel(logger *logrus.Logger, level logrus.Level, dbDriver string) LogWriter {
+	writer := logger.WithField("database", dbDriver).WriterLevel(level)
+	return LogWriter{writer}
 }
 
-func (l logWriter) Print(a ...any) {
+func (l LogWriter) Print(a ...any) {
 	fmt.Fprint(l.Writer, a...)
 }
 
-func (l logWriter) Printf(format string, a ...any) {
+func (l LogWriter) Printf(format string, a ...any) {
 	fmt.Fprintf(l.Writer, format, a...)
 }

plugins/interface.go

@@ -5,6 +5,7 @@ import (
 	"github.com/robfig/cron/v3"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/irc.v4"
+	"gorm.io/gorm"
 
 	"github.com/Luzifer/twitch-bot/v3/pkg/database"
 	"github.com/Luzifer/twitch-bot/v3/pkg/twitch"
@@ -38,6 +39,8 @@ type (
 
 	CronRegistrationFunc func(spec string, cmd func()) (cron.EntryID, error)
 
+	DatabaseCopyFunc func(src, target *gorm.DB) error
+
 	EventHandlerFunc         func(evt string, eventData *FieldCollection) error
 	EventHandlerRegisterFunc func(EventHandlerFunc) error
 
@@ -83,6 +86,10 @@ type (
 		RegisterActorDocumentation ActorDocumentationRegistrationFunc
 		// RegisterAPIRoute registers a new HTTP handler function including documentation
 		RegisterAPIRoute HTTPRouteRegistrationFunc
+		// RegisterCopyDatabaseFunc registers a DatabaseCopyFunc for the
+		// database migration tool. Modules not registering such a func
+		// will not be copied over when migrating to another database.
+		RegisterCopyDatabaseFunc func(name string, fn DatabaseCopyFunc)
 		// RegisterCron is a method to register cron functions in the global cron instance
 		RegisterCron CronRegistrationFunc
 		// RegisterEventHandler is a method to register a handler function receiving ALL events