diff --git a/.circleci/config.yml b/.circleci/config.yml index d36ed9f..dc6163d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -40,8 +40,6 @@ jobs: key: go-mod-v1-{{ arch }}-{{ checksum "go.sum" }} paths: - "/go/pkg/mod" - - run: go get github.com/mattn/goveralls - - run: goveralls -service=circle-ci -coverprofile /tmp/coverage/combined.txt "golang-1_16": <<: *template diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3e60c79..df69cd7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,22 +33,6 @@ jobs: - name: Run test run: make test COVERAGE_DIR=/tmp/coverage - - name: Send goveralls coverage - uses: shogo82148/actions-goveralls@v1 - with: - path-to-profile: /tmp/coverage/combined.txt - flag-name: Go-${{ matrix.go }} - parallel: true - - check-coverage: - name: Check coverage - needs: [test] - runs-on: ubuntu-latest - steps: - - uses: shogo82148/actions-goveralls@v1 - with: - parallel-finished: true - goreleaser: name: Release a new version needs: [lint, test] diff --git a/.travis.yml b/.travis.yml index fdaea8c..f8c6101 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,15 +37,11 @@ before_install: - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0 - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}" -install: - - go get github.com/mattn/goveralls - script: - golangci-lint run - make test COVERAGE_DIR=/tmp/coverage after_success: - - goveralls -service=travis-ci -coverprofile /tmp/coverage/combined.txt - make list-external-deps > dependency_tree.txt && cat dependency_tree.txt - make build-cli - gem install --no-document fpm diff --git a/Makefile b/Makefile index 61e035c..ba94733 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab -DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 +DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 duckdb DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= @@ -7,19 +7,19 @@ REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") COVERAGE_DIR ?= .coverage build: - CGO_ENABLED=0 go build -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' ./cmd/migrate + CGO_ENABLED=1 go build -ldflags='-X main.Version=$(VERSION)' -tags '$(DATABASE) $(SOURCE)' ./cmd/migrate build-docker: - CGO_ENABLED=0 go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$(DATABASE) $(SOURCE)" ./cmd/migrate + CGO_ENABLED=1 go build -a -o build/migrate.linux-386 -ldflags="-s -w -X main.Version=${VERSION}" -tags "$(DATABASE) $(SOURCE)" ./cmd/migrate build-cli: clean -mkdir ./cli/build - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -a -o ../../cli/build/migrate.windows-386.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . - cd ./cmd/migrate && CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -o ../../cli/build/migrate.linux-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=arm GOARM=7 go build -a -o ../../cli/build/migrate.linux-armv7 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=linux GOARCH=arm64 go build -a -o ../../cli/build/migrate.linux-arm64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=darwin GOARCH=amd64 go build -a -o ../../cli/build/migrate.darwin-amd64 -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=windows GOARCH=386 go build -a -o ../../cli/build/migrate.windows-386.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . + cd ./cmd/migrate && CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -o ../../cli/build/migrate.windows-amd64.exe -ldflags='-X main.Version=$(VERSION) -extldflags "-static"' -tags '$(DATABASE) $(SOURCE)' . cd ./cli/build && find . -name 'migrate*' | xargs -I{} tar czf {}.tar.gz {} cd ./cli/build && shasum -a 256 * > sha256sum.txt cat ./cli/build/sha256sum.txt diff --git a/database/duckdb/duckdb.go b/database/duckdb/duckdb.go new file mode 100644 index 0000000..6d4a09f --- /dev/null +++ b/database/duckdb/duckdb.go @@ -0,0 +1,165 @@ +package duckdb + +import ( + "database/sql" + "fmt" + "io" + nurl "net/url" + "strings" + + "go.uber.org/atomic" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/marcboeker/go-duckdb" +) + +func init() { + database.Register("duckdb", &DuckDB{}) +} + +const MigrationTable = "gmg_schema_migrations" + +type DuckDB struct { + db *sql.DB + isLocked atomic.Bool +} + +func (d *DuckDB) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, fmt.Errorf("parsing url: %w", err) + } + dbfile := strings.Replace(migrate.FilterCustomQuery(purl).String(), "duckdb://", "", 1) + db, err := sql.Open("duckdb", dbfile) + if err != nil { + return nil, fmt.Errorf("opening '%s': %w", dbfile, err) + } + + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("pinging: %w", err) + } + d.db = db + + if err := d.ensureVersionTable(); err != nil { + return nil, fmt.Errorf("ensuring version table: %w", err) + } + + return d, nil +} + +func (d *DuckDB) Close() error { + return d.db.Close() +} + +func (d *DuckDB) Lock() error { + if !d.isLocked.CAS(false, true) { + return database.ErrLocked + } + return nil +} + +func (d *DuckDB) Unlock() error { + if !d.isLocked.CAS(true, false) { + return database.ErrNotLocked + } + return nil +} + +func (d *DuckDB) Drop() error { + // FIXME: implement + return fmt.Errorf("drop unimplemented because of duckdb size problems and not enough time during prototyping") +} + +func (d *DuckDB) SetVersion(version int, dirty bool) error { + tx, err := d.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "DELETE FROM " + MigrationTable + if _, err := tx.Exec(query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + // Also re-write the schema version for nil dirty versions to prevent + // empty schema version for failed down migration on the first migration + // See: https://github.com/golang-migrate/migrate/issues/330 + // + // NOTE: Copied from sqlite implementation, unsure if this is necessary for + // duckdb + if version >= 0 || (version == database.NilVersion && dirty) { + query := fmt.Sprintf(`INSERT INTO %s (version, dirty) VALUES (?, ?)`, MigrationTable) + if _, err := tx.Exec(query, version, dirty); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (m *DuckDB) Version() (version int, dirty bool, err error) { + query := "SELECT version, dirty FROM " + MigrationTable + " LIMIT 1" + err = m.db.QueryRow(query).Scan(&version, &dirty) + if err != nil { + return database.NilVersion, false, nil + } + return version, dirty, nil +} + +func (d *DuckDB) Run(migration io.Reader) error { + migr, err := io.ReadAll(migration) + if err != nil { + return fmt.Errorf("reading migration: %w", err) + } + query := string(migr[:]) + + tx, err := d.db.Begin() + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Sqlite type. +func (d *DuckDB) ensureVersionTable() (err error) { + if err = d.Lock(); err != nil { + return err + } + + defer func() { + if e := d.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s (version BIGINT, dirty BOOLEAN);`, MigrationTable) + + if _, err := d.db.Exec(query); err != nil { + return fmt.Errorf("creating version table via '%s': %w", query, err) + } + return nil +} diff --git a/go.mod b/go.mod index da117ac..87898d0 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/jackc/pgx/v5 v5.3.1 github.com/ktrysmt/go-bitbucket v0.6.4 github.com/lib/pq v1.10.2 + github.com/marcboeker/go-duckdb v1.4.1 github.com/markbates/pkger v0.15.1 github.com/mattn/go-sqlite3 v1.14.16 github.com/microsoft/go-mssqldb v1.0.0 @@ -129,7 +130,7 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/mtibben/percent v0.2.1 // indirect diff --git a/go.sum b/go.sum index 700aff2..60deae0 100644 --- a/go.sum +++ b/go.sum @@ -446,6 +446,8 @@ github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/marcboeker/go-duckdb v1.4.1 h1:NJ0kfgtOD8QUADp6Pwe/9f3e4qANet6m/YHXhx+3das= +github.com/marcboeker/go-duckdb v1.4.1/go.mod h1:wm91jO2GNKa6iO9NTcjXIRsW+/ykPoJbQcHSXhdAl28= github.com/markbates/pkger v0.15.1 h1:3MPelV53RnGSW07izx5xGxl4e/sdRD6zqseIk0rMASY= github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -472,8 +474,8 @@ github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcs github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= diff --git a/internal/cli/build_duckdb.go b/internal/cli/build_duckdb.go new file mode 100644 index 0000000..21edf87 --- /dev/null +++ b/internal/cli/build_duckdb.go @@ -0,0 +1,8 @@ +//go:build duckdb +// +build duckdb + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/duckdb" +)