-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature: Initial version of the gomaxscale library
██████ ██████ ███ ███ █████ ██ ██ ███████ ██████ █████ ██ ███████ ██ ██ ██ ████ ████ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ███ ██ ██ ██ ████ ██ ███████ ███ ███████ ██ ███████ ██ █████ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██████ ██████ ██ ██ ██ ██ ██ ██ ███████ ██████ ██ ██ ███████ ███████
- Loading branch information
0 parents
commit 6d6a435
Showing
9 changed files
with
1,166 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
name: Pull Request | ||
on: pull_request | ||
env: | ||
GO_VERSION: "1.17.x" | ||
jobs: | ||
lint: | ||
runs-on: ubuntu-latest | ||
|
||
steps: | ||
- uses: actions/checkout@v2 | ||
with: | ||
persist-credentials: false | ||
|
||
- uses: actions/setup-go@v2 | ||
with: | ||
go-version: ${{ env.GO_VERSION }} | ||
|
||
- uses: golangci/golangci-lint-action@v2 | ||
with: | ||
version: v1.44 | ||
github-token: ${{ secrets.GH_TOKEN }} | ||
skip-go-installation: true | ||
skip-pkg-cache: false | ||
skip-build-cache: false | ||
only-new-issues: true # only check files change in the PR |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
# Binaries for programs and plugins | ||
*.exe | ||
*.exe~ | ||
*.dll | ||
*.so | ||
*.dylib | ||
|
||
# Test binary, built with `go test -c` | ||
*.test | ||
|
||
# Output of the go coverage tool, specifically when used with LiteIDE | ||
*.out | ||
|
||
# Dependency directories (remove the comment below to include it) | ||
# vendor/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,62 @@ | ||
# https://golangci-lint.run/usage/configuration/ | ||
|
||
# options for analysis running | ||
run: | ||
# timeout for analysis, e.g. 30s, 5m, default is 1m | ||
timeout: 4m | ||
|
||
# include test files or not, default is true | ||
tests: true | ||
|
||
# which dirs to skip: issues from them won't be reported; | ||
# can use regexp here: generated.*, regexp is applied on full path; | ||
# default value is empty list, but default dirs are skipped independently | ||
# from this option's value (see skip-dirs-use-default). | ||
# "/" will be replaced by current OS file path separator to properly work | ||
# on Windows. | ||
skip-dirs: | ||
- testdata | ||
|
||
# which files to skip: they will be analyzed, but issues from them | ||
# won't be reported. Default value is empty list, but there is | ||
# no need to include all autogenerated files, we confidently recognize | ||
# autogenerated files. If it's not please let us know. | ||
# "/" will be replaced by current OS file path separator to properly work | ||
# on Windows. | ||
skip-files: | ||
- ".*\\.sql\\.go$" | ||
- ".*\\.html\\.go$" | ||
- ".*\\.peg\\.go$" | ||
- ".*\\_gen\\.go$" | ||
|
||
linters: | ||
disable-all: true | ||
enable: | ||
- govet | ||
- revive | ||
- varcheck | ||
- structcheck | ||
- errcheck | ||
- staticcheck | ||
- ineffassign | ||
- unconvert | ||
- goimports | ||
- misspell | ||
- lll | ||
- nakedret | ||
- gocritic | ||
- deadcode | ||
|
||
# all available settings of specific linters | ||
linters-settings: | ||
lll: | ||
# max line length, lines longer will be reported. Default is 120. | ||
# '\t' is counted as 1 character by default, and can be changed with the tab-width option | ||
line-length: 120 | ||
|
||
issues: | ||
# Independently from option `exclude` we use default exclude patterns, | ||
# it can be disabled by this option. To list all | ||
# excluded by default patterns execute `golangci-lint run --help`. | ||
# Default value for this option is true. | ||
exclude-use-default: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
MIT License | ||
|
||
Copyright (c) 2022 Rafael Dantas Justo | ||
|
||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
|
||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
|
||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,210 @@ | ||
# gomaxscale | ||
|
||
[![Go Reference](https://pkg.go.dev/badge/github.com/rafaeljusto/gomaxscale.svg)](https://pkg.go.dev/github.com/rafaeljusto/gomaxscale) | ||
[![license](http://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/rafaeljusto/gomaxscale/master/LICENSE) | ||
|
||
Go library that allows consuming from [MaxScale](https://mariadb.com/kb/en/maxscale/) | ||
CDC listener. Useful for detecting database changes via binlog. | ||
|
||
This consumer follows the connection protocol defined by MaxScale 6 | ||
[here](https://mariadb.com/kb/en/mariadb-maxscale-6-change-data-capture-cdc-protocol/). | ||
|
||
## Testing with Docker | ||
|
||
For this test environment the following file structure was used: | ||
|
||
* 📂 `mariadb-config` | ||
- 📄 `mariadb.cnf` | ||
* 📂 `mariadb-init` | ||
- 📄 `00_schema.sql` | ||
* 📂 `maxscale-config` | ||
- 📄 `maxscale.cnf` | ||
* 📄 `docker-compose.yml` | ||
* 📄 `consumer.go` | ||
|
||
### mariadb.cnf | ||
|
||
We need to enable replication in the MariaDB master database: | ||
```dosini | ||
[mysqld] | ||
server_id=1 | ||
binlog_format=row | ||
binlog_row_image=full | ||
log-bin=/var/log/mysql/mariadb-bin | ||
``` | ||
|
||
### 00_schema.sql | ||
|
||
A basic schema adding the MaxScale user, and some testing database to play with: | ||
|
||
```sql | ||
RESET MASTER; | ||
|
||
CREATE USER 'maxuser'@'%' IDENTIFIED BY 'maxpwd'; | ||
GRANT REPLICATION SLAVE ON *.* TO 'maxuser'@'%'; | ||
GRANT REPLICATION CLIENT ON *.* TO 'maxuser'@'%'; | ||
-- GRANT FILE ON *.* TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.user TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.db TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.tables_priv TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.columns_priv TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.procs_priv TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.proxies_priv TO 'maxuser'@'%'; | ||
GRANT SELECT ON mysql.roles_mapping TO 'maxuser'@'%'; | ||
GRANT SHOW DATABASES ON *.* TO 'maxuser'@'%'; | ||
-- GRANT SELECT ON *.* TO 'maxuser'@'%'; | ||
|
||
DROP DATABASE IF EXISTS example; | ||
CREATE DATABASE IF NOT EXISTS example; | ||
|
||
USE example; | ||
|
||
DROP TABLE IF EXISTS users; | ||
CREATE TABLE users ( | ||
`id` INT NOT NULL AUTO_INCREMENT, | ||
`name` VARCHAR(255) NOT NULL, | ||
`email` VARCHAR(255) NOT NULL, | ||
PRIMARY KEY (id) | ||
); | ||
|
||
INSERT INTO `users` (`name`, `email`) VALUES ('John Doe', '[email protected]'); | ||
INSERT INTO `users` (`name`, `email`) VALUES ('Jane Doe', '[email protected]'); | ||
``` | ||
|
||
### maxscale.cnf | ||
|
||
MaxScale configuration to configure the [Avro router](https://mariadb.com/kb/en/mariadb-maxscale-6-avrorouter/) | ||
and expose a listener so `gomaxscale` can retrieve the information. | ||
|
||
```dosini | ||
[MaxScale] | ||
threads=1 | ||
admin_secure_gui=false | ||
threads=auto | ||
admin_host=0.0.0.0 | ||
|
||
[server1] | ||
type=server | ||
address=db | ||
port=3306 | ||
protocol=MariaDBBackend | ||
|
||
[cdc-service] | ||
type=service | ||
router=avrorouter | ||
servers=server1 | ||
user=maxuser | ||
password=maxpwd | ||
|
||
[cdc-listener] | ||
type=listener | ||
service=cdc-service | ||
protocol=CDC | ||
port=4001 | ||
|
||
[MariaDB-Monitor] | ||
type=monitor | ||
module=mariadbmon | ||
servers=server1 | ||
user=maxuser | ||
password=maxpwd | ||
monitor_interval=5000 | ||
``` | ||
|
||
### docker-compose.yml | ||
|
||
To setup a MariaDB database and a MaxScale server we will use docker-compose | ||
with the following configuration: | ||
|
||
```yaml | ||
version: '2.4' | ||
services: | ||
db: | ||
container_name: "lab-db" | ||
image: mariadb:10.3.8 | ||
volumes: | ||
- ./mariadb-config:/etc/mysql/conf.d | ||
- ./mariadb-init:/docker-entrypoint-initdb.d | ||
environment: | ||
MYSQL_ROOT_PASSWORD: abc123 | ||
healthcheck: | ||
test: ["CMD", "mysqladmin", "ping", "--silent"] | ||
|
||
dbproxy: | ||
container_name: "lab-maxscale" | ||
image: mariadb/maxscale:6.2 | ||
volumes: | ||
- ./maxscale-config/maxscale.cnf:/etc/maxscale.cnf | ||
ports: | ||
- 4001:4001 | ||
depends_on: | ||
- db | ||
``` | ||
### consumer.go | ||
A local Go file will consume and log the modified items in the database: | ||
```go | ||
package main | ||
|
||
import ( | ||
"fmt" | ||
"log" | ||
"os" | ||
"os/signal" | ||
"syscall" | ||
|
||
"github.com/rafaeljusto/gomaxscale" | ||
) | ||
|
||
func main() { | ||
consumer := gomaxscale.NewConsumer("127.0.0.1:4001", "example", "users", | ||
gomaxscale.WithAuth("maxuser", "maxpwd"), | ||
) | ||
dataStream, err := consumer.Start() | ||
if err != nil { | ||
log.Fatal(err) | ||
} | ||
defer consumer.Close() | ||
|
||
fmt.Printf("started consuming events from database '%s' table '%s'\n", | ||
dataStream.Database, dataStream.Table) | ||
|
||
done := make(chan bool) | ||
go func() { | ||
consumer.Process(func(event gomaxscale.Event) { | ||
fmt.Printf("event '%s' detected\n", event.Type) | ||
}) | ||
done <- true | ||
}() | ||
|
||
signalChanel := make(chan os.Signal, 1) | ||
signal.Notify(signalChanel, syscall.SIGINT, syscall.SIGTERM) | ||
|
||
select { | ||
case <-signalChanel: | ||
case <-done: | ||
} | ||
|
||
fmt.Println("terminating") | ||
} | ||
``` | ||
|
||
### Running | ||
|
||
First, start all services: | ||
``` | ||
% docker-compose up -d | ||
``` | ||
|
||
Then we can start consuming the items: | ||
``` | ||
% go run consumer.go | ||
``` | ||
|
||
To see the magic happening you could do some database changes: | ||
``` | ||
% docker-compose exec db mysql -u root -p abc123 -D example \ | ||
-e "INSERT INTO users (name, email) VALUES ('James Doe', '[email protected]')" | ||
``` |
Oops, something went wrong.