Merge pull request #300 from Slach/main

add additional options for clickhouse support
clidey · Jan 8, 2025 · 1411fcf · 1411fcf
2 parents f5330af + 95026e8
commit 1411fcf
Show file tree

Hide file tree

Showing 11 changed files with 4,624 additions and 3,726 deletions.
diff --git a/core/src/plugins/clickhouse/add.go b/core/src/plugins/clickhouse/add.go
@@ -72,7 +72,7 @@ func (p *ClickHousePlugin) AddStorageUnit(config *engine.PluginConfig, schema st
 		query += fmt.Sprintf("\nSETTINGS %s", strings.Join(settingsClauses, ", "))
 	}
 
-	err = conn.Exec(context.Background(), query)
+	_, err = conn.ExecContext(context.Background(), query)
 	if err != nil {
 		return false, fmt.Errorf("failed to create table: %w (query: %s)", err, query)
 	}
@@ -100,6 +100,6 @@ func (p *ClickHousePlugin) AddRow(config *engine.PluginConfig, schema string, st
 	query := fmt.Sprintf("INSERT INTO %s.%s (%s) VALUES (%s)",
 		schema, storageUnit, strings.Join(columns, ", "), strings.Join(placeholders, ", "))
 
-	err = conn.Exec(context.Background(), query, args...)
+	_, err = conn.ExecContext(context.Background(), query, args...)
 	return err == nil, err
 }
diff --git a/core/src/plugins/clickhouse/clickhouse.go b/core/src/plugins/clickhouse/clickhouse.go
@@ -2,8 +2,8 @@ package clickhouse
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
-	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"strconv"
 
 	"github.com/clidey/whodb/core/src/engine"
@@ -17,7 +17,7 @@ func (p *ClickHousePlugin) IsAvailable(config *engine.PluginConfig) bool {
 		return false
 	}
 	defer conn.Close()
-	return conn.Ping(context.Background()) == nil
+	return conn.PingContext(context.Background()) == nil
 }
 
 func (p *ClickHousePlugin) GetDatabases(config *engine.PluginConfig) ([]string, error) {
@@ -27,7 +27,7 @@ func (p *ClickHousePlugin) GetDatabases(config *engine.PluginConfig) ([]string,
 	}
 	defer conn.Close()
 
-	rows, err := conn.Query(context.Background(), "SHOW DATABASES")
+	rows, err := conn.QueryContext(context.Background(), "SHOW DATABASES")
 	if err != nil {
 		return nil, err
 	}
@@ -66,7 +66,7 @@ func (p *ClickHousePlugin) GetStorageUnits(config *engine.PluginConfig, schema s
 		WHERE database = '%s'
 	`, schema)
 
-	rows, err := conn.Query(context.Background(), query)
+	rows, err := conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return nil, err
 	}
@@ -102,7 +102,7 @@ func (p *ClickHousePlugin) GetStorageUnits(config *engine.PluginConfig, schema s
 	return storageUnits, nil
 }
 
-func getTableSchema(conn driver.Conn, schema string, tableName string) ([]engine.Record, error) {
+func getTableSchema(conn *sql.DB, schema string, tableName string) ([]engine.Record, error) {
 	query := fmt.Sprintf(`
 		SELECT 
 			name,
@@ -112,7 +112,7 @@ func getTableSchema(conn driver.Conn, schema string, tableName string) ([]engine
 		ORDER BY position
 	`, schema, tableName)
 
-	rows, err := conn.Query(context.Background(), query)
+	rows, err := conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return nil, err
 	}

diff --git a/core/src/plugins/clickhouse/db.go b/core/src/plugins/clickhouse/db.go
@@ -2,7 +2,11 @@ package clickhouse
 
 import (
 	"context"
+	"crypto/tls"
+	"database/sql"
 	"fmt"
+	"github.com/clidey/whodb/core/src/log"
+	"strings"
 	"time"
 
 	"github.com/ClickHouse/clickhouse-go/v2"
@@ -11,29 +15,61 @@ import (
 	"github.com/clidey/whodb/core/src/engine"
 )
 
-func DB(config *engine.PluginConfig) (driver.Conn, error) {
-	port := common.GetRecordValueOrDefault(config.Credentials.Advanced, "Port", "9000")
+const (
+	portKey         = "Port"
+	sslModeKey      = "SSL Mode"
+	httpProtocolKey = "HTTP Protocol"
+	readOnlyKey     = "Readonly"
+	debugKey        = "Debug"
+)
+
+func DB(config *engine.PluginConfig) (*sql.DB, error) {
+	port := common.GetRecordValueOrDefault(config.Credentials.Advanced, portKey, "9000")
+	sslMode := common.GetRecordValueOrDefault(config.Credentials.Advanced, sslModeKey, "disable")
+	httpProtocol := common.GetRecordValueOrDefault(config.Credentials.Advanced, httpProtocolKey, "disable")
+	readOnly := common.GetRecordValueOrDefault(config.Credentials.Advanced, readOnlyKey, "disable")
+	debug := common.GetRecordValueOrDefault(config.Credentials.Advanced, debugKey, "disable")
 	options := &clickhouse.Options{
 		Addr: []string{fmt.Sprintf("%s:%s", config.Credentials.Hostname, port)},
 		Auth: clickhouse.Auth{
 			Database: config.Credentials.Database,
 			Username: config.Credentials.Username,
 			Password: config.Credentials.Password,
 		},
-		Settings: clickhouse.Settings{
-			"max_execution_time": 60,
-		},
 		DialTimeout:      time.Second * 30,
-		MaxOpenConns:     5,
-		MaxIdleConns:     5,
-		ConnMaxLifetime:  time.Hour,
 		ConnOpenStrategy: clickhouse.ConnOpenInOrder,
-		Compression: &clickhouse.Compression{
+	}
+	if debug != "disable" {
+		options.Debug = true
+	}
+	if readOnly == "disable" {
+		options.Settings = clickhouse.Settings{
+			"max_execution_time": 60,
+		}
+	}
+	if strings.HasPrefix(port, "8") || httpProtocol != "disable" {
+		options.Protocol = clickhouse.HTTP
+		options.Compression = &clickhouse.Compression{
+			Method: clickhouse.CompressionGZIP,
+		}
+	} else {
+		options.Compression = &clickhouse.Compression{
 			Method: clickhouse.CompressionLZ4,
-		},
+		}
+		options.MaxOpenConns = 5
+		options.MaxIdleConns = 5
+		options.ConnMaxLifetime = time.Hour
+	}
+	if sslMode != "disable" {
+		options.TLS = &tls.Config{InsecureSkipVerify: sslMode == "relaxed" || sslMode == "none"}
 	}
 
-	return clickhouse.Open(options)
+	conn := clickhouse.OpenDB(options)
+	err := conn.PingContext(context.Background())
+	if err != nil {
+		log.Logger.Warnf("clickhouse.Ping() error: %v", err)
+	}
+	return conn, err
 }
 
 func getTableColumns(conn driver.Conn, schema, table string) ([]engine.Record, error) {

diff --git a/core/src/plugins/clickhouse/delete.go b/core/src/plugins/clickhouse/delete.go
@@ -3,12 +3,17 @@ package clickhouse
 import (
 	"context"
 	"fmt"
+	"github.com/clidey/whodb/core/src/common"
 	"strings"
 
 	"github.com/clidey/whodb/core/src/engine"
 )
 
 func (p *ClickHousePlugin) DeleteRow(config *engine.PluginConfig, schema string, storageUnit string, values map[string]string) (bool, error) {
+	readOnly := common.GetRecordValueOrDefault(config.Credentials.Advanced, readOnlyKey, "disable")
+	if readOnly != "disable" {
+		return false, fmt.Errorf("readonly mode don't allow DeleteRow")
+	}
 	conn, err := DB(config)
 	if err != nil {
 		return false, err
@@ -68,7 +73,7 @@ func (p *ClickHousePlugin) DeleteRow(config *engine.PluginConfig, schema string,
 		strings.Join(whereClauses, " AND "))
 
 	// Execute the query
-	err = conn.Exec(context.Background(), query, args...)
+	_, err = conn.ExecContext(context.Background(), query, args...)
 	if err != nil {
 		return false, fmt.Errorf("delete failed: %w (query: %s, args: %+v)", err, query, args)
 	}

diff --git a/core/src/plugins/clickhouse/query.go b/core/src/plugins/clickhouse/query.go
@@ -28,13 +28,16 @@ func (p *ClickHousePlugin) executeQuery(config *engine.PluginConfig, query strin
 	}
 	defer conn.Close()
 
-	rows, err := conn.Query(context.Background(), query, params)
+	rows, err := conn.QueryContext(context.Background(), query, params)
 	if err != nil {
 		return nil, err
 	}
 	defer rows.Close()
 
-	columnTypes := rows.ColumnTypes()
+	columnTypes, err := rows.ColumnTypes()
+	if err != nil {
+		return nil, err
+	}
 	result := &engine.GetRowsResult{
 		Columns: make([]engine.Column, len(columnTypes)),
 		Rows:    [][]string{},

diff --git a/core/src/plugins/clickhouse/update.go b/core/src/plugins/clickhouse/update.go
@@ -3,12 +3,17 @@ package clickhouse
 import (
 	"context"
 	"fmt"
+	"github.com/clidey/whodb/core/src/common"
 	"github.com/clidey/whodb/core/src/engine"
 	"strings"
 	"time"
 )
 
 func (p *ClickHousePlugin) UpdateStorageUnit(config *engine.PluginConfig, schema string, storageUnit string, values map[string]string) (bool, error) {
+	readOnly := common.GetRecordValueOrDefault(config.Credentials.Advanced, readOnlyKey, "disable")
+	if readOnly != "disable" {
+		return false, fmt.Errorf("readonly mode don't allow UpdateStorageUnit")
+	}
 	conn, err := DB(config)
 	if err != nil {
 		return false, err
@@ -99,7 +104,7 @@ func (p *ClickHousePlugin) UpdateStorageUnit(config *engine.PluginConfig, schema
 		strings.Join(setClauses, ", "),
 		strings.Join(whereClauses, " AND "))
 
-	err = conn.Exec(context.Background(), query, args...)
+	_, err = conn.ExecContext(context.Background(), query, args...)
 	if err != nil {
 		return false, fmt.Errorf("update failed: %w", err)
 	}

diff --git a/core/src/plugins/clickhouse/utils.go b/core/src/plugins/clickhouse/utils.go
@@ -2,14 +2,14 @@ package clickhouse
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
-	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
 	"strconv"
 	"strings"
 	"time"
 )
 
-func getColumnTypes(conn driver.Conn, schema string, table string) (map[string]string, error) {
+func getColumnTypes(conn *sql.DB, schema string, table string) (map[string]string, error) {
 	query := fmt.Sprintf(`
 		SELECT 
 			name,
@@ -18,7 +18,7 @@ func getColumnTypes(conn driver.Conn, schema string, table string) (map[string]s
 		WHERE database = '%s' AND table = '%s'`,
 		schema, table)
 
-	rows, err := conn.Query(context.Background(), query)
+	rows, err := conn.QueryContext(context.Background(), query)
 	if err != nil {
 		return nil, err
 	}
@@ -224,14 +224,14 @@ func convertArrayValue(value string, columnType string) (interface{}, error) {
 	return result, nil
 }
 
-func getPrimaryKeyColumns(conn driver.Conn, schema, table string) ([]string, error) {
+func getPrimaryKeyColumns(conn *sql.DB, schema, table string) ([]string, error) {
 	query := `
 		SELECT name
 		FROM system.columns
 		WHERE database = ? AND table = ? AND is_in_primary_key = 1
 	`
 
-	rows, err := conn.Query(context.Background(), query, schema, table)
+	rows, err := conn.QueryContext(context.Background(), query, schema, table)
 	if err != nil {
 		return nil, err
 	}

diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
@@ -89,6 +89,15 @@ services:
       - redis:/bitnami
     networks:
       - db
+  clickhouse:
+    image: clickhouse/clickhouse-server
+    ports:
+      - '8123:8123'
+      - '9000:9000'
+    volumes:
+      - clickhouse:/var/lib/clickhouse
+    networks:
+      - db
 networks:
   db:
     driver: bridge
@@ -99,4 +108,5 @@ volumes:
   mariadb:
   mongo:
   redis:
-  elasticsearch:
+  elasticsearch:
+  clickhouse: