Clickhouse: Supporting replicated merge tree (#22)

.travis.yml

@@ -3,16 +3,13 @@ language: go
 go:
   - 1.19.5
 
-env:
-  global:
-    - GO111MODULE=on
-
 services:
   - mysql
   - postgresql
 
 before_install:
-  - docker run --rm=true -p 9000:9000 --name miga-clickhouse -e CLICKHOUSE_DB=miga -e CLICKHOUSE_USER=user -e CLICKHOUSE_PASSWORD=password -d yandex/clickhouse-server
+  - docker-compose up -d
+  - docker ps -a
   - mysql -e "CREATE DATABASE miga;"
   - mysql -e "CREATE USER user@localhost IDENTIFIED BY 'password'";
   - mysql -e "GRANT ALL ON miga.* TO user@localhost";

Makefile

@@ -26,7 +26,7 @@ release: docker_build
 .PHONY: test
 test:
 	go clean -testcache
-	go test -mod=vendor -v -race ./...
+	go test -v -race ./...
 
 .PHONY: db_up
 db_up: postgres_up mysql_up clickhouse_up vertica_up
@@ -67,14 +67,7 @@ clickhouse_down:
 
 .PHONY: clickhouse_up
 clickhouse_up: clickhouse_down
-	docker run -d \
-		-p 8123:8123 \
-		-p 9000:9000 \
-		-e CLICKHOUSE_DB=miga \
-		-e CLICKHOUSE_USER=user \
-		-e CLICKHOUSE_PASSWORD=password \
-		--name=$(CLICKHOUSE_CONTAINER_NAME) \
-		--ulimit nofile=262144:262144 yandex/clickhouse-server
+	docker-compose up -d
 
 .PHONY: vertica_down
 vertica_down:

README.md

@@ -17,7 +17,7 @@ Aimed to add extra features and hide some limitations of existing golang migrati
 
 PackageName | Version | Postgres            | MySQL    | Clickhouse | Vertica
 ----------- | ------- | ------------------- | -------- | ---------- | ----
-[goose](https://github.com/pressly/goose)       |  3.9.1 ([commit](https://github.com/pressly/goose/commit/ad906520422008360de7c76e1698f6e0a6e4a918))  |  :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:
+[goose](https://github.com/pressly/goose)       |  3.9.1 ([patch](https://github.com/chapsuk/goose/commit/d8dae35e216b5b70d3db4e986884f715b5a280cc)  |  :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark:
 [migrate](https://github.com/golang-migrate/migrate)     |  4.2.5  |  :heavy_check_mark: | :heavy_check_mark: | |
 [impg](https://github.com/im-kulikov/migrate)        |   0.1   |  :heavy_check_mark: | | |
 

config/config.go

@@ -131,12 +131,6 @@ func fillDBConfig(cfg *driver.Config) {
 	if viper.IsSet("clickhouse.dsn") {
 		cfg.Dialect = "clickhouse"
 		cfg.Dsn = viper.GetString("clickhouse.dsn")
-		if viper.IsSet("clickhouse.cluster") {
-			cfg.ClickhouseClusterName = viper.GetString("clickhouse.cluster")
-			cfg.ClickhouseEngine = viper.GetString("clickhouse.engine")
-			cfg.ClickhouseSchema = viper.GetString("clickhouse.schema")
-			cfg.ClickhouseSharded = viper.GetBool("clickhouse.sharded")
-		}
 		return
 	}
 

docker-compose.yml

@@ -0,0 +1,47 @@
+version: '3.9'
+
+services:
+
+  clickhouse0:
+    image: yandex/clickhouse-server
+    container_name: clickhouse0
+    volumes:
+      - ./tests/clickhouse/macros.xml:/etc/clickhouse-server/conf.d/macros.xml
+      - ./tests/clickhouse/zookeeper.xml:/etc/clickhouse-server/conf.d/zookeeper.xml
+      - ./tests/clickhouse/remote_servers.xml:/etc/clickhouse-server/config.d/remote_servers.xml
+    ports:
+      - 8123:8123
+      - 9000:9000
+    environment:
+      CLICKHOUSE_DB: miga
+      CLICKHOUSE_USER: user
+      CLICKHOUSE_PASSWORD: password
+      REPLICA_NAME: clickhouse0
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+
+  clickhouse1:
+    image: yandex/clickhouse-server
+    container_name: clickhouse1
+    volumes:
+      - ./tests/clickhouse/macros.xml:/etc/clickhouse-server/conf.d/macros.xml
+      - ./tests/clickhouse/zookeeper.xml:/etc/clickhouse-server/conf.d/zookeeper.xml
+      - ./tests/clickhouse/remote_servers.xml:/etc/clickhouse-server/config.d/remote_servers.xml
+    environment:
+      CLICKHOUSE_DB: miga
+      CLICKHOUSE_USER: user
+      CLICKHOUSE_PASSWORD: password
+      REPLICA_NAME: clickhouse1
+    healthcheck:
+      test: ["CMD", "wget", "--spider", "-q", "http://localhost:8123/ping"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+
+  zookeeper:
+    image: wurstmeister/zookeeper
+    container_name: zookeeper
+    hostname: zookeeper

driver/driver.go

@@ -62,10 +62,6 @@ func New(cfg *Config) (Interface, error) {
 			cfg.Dsn,
 			cfg.VersionTableName,
 			cfg.Dir,
-			cfg.ClickhouseSchema,
-			cfg.ClickhouseClusterName,
-			cfg.ClickhouseEngine,
-			cfg.ClickhouseSharded,
 		)
 	case Migrate:
 		return migrate.New(

driver/goose/driver.go

@@ -9,21 +9,13 @@ import (
 )
 
 type Goose struct {
-	db                    *sql.DB
-	dir                   string
-	dialect               string
-	versionTableName      string
-	clickhouseSchema      string
-	clickhouseClusterName string
-	clickhouseEngine      string
-	clickhouseSharded     bool
+	db               *sql.DB
+	dir              string
+	dialect          string
+	versionTableName string
 }
 
-func New(
-	dialect, dsn, tableName, dir string,
-	clickhouseSchema, clickhouseClusterName, clickhouseEngine string,
-	clickhouseSharded bool,
-) (*Goose, error) {
+func New(dialect, dsn, tableName, dir string) (*Goose, error) {
 	err := orig.SetDialect(dialect)
 	if err != nil {
 		return nil, err
@@ -32,27 +24,23 @@ func New(
 	orig.SetTableName(tableName)
 	orig.SetLogger(&utils.StdLogger{})
 
+	if dialect == "clickhouse-replicated" {
+		dialect = "clickhouse"
+	}
+
 	db, err := sql.Open(dialect, dsn)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Goose{
-		db:                    db,
-		dir:                   dir,
-		dialect:               dialect,
-		versionTableName:      tableName,
-		clickhouseSchema:      clickhouseSchema,
-		clickhouseClusterName: clickhouseClusterName,
-		clickhouseEngine:      clickhouseEngine,
-		clickhouseSharded:     clickhouseSharded,
+		db:               db,
+		dir:              dir,
+		dialect:          dialect,
+		versionTableName: tableName,
 	}, nil
 }
 
-func (g Goose) isClickhouse() bool {
-	return g.dialect == "clickhouse"
-}
-
 func (g Goose) Close() error {
 	return g.db.Close()
 }

go.mod

@@ -2,6 +2,8 @@ module miga
 
 go 1.19
 
+replace github.com/pressly/goose/v3 v3.9.0 => github.com/chapsuk/goose/v3 v3.0.0-20230127172535-d8dae35e216b
+
 require (
 	github.com/ClickHouse/clickhouse-go/v2 v2.2.0
 	github.com/go-pg/pg v8.0.3+incompatible
@@ -10,7 +12,7 @@ require (
 	github.com/im-kulikov/migrate v0.1.0
 	github.com/lib/pq v1.10.6
 	github.com/pkg/errors v0.9.1
-	github.com/pressly/goose/v3 v3.9.1-0.20230127143906-ad9065204220
+	github.com/pressly/goose/v3 v3.9.0
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spf13/viper v1.3.2
 	github.com/vertica/vertica-sql-go v1.3.1

go.sum

@@ -21,6 +21,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k=
 github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
+github.com/chapsuk/goose/v3 v3.0.0-20230127172535-d8dae35e216b h1:2PabApnk2+DjDuJr2AD4LsWFiD/OgkFUmnMWfCDJT6k=
+github.com/chapsuk/goose/v3 v3.0.0-20230127172535-d8dae35e216b/go.mod h1:MM80AHPkHlmtRxQxxxp2ioe+Yj0suUYOY4JDjQ9Y6Ko=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
@@ -195,8 +197,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/pressly/goose/v3 v3.9.1-0.20230127143906-ad9065204220 h1:G3qR1d2Jv0Jz6g0lUAnWmzo1rP2Bw4QhRLbOiirUJi4=
-github.com/pressly/goose/v3 v3.9.1-0.20230127143906-ad9065204220/go.mod h1:MM80AHPkHlmtRxQxxxp2ioe+Yj0suUYOY4JDjQ9Y6Ko=
 github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=

tests/clickhouse/macros.xml

@@ -0,0 +1,9 @@
+<yandex>
+    <macros>
+        <installation>chapsuk</installation>
+        <all-sharded-shard>0</all-sharded-shard>
+        <cluster>miga</cluster>
+        <shard>0</shard>
+        <replica from_env="REPLICA_NAME" />
+    </macros>
+</yandex>

tests/clickhouse/remote_servers.xml

@@ -0,0 +1,17 @@
+<yandex>
+    <remote_servers>
+        <miga>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>clickhouse0</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>clickhouse1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </miga>
+    </remote_servers>
+</yandex>

tests/clickhouse/zookeeper.xml

@@ -0,0 +1,11 @@
+<yandex>
+    <default_replica_path>/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}</default_replica_path>
+    <default_replica_name>{replica}</default_replica_name>
+
+    <zookeeper>
+        <node>
+            <host>zookeeper</host>
+            <port>2181</port>
+        </node>
+    </zookeeper>
+</yandex>

tests/convert_test.go

@@ -16,7 +16,7 @@ import (
 func TestConvert(t *testing.T) {
 	for driverName, dialects := range drivers {
 		for _, dialect := range dialects {
-			if dialect == "clickhouse" || dialect == "vertica" {
+			if dialect == "clickhouse" || dialect == "clickhouse-replicated" || dialect == "vertica" {
 				continue
 			}
 			for tdriverName := range drivers {
@@ -71,7 +71,7 @@ func TestConvert(t *testing.T) {
 
 							Convey(testCase.Description, func() {
 								testCase.Action(driverInst)
-								testCase.Assert(db)
+								testCase.Assert(db, "")
 							})
 						}
 					})

tests/migrations/goose_clickhouse_replicated/00001_users.sql

@@ -0,0 +1,9 @@
+-- +goose Up
+CREATE TABLE users_replicated ON CLUSTER '{cluster}' (
+    id BIGINT,
+    name VARCHAR(128),
+    migastas BIGINT DEFAULT 0
+) engine=MergeTree() order by id;
+
+-- +goose Down
+DROP TABLE IF EXISTS users_replicated ON CLUSTER '{cluster}';

tests/migrations/goose_clickhouse_replicated/00002_wallets.sql

@@ -0,0 +1,8 @@
+-- +goose Up
+CREATE TABLE wallets_replicated ON CLUSTER '{cluster}' (
+    id      BIGINT,
+    user_id BIGINT
+) engine=MergeTree() order by id;
+
+-- +goose Down
+DROP TABLE IF EXISTS wallets_replicated ON CLUSTER '{cluster}';

tests/migrations/goose_clickhouse_replicated/00003_add_users_email.sql

@@ -0,0 +1,7 @@
+-- +goose Up
+SET replication_alter_partitions_sync = 2;
+ALTER TABLE users_replicated ON CLUSTER '{cluster}' ADD COLUMN email VARCHAR(128);
+
+-- +goose Down
+SET replication_alter_partitions_sync = 2;
+ALTER TABLE users_replicated ON CLUSTER '{cluster}' DROP COLUMN email;

tests/migrations/goose_clickhouse_replicated/00101_wrong.sql

@@ -0,0 +1,5 @@
+-- +goose Up
+ALTER TABLE users_replicated ON CLUSTER '{cluster}' ADD COLUMN email VARCHAR;
+
+-- +goose Down
+ALTER TABLE users_replicated ON CLUSTER '{cluster}' DROP COLUMN IF EXISTS email;

tests/migrations/goose_clickhouse_replicated/00102_never.sql

@@ -0,0 +1,7 @@
+-- +goose Up
+CREATE TABLE never_replicated ON CLUSTER '{cluster}' (
+    id INT
+) engine=Memory;
+
+-- +goose Down
+DROP TABLE IF EXISTS never_replicated ON CLUSTER '{cluster}';