DB Schema migration (#14)

* Add schema migration

* add schema dump script

* Add README for scripts

* Add git hooks for dumping schema

main.go

@@ -10,7 +10,9 @@ import (
 	"github.com/RealImage/QLedger/controllers"
 	"github.com/RealImage/QLedger/middlewares"
 	"github.com/julienschmidt/httprouter"
-	_ "github.com/lib/pq"
+	"github.com/mattes/migrate"
+	"github.com/mattes/migrate/database/postgres"
+	_ "github.com/mattes/migrate/source/file"
 )
 
 func main() {
@@ -19,6 +21,17 @@ func main() {
 		log.Panic("Unable to connect to Database:", err)
 	}
 	log.Println("Successfully established connection to database.")
+
+	log.Println("Starting db schema migration...")
+	driver, _ := postgres.WithInstance(db, &postgres.Config{})
+	m, _ := migrate.NewWithDatabaseInstance(
+		"file://migrations/postgres",
+		"postgres", driver)
+	version, _, _ := m.Version()
+	log.Println("Current schema version:", version)
+	m.Up()
+	version, _, _ = m.Version()
+	log.Println("Migrated schema version:", version)
 	appContext := &ledgerContext.AppContext{DB: db}
 
 	router := httprouter.New()

migrations/postgres/20170607001_create_table_accounts.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS accounts;

migrations/postgres/20170607001_create_table_accounts.up.sql

@@ -0,0 +1,4 @@
+CREATE TABLE accounts (
+    id character varying NOT NULL,
+    data jsonb DEFAULT '{}'::jsonb NOT NULL
+);

migrations/postgres/20170607002_add_contraint_accounts_pkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY accounts
+    DROP CONSTRAINT IF EXISTS accounts_pkey;

migrations/postgres/20170607002_add_contraint_accounts_pkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY accounts
+    ADD CONSTRAINT accounts_pkey PRIMARY KEY (id);

migrations/postgres/20170607003_create_table_account_tags.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS account_tags;

migrations/postgres/20170607003_create_table_account_tags.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE account_tags (
+    account_id character varying NOT NULL,
+    key character varying NOT NULL,
+    value character varying
+);

migrations/postgres/20170607004_add_constraint_account_tags_account_id_fkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY account_tags
+    DROP CONSTRAINT IF EXISTS account_tags_account_id_fkey;

migrations/postgres/20170607004_add_constraint_account_tags_account_id_fkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY account_tags
+    ADD CONSTRAINT account_tags_account_id_fkey FOREIGN KEY (account_id) REFERENCES accounts(id);

migrations/postgres/20170607005_add_index_account_tags_lookup_idx.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS account_tags_lookup_idx;

migrations/postgres/20170607005_add_index_account_tags_lookup_idx.up.sql

@@ -0,0 +1 @@
+CREATE UNIQUE INDEX account_tags_lookup_idx ON account_tags USING btree (value, key, account_id);

migrations/postgres/20170607006_create_table_transactions.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS transactions;

migrations/postgres/20170607006_create_table_transactions.up.sql

@@ -0,0 +1,4 @@
+CREATE TABLE transactions (
+    id character varying NOT NULL,
+    "timestamp" timestamp without time zone NOT NULL
+);

migrations/postgres/20170607007_add_constraint_transactions_pkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY transactions
+    DROP CONSTRAINT IF EXISTS transactions_pkey;

migrations/postgres/20170607007_add_constraint_transactions_pkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY transactions
+    ADD CONSTRAINT transactions_pkey PRIMARY KEY (id);

migrations/postgres/20170607008_add_index_timestamp_idx.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS timestamp_idx;

migrations/postgres/20170607008_add_index_timestamp_idx.up.sql

@@ -0,0 +1 @@
+CREATE INDEX timestamp_idx ON transactions USING brin ("timestamp");

migrations/postgres/20170607009_create_table_transaction_tags.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS transaction_tags;

migrations/postgres/20170607009_create_table_transaction_tags.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE transaction_tags (
+    transaction_id character varying NOT NULL,
+    key character varying NOT NULL,
+    value character varying
+);

migrations/postgres/20170607010_add_constraint_transaction_tags_transaction_id_fkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY transaction_tags
+    DROP CONSTRAINT IF EXISTS transaction_tags_transaction_id_fkey;

migrations/postgres/20170607010_add_constraint_transaction_tags_transaction_id_fkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY transaction_tags
+    ADD CONSTRAINT transaction_tags_transaction_id_fkey FOREIGN KEY (transaction_id) REFERENCES transactions(id);

migrations/postgres/20170607011_add_index_transaction_tags_lookup_idx.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS transaction_tags_lookup_idx;

migrations/postgres/20170607011_add_index_transaction_tags_lookup_idx.up.sql

@@ -0,0 +1 @@
+CREATE UNIQUE INDEX transaction_tags_lookup_idx ON transaction_tags USING btree (value, key, transaction_id);

migrations/postgres/20170607012_create_table_lines.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS lines;

migrations/postgres/20170607012_create_table_lines.up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE lines (
+    id bigint NOT NULL,
+    transaction_id character varying NOT NULL,
+    account_id character varying NOT NULL,
+    delta integer NOT NULL
+);

migrations/postgres/20170607013_add_contraint_lines_pkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY lines
+    DROP CONSTRAINT IF EXISTS lines_pkey;

migrations/postgres/20170607013_add_contraint_lines_pkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY lines
+    ADD CONSTRAINT lines_pkey PRIMARY KEY (id);

migrations/postgres/20170607014_add_contraint_lines_account_id_fkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY lines
+    DROP CONSTRAINT IF EXISTS lines_account_id_fkey;

migrations/postgres/20170607014_add_contraint_lines_account_id_fkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY lines
+    ADD CONSTRAINT lines_account_id_fkey FOREIGN KEY (account_id) REFERENCES accounts(id);

migrations/postgres/20170607015_add_constraint_lines_txn_fkey.down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY lines
+    DROP CONSTRAINT IF EXISTS lines_txn_fkey;

migrations/postgres/20170607015_add_constraint_lines_txn_fkey.up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE ONLY lines
+    ADD CONSTRAINT lines_txn_fkey FOREIGN KEY (transaction_id) REFERENCES transactions(id);

migrations/postgres/20170607016_create_sequence_lines_id_seq.down.sql

@@ -0,0 +1 @@
+DROP SEQUENCE IF EXISTS lines_id_seq;

migrations/postgres/20170607016_create_sequence_lines_id_seq.up.sql

@@ -0,0 +1,7 @@
+CREATE SEQUENCE lines_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1
+    OWNED BY lines.id;

migrations/postgres/20170607017_alter_table_lines_column_id_set_default.down.sql

@@ -0,0 +1 @@
+ALTER TABLE ONLY lines ALTER COLUMN id DROP DEFAULT;

migrations/postgres/20170607017_alter_table_lines_column_id_set_default.up.sql

@@ -0,0 +1 @@
+ALTER TABLE ONLY lines ALTER COLUMN id SET DEFAULT nextval('lines_id_seq'::regclass);

migrations/postgres/20170607018_create_view_current_balances.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS current_balances;

migrations/postgres/20170607018_create_view_current_balances.up.sql

@@ -0,0 +1,5 @@
+CREATE VIEW current_balances AS
+ SELECT lines.account_id,
+    sum(lines.delta) AS balance
+   FROM lines
+  GROUP BY lines.account_id;

migrations/postgres/20170607019_create_view_invalid_transactions.down.sql

@@ -0,0 +1 @@
+DROP VIEW IF EXISTS invalid_transactions;

migrations/postgres/20170607019_create_view_invalid_transactions.up.sql

@@ -0,0 +1,6 @@
+CREATE VIEW invalid_transactions AS
+ SELECT lines.transaction_id,
+    sum(lines.delta) AS sum
+   FROM lines
+  GROUP BY lines.transaction_id
+ HAVING (sum(lines.delta) > 0);

schema.sql

@@ -44,6 +44,10 @@ CREATE SEQUENCE lines_id_seq
     NO MAXVALUE
     CACHE 1;
 ALTER SEQUENCE lines_id_seq OWNED BY lines.id;
+CREATE TABLE schema_migrations (
+    version bigint NOT NULL,
+    dirty boolean NOT NULL
+);
 CREATE TABLE transaction_tags (
     transaction_id character varying NOT NULL,
     key character varying NOT NULL,
@@ -54,14 +58,12 @@ CREATE TABLE transactions (
     "timestamp" timestamp without time zone NOT NULL
 );
 ALTER TABLE ONLY lines ALTER COLUMN id SET DEFAULT nextval('lines_id_seq'::regclass);
-ALTER TABLE ONLY account_tags
-    ADD CONSTRAINT account_tags_pkey PRIMARY KEY (account_id);
 ALTER TABLE ONLY accounts
     ADD CONSTRAINT accounts_pkey PRIMARY KEY (id);
 ALTER TABLE ONLY lines
     ADD CONSTRAINT lines_pkey PRIMARY KEY (id);
-ALTER TABLE ONLY transaction_tags
-    ADD CONSTRAINT transaction_tags_pkey PRIMARY KEY (transaction_id);
+ALTER TABLE ONLY schema_migrations
+    ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version);
 ALTER TABLE ONLY transactions
     ADD CONSTRAINT transactions_pkey PRIMARY KEY (id);
 CREATE UNIQUE INDEX account_tags_lookup_idx ON account_tags USING btree (value, key, account_id);

scripts/README.md

@@ -0,0 +1,12 @@
+## Scripts
+
+#### Install Hooks
+
+```
+cd QLedger
+bash scripts/install_hooks.sh
+```
+
+The above script install Git hooks which does the following tasks:
+
+ - Identify the schema changes and dumps to `schema.sql`

scripts/hooks/git/pre-commit

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+echo "Checking for DB schema changes.."
+DB=`echo $DATABASE_URL|cut -d/ -f4|cut -d? -f1`
+pg_dump -s -x -O $DB | grep -v -e "^--" -e "^$" > schema.sql
+SCHEMA_CHANGED=`git ls-files -m|grep schema.sql`
+if [ "$SCHEMA_CHANGED" != "" ]; then
+  echo "Schema is changed and the changes are dumped to schema.sql"
+  echo "Please add the file to Git and commit"
+  echo "Aborted commit"
+  exit 1
+else
+  echo "No changes in schema!"
+fi

scripts/install_hooks.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+cp scripts/hooks/git/pre-commit .git/hooks/pre-commit
+chmod +x .git/hooks/pre-commit