Merge branch 'pgx' into develop

ImageMonkey · Jan 10, 2020 · 6c917f4 · 6c917f4
2 parents 50afb8a + 64485a9
commit 6c917f4
Show file tree

Hide file tree

Showing 35 changed files with 2,823 additions and 2,435 deletions.
diff --git a/conf/supervisor/imagemonkey-api.conf b/conf/supervisor/imagemonkey-api.conf
@@ -1,6 +1,6 @@
 [program:imagemonkey-api]
 process_name=imagemonkey-api%(process_num)s
-command=/home/imagemonkey/bin/api -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -unverified_donations_dir=/home/imagemonkey/unverified_donations/ -geoip_db=/home/imagemonkey/geoip_database/GeoLite2-Country.mmdb -examples_dir=/home/imagemonkey/label_examples/ -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -avatars_dir=/home/imagemonkey/avatars/ -image_quarantine_dir=/home/imagemonkey/quarantine/ -label_graph_definitions=/home/imagemonkey/wordlists/en/graphdefinitions/ -api_base_url=https://api.imagemonkey.io/ -cors_allow_origin=https://imagemonkey.io -use_sentry -release
+command=/home/imagemonkey/bin/api -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -unverified_donations_dir=/home/imagemonkey/unverified_donations/ -geoip_db=/home/imagemonkey/geoip_database/GeoLite2-Country.mmdb -examples_dir=/home/imagemonkey/label_examples/ -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -avatars_dir=/home/imagemonkey/avatars/ -image_quarantine_dir=/home/imagemonkey/quarantine/ -label_graph_definitions=/home/imagemonkey/wordlists/en/graphdefinitions/ -api_base_url=https://api.imagemonkey.io/ -cors_allow_origin=https://imagemonkey.io -use_sentry -release -db_max_connections=75
 autostart=true
 autorestart=true
 startretries=10
@@ -11,4 +11,4 @@ stdout_logfile=/var/log/imagemonkey-api/out-%(process_num)s.log
 stderr_logfile=/var/log/imagemonkey-api/err-%(process_num)s.log
 stdout_logfile_maxbytes=50MB
 stdout_logfile_backups=10
-numprocs=1
+numprocs=1
diff --git a/conf/supervisor/imagemonkey-web.conf b/conf/supervisor/imagemonkey-web.conf
@@ -1,6 +1,6 @@
 [program:imagemonkey-web]
 process_name=imagemonkey-web%(process_num)s
-command=/home/imagemonkey/bin/web -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -html_dir=/home/imagemonkey/html/templates/ -public_backups_path=/home/imagemonkey/public_backups/public_backups.json -api_base_url=https://api.imagemonkey.io -playground_base_url=https://playground.imagemonkey.io -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -use_sentry -release -local_sentry_dsn=https://sentry:sentry@imagemonkey.io/sentry -labels_repository_url=https://github.com/bbernhard/imagemonkey-labels -trending_labels_repository_url=https://github.com/bbernhard/imagemonkey-trending-labels
+command=/home/imagemonkey/bin/web -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -html_dir=/home/imagemonkey/html/templates/ -public_backups_path=/home/imagemonkey/public_backups/public_backups.json -api_base_url=https://api.imagemonkey.io -playground_base_url=https://playground.imagemonkey.io -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -use_sentry -release -local_sentry_dsn=https://sentry:sentry@imagemonkey.io/sentry -labels_repository_url=https://github.com/bbernhard/imagemonkey-labels -trending_labels_repository_url=https://github.com/bbernhard/imagemonkey-trending-labels -db_max_connections=75
 autostart=true
 autorestart=true
 startretries=10

diff --git a/env/docker/docker-compose.travis.yml b/env/docker/docker-compose.travis.yml
@@ -58,8 +58,8 @@ services:
        dockerfile: env/docker/Dockerfile.api
        cache_from:
          - bbernhard/imagemonkey-api:latest
-    logging:
-      driver: none
+    #logging:
+    #  driver: none
     entrypoint: ./run_api.sh --merge-labels-before-start 
     volumes:
       - ../../geoip_database:/home/imagemonkey/geoip_database

diff --git a/src/api.go b/src/api.go
@@ -513,6 +513,7 @@ func main() {
 	apiBaseUrl := flag.String("api_base_url", "http://127.0.0.1:8081/", "API Base URL")
 	corsAllowOrigin := flag.String("cors_allow_origin", "*", "CORS Access-Control-Allow-Origin")
 	imageHuntAssetsDir := flag.String("imagehunt_assets_dir", "../img/game-assets/", "ImageHunt Game Assets Directory")
+	maxNumOfDatabaseConnections := flag.Int("db_max_connections", 5, "Max. number of database connections")
 
 	sentryEnvironment := "api"
 
@@ -559,7 +560,7 @@ func main() {
 
 	imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING")
 	imageMonkeyDatabase := imagemonkeydb.NewImageMonkeyDatabase()
-	err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString)
+	err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString, int32(*maxNumOfDatabaseConnections))
 	if err != nil {
 		log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error())
 	}
@@ -600,8 +601,8 @@ func main() {
 	psc := redis.PubSubConn{Conn: redisConn}
 	defer psc.Close()
 
-	if err := psc.Subscribe(redis.Args{}.AddFlat([]string{"reloadlabels"})...); err != nil {
-		log.Fatal("Couldn't subscribe to topic 'reloadlabels': ", err.Error())
+	if err := psc.Subscribe(redis.Args{}.AddFlat([]string{"tasks"})...); err != nil {
+		log.Fatal("Couldn't subscribe to topic 'tasks': ", err.Error())
 	}
 
 	done := make(chan error, 1)
@@ -613,25 +614,38 @@ func main() {
 				done <- n
 				return
 			case redis.Message:
-				log.Info("[Main] Reloading labels")
-				err := labelRepository.Load()
-				if err != nil {
-					log.Error("Couldn't read label map: ", err.Error())
-					raven.CaptureError(err, nil)
-				}
-				labelMap = labelRepository.GetMapping()
-				words = labelRepository.GetWords()
-
-				err = metaLabels.Load()
-				if err != nil {
-					log.Error("Couldn't read metalabels map: ", err.Error())
-					raven.CaptureError(err, nil)
-				}
-
-				labelRefinementsMap, err = commons.GetLabelRefinementsMap(*labelRefinementsPath)
-				if err != nil {
-					log.Error("Couldn't read label refinements: ", err.Error())
-					raven.CaptureError(err, nil)
+				if n.Channel == "tasks" {
+					if string(n.Data) == "reloadlabels" {
+						log.Info("[Main] Reloading labels")
+						err := labelRepository.Load()
+						if err != nil {
+							log.Error("Couldn't read label map: ", err.Error())
+							raven.CaptureError(err, nil)
+						}
+						labelMap = labelRepository.GetMapping()
+						words = labelRepository.GetWords()
+
+						err = metaLabels.Load()
+						if err != nil {
+							log.Error("Couldn't read metalabels map: ", err.Error())
+							raven.CaptureError(err, nil)
+						}
+
+						labelRefinementsMap, err = commons.GetLabelRefinementsMap(*labelRefinementsPath)
+						if err != nil {
+							log.Error("Couldn't read label refinements: ", err.Error())
+							raven.CaptureError(err, nil)
+						}
+					} else if string(n.Data) == "reconnectdb" {
+						log.Info("Reconnecting to Database")
+						log.Info(string(n.Data))
+						imageMonkeyDatabase.Close()
+						err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString, int32(*maxNumOfDatabaseConnections))
+						if err != nil {
+							raven.CaptureError(err, nil)
+							log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error())
+						}
+					}
 				}
 			case redis.Subscription:
 				switch n.Count {

diff --git a/src/blogsubscriptionworker.go b/src/blogsubscriptionworker.go
@@ -6,19 +6,20 @@ import (
 	"flag"
 	"github.com/gomodule/redigo/redis"
 	"time"
-	"database/sql"
-	_ "github.com/lib/pq"
+	"github.com/jackc/pgx/v4"
 	"encoding/json"
 	"github.com/getsentry/raven-go"
 	datastructures "github.com/bbernhard/imagemonkey-core/datastructures"
 	commons "github.com/bbernhard/imagemonkey-core/commons"
+	"context"
 )
 
-var db *sql.DB
+var db *pgx.Conn
 
 func subscribe(email string) error {
 	log.Debug("[Main] Got a new subscription: ", email)
-	_,err := db.Exec(`INSERT INTO blog.subscription(email) VALUES ($1)
+	_,err := db.Exec(context.TODO(),
+					 `INSERT INTO blog.subscription(email) VALUES ($1)
 			 		  ON CONFLICT DO NOTHING`, email)
 	if err != nil {
 		raven.CaptureError(err, nil)
@@ -51,16 +52,16 @@ func main(){
 
 	//open database and make sure that we can ping it
 	imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING")
-	db, err = sql.Open("postgres", imageMonkeyDbConnectionString)
+	db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString)
 	if err != nil {
 		log.Fatal("[Main] Couldn't open database: ", err.Error())
 	}
 
-	err = db.Ping()
+	err = db.Ping(context.Background())
 	if err != nil {
 		log.Fatal("[Main] Couldn't ping database: ", err.Error())
 	}
-	defer db.Close()
+	defer db.Close(context.Background())
 
 
 	//create redis pool

diff --git a/src/bot.go b/src/bot.go
@@ -1,20 +1,19 @@
 package main
 
 import (
-	"database/sql"
 	"flag"
 	"fmt"
 	commons "github.com/bbernhard/imagemonkey-core/commons"
 	datastructures "github.com/bbernhard/imagemonkey-core/datastructures"
 	"github.com/getsentry/raven-go"
-	//"github.com/gofrs/uuid"
-	_ "github.com/lib/pq"
+	"github.com/jackc/pgx/v4"
 	log "github.com/sirupsen/logrus"
 	"time"
 	"errors"
+	"context"
 )
 
-var db *sql.DB
+var db *pgx.Conn
 
 func setTrendingLabelBotTaskState(status string, branchName string, jobUrl string, id int64) error {
 	var queryValues []interface{}
@@ -30,12 +29,13 @@ func setTrendingLabelBotTaskState(status string, branchName string, jobUrl strin
 	q := fmt.Sprintf(`UPDATE trending_label_bot_task 
 					  	SET state = $1, branch_name = $2%s
 						WHERE id = $3`, jobUrlStr)
-	_, err := db.Exec(q, queryValues...)
+	_, err := db.Exec(context.TODO(), q, queryValues...)
 	return err
 }
 
 func resetTrendingLabelBotTaskState(id int64) error {
-	_, err := db.Exec(`UPDATE trending_label_bot_task
+	_, err := db.Exec(context.TODO(),
+						`UPDATE trending_label_bot_task
 						SET state = 'accepted', branch_name = null, job_url = null,
 						try = try + 1
 						WHERE id = $1`, id)
@@ -45,7 +45,8 @@ func resetTrendingLabelBotTaskState(id int64) error {
 func getTrendingLabels() ([]datastructures.TrendingLabelBotTask, error) {
 	trendingLabels := []datastructures.TrendingLabelBotTask{}
 
-	rows, err := db.Query(`SELECT s.name, b.id, b.state, COALESCE(b.branch_name, ''), label_type,
+	rows, err := db.Query(context.TODO(),
+						   `SELECT s.name, b.id, b.state, COALESCE(b.branch_name, ''), label_type,
 						   COALESCE(b.plural) as plural, COALESCE(b.description, ''), COALESCE(b.rename_to, '')
 					  	   FROM trending_label_suggestion t
 					  	   JOIN trending_label_bot_task b ON b.trending_label_suggestion_id = t.id 
@@ -75,7 +76,8 @@ func getTrendingLabels() ([]datastructures.TrendingLabelBotTask, error) {
 }
 
 func labelAlreadyExistsInPipeline(label string, trendingLabelBotTaskId int64) (bool, error) {
-	rows, err := db.Query(`SELECT count(*)
+	rows, err := db.Query(context.TODO(),
+						   `SELECT count(*)
 			  				FROM trending_label_suggestion t
 			  				JOIN trending_label_bot_task b ON b.trending_label_suggestion_id = t.id
 			  				WHERE b.rename_to = $1 AND b.id != $2`, label, trendingLabelBotTaskId)
@@ -135,18 +137,18 @@ func main() {
 	//open database and make sure that we can ping it
 	var err error
 	imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING")
-	db, err = sql.Open("postgres", imageMonkeyDbConnectionString)
+	db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString)
 	if err != nil {
 		raven.CaptureError(err, nil)
 		log.Fatal("Couldn't open database: ", err.Error())
 	}
 
-	err = db.Ping()
+	err = db.Ping(context.Background())
 	if err != nil {
 		raven.CaptureError(err, nil)
 		log.Fatal("Couldn't ping database: ", err.Error())
 	}
-	defer db.Close()
+	defer db.Close(context.Background())
 
 	labelsRepository := commons.NewLabelsRepository(*labelsRepositoryOwner, *labelsRepositoryName, *gitCheckoutDir)
 	labelsRepository.SetToken(imageMonkeyBotGithubApiToken)