diff --git a/conf/supervisor/imagemonkey-api.conf b/conf/supervisor/imagemonkey-api.conf index 2cbd21d3..962986e8 100644 --- a/conf/supervisor/imagemonkey-api.conf +++ b/conf/supervisor/imagemonkey-api.conf @@ -1,6 +1,6 @@ [program:imagemonkey-api] process_name=imagemonkey-api%(process_num)s -command=/home/imagemonkey/bin/api -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -unverified_donations_dir=/home/imagemonkey/unverified_donations/ -geoip_db=/home/imagemonkey/geoip_database/GeoLite2-Country.mmdb -examples_dir=/home/imagemonkey/label_examples/ -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -avatars_dir=/home/imagemonkey/avatars/ -image_quarantine_dir=/home/imagemonkey/quarantine/ -label_graph_definitions=/home/imagemonkey/wordlists/en/graphdefinitions/ -api_base_url=https://api.imagemonkey.io/ -cors_allow_origin=https://imagemonkey.io -use_sentry -release +command=/home/imagemonkey/bin/api -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -unverified_donations_dir=/home/imagemonkey/unverified_donations/ -geoip_db=/home/imagemonkey/geoip_database/GeoLite2-Country.mmdb -examples_dir=/home/imagemonkey/label_examples/ -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -avatars_dir=/home/imagemonkey/avatars/ -image_quarantine_dir=/home/imagemonkey/quarantine/ -label_graph_definitions=/home/imagemonkey/wordlists/en/graphdefinitions/ -api_base_url=https://api.imagemonkey.io/ -cors_allow_origin=https://imagemonkey.io -use_sentry -release -db_max_connections=75 autostart=true autorestart=true startretries=10 @@ -11,4 +11,4 @@ stdout_logfile=/var/log/imagemonkey-api/out-%(process_num)s.log stderr_logfile=/var/log/imagemonkey-api/err-%(process_num)s.log stdout_logfile_maxbytes=50MB stdout_logfile_backups=10 -numprocs=1 \ No newline at end of file +numprocs=1 diff --git a/conf/supervisor/imagemonkey-web.conf b/conf/supervisor/imagemonkey-web.conf index 28f11e22..c01f0694 100644 --- a/conf/supervisor/imagemonkey-web.conf +++ b/conf/supervisor/imagemonkey-web.conf @@ -1,6 +1,6 @@ [program:imagemonkey-web] process_name=imagemonkey-web%(process_num)s -command=/home/imagemonkey/bin/web -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -html_dir=/home/imagemonkey/html/templates/ -public_backups_path=/home/imagemonkey/public_backups/public_backups.json -api_base_url=https://api.imagemonkey.io -playground_base_url=https://playground.imagemonkey.io -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -use_sentry -release -local_sentry_dsn=https://sentry:sentry@imagemonkey.io/sentry -labels_repository_url=https://github.com/bbernhard/imagemonkey-labels -trending_labels_repository_url=https://github.com/bbernhard/imagemonkey-trending-labels +command=/home/imagemonkey/bin/web -wordlist=/home/imagemonkey/wordlists/en/labels.jsonnet -donations_dir=/home/imagemonkey/donations/ -html_dir=/home/imagemonkey/html/templates/ -public_backups_path=/home/imagemonkey/public_backups/public_backups.json -api_base_url=https://api.imagemonkey.io -playground_base_url=https://playground.imagemonkey.io -maintenance_mode_file=/home/imagemonkey/maintenance.tmp -use_sentry -release -local_sentry_dsn=https://sentry:sentry@imagemonkey.io/sentry -labels_repository_url=https://github.com/bbernhard/imagemonkey-labels -trending_labels_repository_url=https://github.com/bbernhard/imagemonkey-trending-labels -db_max_connections=75 autostart=true autorestart=true startretries=10 diff --git a/env/docker/docker-compose.travis.yml b/env/docker/docker-compose.travis.yml index 612008a8..2b5baf12 100644 --- a/env/docker/docker-compose.travis.yml +++ b/env/docker/docker-compose.travis.yml @@ -58,8 +58,8 @@ services: dockerfile: env/docker/Dockerfile.api cache_from: - bbernhard/imagemonkey-api:latest - logging: - driver: none + #logging: + # driver: none entrypoint: ./run_api.sh --merge-labels-before-start volumes: - ../../geoip_database:/home/imagemonkey/geoip_database diff --git a/src/api.go b/src/api.go index d33acfb5..70ad56df 100644 --- a/src/api.go +++ b/src/api.go @@ -513,6 +513,7 @@ func main() { apiBaseUrl := flag.String("api_base_url", "http://127.0.0.1:8081/", "API Base URL") corsAllowOrigin := flag.String("cors_allow_origin", "*", "CORS Access-Control-Allow-Origin") imageHuntAssetsDir := flag.String("imagehunt_assets_dir", "../img/game-assets/", "ImageHunt Game Assets Directory") + maxNumOfDatabaseConnections := flag.Int("db_max_connections", 5, "Max. number of database connections") sentryEnvironment := "api" @@ -559,7 +560,7 @@ func main() { imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") imageMonkeyDatabase := imagemonkeydb.NewImageMonkeyDatabase() - err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString) + err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString, int32(*maxNumOfDatabaseConnections)) if err != nil { log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error()) } @@ -600,8 +601,8 @@ func main() { psc := redis.PubSubConn{Conn: redisConn} defer psc.Close() - if err := psc.Subscribe(redis.Args{}.AddFlat([]string{"reloadlabels"})...); err != nil { - log.Fatal("Couldn't subscribe to topic 'reloadlabels': ", err.Error()) + if err := psc.Subscribe(redis.Args{}.AddFlat([]string{"tasks"})...); err != nil { + log.Fatal("Couldn't subscribe to topic 'tasks': ", err.Error()) } done := make(chan error, 1) @@ -613,25 +614,38 @@ func main() { done <- n return case redis.Message: - log.Info("[Main] Reloading labels") - err := labelRepository.Load() - if err != nil { - log.Error("Couldn't read label map: ", err.Error()) - raven.CaptureError(err, nil) - } - labelMap = labelRepository.GetMapping() - words = labelRepository.GetWords() - - err = metaLabels.Load() - if err != nil { - log.Error("Couldn't read metalabels map: ", err.Error()) - raven.CaptureError(err, nil) - } - - labelRefinementsMap, err = commons.GetLabelRefinementsMap(*labelRefinementsPath) - if err != nil { - log.Error("Couldn't read label refinements: ", err.Error()) - raven.CaptureError(err, nil) + if n.Channel == "tasks" { + if string(n.Data) == "reloadlabels" { + log.Info("[Main] Reloading labels") + err := labelRepository.Load() + if err != nil { + log.Error("Couldn't read label map: ", err.Error()) + raven.CaptureError(err, nil) + } + labelMap = labelRepository.GetMapping() + words = labelRepository.GetWords() + + err = metaLabels.Load() + if err != nil { + log.Error("Couldn't read metalabels map: ", err.Error()) + raven.CaptureError(err, nil) + } + + labelRefinementsMap, err = commons.GetLabelRefinementsMap(*labelRefinementsPath) + if err != nil { + log.Error("Couldn't read label refinements: ", err.Error()) + raven.CaptureError(err, nil) + } + } else if string(n.Data) == "reconnectdb" { + log.Info("Reconnecting to Database") + log.Info(string(n.Data)) + imageMonkeyDatabase.Close() + err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString, int32(*maxNumOfDatabaseConnections)) + if err != nil { + raven.CaptureError(err, nil) + log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error()) + } + } } case redis.Subscription: switch n.Count { diff --git a/src/blogsubscriptionworker.go b/src/blogsubscriptionworker.go index e291f54a..794dc610 100644 --- a/src/blogsubscriptionworker.go +++ b/src/blogsubscriptionworker.go @@ -6,19 +6,20 @@ import ( "flag" "github.com/gomodule/redigo/redis" "time" - "database/sql" - _ "github.com/lib/pq" + "github.com/jackc/pgx/v4" "encoding/json" "github.com/getsentry/raven-go" datastructures "github.com/bbernhard/imagemonkey-core/datastructures" commons "github.com/bbernhard/imagemonkey-core/commons" + "context" ) -var db *sql.DB +var db *pgx.Conn func subscribe(email string) error { log.Debug("[Main] Got a new subscription: ", email) - _,err := db.Exec(`INSERT INTO blog.subscription(email) VALUES ($1) + _,err := db.Exec(context.TODO(), + `INSERT INTO blog.subscription(email) VALUES ($1) ON CONFLICT DO NOTHING`, email) if err != nil { raven.CaptureError(err, nil) @@ -51,16 +52,16 @@ func main(){ //open database and make sure that we can ping it imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") - db, err = sql.Open("postgres", imageMonkeyDbConnectionString) + db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString) if err != nil { log.Fatal("[Main] Couldn't open database: ", err.Error()) } - err = db.Ping() + err = db.Ping(context.Background()) if err != nil { log.Fatal("[Main] Couldn't ping database: ", err.Error()) } - defer db.Close() + defer db.Close(context.Background()) //create redis pool diff --git a/src/bot.go b/src/bot.go index 8417d6f0..099e979e 100644 --- a/src/bot.go +++ b/src/bot.go @@ -1,20 +1,19 @@ package main import ( - "database/sql" "flag" "fmt" commons "github.com/bbernhard/imagemonkey-core/commons" datastructures "github.com/bbernhard/imagemonkey-core/datastructures" "github.com/getsentry/raven-go" - //"github.com/gofrs/uuid" - _ "github.com/lib/pq" + "github.com/jackc/pgx/v4" log "github.com/sirupsen/logrus" "time" "errors" + "context" ) -var db *sql.DB +var db *pgx.Conn func setTrendingLabelBotTaskState(status string, branchName string, jobUrl string, id int64) error { var queryValues []interface{} @@ -30,12 +29,13 @@ func setTrendingLabelBotTaskState(status string, branchName string, jobUrl strin q := fmt.Sprintf(`UPDATE trending_label_bot_task SET state = $1, branch_name = $2%s WHERE id = $3`, jobUrlStr) - _, err := db.Exec(q, queryValues...) + _, err := db.Exec(context.TODO(), q, queryValues...) return err } func resetTrendingLabelBotTaskState(id int64) error { - _, err := db.Exec(`UPDATE trending_label_bot_task + _, err := db.Exec(context.TODO(), + `UPDATE trending_label_bot_task SET state = 'accepted', branch_name = null, job_url = null, try = try + 1 WHERE id = $1`, id) @@ -45,7 +45,8 @@ func resetTrendingLabelBotTaskState(id int64) error { func getTrendingLabels() ([]datastructures.TrendingLabelBotTask, error) { trendingLabels := []datastructures.TrendingLabelBotTask{} - rows, err := db.Query(`SELECT s.name, b.id, b.state, COALESCE(b.branch_name, ''), label_type, + rows, err := db.Query(context.TODO(), + `SELECT s.name, b.id, b.state, COALESCE(b.branch_name, ''), label_type, COALESCE(b.plural) as plural, COALESCE(b.description, ''), COALESCE(b.rename_to, '') FROM trending_label_suggestion t JOIN trending_label_bot_task b ON b.trending_label_suggestion_id = t.id @@ -75,7 +76,8 @@ func getTrendingLabels() ([]datastructures.TrendingLabelBotTask, error) { } func labelAlreadyExistsInPipeline(label string, trendingLabelBotTaskId int64) (bool, error) { - rows, err := db.Query(`SELECT count(*) + rows, err := db.Query(context.TODO(), + `SELECT count(*) FROM trending_label_suggestion t JOIN trending_label_bot_task b ON b.trending_label_suggestion_id = t.id WHERE b.rename_to = $1 AND b.id != $2`, label, trendingLabelBotTaskId) @@ -135,18 +137,18 @@ func main() { //open database and make sure that we can ping it var err error imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") - db, err = sql.Open("postgres", imageMonkeyDbConnectionString) + db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString) if err != nil { raven.CaptureError(err, nil) log.Fatal("Couldn't open database: ", err.Error()) } - err = db.Ping() + err = db.Ping(context.Background()) if err != nil { raven.CaptureError(err, nil) log.Fatal("Couldn't ping database: ", err.Error()) } - defer db.Close() + defer db.Close(context.Background()) labelsRepository := commons.NewLabelsRepository(*labelsRepositoryOwner, *labelsRepositoryName, *gitCheckoutDir) labelsRepository.SetToken(imageMonkeyBotGithubApiToken) diff --git a/src/clients/labels_populator.go b/src/clients/labels_populator.go index e8fe82fa..9d95570f 100644 --- a/src/clients/labels_populator.go +++ b/src/clients/labels_populator.go @@ -4,9 +4,9 @@ import ( "errors" datastructures "github.com/bbernhard/imagemonkey-core/datastructures" commons "github.com/bbernhard/imagemonkey-core/commons" - "database/sql" - _ "github.com/lib/pq" + "github.com/jackc/pgx/v4" log "github.com/sirupsen/logrus" + "context" ) type LabelsPopulatorClient struct { @@ -17,7 +17,7 @@ type LabelsPopulatorClient struct { labelRefinements map[string]datastructures.LabelMapRefinementEntry metalabels *commons.MetaLabels dbConnectionString string - db *sql.DB + db *pgx.Conn } func NewLabelsPopulatorClient(dbConnectionString string, labelsPath string, labelRefinementsPath string, metalabelsPath string) *LabelsPopulatorClient { @@ -47,7 +47,7 @@ func (p *LabelsPopulatorClient) Load() error { return errors.New("Couldn't get meta labels: " + err.Error()) } - p.db, err = sql.Open("postgres", p.dbConnectionString) + p.db, err = pgx.Connect(context.TODO(), p.dbConnectionString) if err != nil { return err } @@ -57,7 +57,7 @@ func (p *LabelsPopulatorClient) Load() error { } func (p *LabelsPopulatorClient) Populate(dryRun bool) error { - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { return errors.New("Couldn't start transaction: " + err.Error()) } @@ -76,33 +76,34 @@ func (p *LabelsPopulatorClient) Populate(dryRun bool) error { } if ! dryRun { - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { return errors.New("Couldn't commit changes: " + err.Error()) } } else { - tx.Rollback() + tx.Rollback(context.TODO()) log.Info("Rolling back transaction...it was only a dry run.") } return nil } -func getLabelId(tx *sql.Tx, label string, sublabel string) (int64, error) { +func getLabelId(tx pgx.Tx, label string, sublabel string) (int64, error) { var labelId int64 labelId = -1 if sublabel == "" { - err := tx.QueryRow(`SELECT id FROM label WHERE name = $1 and parent_id is null`, label).Scan(&labelId) + err := tx.QueryRow(context.TODO(), `SELECT id FROM label WHERE name = $1 and parent_id is null`, label).Scan(&labelId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return labelId, errors.New("Couldn't get label id: " + err.Error()) } } else { - err := tx.QueryRow(`SELECT l.id FROM label l + err := tx.QueryRow(context.TODO(), + `SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE l.name = $1 and pl.name = $2`, sublabel, label).Scan(&labelId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return labelId, errors.New("Couldn't get label id: " + label + "," + sublabel) } } @@ -110,10 +111,11 @@ func getLabelId(tx *sql.Tx, label string, sublabel string) (int64, error) { return labelId, nil } -func addAccessor(tx *sql.Tx, labelId int64, accessor string) error { +func addAccessor(tx pgx.Tx, labelId int64, accessor string) error { var insertedId int64 insertedId = -1 - err := tx.QueryRow(`INSERT INTO label_accessor(label_id, accessor) VALUES($1, $2) + err := tx.QueryRow(context.TODO(), + `INSERT INTO label_accessor(label_id, accessor) VALUES($1, $2) ON CONFLICT (label_id, accessor) DO NOTHING RETURNING id`, labelId, accessor).Scan(&insertedId) if insertedId != -1 { @@ -125,7 +127,7 @@ func addAccessor(tx *sql.Tx, labelId int64, accessor string) error { return err } -func addAccessors(tx *sql.Tx, label string, val datastructures.LabelMapEntry) error { +func addAccessors(tx pgx.Tx, label string, val datastructures.LabelMapEntry) error { for _, accessor := range val.Accessors { labelId, err := getLabelId(tx, label, "") @@ -140,7 +142,7 @@ func addAccessors(tx *sql.Tx, label string, val datastructures.LabelMapEntry) er err = addAccessor(tx, labelId, accessorStr) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't add accessor: " + err.Error()) } @@ -160,7 +162,7 @@ func addAccessors(tx *sql.Tx, label string, val datastructures.LabelMapEntry) er err = addAccessor(tx, labelId, subaccessorStr) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't add accessor for sublabel: " + err.Error()) } } @@ -187,7 +189,7 @@ func addAccessors(tx *sql.Tx, label string, val datastructures.LabelMapEntry) er err = addAccessor(tx, labelId, quizEntryAccessorStr) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't add accessor for quiz entry: " + err.Error()) } @@ -199,9 +201,10 @@ func addAccessors(tx *sql.Tx, label string, val datastructures.LabelMapEntry) er return nil } -func addQuizQuestion(tx *sql.Tx, parentLabelUuid string, val datastructures.LabelMapEntry) error { +func addQuizQuestion(tx pgx.Tx, parentLabelUuid string, val datastructures.LabelMapEntry) error { for _, quizEntry := range val.Quiz { - _, err := tx.Exec(`INSERT INTO quiz_question(question, refines_label_id, recommended_control, + _, err := tx.Exec(context.TODO(), + `INSERT INTO quiz_question(question, refines_label_id, recommended_control, allow_unknown, allow_other, browse_by_example, multiselect, uuid) SELECT $1, id, $2, $3, $4, $5, $6, $8 FROM label WHERE uuid = $7 ON CONFLICT(uuid) DO NOTHING`, @@ -209,22 +212,23 @@ func addQuizQuestion(tx *sql.Tx, parentLabelUuid string, val datastructures.Labe quizEntry.AllowOther, quizEntry.BrowseByExample, quizEntry.Multiselect, parentLabelUuid, quizEntry.Uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't add quiz question " + err.Error()) } } return nil } -func addQuizAnswers(tx *sql.Tx, parentLabelUuid string, val datastructures.LabelMapEntry) error { +func addQuizAnswers(tx pgx.Tx, parentLabelUuid string, val datastructures.LabelMapEntry) error { //quiz answers for _, quizEntry := range val.Quiz { for _, answer := range quizEntry.Answers { - rows, err := tx.Query(`INSERT INTO label(name, parent_id, uuid, label_type) + rows, err := tx.Query(context.TODO(), + `INSERT INTO label(name, parent_id, uuid, label_type) SELECT $1, id, $3, 'refinement' FROM label WHERE uuid = $2 ON CONFLICT(uuid) DO NOTHING RETURNING id`, answer.Name, parentLabelUuid, answer.Uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't add quiz answer label " + err.Error()) } @@ -236,13 +240,14 @@ func addQuizAnswers(tx *sql.Tx, parentLabelUuid string, val datastructures.Label rows.Close() - rows, err = tx.Query(`INSERT INTO quiz_answer(quiz_question_id, label_id) + rows, err = tx.Query(context.TODO(), + `INSERT INTO quiz_answer(quiz_question_id, label_id) SELECT (SELECT q.id FROM quiz_question q WHERE q.uuid = $1), (SELECT l.id FROM label l WHERE l.uuid = $2) ON CONFLICT DO NOTHING RETURNING id`, quizEntry.Uuid, answer.Uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't add quiz answer entry " + err.Error()) } @@ -258,21 +263,21 @@ func addQuizAnswers(tx *sql.Tx, parentLabelUuid string, val datastructures.Label return nil } -func addLabel(tx *sql.Tx, uuid string, label string) error { - rows, err := tx.Query("SELECT COUNT(id) FROM label WHERE uuid = $1", uuid) +func addLabel(tx pgx.Tx, uuid string, label string) error { + rows, err := tx.Query(context.TODO(), "SELECT COUNT(id) FROM label WHERE uuid = $1", uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } if !rows.Next() { - tx.Rollback() + tx.Rollback(context.TODO()) return err } numOfLabels := 0 err = rows.Scan(&numOfLabels) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -280,9 +285,9 @@ func addLabel(tx *sql.Tx, uuid string, label string) error { if numOfLabels == 0 { log.Info("Adding label ", label) - _,err := tx.Exec("INSERT INTO label(name, uuid, label_type) VALUES($1, $2, 'normal')", label, uuid) + _,err := tx.Exec(context.TODO(), "INSERT INTO label(name, uuid, label_type) VALUES($1, $2, 'normal')", label, uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } } else { @@ -292,21 +297,21 @@ func addLabel(tx *sql.Tx, uuid string, label string) error { return nil } -func addSublabel(tx *sql.Tx, uuid string, label string, sublabel string) error { - rows, err := tx.Query("SELECT count(*) FROM label WHERE uuid = $1", uuid) +func addSublabel(tx pgx.Tx, uuid string, label string, sublabel string) error { + rows, err := tx.Query(context.TODO(), "SELECT count(*) FROM label WHERE uuid = $1", uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } if !rows.Next() { - tx.Rollback() + tx.Rollback(context.TODO()) return err } numOfLabels := 0 err = rows.Scan(&numOfLabels) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -314,11 +319,12 @@ func addSublabel(tx *sql.Tx, uuid string, label string, sublabel string) error { if numOfLabels == 0 { log.Info("Adding label ", sublabel, " (parent: ", label, " )") - _,err := tx.Exec(`INSERT INTO label(name, parent_id, uuid, label_type) + _,err := tx.Exec(context.TODO(), + `INSERT INTO label(name, parent_id, uuid, label_type) SELECT $1, l.id, $3, 'normal' FROM label l WHERE l.name = $2 AND l.parent_id is null`, sublabel, label, uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } } else { @@ -328,17 +334,18 @@ func addSublabel(tx *sql.Tx, uuid string, label string, sublabel string) error { return nil } -func addLabelRefinements(tx *sql.Tx, labelMapRefinementEntries map[string]datastructures.LabelMapRefinementEntry) error { +func addLabelRefinements(tx pgx.Tx, labelMapRefinementEntries map[string]datastructures.LabelMapRefinementEntry) error { for k, v := range labelMapRefinementEntries { if v.Uuid == "" { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("refinement type uuid is empty!") } - rows, err := tx.Query(`INSERT INTO label(name, parent_id, uuid, label_type) VALUES ($1, null, $2, 'refinement_category') + rows, err := tx.Query(context.TODO(), + `INSERT INTO label(name, parent_id, uuid, label_type) VALUES ($1, null, $2, 'refinement_category') ON CONFLICT DO NOTHING RETURNING id`, k, v.Uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -352,17 +359,18 @@ func addLabelRefinements(tx *sql.Tx, labelMapRefinementEntries map[string]datast for k1, v1 := range v.Values { if v1.Uuid == "" { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("refinement label uuid is empty!") } //insert label if not exists - rows, err = tx.Query(`INSERT INTO label(name, parent_id, uuid, label_type) + rows, err = tx.Query(context.TODO(), + `INSERT INTO label(name, parent_id, uuid, label_type) SELECT $1, l.id, $2, 'refinement' FROM label l WHERE l.uuid = $3 ON CONFLICT DO NOTHING RETURNING id`, k1, v1.Uuid, v.Uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -385,18 +393,19 @@ func addLabelRefinements(tx *sql.Tx, labelMapRefinementEntries map[string]datast -func addMetaLabelAccessors(tx *sql.Tx, labelUuid string, label string, accessors []string) error { +func addMetaLabelAccessors(tx pgx.Tx, labelUuid string, label string, accessors []string) error { for _, acc := range accessors { accessor := acc if accessor == "." { accessor = label } - rows, err := tx.Query(`INSERT INTO label_accessor(label_id, accessor) + rows, err := tx.Query(context.TODO(), + `INSERT INTO label_accessor(label_id, accessor) SELECT l.id, $2 FROM label l WHERE uuid = $1 ON CONFLICT DO NOTHING RETURNING id`, labelUuid, accessor) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -411,7 +420,7 @@ func addMetaLabelAccessors(tx *sql.Tx, labelUuid string, label string, accessors return nil } -func addLabels(tx *sql.Tx, labelMap map[string]datastructures.LabelMapEntry) error { +func addLabels(tx pgx.Tx, labelMap map[string]datastructures.LabelMapEntry) error { for k := range labelMap { val := labelMap[k] @@ -447,17 +456,18 @@ func addLabels(tx *sql.Tx, labelMap map[string]datastructures.LabelMapEntry) err return nil } -func addMetaLabels(tx *sql.Tx, metaLabels datastructures.MetaLabelMap) error { +func addMetaLabels(tx pgx.Tx, metaLabels datastructures.MetaLabelMap) error { for k, v := range metaLabels.MetaLabelMapEntries { if v.Uuid == "" { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("metalabels uuid is empty!") } - rows, err := tx.Query(`INSERT INTO label(name, parent_id, uuid, label_type) VALUES ($1, null, $2, 'meta') + rows, err := tx.Query(context.TODO(), + `INSERT INTO label(name, parent_id, uuid, label_type) VALUES ($1, null, $2, 'meta') ON CONFLICT DO NOTHING RETURNING id`, k, v.Uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -477,18 +487,19 @@ func addMetaLabels(tx *sql.Tx, metaLabels datastructures.MetaLabelMap) error { return nil } -func addLabelRefinementAccessors(tx *sql.Tx, labelUuid string, label string, accessors []string) error { +func addLabelRefinementAccessors(tx pgx.Tx, labelUuid string, label string, accessors []string) error { for _, acc := range accessors { accessor := acc if accessor == "." { accessor = label } - rows, err := tx.Query(`INSERT INTO label_accessor(label_id, accessor) + rows, err := tx.Query(context.TODO(), + `INSERT INTO label_accessor(label_id, accessor) SELECT l.id, $2 FROM label l WHERE uuid = $1 ON CONFLICT DO NOTHING RETURNING id`, labelUuid, accessor) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } diff --git a/src/clients/make_labels_productive.go b/src/clients/make_labels_productive.go index 337ab02a..c9f7e654 100644 --- a/src/clients/make_labels_productive.go +++ b/src/clients/make_labels_productive.go @@ -1,7 +1,6 @@ package clients import ( - "database/sql" "golang.org/x/oauth2" "errors" "context" @@ -11,6 +10,7 @@ import ( commons "github.com/bbernhard/imagemonkey-core/commons" imagemonkeydb "github.com/bbernhard/imagemonkey-core/database" log "github.com/sirupsen/logrus" + "github.com/jackc/pgx/v4" ) type MakeLabelsProductiveClient struct { @@ -24,7 +24,7 @@ type MakeLabelsProductiveClient struct { strict bool labels *commons.LabelRepository metalabels *commons.MetaLabels - db *sql.DB + db *pgx.Conn } func NewMakeLabelsProductiveClient(dbConnectionString string, labelsPath string, metalabelsPath string, strict bool, autoCloseIssue bool) *MakeLabelsProductiveClient { @@ -63,12 +63,12 @@ func (p *MakeLabelsProductiveClient) Load() error { return err } - p.db, err = sql.Open("postgres", p.dbConnectionString) + p.db, err = pgx.Connect(context.TODO(), p.dbConnectionString) if err != nil { return err } - err = p.db.Ping() + err = p.db.Ping(context.TODO()) if err != nil { return err } @@ -85,7 +85,7 @@ func (p *MakeLabelsProductiveClient) DoIt(trendingLabel string, renameTo string, return errors.New("Please provide a trending label!") } - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { return errors.New("Couldn't begin trensaction: " + err.Error()) } @@ -94,7 +94,7 @@ func (p *MakeLabelsProductiveClient) DoIt(trendingLabel string, renameTo string, if p.strict { exists, err := trendingLabelExists(trendingLabel, tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't determine whether trending label exists: " + err.Error()) } @@ -104,7 +104,7 @@ func (p *MakeLabelsProductiveClient) DoIt(trendingLabel string, renameTo string, } else { nonStrictLabels, err := getNonStrictLabels(tx, trendingLabel) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't get non strict labels: " + err.Error()) } trendingLabels = nonStrictLabels @@ -123,48 +123,48 @@ func (p *MakeLabelsProductiveClient) DoIt(trendingLabel string, renameTo string, labelId, err := _getLabelId(labelToCheck, tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't determine whether label exists: " + err.Error()) } if labelId == -1 { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("label " + labelToCheck + " doesn't exist in database - please add it via the populate_labels script.") } labelMeEntry, err := makeLabelMeEntry(tx, labelToCheck) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't create label entry - is UUID valid?") } if !isLabelInLabelsMap(p.labels.GetMapping(), p.metalabels, labelMeEntry) && renameTo == "" { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Label doesn't exist in labels map - please add it first!") } for _, trendingLabel := range trendingLabels { err = makeTrendingLabelProductive(trendingLabel, labelMeEntry, labelId, tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't make trending label " + trendingLabel + " productive: " + err.Error()) } err = removeTrendingLabelEntries(trendingLabel, tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't remove trending label entries: " + err.Error()) } } err = imagemonkeydb.MakeAnnotationsProductive(tx, trendingLabel, labelId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't make annotations productive: " + err.Error()) } if dryRun { - err = tx.Rollback() + err = tx.Rollback(context.TODO()) if err != nil { return errors.New("Couldn't rollback transaction: " + err.Error()) } @@ -176,13 +176,13 @@ func (p *MakeLabelsProductiveClient) DoIt(trendingLabel string, renameTo string, for _, trendingLabel := range trendingLabels { err := closeGithubIssue(trendingLabel, p.githubRepository, p.githubRepositoryOwner, p.githubApiToken, tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("Couldn't get github issue id to close issue!") } } } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { return errors.New("Couldn't commit transaction: " + err.Error()) } @@ -204,9 +204,10 @@ func (p *MakeLabelsProductiveClient) DoIt(trendingLabel string, renameTo string, return nil } -func trendingLabelExists(label string, tx *sql.Tx) (bool, error) { +func trendingLabelExists(label string, tx pgx.Tx) (bool, error) { var numOfRows int32 - err := tx.QueryRow(`SELECT COUNT(*) FROM trending_label_suggestion t + err := tx.QueryRow(context.TODO(), + `SELECT COUNT(*) FROM trending_label_suggestion t RIGHT JOIN label_suggestion l ON t.label_suggestion_id = l.id WHERE l.name = $1`, label).Scan(&numOfRows) if err != nil { @@ -220,18 +221,20 @@ func trendingLabelExists(label string, tx *sql.Tx) (bool, error) { return false, nil } -func _getLabelId(labelIdentifier string, tx *sql.Tx) (int64, error) { +func _getLabelId(labelIdentifier string, tx pgx.Tx) (int64, error) { var labelId int64 = -1 var err error - var rows *sql.Rows + var rows pgx.Rows _, err = uuid.FromString(labelIdentifier) if err == nil { //is UUID - rows, err = tx.Query(`SELECT l.id + rows, err = tx.Query(context.TODO(), + `SELECT l.id FROM label l WHERE l.uuid::text = $1`, labelIdentifier) } else { - rows, err = tx.Query(`SELECT l.id + rows, err = tx.Query(context.TODO(), + `SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null`, labelIdentifier) } @@ -250,10 +253,11 @@ func _getLabelId(labelIdentifier string, tx *sql.Tx) (int64, error) { return labelId, nil } -func getLabelMeEntryFromUuid(tx *sql.Tx, uuid string) (datastructures.LabelMeEntry, error) { +func getLabelMeEntryFromUuid(tx pgx.Tx, uuid string) (datastructures.LabelMeEntry, error) { var labelMeEntry datastructures.LabelMeEntry - rows, err := tx.Query(`SELECT l.name, COALESCE(pl.name, '') + rows, err := tx.Query(context.TODO(), + `SELECT l.name, COALESCE(pl.name, '') FROM label l LEFT JOIN label pl ON pl.id = l.parent_id WHERE l.uuid::text = $1`, uuid) @@ -282,8 +286,9 @@ func getLabelMeEntryFromUuid(tx *sql.Tx, uuid string) (datastructures.LabelMeEnt return labelMeEntry, nil } -func removeTrendingLabelEntries(trendingLabel string, tx *sql.Tx) (error) { - _, err := tx.Exec(`DELETE FROM image_label_suggestion s +func removeTrendingLabelEntries(trendingLabel string, tx pgx.Tx) (error) { + _, err := tx.Exec(context.TODO(), + `DELETE FROM image_label_suggestion s WHERE s.label_suggestion_id IN ( SELECT l.id FROM label_suggestion l WHERE l.name = $1 )`, trendingLabel) @@ -292,7 +297,8 @@ func removeTrendingLabelEntries(trendingLabel string, tx *sql.Tx) (error) { return err } - _, err = tx.Exec(`UPDATE trending_label_suggestion t + _, err = tx.Exec(context.TODO(), + `UPDATE trending_label_suggestion t SET closed = true FROM label_suggestion AS l WHERE label_suggestion_id = l.id AND l.name = $1`, trendingLabel) @@ -300,8 +306,9 @@ func removeTrendingLabelEntries(trendingLabel string, tx *sql.Tx) (error) { return err } -func closeGithubIssue(trendingLabel string, repository string, githubProjectOwner string, githubApiToken string, tx *sql.Tx) error { - rows, err := tx.Query(`SELECT t.github_issue_id, t.closed +func closeGithubIssue(trendingLabel string, repository string, githubProjectOwner string, githubApiToken string, tx pgx.Tx) error { + rows, err := tx.Query(context.TODO(), + `SELECT t.github_issue_id, t.closed FROM trending_label_suggestion t JOIN label_suggestion l ON t.label_suggestion_id = l.id WHERE l.name = $1`, trendingLabel) @@ -359,13 +366,14 @@ func closeGithubIssue(trendingLabel string, repository string, githubProjectOwne func makeTrendingLabelProductive(trendingLabel string, label datastructures.LabelMeEntry, - labelId int64, tx *sql.Tx) error { + labelId int64, tx pgx.Tx) error { type Result struct { ImageId string Annotatable bool } - rows, err := tx.Query(`SELECT i.key, annotatable + rows, err := tx.Query(context.TODO(), + `SELECT i.key, annotatable FROM label_suggestion l JOIN image_label_suggestion isg on isg.label_suggestion_id =l.id JOIN image i on i.id = isg.image_id @@ -408,7 +416,8 @@ func makeTrendingLabelProductive(trendingLabel string, label datastructures.Labe } } - _, err = tx.Exec(`UPDATE trending_label_suggestion t + _, err = tx.Exec(context.TODO(), + `UPDATE trending_label_suggestion t SET productive_label_id = $2 FROM label_suggestion l WHERE t.label_suggestion_id = l.id AND l.name = $1`, trendingLabel, labelId) @@ -419,7 +428,7 @@ func makeTrendingLabelProductive(trendingLabel string, label datastructures.Labe return nil } -func makeLabelMeEntry(tx *sql.Tx, name string) (datastructures.LabelMeEntry, error) { +func makeLabelMeEntry(tx pgx.Tx, name string) (datastructures.LabelMeEntry, error) { _, err := uuid.FromString(name) if err == nil { //is UUID entry, err := getLabelMeEntryFromUuid(tx, name) @@ -442,10 +451,10 @@ func isLabelInLabelsMap(labelMap map[string]datastructures.LabelMapEntry, metala } -func getNonStrictLabels(tx *sql.Tx, name string) ([]string, error) { +func getNonStrictLabels(tx pgx.Tx, name string) ([]string, error) { names := []string{} - rows, err := tx.Query("SELECT l.name FROM label_suggestion l WHERE l.name ~ ('^[ ]*'||$1||'[ ]*$')", name) + rows, err := tx.Query(context.TODO(), "SELECT l.name FROM label_suggestion l WHERE l.name ~ ('^[ ]*'||$1||'[ ]*$')", name) if err != nil { return names, err } diff --git a/src/data_processor.go b/src/data_processor.go index 0a375acf..d0ef8495 100644 --- a/src/data_processor.go +++ b/src/data_processor.go @@ -1,23 +1,23 @@ package main import ( - "time" - log "github.com/sirupsen/logrus" - "database/sql" - _ "github.com/lib/pq" - "github.com/getsentry/raven-go" + "context" + "encoding/json" "flag" "fmt" + commons "github.com/bbernhard/imagemonkey-core/commons" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "github.com/getsentry/raven-go" "github.com/gomodule/redigo/redis" + "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" "os" - "encoding/json" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" - commons "github.com/bbernhard/imagemonkey-core/commons" + "time" ) -var db *sql.DB +var db *pgx.Conn -func removeOldImageAnnotationCoverage(updateAnnotationCoverageRequest datastructures.UpdateAnnotationCoverageRequest, tx *sql.Tx) error { +func removeOldImageAnnotationCoverage(updateAnnotationCoverageRequest datastructures.UpdateAnnotationCoverageRequest, tx pgx.Tx) error { var queryValues []interface{} q1 := "" if updateAnnotationCoverageRequest.Uuid != "" { @@ -30,15 +30,15 @@ func removeOldImageAnnotationCoverage(updateAnnotationCoverageRequest datastruct } q := fmt.Sprintf(`DELETE FROM image_annotation_coverage c %s`, q1) - _, err := tx.Exec(q, queryValues...) - if err != nil { - tx.Rollback() - log.Error("[Removing old Image Annotation coverage] Couldn't remove old image annotation coverage: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - return nil + _, err := tx.Exec(context.TODO(), q, queryValues...) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Removing old Image Annotation coverage] Couldn't remove old image annotation coverage: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + return nil } func calculateImageAnnotationCoverage(updateAnnotationCoverageRequest datastructures.UpdateAnnotationCoverageRequest) error { @@ -60,35 +60,34 @@ func calculateImageAnnotationCoverage(updateAnnotationCoverageRequest datastruct SELECT image_id, area, annotated_percentage FROM sp_get_image_annotation_coverage(%s)`, q1) - tx, err := db.Begin() - if err != nil { - log.Error("[Calculating Image Annotation coverage] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + tx, err := db.Begin(context.TODO()) + if err != nil { + log.Error("[Calculating Image Annotation coverage] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - err = removeOldImageAnnotationCoverage(updateAnnotationCoverageRequest, tx) + err = removeOldImageAnnotationCoverage(updateAnnotationCoverageRequest, tx) if err != nil { //transaction already rolled back, so nothing to do here return err } - - _, err = tx.Exec(q, queryValues...) + _, err = tx.Exec(context.TODO(), q, queryValues...) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Calculating Image Annotation coverage] Couldn't calculate image annotation coverage", err.Error()) raven.CaptureError(err, nil) return err } - err = tx.Commit() - if err != nil { - log.Error("[Calculating Image Annotation coverage] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + err = tx.Commit(context.TODO()) + if err != nil { + log.Error("[Calculating Image Annotation coverage] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - return nil + return nil } func main() { @@ -108,18 +107,18 @@ func main() { imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") var err error - db, err = sql.Open("postgres", imageMonkeyDbConnectionString) + db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString) if err != nil { raven.CaptureError(err, nil) log.Fatal(err) } - err = db.Ping() + err = db.Ping(context.Background()) if err != nil { raven.CaptureError(err, nil) log.Fatal("[Main] Couldn't ping database: ", err.Error()) } - defer db.Close() + defer db.Close(context.Background()) //create redis pool redisPool := redis.NewPool(func() (redis.Conn, error) { @@ -146,7 +145,7 @@ func main() { //on startup, do a full re-calculate for { err := calculateImageAnnotationCoverage(datastructures.UpdateAnnotationCoverageRequest{}) - if err == nil { + if err == nil { log.Info("[Main] Completely re-calculated image annotation coverage") break } @@ -162,36 +161,35 @@ func main() { redisConn := redisPool.Get() data, err := redis.Bytes(redisConn.Do("LPOP", commons.UPDATE_IMAGE_ANNOTATION_COVERAGE_TOPIC)) - if err == nil { //data available - retryImmediately = true //in case there is data available, try it again immediatelly to get more data - - var updateAnnotationCoverageRequest datastructures.UpdateAnnotationCoverageRequest - err = json.Unmarshal(data, &updateAnnotationCoverageRequest) - if err != nil{ - retryImmediately = false //in case of an error, wait a bit (maybe it recovers in the meanwhile) - log.Error("[Main] Couldn't unmarshal request: ", err.Error()) - raven.CaptureError(err, nil) - } else { - err := calculateImageAnnotationCoverage(updateAnnotationCoverageRequest) - if err == nil { - retryImmediately = false //in case of an error, wait a bit (maybe it recovers in the meanwhile) - log.Info("[Main] Re-calculated image annotation coverage for ", - updateAnnotationCoverageRequest.Type, " with id: ", updateAnnotationCoverageRequest.Uuid) - } - } - } - - redisConn.Close() + if err == nil { //data available + retryImmediately = true //in case there is data available, try it again immediatelly to get more data + + var updateAnnotationCoverageRequest datastructures.UpdateAnnotationCoverageRequest + err = json.Unmarshal(data, &updateAnnotationCoverageRequest) + if err != nil { + retryImmediately = false //in case of an error, wait a bit (maybe it recovers in the meanwhile) + log.Error("[Main] Couldn't unmarshal request: ", err.Error()) + raven.CaptureError(err, nil) + } else { + err := calculateImageAnnotationCoverage(updateAnnotationCoverageRequest) + if err == nil { + retryImmediately = false //in case of an error, wait a bit (maybe it recovers in the meanwhile) + log.Info("[Main] Re-calculated image annotation coverage for ", + updateAnnotationCoverageRequest.Type, " with id: ", updateAnnotationCoverageRequest.Uuid) + } + } + } + + redisConn.Close() if !retryImmediately { time.Sleep((time.Second * 2)) //sleep for two seconds } - } - + } else { - select{} //sleep forever, without eating CPU + select {} //sleep forever, without eating CPU } } diff --git a/src/database/access_token.go b/src/database/access_token.go index 5cb48cee..5145e70f 100644 --- a/src/database/access_token.go +++ b/src/database/access_token.go @@ -1,56 +1,57 @@ package imagemonkeydb import ( - "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - "errors" + "context" + "errors" + "github.com/getsentry/raven-go" + log "github.com/sirupsen/logrus" ) func (p *ImageMonkeyDatabase) AddAccessToken(username string, accessToken string, expirationTime int64) error { - var insertedId int64 + var insertedId int64 - insertedId = 0 - err := p.db.QueryRow(`INSERT INTO access_token(user_id, token, expiration_time) + insertedId = 0 + err := p.db.QueryRow(context.TODO(), `INSERT INTO access_token(user_id, token, expiration_time) SELECT id, $2, $3 FROM account a WHERE a.name = $1 RETURNING id`, username, accessToken, expirationTime).Scan(&insertedId) - if err != nil { - log.Debug("[Add Access Token] Couldn't add access token: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - if insertedId == 0 { - log.Debug("[Add Access Token] Nothing inserted") - return errors.New("Nothing inserted") - } - - return nil + if err != nil { + log.Debug("[Add Access Token] Couldn't add access token: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + if insertedId == 0 { + log.Debug("[Add Access Token] Nothing inserted") + return errors.New("Nothing inserted") + } + + return nil } func (p *ImageMonkeyDatabase) RemoveAccessToken(accessToken string) error { - _, err := p.db.Exec(`DELETE FROM access_token WHERE token = $1`, accessToken) - if err != nil { - log.Debug("[Remove Access Token] Couldn't remove access token: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - return nil + _, err := p.db.Exec(context.TODO(), `DELETE FROM access_token WHERE token = $1`, accessToken) + if err != nil { + log.Debug("[Remove Access Token] Couldn't remove access token: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + return nil } func (p *ImageMonkeyDatabase) AccessTokenExists(accessToken string) bool { - var numOfAccessTokens int32 + var numOfAccessTokens int32 - numOfAccessTokens = 0 - err := p.db.QueryRow("SELECT count(*) FROM access_token WHERE token = $1", accessToken).Scan(&numOfAccessTokens) - if err != nil { - log.Debug("[Add Access Token] Couldn't add access token: ", err.Error()) - raven.CaptureError(err, nil) - return false - } + numOfAccessTokens = 0 + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM access_token WHERE token = $1", accessToken).Scan(&numOfAccessTokens) + if err != nil { + log.Debug("[Add Access Token] Couldn't add access token: ", err.Error()) + raven.CaptureError(err, nil) + return false + } - if numOfAccessTokens == 0 { - return false - } + if numOfAccessTokens == 0 { + return false + } - return true + return true } diff --git a/src/database/api_token.go b/src/database/api_token.go index c8e91f13..9fc44fb4 100644 --- a/src/database/api_token.go +++ b/src/database/api_token.go @@ -1,124 +1,127 @@ package imagemonkeydb import ( - "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - "github.com/dgrijalva/jwt-go" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "context" + "errors" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "github.com/dgrijalva/jwt-go" + "github.com/getsentry/raven-go" + log "github.com/sirupsen/logrus" "time" - "errors" ) func (p *ImageMonkeyDatabase) GetApiTokens(username string) ([]datastructures.APIToken, error) { - var apiTokens []datastructures.APIToken - rows, err := p.db.Query(`SELECT token, issued_at, description, revoked + var apiTokens []datastructures.APIToken + rows, err := p.db.Query(context.TODO(), + `SELECT token, issued_at, description, revoked FROM api_token a JOIN account a1 ON a1.id = a.account_id WHERE a1.name = $1`, username) - if err != nil { - log.Debug("[Get API Tokens] Couldn't get rows: ", err.Error()) - raven.CaptureError(err, nil) - return apiTokens, err - } - - defer rows.Close() - - for rows.Next() { - var apiToken datastructures.APIToken - err = rows.Scan(&apiToken.Token, &apiToken.IssuedAtUnixTimestamp, &apiToken.Description, &apiToken.Revoked) - if err != nil { - log.Debug("[Get API Tokens] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return apiTokens, err - } - - apiTokens = append(apiTokens, apiToken) - } - - return apiTokens, nil + if err != nil { + log.Debug("[Get API Tokens] Couldn't get rows: ", err.Error()) + raven.CaptureError(err, nil) + return apiTokens, err + } + + defer rows.Close() + + for rows.Next() { + var apiToken datastructures.APIToken + err = rows.Scan(&apiToken.Token, &apiToken.IssuedAtUnixTimestamp, &apiToken.Description, &apiToken.Revoked) + if err != nil { + log.Debug("[Get API Tokens] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return apiTokens, err + } + + apiTokens = append(apiTokens, apiToken) + } + + return apiTokens, nil } func (p *ImageMonkeyDatabase) IsApiTokenRevoked(token string) (bool, error) { - var revoked bool = false - rows, err := p.db.Query("SELECT revoked FROM api_token WHERE token = $1", token) - if err != nil { - log.Error("[Is API Token revoked] Couldn't determine whether API token is revoked: ", err.Error()) - raven.CaptureError(err, nil) - return false, err - } - defer rows.Close() - - if rows.Next() { - err = rows.Scan(&revoked) - if err != nil { - log.Error("[Is API Token revoked] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return false, err - } - - return revoked, nil - } - - return revoked, errors.New("[Is API Token revoked] Invalid result set") + var revoked bool = false + rows, err := p.db.Query(context.TODO(), "SELECT revoked FROM api_token WHERE token = $1", token) + if err != nil { + log.Error("[Is API Token revoked] Couldn't determine whether API token is revoked: ", err.Error()) + raven.CaptureError(err, nil) + return false, err + } + defer rows.Close() + + if rows.Next() { + err = rows.Scan(&revoked) + if err != nil { + log.Error("[Is API Token revoked] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return false, err + } + + return revoked, nil + } + + return revoked, errors.New("[Is API Token revoked] Invalid result set") } func (p *ImageMonkeyDatabase) GenerateApiToken(jwtSecret string, username string, description string) (datastructures.APIToken, error) { - type MyCustomClaims struct { - Username string `json:"username"` - Created int64 `json:"created"` - jwt.StandardClaims - } - - var apiToken datastructures.APIToken - - issuedAt := time.Now() - expiresAt := issuedAt.Add(time.Hour * 24 * 365 * 10) //10 years - - claims := MyCustomClaims { - username, - issuedAt.Unix(), - jwt.StandardClaims{ - ExpiresAt: expiresAt.Unix(), - Issuer: "imagemonkey-api", - }, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) - - tokenString, err := token.SignedString([]byte(jwtSecret)) - if err != nil { - return apiToken, err - } - - - _, err = p.db.Exec(`INSERT INTO api_token(account_id, issued_at, description, revoked, token, expires_at) - SELECT id, $2, $3, $4, $5, $6 FROM account WHERE name = $1`, - username, issuedAt.Unix(), description, false, tokenString, expiresAt.Unix()) - if err != nil { - log.Debug("[Generate API Token] Couldn't insert entry: ", err.Error()) - raven.CaptureError(err, nil) - return apiToken, err - } - - apiToken.Description = description - apiToken.Token = tokenString - apiToken.IssuedAtUnixTimestamp = issuedAt.Unix() - - return apiToken, nil + type MyCustomClaims struct { + Username string `json:"username"` + Created int64 `json:"created"` + jwt.StandardClaims + } + + var apiToken datastructures.APIToken + + issuedAt := time.Now() + expiresAt := issuedAt.Add(time.Hour * 24 * 365 * 10) //10 years + + claims := MyCustomClaims{ + username, + issuedAt.Unix(), + jwt.StandardClaims{ + ExpiresAt: expiresAt.Unix(), + Issuer: "imagemonkey-api", + }, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + + tokenString, err := token.SignedString([]byte(jwtSecret)) + if err != nil { + return apiToken, err + } + + _, err = p.db.Exec(context.TODO(), + `INSERT INTO api_token(account_id, issued_at, description, revoked, token, expires_at) + SELECT id, $2, $3, $4, $5, $6 FROM account WHERE name = $1`, + username, issuedAt.Unix(), description, false, tokenString, expiresAt.Unix()) + if err != nil { + log.Debug("[Generate API Token] Couldn't insert entry: ", err.Error()) + raven.CaptureError(err, nil) + return apiToken, err + } + + apiToken.Description = description + apiToken.Token = tokenString + apiToken.IssuedAtUnixTimestamp = issuedAt.Unix() + + return apiToken, nil } func (p *ImageMonkeyDatabase) RevokeApiToken(username string, apiToken string) (bool, error) { - var modifiedId int64 - err := p.db.QueryRow(`UPDATE api_token AS a + var modifiedId int64 + err := p.db.QueryRow(context.TODO(), + `UPDATE api_token AS a SET revoked = true FROM account AS acc WHERE acc.id = a.account_id AND acc.name = $1 AND a.token = $2 RETURNING a.id`, username, apiToken).Scan(&modifiedId) - if err != nil { - log.Debug("[Revoke API Token] Couldn't revoke token: ", err.Error()) - raven.CaptureError(err, nil) - return false, err - } + if err != nil { + log.Debug("[Revoke API Token] Couldn't revoke token: ", err.Error()) + raven.CaptureError(err, nil) + return false, err + } - return true, nil + return true, nil } diff --git a/src/database/database.go b/src/database/database.go index e51163d9..6f9548fd 100644 --- a/src/database/database.go +++ b/src/database/database.go @@ -1,30 +1,37 @@ package imagemonkeydb import ( - "database/sql" + "context" "github.com/getsentry/raven-go" + "github.com/jackc/pgx/v4/pgxpool" ) type ImageMonkeyDatabase struct { - db *sql.DB + db *pgxpool.Pool } func NewImageMonkeyDatabase() *ImageMonkeyDatabase { - return &ImageMonkeyDatabase{} + return &ImageMonkeyDatabase{} } -func (p *ImageMonkeyDatabase) Open(connectionString string) error { - var err error - p.db, err = sql.Open("postgres", connectionString) +func (p *ImageMonkeyDatabase) Open(connectionString string, maxNumConnections int32) error { + cfg, err := pgxpool.ParseConfig(connectionString) if err != nil { return err } - err = p.db.Ping() + cfg.MaxConns = maxNumConnections + + p.db, err = pgxpool.ConnectConfig(context.Background(), cfg) if err != nil { return err } + /*err = p.db.Ping(context.Background()) + if err != nil { + return err + }*/ + return nil } @@ -35,4 +42,4 @@ func (p *ImageMonkeyDatabase) InitializeSentry(sentryDSN string, environment str func (p *ImageMonkeyDatabase) Close() { p.db.Close() -} \ No newline at end of file +} diff --git a/src/database/game.go b/src/database/game.go index d0bd3e9b..ce797be6 100644 --- a/src/database/game.go +++ b/src/database/game.go @@ -1,22 +1,19 @@ package imagemonkeydb import ( - "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "context" + commons "github.com/bbernhard/imagemonkey-core/commons" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "github.com/getsentry/raven-go" + log "github.com/sirupsen/logrus" "time" - commons "github.com/bbernhard/imagemonkey-core/commons" - //parser "../parser" - /*"fmt" - "errors" - "encoding/json" - "github.com/lib/pq"*/ ) func (p *ImageMonkeyDatabase) GetImageHuntTasks(apiUser datastructures.APIUser, apiBaseUrl string) ([]datastructures.ImageHuntTask, error) { imageHuntTasks := []datastructures.ImageHuntTask{} - rows, err := p.db.Query(`SELECT q.image_width, q.image_height, q.image_unlocked, q.image_key, q.label_accessor, q.label + rows, err := p.db.Query(context.TODO(), + `SELECT q.image_width, q.image_height, q.image_unlocked, q.image_key, q.label_accessor, q.label FROM ( SELECT i.width as image_width, i.height as image_height, i.unlocked as image_unlocked, @@ -53,22 +50,22 @@ func (p *ImageMonkeyDatabase) GetImageHuntTasks(apiUser datastructures.APIUser, `, apiUser.Name) if err != nil { log.Error("[Get ImageHunt Tasks] Couldn't get tasks: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntTasks, err + raven.CaptureError(err, nil) + return imageHuntTasks, err } - defer rows.Close() + defer rows.Close() for rows.Next() { var imageHuntTask datastructures.ImageHuntTask var imageHuntTaskImage datastructures.ImageHuntTaskImage - err = rows.Scan(&imageHuntTaskImage.Width, &imageHuntTaskImage.Height, &imageHuntTaskImage.Unlocked, - &imageHuntTaskImage.Id, &imageHuntTask.Label.Accessor, &imageHuntTask.Label.Name) + err = rows.Scan(&imageHuntTaskImage.Width, &imageHuntTaskImage.Height, &imageHuntTaskImage.Unlocked, + &imageHuntTaskImage.Id, &imageHuntTask.Label.Accessor, &imageHuntTask.Label.Name) if err != nil { log.Error("[Get ImageHunt Tasks] Couldn't scan tasks: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntTasks, err + raven.CaptureError(err, nil) + return imageHuntTasks, err } if imageHuntTaskImage.Id != "" { @@ -99,10 +96,11 @@ func isValidationValid(numOfValid int, numOfInvalid int) bool { } func (p *ImageMonkeyDatabase) GetImageHuntStats(apiUser datastructures.APIUser, apiBaseUrl string, - numOfAvailableLabels int, utcOffset int64) (datastructures.ImageHuntStats, error) { + numOfAvailableLabels int, utcOffset int64) (datastructures.ImageHuntStats, error) { var imageHuntStats datastructures.ImageHuntStats - rows, err := p.db.Query(`SELECT count(*) + rows, err := p.db.Query(context.TODO(), + `SELECT count(*) FROM imagehunt_task h JOIN image_validation v ON v.id = h.image_validation_id JOIN user_image u ON u.image_id = v.image_id @@ -111,8 +109,8 @@ func (p *ImageMonkeyDatabase) GetImageHuntStats(apiUser datastructures.APIUser, if err != nil { log.Error("[Get ImageHunt Stats] Couldn't get stats: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntStats, err + raven.CaptureError(err, nil) + return imageHuntStats, err } defer rows.Close() @@ -121,15 +119,15 @@ func (p *ImageMonkeyDatabase) GetImageHuntStats(apiUser datastructures.APIUser, err = rows.Scan(&imageHuntStats.Stars) if err != nil { log.Error("[Get ImageHunt Stats] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntStats, err + raven.CaptureError(err, nil) + return imageHuntStats, err } } rows.Close() - - rows, err = p.db.Query(`SELECT h.created, v.num_of_valid, v.num_of_invalid + rows, err = p.db.Query(context.TODO(), + `SELECT h.created, v.num_of_valid, v.num_of_invalid FROM image_validation v JOIN imagehunt_task h ON h.image_validation_id = v.id JOIN user_image u ON u.image_id = v.image_id @@ -139,8 +137,8 @@ func (p *ImageMonkeyDatabase) GetImageHuntStats(apiUser datastructures.APIUser, if err != nil { log.Error("[Get ImageHunt Stats] Couldn't get detailed stats: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntStats, err + raven.CaptureError(err, nil) + return imageHuntStats, err } defer rows.Close() @@ -154,29 +152,27 @@ func (p *ImageMonkeyDatabase) GetImageHuntStats(apiUser datastructures.APIUser, err = rows.Scan(&created, &numOfValid, &numOfInvalid) if err != nil { log.Error("[Get ImageHunt Stats] Couldn't scan detailed row: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntStats, err + raven.CaptureError(err, nil) + return imageHuntStats, err } - t := time.Unix(created, 0) //unix timestamp -> time - t.Add(time.Duration(utcOffset * 10^9)) //add utc offset (in ns) + t := time.Unix(created, 0) //unix timestamp -> time + t.Add(time.Duration(utcOffset*10 ^ 9)) //add utc offset (in ns) isValid := isValidationValid(numOfValid, numOfInvalid) if isValid { achievementsGenerator.Add(t) - } - + } imageHuntStats.Achievements, err = achievementsGenerator.GetAchievements(apiBaseUrl) if err != nil { log.Error("[Get ImageHunt Stats] Couldn't get achievements: ", err.Error()) - raven.CaptureError(err, nil) - return imageHuntStats, err + raven.CaptureError(err, nil) + return imageHuntStats, err } - return imageHuntStats, nil } diff --git a/src/database/image.go b/src/database/image.go index 652b5f44..e86d3473 100644 --- a/src/database/image.go +++ b/src/database/image.go @@ -1,7 +1,6 @@ package imagemonkeydb import ( - "database/sql" "encoding/json" "errors" "fmt" @@ -11,6 +10,8 @@ import ( "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" "time" + "context" + "github.com/jackc/pgx/v4" ) type ImageDonationErrorType int @@ -30,11 +31,11 @@ func sublabelsToStringlist(sublabels []datastructures.Sublabel) []string { return s } -func _addImageSource(imageId int64, imageSource datastructures.ImageSource, tx *sql.Tx) (int64, error) { +func _addImageSource(imageId int64, imageSource datastructures.ImageSource, tx pgx.Tx) (int64, error) { var insertedId int64 - err := tx.QueryRow("INSERT INTO image_source(image_id, url) VALUES($1, $2) RETURNING id", imageId, imageSource.Url).Scan(&insertedId) + err := tx.QueryRow(context.TODO(), "INSERT INTO image_source(image_id, url) VALUES($1, $2) RETURNING id", imageId, imageSource.Url).Scan(&insertedId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Add image source] Couldn't add image source: ", err.Error()) raven.CaptureError(err, nil) return insertedId, err @@ -47,7 +48,7 @@ func _addImageSource(imageId int64, imageSource datastructures.ImageSource, tx * func (p *ImageMonkeyDatabase) GetRandomGroupedImages(label string, limit int) ([]datastructures.ValidationImage, error) { var images []datastructures.ValidationImage - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[Random grouped images] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -60,13 +61,14 @@ func (p *ImageMonkeyDatabase) GetRandomGroupedImages(label string, limit int) ([ //TODO: the following SQL query is a potential candidate for improvement, as it probably gets slow if there //are ten thousands of rows in the DB. var numOfRows int - err = tx.QueryRow(`SELECT count(*) FROM image i + err = tx.QueryRow(context.TODO(), + `SELECT count(*) FROM image i JOIN image_provider p ON i.image_provider_id = p.id JOIN image_validation v ON v.image_id = i.id JOIN label l ON v.label_id = l.id WHERE i.unlocked = true AND p.name = 'donation' AND l.name = $1 AND l.parent_id is null`, label).Scan(&numOfRows) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Random grouped images] Couldn't get num of rows: ", err.Error()) raven.CaptureError(err, nil) return images, err @@ -83,7 +85,8 @@ func (p *ImageMonkeyDatabase) GetRandomGroupedImages(label string, limit int) ([ } //fetch images - rows, err := p.db.Query(`SELECT i.key, l.name, v.num_of_valid, v.num_of_invalid, v.uuid FROM image i + rows, err := p.db.Query(context.TODO(), + `SELECT i.key, l.name, v.num_of_valid, v.num_of_invalid, v.uuid FROM image i JOIN image_provider p ON i.image_provider_id = p.id JOIN image_validation v ON v.image_id = i.id JOIN label l ON v.label_id = l.id @@ -91,7 +94,7 @@ func (p *ImageMonkeyDatabase) GetRandomGroupedImages(label string, limit int) ([ OFFSET $2 LIMIT $3`, label, randomNumber, limit) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Random grouped images] Couldn't get images: ", err.Error()) raven.CaptureError(err, nil) return images, err @@ -104,7 +107,7 @@ func (p *ImageMonkeyDatabase) GetRandomGroupedImages(label string, limit int) ([ image.Provider = "donation" err = rows.Scan(&image.Id, &image.Label, &image.Validation.NumOfValid, &image.Validation.NumOfInvalid, &image.Validation.Id) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Fetch random grouped image] Couldn't scan row: ", err.Error()) raven.CaptureError(err, nil) return images, err @@ -113,11 +116,11 @@ func (p *ImageMonkeyDatabase) GetRandomGroupedImages(label string, limit int) ([ images = append(images, image) } - return images, tx.Commit() + return images, tx.Commit(context.TODO()) } func (p *ImageMonkeyDatabase) UnlockImage(imageId string) error { - _, err := p.db.Exec("UPDATE image SET unlocked = true WHERE key = $1", imageId) + _, err := p.db.Exec(context.TODO(), "UPDATE image SET unlocked = true WHERE key = $1", imageId) if err != nil { log.Debug("[Unlock Image] Couldn't unlock image: ", err.Error()) raven.CaptureError(err, nil) @@ -128,7 +131,7 @@ func (p *ImageMonkeyDatabase) UnlockImage(imageId string) error { } func (p *ImageMonkeyDatabase) PutImageInQuarantine(imageId string) error { - _, err := p.db.Exec(`INSERT INTO image_quarantine(image_id) + _, err := p.db.Exec(context.TODO(), `INSERT INTO image_quarantine(image_id) SELECT id FROM image WHERE key = $1 ON CONFLICT(image_id) DO NOTHING`, imageId) if err != nil { @@ -143,7 +146,7 @@ func (p *ImageMonkeyDatabase) PutImageInQuarantine(imageId string) error { func (p *ImageMonkeyDatabase) IsImageUnlocked(uuid string) (bool, error) { var unlocked bool unlocked = false - rows, err := p.db.Query("SELECT unlocked FROM image WHERE key = $1", uuid) + rows, err := p.db.Query(context.TODO(), "SELECT unlocked FROM image WHERE key = $1", uuid) if err != nil { log.Debug("[Is Image Unlocked] Couldn't get row: ", err.Error()) raven.CaptureError(err, nil) @@ -166,7 +169,7 @@ func (p *ImageMonkeyDatabase) IsImageUnlocked(uuid string) (bool, error) { func (p *ImageMonkeyDatabase) ImageExists(hash uint64) (bool, error) { //PostgreSQL can't handle unsigned 64bit, so we are casting the hash to a signed 64bit value when comparing against the stored hash (so values above maxuint64/2 are negative). - rows, err := p.db.Query("SELECT COUNT(hash) FROM image where hash = $1", int64(hash)) + rows, err := p.db.Query(context.TODO(), "SELECT COUNT(hash) FROM image where hash = $1", int64(hash)) if err != nil { log.Debug("[Checking if photo exists] Couldn't get hash: ", err.Error()) raven.CaptureError(err, nil) @@ -219,7 +222,7 @@ func (p *ImageMonkeyDatabase) ImageExistsForUser(imageId string, username string q := fmt.Sprintf(`SELECT COUNT(i.id) FROM image i WHERE i.key = $1 AND (i.unlocked = true %s)`, includeOwnImageDonations) var num int = 0 - err := p.db.QueryRow(q, queryValues...).Scan(&num) + err := p.db.QueryRow(context.TODO(), q, queryValues...).Scan(&num) if err != nil { log.Error("[Image exists for user] Couldn't determine whether image exists: ", err.Error()) raven.CaptureError(err, nil) @@ -235,7 +238,7 @@ func (p *ImageMonkeyDatabase) ImageExistsForUser(imageId string, username string func (p *ImageMonkeyDatabase) AddDonatedPhoto(apiUser datastructures.APIUser, imageInfo datastructures.ImageInfo, autoUnlock bool, labels []datastructures.LabelMeEntry, imageCollectionName string, labelMap map[string]datastructures.LabelMapEntry, metalabels *commons.MetaLabels) error { - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[Adding donated photo] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -250,14 +253,15 @@ func (p *ImageMonkeyDatabase) AddDonatedPhoto(apiUser datastructures.APIUser, im //PostgreSQL can't store unsigned 64bit, so we are casting the hash to a signed 64bit value when storing the hash (so values above maxuint64/2 are negative). //this should be ok, as we do not need to order those values, but just need to check if a hash exists. So it should be fine var imageId int64 - err = tx.QueryRow(`INSERT INTO image(key, unlocked, image_provider_id, hash, width, height, sys_period) + err = tx.QueryRow(context.TODO(), + `INSERT INTO image(key, unlocked, image_provider_id, hash, width, height, sys_period) SELECT $1, $2, p.id, $3, $5, $6, '["now()",]'::tstzrange FROM image_provider p WHERE p.name = $4 RETURNING id`, imageInfo.Name, autoUnlock, int64(imageInfo.Hash), imageProvider, imageInfo.Width, imageInfo.Height).Scan(&imageId) if err != nil { log.Debug("[Adding donated photo] Couldn't insert image: ", err.Error()) raven.CaptureError(err, nil) - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -291,10 +295,11 @@ func (p *ImageMonkeyDatabase) AddDonatedPhoto(apiUser datastructures.APIUser, im //in case a username is provided, link image to user account if apiUser.Name != "" { - _, err := tx.Exec(`INSERT INTO user_image(image_id, account_id) + _, err := tx.Exec(context.TODO(), + `INSERT INTO user_image(image_id, account_id) SELECT $1, id FROM account WHERE name = $2`, imageId, apiUser.Name) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Add user image entry] Couldn't add entry: ", err.Error()) raven.CaptureError(err, nil) return err @@ -326,24 +331,25 @@ func (p *ImageMonkeyDatabase) AddDonatedPhoto(apiUser datastructures.APIUser, im if imageInfo.Source.Provider == "imagehunt" { if len(insertedValidationIds) != 1 { - tx.Rollback() + tx.Rollback(context.TODO()) err = errors.New("Couldn't create imagehunt entry due to missing or invalid label") log.Error("[Create ImageHunt entry for donated image]", err.Error()) raven.CaptureError(err, nil) return err } - _, err := tx.Exec(`INSERT INTO imagehunt_task(image_validation_id, created) + _, err := tx.Exec(context.TODO(), + `INSERT INTO imagehunt_task(image_validation_id, created) VALUES($1, $2)`, insertedValidationIds[0], time.Now().Unix()) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Create ImageHunt entry for donated image] Couldn't create entry: ", err.Error()) raven.CaptureError(err, nil) return err } } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Add donated Image] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -355,7 +361,8 @@ func (p *ImageMonkeyDatabase) AddDonatedPhoto(apiUser datastructures.APIUser, im func (p *ImageMonkeyDatabase) IsOwnDonation(imageId string, username string) (bool, error) { isOwnDonation := false - rows, err := p.db.Query(`SELECT count(*) + rows, err := p.db.Query(context.TODO(), + `SELECT count(*) FROM image i WHERE i.key = $1 AND EXISTS ( @@ -397,7 +404,7 @@ func (p *ImageMonkeyDatabase) IsOwnDonation(imageId string, username string) (bo func (p *ImageMonkeyDatabase) ReportImage(imageId string, reason string) error { insertedId := 0 - err := p.db.QueryRow("INSERT INTO image_report(image_id, reason) SELECT i.id, $2 FROM image i WHERE i.key = $1 RETURNING id", + err := p.db.QueryRow(context.TODO(), "INSERT INTO image_report(image_id, reason) SELECT i.id, $2 FROM image i WHERE i.key = $1 RETURNING id", imageId, reason).Scan(&insertedId) if err != nil { log.Debug("[Report image] Couldn't add report: ", err.Error()) @@ -465,16 +472,16 @@ func (p *ImageMonkeyDatabase) GetAllUnverifiedImages(imageProvider string, shuff %s AND q.unlocked = false`, q1) var err error - var rows *sql.Rows + var rows pgx.Rows - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[Fetch unverified images] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) return lockedImages, err } - rows, err = tx.Query(q, queryValues...) + rows, err = tx.Query(context.TODO(), q, queryValues...) if err != nil { log.Debug("[Fetch unverified images] Couldn't fetch unverified images: ", err.Error()) @@ -496,14 +503,14 @@ func (p *ImageMonkeyDatabase) GetAllUnverifiedImages(imageProvider string, shuff lockedImages.Images = append(lockedImages.Images, image) } - err = tx.QueryRow(totalImagesQuery).Scan(&lockedImages.Total) + err = tx.QueryRow(context.TODO(), totalImagesQuery).Scan(&lockedImages.Total) if err != nil { log.Debug("[Fetch unverified images] Couldn't get number of images: ", err.Error()) raven.CaptureError(err, nil) return lockedImages, err } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Debug("[Fetch unverified images] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -516,64 +523,68 @@ func (p *ImageMonkeyDatabase) GetAllUnverifiedImages(imageProvider string, shuff func (p *ImageMonkeyDatabase) DeleteImage(uuid string) error { var imageId int64 - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[Delete image] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) return err } - _, err = tx.Exec(`DELETE FROM user_image + _, err = tx.Exec(context.TODO(), + `DELETE FROM user_image WHERE image_id IN ( SELECT id FROM image WHERE key = $1 )`, uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Delete image] Couldn't delete user_image entry: ", err.Error()) raven.CaptureError(err, nil) return err } - _, err = tx.Exec(`DELETE FROM image_validation + _, err = tx.Exec(context.TODO(), + `DELETE FROM image_validation WHERE image_id IN ( SELECT id FROM image WHERE key = $1 )`, uuid) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Delete image] Couldn't delete image_validation entry: ", err.Error()) raven.CaptureError(err, nil) return err } imageId = -1 - err = tx.QueryRow(`DELETE FROM image i WHERE key = $1 + err = tx.QueryRow(context.TODO(), + `DELETE FROM image i WHERE key = $1 RETURNING i.id`, uuid).Scan(&imageId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Delete image] Couldn't delete image entry: ", err.Error()) raven.CaptureError(err, nil) return err } if imageId == -1 { - tx.Rollback() + tx.Rollback(context.TODO()) err = errors.New("nothing deleted") log.Debug("[Delete image] Couldn't delete image entry: ", err.Error()) raven.CaptureError(err, nil) return err } - _, err = tx.Exec(`DELETE FROM image_label_suggestion s + _, err = tx.Exec(context.TODO(), + `DELETE FROM image_label_suggestion s WHERE image_id = $1`, imageId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Delete image] Couldn't delete image_label_suggestion entry: ", err.Error()) raven.CaptureError(err, nil) return err } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Debug("[Delete image] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -645,7 +656,7 @@ func (p *ImageMonkeyDatabase) Export(parseResult parser.ParseResult, annotations WHERE i.unlocked = true GROUP BY i.key, q3.validations, i.width, i.height`, identifier, q1, parseResult.Query, identifier, q2, parseResult.Query, joinType, identifier, q3, parseResult.Query) - rows, err := p.db.Query(q, parseResult.QueryValues...) + rows, err := p.db.Query(context.TODO(), q, parseResult.QueryValues...) if err != nil { log.Debug("[Export] Couldn't export data: ", err.Error()) raven.CaptureError(err, nil) diff --git a/src/database/image_annotation.go b/src/database/image_annotation.go index d1cbb54c..375b8571 100644 --- a/src/database/image_annotation.go +++ b/src/database/image_annotation.go @@ -1,29 +1,28 @@ package imagemonkeydb import ( - "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" - parser "github.com/bbernhard/imagemonkey-core/parser/v2" - commons "github.com/bbernhard/imagemonkey-core/commons" + "context" "encoding/json" - "errors" - "fmt" - "database/sql" - "github.com/lib/pq" - "image" - "github.com/gofrs/uuid" - //"github.com/francoispqt/gojay" - //"bytes" + "errors" + "fmt" + commons "github.com/bbernhard/imagemonkey-core/commons" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + parser "github.com/bbernhard/imagemonkey-core/parser/v2" + "github.com/getsentry/raven-go" + "github.com/gofrs/uuid" + "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" + "image" ) -func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) error { +func MakeAnnotationsProductive(tx pgx.Tx, trendingLabel string, labelId int64) error { //safety check //the below code was written given a specific database table layout. If someone changes //the database schema (e.g add/remove a column), the code below needs to be adapted. //so in order to prevent that someone adds/removes a column to these tables, but forgets //to change the code below we strictly check for the number of columns here - rows, err := tx.Query(`SELECT table_name, count(*) as columns + rows, err := tx.Query(context.TODO(), + `SELECT table_name, count(*) as columns FROM information_schema.columns WHERE table_name='image_annotation' OR table_name = 'image_annotation_revision' or @@ -51,7 +50,6 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) tableToColumnsMapping[name] = cols } rows.Close() - if tableToColumnsMapping["image_annotation"] != 10 || tableToColumnsMapping["image_annotation_suggestion"] != 10 { return errors.New("either the image_annotation or the image_annotation_suggestion table has more columns than expected!") @@ -59,34 +57,34 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) if tableToColumnsMapping["annotation_data"] != 6 || tableToColumnsMapping["annotation_suggestion_data"] != 6 { return errors.New("either the annotation_data or the annotation_suggestion_data table has more columns than expected!") - } + } if tableToColumnsMapping["user_image_annotation"] != 4 || tableToColumnsMapping["user_image_annotation_suggestion"] != 4 { return errors.New("either the user_image_annotation or the user_image_annotation_suggestion table has more columns than expected!") } - + if tableToColumnsMapping["image_annotation_revision"] != 3 || tableToColumnsMapping["image_annotation_suggestion_revision"] != 3 { return errors.New("either the image_annotation_revision or the image_annotation_suggestion_revision table has more columns than expected!") } - - tempTables := []string{"temp_image_annotation_mapping", "temp_annotation_data_mapping", "temp_image_annotation_revision_mapping", - "temp_annotation_data_revision_mapping"} - + "temp_annotation_data_revision_mapping"} + for _, tempTable := range tempTables { - _, err := tx.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tempTable)) //controlled input, so no sql injection possible + _, err := tx.Exec(context.TODO(), fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tempTable)) //controlled input, so no sql injection possible if err != nil { return err } } - _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_image_annotation_mapping(old_image_annotation_id bigint, new_image_annotation_id bigint)`) + _, err = tx.Exec(context.TODO(), + `CREATE TEMPORARY TABLE temp_image_annotation_mapping(old_image_annotation_id bigint, new_image_annotation_id bigint)`) if err != nil { return err } - _, err = tx.Exec(`INSERT INTO temp_image_annotation_mapping(old_image_annotation_id, new_image_annotation_id) + _, err = tx.Exec(context.TODO(), + `INSERT INTO temp_image_annotation_mapping(old_image_annotation_id, new_image_annotation_id) SELECT a.id as old_image_annotation_id, nextval('image_annotation_id_seq') as new_image_annotation_id FROM image_annotation_suggestion a JOIN label_suggestion s ON s.id = a.label_suggestion_id @@ -95,14 +93,16 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return err } - _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_annotation_data_mapping(old_annotation_data_id bigint, + _, err = tx.Exec(context.TODO(), + `CREATE TEMPORARY TABLE temp_annotation_data_mapping(old_annotation_data_id bigint, old_image_annotation_id bigint, new_annotation_data_id bigint, new_image_annotation_id bigint, old_image_annotation_revision_id bigint)`) if err != nil { return err } - _, err = tx.Exec(`INSERT INTO temp_annotation_data_mapping(old_annotation_data_id, old_image_annotation_id, new_annotation_data_id, + _, err = tx.Exec(context.TODO(), + `INSERT INTO temp_annotation_data_mapping(old_annotation_data_id, old_image_annotation_id, new_annotation_data_id, new_image_annotation_id, old_image_annotation_revision_id) SELECT d.id as old_annotation_data_id, m.old_image_annotation_id as old_image_annotation_id, nextval('image_annotation_data_id_seq') as new_annotation_data_id, m.new_image_annotation_id as new_image_annotation_id, @@ -114,13 +114,15 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return err } - _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_image_annotation_revision_mapping(old_image_annotation_revision_id bigint, + _, err = tx.Exec(context.TODO(), + `CREATE TEMPORARY TABLE temp_image_annotation_revision_mapping(old_image_annotation_revision_id bigint, old_image_annotation_id bigint, new_image_annotation_revision_id bigint, new_image_annotation_id bigint)`) if err != nil { return err } - _, err = tx.Exec(`INSERT INTO temp_image_annotation_revision_mapping(old_image_annotation_revision_id, old_image_annotation_id, new_image_annotation_revision_id, + _, err = tx.Exec(context.TODO(), + `INSERT INTO temp_image_annotation_revision_mapping(old_image_annotation_revision_id, old_image_annotation_id, new_image_annotation_revision_id, new_image_annotation_id) SELECT r.id as old_image_annotation_revision_id, r.image_annotation_suggestion_id as old_image_annotation_id, nextval('image_annotation_revision_id_seq') as new_image_annotation_revision_id, m.new_image_annotation_id as new_image_annotation_id @@ -130,15 +132,16 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return err } - - _, err = tx.Exec(`CREATE TEMPORARY TABLE temp_annotation_data_revision_mapping(old_annotation_data_id bigint, + _, err = tx.Exec(context.TODO(), + `CREATE TEMPORARY TABLE temp_annotation_data_revision_mapping(old_annotation_data_id bigint, old_image_annotation_id bigint, new_annotation_data_id bigint, new_image_annotation_id bigint, old_image_annotation_revision_id bigint)`) if err != nil { return err } - _, err = tx.Exec(`INSERT INTO temp_annotation_data_revision_mapping(old_annotation_data_id, old_image_annotation_id, new_annotation_data_id, + _, err = tx.Exec(context.TODO(), + `INSERT INTO temp_annotation_data_revision_mapping(old_annotation_data_id, old_image_annotation_id, new_annotation_data_id, new_image_annotation_id, old_image_annotation_revision_id) SELECT d.id as old_annotation_data_id, m.old_image_annotation_id as old_image_annotation_id, nextval('image_annotation_data_id_seq') as new_annotation_data_id, m.new_image_annotation_id as new_image_annotation_id, @@ -152,7 +155,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) //insert var insertedImageAnnotationRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( INSERT INTO image_annotation (id, image_id, num_of_valid, num_of_invalid, fingerprint_of_last_modification, sys_period, label_id, uuid, auto_generated, revision) SELECT m.new_image_annotation_id, a.image_id, a.num_of_valid, a.num_of_invalid, a.fingerprint_of_last_modification, a.sys_period, @@ -168,7 +172,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) } var insertedAnnotationDataRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( INSERT INTO annotation_data(id, image_annotation_id, annotation, annotation_type_id, image_annotation_revision_id, uuid) SELECT m.new_annotation_data_id, m.new_image_annotation_id, d.annotation, d.annotation_type_id, m1.new_image_annotation_revision_id, d.uuid @@ -185,7 +190,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) } var insertedImageAnnotationRevisionRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( INSERT INTO image_annotation_revision(id, image_annotation_id, revision) SELECT m.new_image_annotation_revision_id, m.new_image_annotation_id, r.revision FROM temp_image_annotation_revision_mapping m @@ -199,7 +205,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) } var insertedAnnotationDataRevisionRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( INSERT INTO annotation_data(id, image_annotation_id, annotation, annotation_type_id, image_annotation_revision_id, uuid) SELECT m.new_annotation_data_id, null, d.annotation, d.annotation_type_id, m1.new_image_annotation_revision_id, d.uuid @@ -216,7 +223,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) } var insertedUserImageAnnotationRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( INSERT INTO user_image_annotation(image_annotation_id, account_id, timestamp) SELECT m.new_image_annotation_id, u.account_id, u.timestamp FROM temp_image_annotation_mapping m @@ -226,9 +234,10 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) ) SELECT count(*) FROM rows`).Scan(&insertedUserImageAnnotationRows) - //delete + //delete var deletedUserImageAnnotationSuggestionRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( DELETE FROM user_image_annotation_suggestion WHERE image_annotation_suggestion_id IN (SELECT old_image_annotation_id FROM temp_image_annotation_mapping) @@ -242,9 +251,10 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) if insertedUserImageAnnotationRows != deletedUserImageAnnotationSuggestionRows { return errors.New("inserted user_image_annotation rows differ from deleted user_image_annoation_suggestion rows!") } - + var deletedAnnotationSuggestionDataRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( DELETE FROM annotation_suggestion_data WHERE id IN (SELECT old_annotation_data_id FROM temp_annotation_data_mapping) @@ -259,9 +269,9 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return errors.New("inserted annotation_data rows differ from deleted annotation_suggestion_data rows!") } - var deletedAnnotationSuggestionDataRevisionRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( DELETE FROM annotation_suggestion_data WHERE id IN (SELECT old_annotation_data_id FROM temp_annotation_data_revision_mapping) @@ -275,9 +285,9 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return errors.New("inserted annotation_data revision rows differ from deleted annotation_suggestion_data revision rows!") } - var deletedImageAnnotationSuggestionRevisionRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( DELETE FROM image_annotation_suggestion_revision WHERE id IN (SELECT old_image_annotation_revision_id FROM temp_image_annotation_revision_mapping) @@ -293,7 +303,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) } var deletedImageAnnotationSuggestionRows int64 = 0 - err = tx.QueryRow(`WITH rows AS ( + err = tx.QueryRow(context.TODO(), + `WITH rows AS ( DELETE FROM image_annotation_suggestion WHERE id IN (SELECT old_image_annotation_id FROM temp_image_annotation_mapping) @@ -308,9 +319,8 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return errors.New("inserted image_annotation rows differ from deleted image_annotation_suggestion rows!") } - for _, tempTable := range tempTables { - _, err = tx.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tempTable)) //controlled input, so no sql injection possible + _, err = tx.Exec(context.TODO(), fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tempTable)) //controlled input, so no sql injection possible if err != nil { return err } @@ -319,84 +329,83 @@ func MakeAnnotationsProductive(tx *sql.Tx, trendingLabel string, labelId int64) return nil } - -func updateAnnotationInTransaction2(tx *sql.Tx, apiUser datastructures.APIUser, label string, sublabel string, - annotationsContainer datastructures.AnnotationsContainer) error { - var queryValues []interface{} - query := "" - if label != "" && sublabel != "" { - if annotationsContainer.IsSuggestion { - tx.Rollback() +func updateAnnotationInTransaction2(tx pgx.Tx, apiUser datastructures.APIUser, label string, sublabel string, + annotationsContainer datastructures.AnnotationsContainer) error { + var queryValues []interface{} + query := "" + if label != "" && sublabel != "" { + if annotationsContainer.IsSuggestion { + tx.Rollback(context.TODO()) return errors.New("Unexpected sublabel set for annotation suggestion") } else { query = `SELECT a.uuid FROM image_annotation a JOIN label l ON l.id = a.label_id JOIN label pl ON l.parent_id = pl.id - WHERE l.name = $1 AND pl.name = $2` - queryValues = append(queryValues, label, sublabel) + WHERE l.name = $1 AND pl.name = $2` + queryValues = append(queryValues, label, sublabel) //queryValues = append(queryValues, label) - //queryValues = append(queryValues, sublabel) + //queryValues = append(queryValues, sublabel) } - } else { + } else { if annotationsContainer.IsSuggestion { - query = `SELECT a.uuid + query = `SELECT a.uuid FROM image_annotation_suggestion a JOIN label_suggestion l ON l.id = a.label_suggestion_id - WHERE l.name = $1` + WHERE l.name = $1` } else { query = `SELECT a.uuid FROM image_annotation a JOIN label l ON l.id = a.label_id WHERE l.name = $1` - } + } queryValues = append(queryValues, label) //queryValues = append(queryValues, label) } - rows, err := tx.Query(query, queryValues...) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't get annotation id: ", err.Error()) - return err - } - - if rows.Next() { - var annotationId string - err = rows.Scan(&annotationId) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't scan annotation id: ", err.Error()) - return err - } - - rows.Close() - - return updateAnnotationInTransaction(tx, apiUser, annotationId, annotationsContainer) - } - tx.Rollback() - return errors.New("[Update Annotation] Couldn't get uuid for label") + rows, err := tx.Query(context.TODO(), query, queryValues...) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't get annotation id: ", err.Error()) + return err + } + + if rows.Next() { + var annotationId string + err = rows.Scan(&annotationId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't scan annotation id: ", err.Error()) + return err + } + + rows.Close() + + return updateAnnotationInTransaction(tx, apiUser, annotationId, annotationsContainer) + } + tx.Rollback(context.TODO()) + return errors.New("[Update Annotation] Couldn't get uuid for label") } -func updateAnnotationInTransaction(tx *sql.Tx, apiUser datastructures.APIUser, annotationId string, - annotationsContainer datastructures.AnnotationsContainer) error { - byt, err := json.Marshal(annotationsContainer.Annotations.Annotations) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't create byte array: ", err.Error()) - return err - } +func updateAnnotationInTransaction(tx pgx.Tx, apiUser datastructures.APIUser, annotationId string, + annotationsContainer datastructures.AnnotationsContainer) error { + byt, err := json.Marshal(annotationsContainer.Annotations.Annotations) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't create byte array: ", err.Error()) + return err + } if annotationsContainer.IsSuggestion { if apiUser.Name == "" { - tx.Rollback() + tx.Rollback(context.TODO()) return &AuthenticationRequiredError{Description: "Couldn't process request - you need to be authenticated to perform this action"} } } - var imageAnnotationRevisionId int64 + var imageAnnotationRevisionId int64 - //add entry to image_annotation_revision table + //add entry to image_annotation_revision table insertImageAnnotationRevisionQuery := "" if annotationsContainer.IsSuggestion { insertImageAnnotationRevisionQuery = `INSERT INTO image_annotation_suggestion_revision(image_annotation_suggestion_id, revision) @@ -410,13 +419,13 @@ func updateAnnotationInTransaction(tx *sql.Tx, apiUser datastructures.APIUser, a RETURNING id` } - err = tx.QueryRow(insertImageAnnotationRevisionQuery, annotationId).Scan(&imageAnnotationRevisionId) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't insert to annotation revision: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + err = tx.QueryRow(context.TODO(), insertImageAnnotationRevisionQuery, annotationId).Scan(&imageAnnotationRevisionId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't insert to annotation revision: ", err.Error()) + raven.CaptureError(err, nil) + return err + } updateAnnotationDataQuery := "" if annotationsContainer.IsSuggestion { @@ -431,16 +440,16 @@ func updateAnnotationInTransaction(tx *sql.Tx, apiUser datastructures.APIUser, a AND a.id = image_annotation_id` } - _, err = tx.Exec(updateAnnotationDataQuery, annotationId, imageAnnotationRevisionId) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't update annotation data: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + _, err = tx.Exec(context.TODO(), updateAnnotationDataQuery, annotationId, imageAnnotationRevisionId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't update annotation data: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + var imageAnnotationId int64 - var imageAnnotationId int64 - updateImageAnnotationQuery := "" if annotationsContainer.IsSuggestion { updateImageAnnotationQuery = `UPDATE image_annotation_suggestion a SET num_of_valid = 0, num_of_invalid = 0, revision = revision + 1 @@ -452,16 +461,15 @@ func updateAnnotationInTransaction(tx *sql.Tx, apiUser datastructures.APIUser, a RETURNING id` } - err = tx.QueryRow(updateImageAnnotationQuery, annotationId).Scan(&imageAnnotationId) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't update annotation: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - + err = tx.QueryRow(context.TODO(), updateImageAnnotationQuery, annotationId).Scan(&imageAnnotationId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't update annotation: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - //insertes annotation data; 'type' and 'refinements' are removed removed before inserting data + //insertes annotation data; 'type' and 'refinements' are removed removed before inserting data insertAnnotationDataQuery := "" if annotationsContainer.IsSuggestion { insertAnnotationDataQuery = `INSERT INTO annotation_suggestion_data(image_annotation_suggestion_id, uuid, annotation, annotation_type_id) @@ -477,111 +485,110 @@ func updateAnnotationInTransaction(tx *sql.Tx, apiUser datastructures.APIUser, a RETURNING uuid` } - var rows *sql.Rows - rows, err = tx.Query(insertAnnotationDataQuery, imageAnnotationId, byt) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't add annotations: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - defer rows.Close() - annotationDataIds := make(map[int]string) - i := 0 - for rows.Next() { - var annotationDataId string - err = rows.Scan(&annotationDataId) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't scan annotation data ids: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - annotationDataIds[i] = annotationDataId - i += 1 - } - rows.Close() - - if len(annotationsContainer.AllowedRefinements) != len(annotationDataIds) { - tx.Rollback() - err = errors.New("Num of annotation refinements do not match num of annotation data ids!") - log.Error("[Update Annotation] Couldn't add annotations : ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - for i, refinements := range annotationsContainer.AllowedRefinements { - if val, ok := annotationDataIds[i]; ok { - err = addOrUpdateRefinementsInTransaction(tx, annotationId, val, refinements, apiUser.ClientFingerprint, annotationsContainer.IsSuggestion) - if err != nil { //transaction already rolled back, so we can return here - return err - } - } - } - return nil + var rows pgx.Rows + rows, err = tx.Query(context.TODO(), insertAnnotationDataQuery, imageAnnotationId, byt) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't add annotations: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + defer rows.Close() + annotationDataIds := make(map[int]string) + i := 0 + for rows.Next() { + var annotationDataId string + err = rows.Scan(&annotationDataId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't scan annotation data ids: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + annotationDataIds[i] = annotationDataId + i += 1 + } + rows.Close() + + if len(annotationsContainer.AllowedRefinements) != len(annotationDataIds) { + tx.Rollback(context.TODO()) + err = errors.New("Num of annotation refinements do not match num of annotation data ids!") + log.Error("[Update Annotation] Couldn't add annotations : ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + for i, refinements := range annotationsContainer.AllowedRefinements { + if val, ok := annotationDataIds[i]; ok { + err = addOrUpdateRefinementsInTransaction(tx, annotationId, val, refinements, apiUser.ClientFingerprint, annotationsContainer.IsSuggestion) + if err != nil { //transaction already rolled back, so we can return here + return err + } + } + } + return nil } -func (p *ImageMonkeyDatabase) UpdateAnnotation(apiUser datastructures.APIUser, annotationId string, - annotationsContainer datastructures.AnnotationsContainer) error { +func (p *ImageMonkeyDatabase) UpdateAnnotation(apiUser datastructures.APIUser, annotationId string, + annotationsContainer datastructures.AnnotationsContainer) error { - tx, err := p.db.Begin() - if err != nil { - log.Debug("[Update Annotation] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Debug("[Update Annotation] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - err = updateAnnotationInTransaction(tx, apiUser, annotationId, annotationsContainer) - if err != nil { //transaction already rolled back, so we can return here + err = updateAnnotationInTransaction(tx, apiUser, annotationId, annotationsContainer) + if err != nil { //transaction already rolled back, so we can return here log.Error(err.Error()) - return err - } + return err + } - err = tx.Commit() - if err != nil { - log.Debug("[Update Annotation] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + err = tx.Commit(context.TODO()) + if err != nil { + log.Debug("[Update Annotation] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - return nil + return nil } -func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, imageId string, - annotations []datastructures.AnnotationsContainer) ([]string, error) { +func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, imageId string, + annotations []datastructures.AnnotationsContainer) ([]string, error) { - annotationIds := []string{} + annotationIds := []string{} - tx, err := p.db.Begin() - if err != nil { - log.Error("[Add Annotation] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return annotationIds, err - } + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Error("[Add Annotation] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return annotationIds, err + } - //currently there is a uniqueness constraint on the image_id column to ensure that we only have - //one image annotation per image. That means that the below query can fail with a unique constraint error. - //at the moment the uniqueness constraint errors are handled gracefully - that means we return nil. - //we might want to change that in the future to support multiple annotations per image (if there is a use case for it), - //but for now it should be fine. + //currently there is a uniqueness constraint on the image_id column to ensure that we only have + //one image annotation per image. That means that the below query can fail with a unique constraint error. + //at the moment the uniqueness constraint errors are handled gracefully - that means we return nil. + //we might want to change that in the future to support multiple annotations per image (if there is a use case for it), + //but for now it should be fine. - for _, annotation := range annotations { + for _, annotation := range annotations { if annotation.IsSuggestion { //label is not known to us (i.e non-productive) if apiUser.Name == "" { - tx.Rollback() + tx.Rollback(context.TODO()) return annotationIds, &AuthenticationRequiredError{Description: "you need to be authenticated to perform this action"} } } - - + byt, err := json.Marshal(annotation.Annotations.Annotations) - if err != nil { - tx.Rollback() - log.Error("[Add Annotation] Couldn't create byte array: ", err.Error()) - return annotationIds, err - } - - var idRows *sql.Rows + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Add Annotation] Couldn't create byte array: ", err.Error()) + return annotationIds, err + } + + var idRows pgx.Rows var insertImageAnnotationQueryValues []interface{} insertImageAnnotationQuery := "" if annotation.IsSuggestion { @@ -592,10 +599,10 @@ func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, ima (SELECT i.id FROM image i WHERE i.key = $1), uuid_generate_v4(), $6, $7 ON CONFLICT DO NOTHING RETURNING id, uuid` - - insertImageAnnotationQueryValues = append(insertImageAnnotationQueryValues, imageId, 0, 0, apiUser.ClientFingerprint, - annotation.Annotations.Label, annotation.AutoGenerated, 1) - + + insertImageAnnotationQueryValues = append(insertImageAnnotationQueryValues, imageId, 0, 0, apiUser.ClientFingerprint, + annotation.Annotations.Label, annotation.AutoGenerated, 1) + } else { if annotation.Annotations.Sublabel == "" { insertImageAnnotationQuery = `INSERT INTO image_annotation(label_id, num_of_valid, num_of_invalid, @@ -605,9 +612,9 @@ func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, ima (SELECT i.id FROM image i WHERE i.key = $1), uuid_generate_v4(), $6, $7 ON CONFLICT DO NOTHING RETURNING id, uuid` - - insertImageAnnotationQueryValues = append(insertImageAnnotationQueryValues, imageId, 0, 0, apiUser.ClientFingerprint, - annotation.Annotations.Label, annotation.AutoGenerated, 1) + + insertImageAnnotationQueryValues = append(insertImageAnnotationQueryValues, imageId, 0, 0, apiUser.ClientFingerprint, + annotation.Annotations.Label, annotation.AutoGenerated, 1) } else { insertImageAnnotationQuery = `INSERT INTO image_annotation(label_id, num_of_valid, num_of_invalid, @@ -617,45 +624,45 @@ func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, ima (SELECT i.id FROM image i WHERE i.key = $1), uuid_generate_v4(), $7, $8 ON CONFLICT DO NOTHING RETURNING id, uuid` - - insertImageAnnotationQueryValues = append(insertImageAnnotationQueryValues, imageId, 0, 0, apiUser.ClientFingerprint, - annotation.Annotations.Sublabel, annotation.Annotations.Label, annotation.AutoGenerated, 1) + + insertImageAnnotationQueryValues = append(insertImageAnnotationQueryValues, imageId, 0, 0, apiUser.ClientFingerprint, + annotation.Annotations.Sublabel, annotation.Annotations.Label, annotation.AutoGenerated, 1) + } + } + + idRows, err = tx.Query(context.TODO(), insertImageAnnotationQuery, insertImageAnnotationQueryValues...) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't add image annotation: ", err.Error()) + return annotationIds, err + } + + defer idRows.Close() + + var annotationId string + var insertedId int64 + + if !idRows.Next() { //we get no result set in case there already exists an entry + //in that case, just update the annotation + idRows.Close() + err = updateAnnotationInTransaction2(tx, apiUser, annotation.Annotations.Label, annotation.Annotations.Sublabel, annotation) + if err != nil { //transaction already rolled back, so we can return here + return annotationIds, err + } + continue + } else { //image annotation successfully added, get inserted ids + err = idRows.Scan(&insertedId, &annotationId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Update Annotation] Couldn't scan image annotation row: ", err.Error()) + return annotationIds, err } - } - - idRows, err = tx.Query(insertImageAnnotationQuery, insertImageAnnotationQueryValues...) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't add image annotation: ", err.Error()) - return annotationIds, err - } - - defer idRows.Close() - - var annotationId string - var insertedId int64 - - if !idRows.Next() { //we get no result set in case there already exists an entry - //in that case, just update the annotation - idRows.Close() - err = updateAnnotationInTransaction2(tx, apiUser, annotation.Annotations.Label, annotation.Annotations.Sublabel, annotation) - if err != nil { //transaction already rolled back, so we can return here - return annotationIds, err - } - continue - } else { //image annotation successfully added, get inserted ids - err = idRows.Scan(&insertedId, &annotationId) - if err != nil { - tx.Rollback() - log.Error("[Update Annotation] Couldn't scan image annotation row: ", err.Error()) - return annotationIds, err - } - } - - idRows.Close() - - //insertes annotation data; 'type' and 'refinements' are removed removed before inserting data - var rows *sql.Rows + } + + idRows.Close() + + //insertes annotation data; 'type' and 'refinements' are removed removed before inserting data + var rows pgx.Rows insertAnnotationDataQuery := "" if annotation.IsSuggestion { insertAnnotationDataQuery = `INSERT INTO annotation_suggestion_data(image_annotation_suggestion_id, uuid, annotation, annotation_type_id) @@ -670,54 +677,54 @@ func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, ima ((q.*)::jsonb - 'type' - 'refinements'), (SELECT id FROM annotation_type where name = ((q.*)->>'type')::text) FROM json_array_elements($2) q - RETURNING uuid` + RETURNING uuid` + } + + rows, err = tx.Query(context.TODO(), insertAnnotationDataQuery, insertedId, byt) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Add Annotation] Couldn't add annotations: ", err.Error()) + raven.CaptureError(err, nil) + return annotationIds, err + } + defer rows.Close() + annotationDataIds := make(map[int]string) + i := 0 + for rows.Next() { + var annotationDataId string + err = rows.Scan(&annotationDataId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Add Annotation] Couldn't scan annotation data ids: ", err.Error()) + raven.CaptureError(err, nil) + return annotationIds, err + } + annotationDataIds[i] = annotationDataId + i += 1 + } + rows.Close() + + if len(annotation.AllowedRefinements) != len(annotationDataIds) { + tx.Rollback(context.TODO()) + err = errors.New("Num of annotation refinements do not match num of annotation data ids!") + log.Error("[Add Annotation] Couldn't add annotations : ", err.Error()) + raven.CaptureError(err, nil) + return annotationIds, err + } + + for i, refinements := range annotation.AllowedRefinements { + if val, ok := annotationDataIds[i]; ok { + err = addOrUpdateRefinementsInTransaction(tx, annotationId, val, refinements, apiUser.ClientFingerprint, annotation.IsSuggestion) + if err != nil { //transaction already rolled back, so we can return here + return annotationIds, err + } + } } - rows, err = tx.Query(insertAnnotationDataQuery, insertedId, byt) - if err != nil { - tx.Rollback() - log.Error("[Add Annotation] Couldn't add annotations: ", err.Error()) - raven.CaptureError(err, nil) - return annotationIds, err - } - defer rows.Close() - annotationDataIds := make(map[int]string) - i := 0 - for rows.Next() { - var annotationDataId string - err = rows.Scan(&annotationDataId) - if err != nil { - tx.Rollback() - log.Error("[Add Annotation] Couldn't scan annotation data ids: ", err.Error()) - raven.CaptureError(err, nil) - return annotationIds, err - } - annotationDataIds[i] = annotationDataId - i += 1 - } - rows.Close() - - if len(annotation.AllowedRefinements) != len(annotationDataIds) { - tx.Rollback() - err = errors.New("Num of annotation refinements do not match num of annotation data ids!") - log.Error("[Add Annotation] Couldn't add annotations : ", err.Error()) - raven.CaptureError(err, nil) - return annotationIds, err - } - - for i, refinements := range annotation.AllowedRefinements { - if val, ok := annotationDataIds[i]; ok { - err = addOrUpdateRefinementsInTransaction(tx, annotationId, val, refinements, apiUser.ClientFingerprint, annotation.IsSuggestion) - if err != nil { //transaction already rolled back, so we can return here - return annotationIds, err - } - } - } - - if apiUser.Name != "" { - var id int64 - - id = 0 + if apiUser.Name != "" { + var id int64 + + id = 0 userImageAnnotationQuery := "" if annotation.IsSuggestion { userImageAnnotationQuery = `INSERT INTO user_image_annotation_suggestion(image_annotation_suggestion_id, account_id, timestamp) @@ -730,42 +737,41 @@ func (p *ImageMonkeyDatabase) AddAnnotations(apiUser datastructures.APIUser, ima FROM account a WHERE a.name = $2 RETURNING id` } - err = tx.QueryRow(userImageAnnotationQuery, insertedId, apiUser.Name).Scan(&id) - if err != nil { - tx.Rollback() - log.Error("[Add User Annotation] Couldn't add user annotation entry: ", err.Error()) - raven.CaptureError(err, nil) - return annotationIds, err - } - - if id == 0 { - tx.Rollback() - log.Error("[Add User Annotation] Nothing inserted") - return annotationIds, errors.New("nothing inserted") - } - } - - annotationIds = append(annotationIds, annotationId) - } - - - err = tx.Commit() - if err != nil { - log.Error("[Add Annotation] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return annotationIds, err - } - - return annotationIds, nil + err = tx.QueryRow(context.TODO(), userImageAnnotationQuery, insertedId, apiUser.Name).Scan(&id) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Add User Annotation] Couldn't add user annotation entry: ", err.Error()) + raven.CaptureError(err, nil) + return annotationIds, err + } + + if id == 0 { + tx.Rollback(context.TODO()) + log.Error("[Add User Annotation] Nothing inserted") + return annotationIds, errors.New("nothing inserted") + } + } + + annotationIds = append(annotationIds, annotationId) + } + + err = tx.Commit(context.TODO()) + if err != nil { + log.Error("[Add Annotation] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return annotationIds, err + } + + return annotationIds, nil } -func (p *ImageMonkeyDatabase) _getImageForAnnotationFromValidationId(username string, validationId string, - addAutoAnnotations bool) (datastructures.UnannotatedImage, error) { - var unannotatedImage datastructures.UnannotatedImage +func (p *ImageMonkeyDatabase) _getImageForAnnotationFromValidationId(username string, validationId string, + addAutoAnnotations bool) (datastructures.UnannotatedImage, error) { + var unannotatedImage datastructures.UnannotatedImage - includeOwnImageDonations := "" - if username != "" { - includeOwnImageDonations = `OR ( + includeOwnImageDonations := "" + if username != "" { + includeOwnImageDonations = `OR ( EXISTS ( SELECT 1 @@ -780,10 +786,10 @@ func (p *ImageMonkeyDatabase) _getImageForAnnotationFromValidationId(username st WHERE q.image_id = i.id ) )` - - } - q := fmt.Sprintf(`SELECT i.key, label_name, parent_label_name, q2.label_accessor, i.width, i.height, validation_uuid, + } + + q := fmt.Sprintf(`SELECT i.key, label_name, parent_label_name, q2.label_accessor, i.width, i.height, validation_uuid, json_agg(q1.annotation || ('{"type":"' || q1.name || '"}')::jsonb)::jsonb as auto_annotations, i.unlocked FROM image i @@ -832,100 +838,99 @@ func (p *ImageMonkeyDatabase) _getImageForAnnotationFromValidationId(username st GROUP BY i.key, q2.label_name, q2.parent_label_name, q2.label_accessor, i.width, i.height, q2.validation_uuid, i.unlocked`, includeOwnImageDonations) - //we do not check, whether there already exists a annotation for the given validation id. - //there is anyway only one annotation per validation allowed, so if someone tries to push another annotation, the corresponding POST request - //would fail - var rows *sql.Rows - var err error - - if username == "" { - rows, err = p.db.Query(q, validationId) - } else { - rows, err = p.db.Query(q, validationId, username) - } - - if err != nil { - log.Debug("[Get specific Image for Annotation] Couldn't get annotation ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedImage, err - } - - defer rows.Close() - - var label1 string - var label2 string - var autoAnnotationBytes []byte - if rows.Next() { - unannotatedImage.Provider = "donation" - - err = rows.Scan(&unannotatedImage.Id, &label1, &label2, &unannotatedImage.Label.Accessor, - &unannotatedImage.Width, &unannotatedImage.Height, &unannotatedImage.Validation.Id, - &autoAnnotationBytes, &unannotatedImage.Unlocked) - if err != nil { - log.Debug("[Get specific Image for Annotation] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedImage, err - } - - if addAutoAnnotations { - if len(autoAnnotationBytes) > 0 { - err = json.Unmarshal(autoAnnotationBytes, &unannotatedImage.AutoAnnotations) - if err != nil { - log.Debug("[Get specific Image for Annotation] Couldn't unmarshal auto annotations: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedImage, err - } - } - } - - if label2 == "" { - unannotatedImage.Label.Label = label1 - unannotatedImage.Label.Sublabel = "" - } else { - unannotatedImage.Label.Label = label2 - unannotatedImage.Label.Sublabel = label1 - } - } - - return unannotatedImage, nil -} + //we do not check, whether there already exists a annotation for the given validation id. + //there is anyway only one annotation per validation allowed, so if someone tries to push another annotation, the corresponding POST request + //would fail + var rows pgx.Rows + var err error -func (p *ImageMonkeyDatabase) GetImageForAnnotation(username string, addAutoAnnotations bool, - validationId string, labelId string) (datastructures.UnannotatedImage, error) { - //if a validation id is provided, use a different code path. - //selecting a single image given a validation id is totally different from selecting a random image - //so it makes sense to use a different code path here. - if validationId != "" { - return p._getImageForAnnotationFromValidationId(username, validationId, addAutoAnnotations) - } + if username == "" { + rows, err = p.db.Query(context.TODO(), q, validationId) + } else { + rows, err = p.db.Query(context.TODO(), q, validationId, username) + } + if err != nil { + log.Debug("[Get specific Image for Annotation] Couldn't get annotation ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedImage, err + } - var unannotatedImage datastructures.UnannotatedImage + defer rows.Close() - //specify the max. number of not-annotatables before we skip the annotation task - maxNumNotAnnotatable := 3 + var label1 string + var label2 string + var autoAnnotationBytes []byte + if rows.Next() { + unannotatedImage.Provider = "donation" - q1 := "" - posNum := 1 - if labelId != "" { - q1 = "AND l.uuid = $1" - posNum = 2 - } + err = rows.Scan(&unannotatedImage.Id, &label1, &label2, &unannotatedImage.Label.Accessor, + &unannotatedImage.Width, &unannotatedImage.Height, &unannotatedImage.Validation.Id, + &autoAnnotationBytes, &unannotatedImage.Unlocked) + if err != nil { + log.Debug("[Get specific Image for Annotation] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedImage, err + } - q3 := fmt.Sprintf("AND v.num_of_not_annotatable < $%d", posNum) - posNum += 1 + if addAutoAnnotations { + if len(autoAnnotationBytes) > 0 { + err = json.Unmarshal(autoAnnotationBytes, &unannotatedImage.AutoAnnotations) + if err != nil { + log.Debug("[Get specific Image for Annotation] Couldn't unmarshal auto annotations: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedImage, err + } + } + } + + if label2 == "" { + unannotatedImage.Label.Label = label1 + unannotatedImage.Label.Sublabel = "" + } else { + unannotatedImage.Label.Label = label2 + unannotatedImage.Label.Sublabel = label1 + } + } + + return unannotatedImage, nil +} + +func (p *ImageMonkeyDatabase) GetImageForAnnotation(username string, addAutoAnnotations bool, + validationId string, labelId string) (datastructures.UnannotatedImage, error) { + //if a validation id is provided, use a different code path. + //selecting a single image given a validation id is totally different from selecting a random image + //so it makes sense to use a different code path here. + if validationId != "" { + return p._getImageForAnnotationFromValidationId(username, validationId, addAutoAnnotations) + } + + var unannotatedImage datastructures.UnannotatedImage + + //specify the max. number of not-annotatables before we skip the annotation task + maxNumNotAnnotatable := 3 - includeOwnImageDonations := "" - q2 := "" - if username != "" { - q2 = fmt.Sprintf(`AND NOT EXISTS + q1 := "" + posNum := 1 + if labelId != "" { + q1 = "AND l.uuid = $1" + posNum = 2 + } + + q3 := fmt.Sprintf("AND v.num_of_not_annotatable < $%d", posNum) + posNum += 1 + + includeOwnImageDonations := "" + q2 := "" + if username != "" { + q2 = fmt.Sprintf(`AND NOT EXISTS ( SELECT 1 FROM user_annotation_blacklist bl JOIN account acc ON acc.id = bl.account_id WHERE bl.image_validation_id = v.id AND acc.name = $%d )`, posNum) - includeOwnImageDonations = fmt.Sprintf(`OR ( + includeOwnImageDonations = fmt.Sprintf(`OR ( EXISTS ( SELECT 1 @@ -939,12 +944,11 @@ func (p *ImageMonkeyDatabase) GetImageForAnnotation(username string, addAutoAnno FROM image_quarantine q WHERE q.image_id = i.id ) - )`, posNum) - - } + )`, posNum) + } - q := fmt.Sprintf(`SELECT q.image_key, q.label, q.parent_label, q.accessor, q.image_width, q.image_height, q.validation_uuid, + q := fmt.Sprintf(`SELECT q.image_key, q.label, q.parent_label, q.accessor, q.image_width, q.image_height, q.validation_uuid, CASE WHEN json_agg(q1.annotation)::jsonb = '[null]'::jsonb THEN '[]' ELSE json_agg(q1.annotation || ('{"type":"' || q1.annotation_type || '"}')::jsonb)::jsonb END as auto_annotations, q.image_unlocked FROM @@ -995,78 +999,78 @@ func (p *ImageMonkeyDatabase) GetImageForAnnotation(username string, addAutoAnno WHERE a.auto_generated = true ) q1 ON q.label_id = q1.label_id AND q.image_id = q1.image_id GROUP BY q.image_key, q.label, q.parent_label, q.accessor, - q.image_width, q.image_height, q.validation_uuid, q.image_unlocked`, - includeOwnImageDonations, q1, q2, q3, includeOwnImageDonations, q1, q2, q3) - - //select all images that aren't already annotated and have a label correctness probability of >= 0.8 - var rows *sql.Rows - var err error - if labelId == "" { - if username != "" { - rows, err = p.db.Query(q, maxNumNotAnnotatable, username) - } else { - rows, err = p.db.Query(q, maxNumNotAnnotatable) - } - } else { - if username != "" { - rows, err = p.db.Query(q, labelId, maxNumNotAnnotatable, username) - } else { - rows, err = p.db.Query(q, labelId, maxNumNotAnnotatable) - } - } - - if err != nil { - log.Debug("[Get Random Un-annotated Image] Couldn't fetch result: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedImage, err - } - - defer rows.Close() - - var label1 string - var label2 string - var autoAnnotationBytes []byte - if rows.Next() { - unannotatedImage.Provider = "donation" - - err = rows.Scan(&unannotatedImage.Id, &label1, &label2, &unannotatedImage.Label.Accessor, - &unannotatedImage.Width, &unannotatedImage.Height, &unannotatedImage.Validation.Id, - &autoAnnotationBytes, &unannotatedImage.Unlocked) - if err != nil { - log.Debug("[Get Random Un-annotated Image] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedImage, err - } - - if addAutoAnnotations { - if len(autoAnnotationBytes) > 0 { - err = json.Unmarshal(autoAnnotationBytes, &unannotatedImage.AutoAnnotations) - if err != nil { - log.Debug("[Get Random Un-annotated Image] Couldn't unmarshal auto annotations: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedImage, err - } - } - } - - if label2 == "" { - unannotatedImage.Label.Label = label1 - unannotatedImage.Label.Sublabel = "" - } else { - unannotatedImage.Label.Label = label2 - unannotatedImage.Label.Sublabel = label1 - } - } - - return unannotatedImage, nil + q.image_width, q.image_height, q.validation_uuid, q.image_unlocked`, + includeOwnImageDonations, q1, q2, q3, includeOwnImageDonations, q1, q2, q3) + + //select all images that aren't already annotated and have a label correctness probability of >= 0.8 + var rows pgx.Rows + var err error + if labelId == "" { + if username != "" { + rows, err = p.db.Query(context.TODO(), q, maxNumNotAnnotatable, username) + } else { + rows, err = p.db.Query(context.TODO(), q, maxNumNotAnnotatable) + } + } else { + if username != "" { + rows, err = p.db.Query(context.TODO(), q, labelId, maxNumNotAnnotatable, username) + } else { + rows, err = p.db.Query(context.TODO(), q, labelId, maxNumNotAnnotatable) + } + } + + if err != nil { + log.Debug("[Get Random Un-annotated Image] Couldn't fetch result: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedImage, err + } + + defer rows.Close() + + var label1 string + var label2 string + var autoAnnotationBytes []byte + if rows.Next() { + unannotatedImage.Provider = "donation" + + err = rows.Scan(&unannotatedImage.Id, &label1, &label2, &unannotatedImage.Label.Accessor, + &unannotatedImage.Width, &unannotatedImage.Height, &unannotatedImage.Validation.Id, + &autoAnnotationBytes, &unannotatedImage.Unlocked) + if err != nil { + log.Debug("[Get Random Un-annotated Image] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedImage, err + } + + if addAutoAnnotations { + if len(autoAnnotationBytes) > 0 { + err = json.Unmarshal(autoAnnotationBytes, &unannotatedImage.AutoAnnotations) + if err != nil { + log.Debug("[Get Random Un-annotated Image] Couldn't unmarshal auto annotations: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedImage, err + } + } + } + + if label2 == "" { + unannotatedImage.Label.Label = label1 + unannotatedImage.Label.Sublabel = "" + } else { + unannotatedImage.Label.Label = label2 + unannotatedImage.Label.Sublabel = label1 + } + } + + return unannotatedImage, nil } -func (p *ImageMonkeyDatabase) GetAnnotatedImage(apiUser datastructures.APIUser, annotationId string, - autoGenerated bool, revision int32) (datastructures.AnnotatedImage, error) { - var annotatedImage datastructures.AnnotatedImage +func (p *ImageMonkeyDatabase) GetAnnotatedImage(apiUser datastructures.APIUser, annotationId string, + autoGenerated bool, revision int32) (datastructures.AnnotatedImage, error) { + var annotatedImage datastructures.AnnotatedImage - includeOwnImageDonations := "" - includeOwnImageDonationsStr := `OR ( + includeOwnImageDonations := "" + includeOwnImageDonationsStr := `OR ( EXISTS ( SELECT 1 @@ -1080,16 +1084,16 @@ func (p *ImageMonkeyDatabase) GetAnnotatedImage(apiUser datastructures.APIUser, FROM image_quarantine q WHERE q.image_id = i.id ) - )` + )` - q := "" - if revision != -1 && annotationId != "" { - - if apiUser.Name != "" { - includeOwnImageDonations = fmt.Sprintf(includeOwnImageDonationsStr, 3) - } + q := "" + if revision != -1 && annotationId != "" { + + if apiUser.Name != "" { + includeOwnImageDonations = fmt.Sprintf(includeOwnImageDonationsStr, 3) + } - q = fmt.Sprintf(`SELECT q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, json_agg(q2.annotation), + q = fmt.Sprintf(`SELECT q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, json_agg(q2.annotation), q2.num_of_valid, q2.num_of_invalid, q2.image_width, q2.image_height, q2.image_unlocked, q2.is_suggestion FROM ( @@ -1169,21 +1173,21 @@ func (p *ImageMonkeyDatabase) GetAnnotatedImage(apiUser datastructures.APIUser, GROUP BY q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, q2.num_of_valid, q2.num_of_invalid, q2.image_width, q2.image_height, q2.image_unlocked, q2.is_suggestion`, includeOwnImageDonations) - } else { - q1 := "" - if annotationId != "" { - q1 = "AND a.uuid::text = $2" - - if apiUser.Name != "" { - includeOwnImageDonations = fmt.Sprintf(includeOwnImageDonationsStr, 3) - } - - } else { - if apiUser.Name != "" { - includeOwnImageDonations = fmt.Sprintf(includeOwnImageDonationsStr, 2) - } - - q1 = fmt.Sprintf(`OFFSET floor( + } else { + q1 := "" + if annotationId != "" { + q1 = "AND a.uuid::text = $2" + + if apiUser.Name != "" { + includeOwnImageDonations = fmt.Sprintf(includeOwnImageDonationsStr, 3) + } + + } else { + if apiUser.Name != "" { + includeOwnImageDonations = fmt.Sprintf(includeOwnImageDonationsStr, 2) + } + + q1 = fmt.Sprintf(`OFFSET floor( random() * ( SELECT count(*) FROM image i @@ -1193,9 +1197,9 @@ func (p *ImageMonkeyDatabase) GetAnnotatedImage(apiUser datastructures.APIUser, ) ) LIMIT 1`, includeOwnImageDonations) - } + } - q = fmt.Sprintf(`SELECT q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, json_agg(q2.annotation), q2.num_of_valid, + q = fmt.Sprintf(`SELECT q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, json_agg(q2.annotation), q2.num_of_valid, q2.num_of_invalid, q2.image_width, q2.image_height, q2.image_unlocked, q2.is_suggestion FROM ( @@ -1268,232 +1272,235 @@ func (p *ImageMonkeyDatabase) GetAnnotatedImage(apiUser datastructures.APIUser, q2.num_of_invalid, q2.image_width, q2.image_height, q2.image_unlocked, q2.is_suggestion`, includeOwnImageDonations, q1) } - var err error - - tx, err := p.db.Begin() - if err != nil { - log.Debug("[Get Annotated Image] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImage, err - } - - var rows *sql.Rows - - if revision != -1 && annotationId != "" { - if apiUser.Name == "" { - rows, err = tx.Query(q, revision, annotationId) - } else { - rows, err = tx.Query(q, revision, annotationId, apiUser.Name) - } - } else { - if annotationId == "" { - if apiUser.Name == "" { - rows, err = p.db.Query(q, autoGenerated) - } else { - rows, err = p.db.Query(q, autoGenerated, apiUser.Name) - } - } else { - if apiUser.Name == "" { - rows, err = p.db.Query(q, autoGenerated, annotationId) - } else { - rows, err = p.db.Query(q, autoGenerated, annotationId, apiUser.Name) - } - } - } - - if err != nil { - tx.Rollback() - log.Debug("[Get Annotated Image] Couldn't get annotated image: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImage, err - } - - defer rows.Close() - - var label1 string - var label2 string + var err error + + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Debug("[Get Annotated Image] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImage, err + } + + var rows pgx.Rows + + if revision != -1 && annotationId != "" { + if apiUser.Name == "" { + rows, err = tx.Query(context.TODO(), q, revision, annotationId) + } else { + rows, err = tx.Query(context.TODO(), q, revision, annotationId, apiUser.Name) + } + } else { + if annotationId == "" { + if apiUser.Name == "" { + rows, err = p.db.Query(context.TODO(), q, autoGenerated) + } else { + rows, err = p.db.Query(context.TODO(), q, autoGenerated, apiUser.Name) + } + } else { + if apiUser.Name == "" { + rows, err = p.db.Query(context.TODO(), q, autoGenerated, annotationId) + } else { + rows, err = p.db.Query(context.TODO(), q, autoGenerated, annotationId, apiUser.Name) + } + } + } + + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Get Annotated Image] Couldn't get annotated image: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImage, err + } + + defer rows.Close() + + var label1 string + var label2 string var isSuggestion bool = false - if rows.Next() { - var annotations []byte - annotatedImage.Image.Provider = "donation" - - err = rows.Scan(&annotatedImage.Image.Id, &label1, &label2, &annotatedImage.Id, - &annotations, &annotatedImage.NumOfValid, &annotatedImage.NumOfInvalid, - &annotatedImage.Image.Width, &annotatedImage.Image.Height, &annotatedImage.Image.Unlocked, &isSuggestion) - if err != nil { - tx.Rollback() - log.Debug("[Get Annotated Image] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImage, err - } - - err := json.Unmarshal(annotations, &annotatedImage.Annotations) - if err != nil { - tx.Rollback() - log.Debug("[Get Annotated Image] Couldn't unmarshal: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImage, err - } - - if label2 == "" { - annotatedImage.Validation.Label = label1 - annotatedImage.Validation.Sublabel = "" - } else { - annotatedImage.Validation.Label = label2 - annotatedImage.Validation.Sublabel = label1 - } + if rows.Next() { + var annotations []byte + annotatedImage.Image.Provider = "donation" + + err = rows.Scan(&annotatedImage.Image.Id, &label1, &label2, &annotatedImage.Id, + &annotations, &annotatedImage.NumOfValid, &annotatedImage.NumOfInvalid, + &annotatedImage.Image.Width, &annotatedImage.Image.Height, &annotatedImage.Image.Unlocked, &isSuggestion) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Get Annotated Image] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImage, err + } + + err := json.Unmarshal(annotations, &annotatedImage.Annotations) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Get Annotated Image] Couldn't unmarshal: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImage, err + } + + if label2 == "" { + annotatedImage.Validation.Label = label1 + annotatedImage.Validation.Sublabel = "" + } else { + annotatedImage.Validation.Label = label2 + annotatedImage.Validation.Sublabel = label1 + } annotatedImage.Validation.Unlocked = !isSuggestion - } + } - if annotationId != "" { - rows.Close() - if isSuggestion { - err = tx.QueryRow(`SELECT (SUM(CASE WHEN r.id is null THEN 0 ELSE 1 END) + 1)::integer as num + if annotationId != "" { + rows.Close() + if isSuggestion { + err = tx.QueryRow(context.TODO(), + `SELECT (SUM(CASE WHEN r.id is null THEN 0 ELSE 1 END) + 1)::integer as num FROM image_annotation_suggestion a LEFT JOIN image_annotation_suggestion_revision r ON r.image_annotation_suggestion_id = a.id WHERE a.uuid::text = $1`, annotationId).Scan(&annotatedImage.NumRevisions) } else { - err = tx.QueryRow(`SELECT (SUM(CASE WHEN r.id is null THEN 0 ELSE 1 END) + 1)::integer as num + err = tx.QueryRow(context.TODO(), + `SELECT (SUM(CASE WHEN r.id is null THEN 0 ELSE 1 END) + 1)::integer as num FROM image_annotation a LEFT JOIN image_annotation_revision r ON r.image_annotation_id = a.id WHERE a.uuid::text = $1`, annotationId).Scan(&annotatedImage.NumRevisions) } - if err != nil { - tx.Rollback() - log.Debug("[Get Annotated Image] Couldn't get number of annotation revisions: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImage, err - } + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Get Annotated Image] Couldn't get number of annotation revisions: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImage, err + } - annotatedImage.Revision = revision - } + annotatedImage.Revision = revision + } - err = tx.Commit() - if err != nil { - log.Debug("[Get Annotated Image] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImage, err - } + err = tx.Commit(context.TODO()) + if err != nil { + log.Debug("[Get Annotated Image] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImage, err + } - return annotatedImage, nil + return annotatedImage, nil } -func (p *ImageMonkeyDatabase) ValidateAnnotatedImage(clientFingerprint string, annotationId string, - labelValidationEntry datastructures.LabelValidationEntry, valid bool) error { - if valid { - var err error - if labelValidationEntry.Sublabel == "" { - _, err = p.db.Exec(`UPDATE image_annotation AS a +func (p *ImageMonkeyDatabase) ValidateAnnotatedImage(clientFingerprint string, annotationId string, + labelValidationEntry datastructures.LabelValidationEntry, valid bool) error { + if valid { + var err error + if labelValidationEntry.Sublabel == "" { + _, err = p.db.Exec(context.TODO(), + `UPDATE image_annotation AS a SET num_of_valid = num_of_valid + 1, fingerprint_of_last_modification = $1 - WHERE a.uuid = $2 AND a.label_id = (SELECT id FROM label WHERE name = $3 AND parent_id is null)`, - clientFingerprint, annotationId, labelValidationEntry.Label) - } else { - _, err = p.db.Exec(`UPDATE image_annotation AS a + WHERE a.uuid = $2 AND a.label_id = (SELECT id FROM label WHERE name = $3 AND parent_id is null)`, + clientFingerprint, annotationId, labelValidationEntry.Label) + } else { + _, err = p.db.Exec(context.TODO(), + `UPDATE image_annotation AS a SET num_of_valid = num_of_valid + 1, fingerprint_of_last_modification = $1 WHERE a.uuid = $2 AND a.label_id = ( SELECT l.id FROM label l JOIN label pl ON l.parent_id = pl.id WHERE l.name = $3 AND pl.name = $4 - )`, - clientFingerprint, annotationId, labelValidationEntry.Sublabel, labelValidationEntry.Label) - } - - - if err != nil { - log.Debug("[Validating annotated photo] Couldn't increase num_of_valid: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - } else { - var err error - if labelValidationEntry.Sublabel == "" { - _,err = p.db.Exec(`UPDATE image_annotation AS a + )`, + clientFingerprint, annotationId, labelValidationEntry.Sublabel, labelValidationEntry.Label) + } + + if err != nil { + log.Debug("[Validating annotated photo] Couldn't increase num_of_valid: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + } else { + var err error + if labelValidationEntry.Sublabel == "" { + _, err = p.db.Exec(context.TODO(), + `UPDATE image_annotation AS a SET num_of_invalid = num_of_invalid + 1, fingerprint_of_last_modification = $1 WHERE a.uuid = $2 AND a.label_id = ( SELECT id FROM label WHERE name = $3 AND parent_id is null - )`, - clientFingerprint, annotationId, labelValidationEntry.Label) - } else { - _,err = p.db.Exec(`UPDATE image_annotation AS a + )`, + clientFingerprint, annotationId, labelValidationEntry.Label) + } else { + _, err = p.db.Exec(context.TODO(), + `UPDATE image_annotation AS a SET num_of_invalid = num_of_invalid + 1, fingerprint_of_last_modification = $1 WHERE a.uuid = $2 AND a.label_id = ( SELECT l.id FROM label l JOIN label pl ON l.parent_id = pl.id WHERE l.name = $3 AND pl.name = $4 - )`, - clientFingerprint, annotationId, labelValidationEntry.Sublabel, labelValidationEntry.Label) - } - + )`, + clientFingerprint, annotationId, labelValidationEntry.Sublabel, labelValidationEntry.Label) + } - if err != nil { - log.Debug("[Validating annotated photo] Couldn't increase num_of_invalid: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - } + if err != nil { + log.Debug("[Validating annotated photo] Couldn't increase num_of_invalid: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + } - return nil + return nil } func (p *ImageMonkeyDatabase) GetAnnotationCoverage(imageId string) ([]datastructures.ImageAnnotationCoverage, error) { - var imageAnnotationCoverages []datastructures.ImageAnnotationCoverage + var imageAnnotationCoverages []datastructures.ImageAnnotationCoverage - q1 := "" - var queryValues []interface{} - if imageId != "" { - q1 = "WHERE i.key = $1" - queryValues = append(queryValues, imageId) - } + q1 := "" + var queryValues []interface{} + if imageId != "" { + q1 = "WHERE i.key = $1" + queryValues = append(queryValues, imageId) + } - q := fmt.Sprintf(`SELECT i.key, i.width, i.height, c.annotated_percentage + q := fmt.Sprintf(`SELECT i.key, i.width, i.height, c.annotated_percentage FROM image_annotation_coverage c JOIN image i ON c.image_id = i.id %s`, q1) - rows, err := p.db.Query(q, queryValues...) - if err != nil { - log.Debug("[Get annotation coverage] Couldn't get annotation coverage: ", err.Error()) - raven.CaptureError(err, nil) - return imageAnnotationCoverages, err - } + rows, err := p.db.Query(context.TODO(), q, queryValues...) + if err != nil { + log.Debug("[Get annotation coverage] Couldn't get annotation coverage: ", err.Error()) + raven.CaptureError(err, nil) + return imageAnnotationCoverages, err + } - defer rows.Close() + defer rows.Close() - for rows.Next() { - var imageAnnotationCoverage datastructures.ImageAnnotationCoverage + for rows.Next() { + var imageAnnotationCoverage datastructures.ImageAnnotationCoverage - err = rows.Scan(&imageAnnotationCoverage.Image.Id, &imageAnnotationCoverage.Image.Width, - &imageAnnotationCoverage.Image.Height, &imageAnnotationCoverage.Coverage) - if err != nil { - log.Debug("[Get annotation coverage] Couldn't scan rows: ", err.Error()) - raven.CaptureError(err, nil) - return imageAnnotationCoverages, err - } + err = rows.Scan(&imageAnnotationCoverage.Image.Id, &imageAnnotationCoverage.Image.Width, + &imageAnnotationCoverage.Image.Height, &imageAnnotationCoverage.Coverage) + if err != nil { + log.Debug("[Get annotation coverage] Couldn't scan rows: ", err.Error()) + raven.CaptureError(err, nil) + return imageAnnotationCoverages, err + } - imageAnnotationCoverages = append(imageAnnotationCoverages, imageAnnotationCoverage) - } + imageAnnotationCoverages = append(imageAnnotationCoverages, imageAnnotationCoverage) + } - return imageAnnotationCoverages, nil + return imageAnnotationCoverages, nil } +func (p *ImageMonkeyDatabase) GetAnnotationsForRefinement(parseResult parser.ParseResult, apiBaseUrl string, + annotationDataId string) ([]datastructures.AnnotationRefinementTask, error) { + var annotationRefinementTasks []datastructures.AnnotationRefinementTask -func (p *ImageMonkeyDatabase) GetAnnotationsForRefinement(parseResult parser.ParseResult, apiBaseUrl string, - annotationDataId string) ([]datastructures.AnnotationRefinementTask, error) { - var annotationRefinementTasks []datastructures.AnnotationRefinementTask - - q1 := "" - if annotationDataId != "" { - q1 = fmt.Sprintf("WHERE d.uuid::text = $%d", len(parseResult.QueryValues) + 1) - } + q1 := "" + if annotationDataId != "" { + q1 = fmt.Sprintf("WHERE d.uuid::text = $%d", len(parseResult.QueryValues)+1) + } - q2 := "" - if len(parseResult.QueryValues) > 0 { - q2 = fmt.Sprintf("WHERE %s", parseResult.Query) - } + q2 := "" + if len(parseResult.QueryValues) > 0 { + q2 = fmt.Sprintf("WHERE %s", parseResult.Query) + } - q := fmt.Sprintf(`WITH + q := fmt.Sprintf(`WITH productive_image_annotation_data_entries AS ( SELECT q.annotation_data_id, array_agg(q.label)::text[] as accessors FROM ( @@ -1546,72 +1553,69 @@ func (p *ImageMonkeyDatabase) GetAnnotationsForRefinement(parseResult parser.Par %s GROUP BY i.key, i.unlocked, i.width, i.height, a.uuid, d.annotation, d.uuid, t.name`, q2, q1) - if annotationDataId != "" { - parseResult.QueryValues = append(parseResult.QueryValues, annotationDataId) - } - - rows, err := p.db.Query(q, parseResult.QueryValues...) - if err != nil { - log.Debug("[Get Annotations For Refinement] Couldn't get annotations for refinement: ", err.Error()) - raven.CaptureError(err, nil) - return annotationRefinementTasks, err - } - - defer rows.Close() - - for rows.Next() { - var annotationBytes []byte - var labelAccessorsBytes []byte - var annotationRefinementTask datastructures.AnnotationRefinementTask - rows.Scan(&annotationRefinementTask.Image.Id, &annotationRefinementTask.Image.Unlocked, - &annotationRefinementTask.Image.Width, &annotationRefinementTask.Image.Height, - &annotationRefinementTask.Annotation.Id, &annotationBytes, &labelAccessorsBytes) - - err = json.Unmarshal(annotationBytes, &annotationRefinementTask.Annotation.Data) - if err != nil { - log.Debug("[Get Annotations For Refinement] Couldn't unmarshal annotation: ", err.Error()) - raven.CaptureError(err, nil) - return annotationRefinementTasks, err - } - - err = json.Unmarshal(labelAccessorsBytes, &annotationRefinementTask.Refinements) - if err != nil { - log.Debug("[Get Annotations For Refinement] Couldn't unmarshal labels: ", err.Error()) - raven.CaptureError(err, nil) - return annotationRefinementTasks, err - } - - - annotationRefinementTask.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, annotationRefinementTask.Image.Id, - annotationRefinementTask.Image.Unlocked) - - annotationRefinementTasks = append(annotationRefinementTasks, annotationRefinementTask) - } - - return annotationRefinementTasks, nil -} + if annotationDataId != "" { + parseResult.QueryValues = append(parseResult.QueryValues, annotationDataId) + } + + rows, err := p.db.Query(context.TODO(), q, parseResult.QueryValues...) + if err != nil { + log.Debug("[Get Annotations For Refinement] Couldn't get annotations for refinement: ", err.Error()) + raven.CaptureError(err, nil) + return annotationRefinementTasks, err + } + + defer rows.Close() + + for rows.Next() { + var annotationBytes []byte + var labelAccessorsBytes []byte + var annotationRefinementTask datastructures.AnnotationRefinementTask + rows.Scan(&annotationRefinementTask.Image.Id, &annotationRefinementTask.Image.Unlocked, + &annotationRefinementTask.Image.Width, &annotationRefinementTask.Image.Height, + &annotationRefinementTask.Annotation.Id, &annotationBytes, &labelAccessorsBytes) + + err = json.Unmarshal(annotationBytes, &annotationRefinementTask.Annotation.Data) + if err != nil { + log.Debug("[Get Annotations For Refinement] Couldn't unmarshal annotation: ", err.Error()) + raven.CaptureError(err, nil) + return annotationRefinementTasks, err + } + err = json.Unmarshal(labelAccessorsBytes, &annotationRefinementTask.Refinements) + if err != nil { + log.Debug("[Get Annotations For Refinement] Couldn't unmarshal labels: ", err.Error()) + raven.CaptureError(err, nil) + return annotationRefinementTasks, err + } + + annotationRefinementTask.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, annotationRefinementTask.Image.Id, + annotationRefinementTask.Image.Unlocked) -func (p *ImageMonkeyDatabase) GetAnnotations(apiUser datastructures.APIUser, parseResult parser.ParseResult, - imageId string, apiBaseUrl string) ([]datastructures.AnnotatedImage, error) { - annotatedImages := []datastructures.AnnotatedImage{} - var queryValues []interface{} + annotationRefinementTasks = append(annotationRefinementTasks, annotationRefinementTask) + } + + return annotationRefinementTasks, nil +} +func (p *ImageMonkeyDatabase) GetAnnotations(apiUser datastructures.APIUser, parseResult parser.ParseResult, + imageId string, apiBaseUrl string) ([]datastructures.AnnotatedImage, error) { + annotatedImages := []datastructures.AnnotatedImage{} + var queryValues []interface{} - q1 := "" - if imageId == "" { - q1 = "WHERE " + parseResult.Query - queryValues = parseResult.QueryValues - } else { - q1 = "WHERE image_key = $1" - queryValues = append(queryValues, imageId) - } + q1 := "" + if imageId == "" { + q1 = "WHERE " + parseResult.Query + queryValues = parseResult.QueryValues + } else { + q1 = "WHERE image_key = $1" + queryValues = append(queryValues, imageId) + } q2 := "acc.name is null" - includeOwnImageDonations := "" - if apiUser.Name != "" { - q2 = fmt.Sprintf(`acc.name = $%d`, len(queryValues) + 1) - + includeOwnImageDonations := "" + if apiUser.Name != "" { + q2 = fmt.Sprintf(`acc.name = $%d`, len(queryValues)+1) + includeOwnImageDonations = fmt.Sprintf(`OR ( EXISTS ( @@ -1626,11 +1630,11 @@ func (p *ImageMonkeyDatabase) GetAnnotations(apiUser datastructures.APIUser, par FROM image_quarantine q WHERE q.image_id = i.id ) - )`, len(queryValues) + 1) - queryValues = append(queryValues, apiUser.Name) - } + )`, len(queryValues)+1) + queryValues = append(queryValues, apiUser.Name) + } - q := fmt.Sprintf(`SELECT q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, json_agg(q2.annotation), + q := fmt.Sprintf(`SELECT q2.image_key, q2.label_name, q2.parent_label_name, q2.annotation_uuid, json_agg(q2.annotation), q2.num_of_valid, q2.num_of_invalid, q2.image_width, q2.image_height, q2.image_unlocked, q2.is_suggestion FROM ( @@ -1719,65 +1723,64 @@ func (p *ImageMonkeyDatabase) GetAnnotations(apiUser datastructures.APIUser, par q2.num_of_valid, q2.num_of_invalid, q2.image_width, q2.image_height, q2.image_unlocked, q2.is_suggestion `, q2, includeOwnImageDonations, q1) - rows, err := p.db.Query(q, queryValues...) - if err != nil { - log.Debug("[Get Annotated Images] Couldn't get annotations: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImages, err - } + rows, err := p.db.Query(context.TODO(), q, queryValues...) + if err != nil { + log.Debug("[Get Annotated Images] Couldn't get annotations: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImages, err + } - defer rows.Close() + defer rows.Close() - var label1 string - var label2 string - var annotations []byte + var label1 string + var label2 string + var annotations []byte var isSuggestion bool - for rows.Next() { - var annotatedImage datastructures.AnnotatedImage - annotatedImage.Image.Provider = "donation" - - err = rows.Scan(&annotatedImage.Image.Id, &label1, &label2, &annotatedImage.Id, - &annotations, &annotatedImage.NumOfValid, &annotatedImage.NumOfInvalid, - &annotatedImage.Image.Width, &annotatedImage.Image.Height, &annotatedImage.Image.Unlocked, &isSuggestion) - if err != nil { - log.Debug("[Get Annotated Images] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImages, err - } - - err := json.Unmarshal(annotations, &annotatedImage.Annotations) - if err != nil { - log.Debug("[Get Annotated Images] Couldn't unmarshal: ", err.Error()) - raven.CaptureError(err, nil) - return annotatedImages, err - } - - if label2 == "" { - annotatedImage.Validation.Label = label1 - annotatedImage.Validation.Sublabel = "" - } else { - annotatedImage.Validation.Label = label2 - annotatedImage.Validation.Sublabel = label1 - } + for rows.Next() { + var annotatedImage datastructures.AnnotatedImage + annotatedImage.Image.Provider = "donation" + + err = rows.Scan(&annotatedImage.Image.Id, &label1, &label2, &annotatedImage.Id, + &annotations, &annotatedImage.NumOfValid, &annotatedImage.NumOfInvalid, + &annotatedImage.Image.Width, &annotatedImage.Image.Height, &annotatedImage.Image.Unlocked, &isSuggestion) + if err != nil { + log.Debug("[Get Annotated Images] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImages, err + } + + err := json.Unmarshal(annotations, &annotatedImage.Annotations) + if err != nil { + log.Debug("[Get Annotated Images] Couldn't unmarshal: ", err.Error()) + raven.CaptureError(err, nil) + return annotatedImages, err + } + + if label2 == "" { + annotatedImage.Validation.Label = label1 + annotatedImage.Validation.Sublabel = "" + } else { + annotatedImage.Validation.Label = label2 + annotatedImage.Validation.Sublabel = label1 + } annotatedImage.Validation.Unlocked = !isSuggestion - annotatedImage.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, annotatedImage.Image.Id, annotatedImage.Image.Unlocked) + annotatedImage.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, annotatedImage.Image.Id, annotatedImage.Image.Unlocked) - annotatedImages = append(annotatedImages, annotatedImage) + annotatedImages = append(annotatedImages, annotatedImage) - } - return annotatedImages, nil + } + return annotatedImages, nil } -func (p *ImageMonkeyDatabase) GetAvailableAnnotationTasks(apiUser datastructures.APIUser, parseResult parser.ParseResult, - orderRandomly bool, apiBaseUrl string, includeImageSuggestions bool) ([]datastructures.AnnotationTask, error) { - var annotationTasks []datastructures.AnnotationTask +func (p *ImageMonkeyDatabase) GetAvailableAnnotationTasks(apiUser datastructures.APIUser, parseResult parser.ParseResult, + orderRandomly bool, apiBaseUrl string, includeImageSuggestions bool) ([]datastructures.AnnotationTask, error) { + var annotationTasks []datastructures.AnnotationTask - - includeOwnImageDonations := "" - if apiUser.Name != "" { - includeOwnImageDonations = fmt.Sprintf(`OR ( + includeOwnImageDonations := "" + if apiUser.Name != "" { + includeOwnImageDonations = fmt.Sprintf(`OR ( EXISTS ( SELECT 1 @@ -1791,25 +1794,25 @@ func (p *ImageMonkeyDatabase) GetAvailableAnnotationTasks(apiUser datastructures FROM image_quarantine q WHERE q.image_id = i.id ) - )`, len(parseResult.QueryValues) + 1) - } + )`, len(parseResult.QueryValues)+1) + } - orderBy := "" - if orderRandomly { - orderBy = " ORDER BY RANDOM()" - } + orderBy := "" + if orderRandomly { + orderBy = " ORDER BY RANDOM()" + } - q2 := "" - q3 := "acc.name is null" - if apiUser.Name != "" { + q2 := "" + q3 := "acc.name is null" + if apiUser.Name != "" { q2 = fmt.Sprintf(` AND NOT EXISTS ( SELECT 1 FROM user_annotation_blacklist bl JOIN account acc ON acc.id = bl.account_id WHERE bl.image_validation_id = v.id AND acc.name = $%d - )`, len(parseResult.QueryValues) + 1) - - q3 = fmt.Sprintf(`acc.name = $%d`, len(parseResult.QueryValues) + 1) + )`, len(parseResult.QueryValues)+1) + + q3 = fmt.Sprintf(`acc.name = $%d`, len(parseResult.QueryValues)+1) } q4 := "" @@ -1824,7 +1827,7 @@ func (p *ImageMonkeyDatabase) GetAvailableAnnotationTasks(apiUser datastructures SELECT 1 FROM image_annotation_suggestion a WHERE a.label_suggestion_id = s.label_suggestion_id AND a.image_id = s.image_id ) - ` + ` } //in case no subquery is provided, set 1=1 to "catch all". if we won't do that, the query @@ -1833,7 +1836,7 @@ func (p *ImageMonkeyDatabase) GetAvailableAnnotationTasks(apiUser datastructures parseResult.Subquery = "1 = 1" } - q := fmt.Sprintf(`SELECT qqq.image_key, qqq.image_width, qqq.image_height, qqq.validation_uuid, qqq.image_unlocked, + q := fmt.Sprintf(`SELECT qqq.image_key, qqq.image_width, qqq.image_height, qqq.validation_uuid, qqq.image_unlocked, accessor FROM ( @@ -1883,56 +1886,57 @@ func (p *ImageMonkeyDatabase) GetAvailableAnnotationTasks(apiUser datastructures )qq ) qqq GROUP BY image_key, image_width, image_height, validation_uuid, image_unlocked, accessor - %s`, parseResult.Subquery, parseResult.Subquery, parseResult.Subquery, - q2, q4, q3, includeOwnImageDonations, parseResult.Query, orderBy) - - //first item in query value is the label we want to annotate - //parseResult.queryValues = append([]interface{}{parseResult.queryValues[0]}, parseResult.queryValues...) - - var rows *sql.Rows - var err error - if apiUser.Name == "" { - rows, err = p.db.Query(q, parseResult.QueryValues...) - } else { - parseResult.QueryValues = append(parseResult.QueryValues, apiUser.Name) - rows, err = p.db.Query(q, parseResult.QueryValues...) - } - if err != nil { - log.Debug("[Annotation Tasks] Couldn't get available annotation tasks: ", err.Error()) - raven.CaptureError(err, nil) - return annotationTasks, err - } - - defer rows.Close() - - for rows.Next() { - var annotationTask datastructures.AnnotationTask - err = rows.Scan(&annotationTask.Image.Id, &annotationTask.Image.Width, &annotationTask.Image.Height, - &annotationTask.Id, &annotationTask.Image.Unlocked, &annotationTask.Label.Accessor) - if err != nil { - log.Debug("[Annotation Tasks] Couldn't get available annotation tasks: ", err.Error()) - raven.CaptureError(err, nil) - return annotationTasks, err - } - - if annotationTask.Id == "" { - continue - } - - annotationTask.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, annotationTask.Image.Id, annotationTask.Image.Unlocked) - - annotationTasks = append(annotationTasks, annotationTask) - } - - return annotationTasks, nil + %s`, parseResult.Subquery, parseResult.Subquery, parseResult.Subquery, + q2, q4, q3, includeOwnImageDonations, parseResult.Query, orderBy) + + //first item in query value is the label we want to annotate + //parseResult.queryValues = append([]interface{}{parseResult.queryValues[0]}, parseResult.queryValues...) + + var rows pgx.Rows + var err error + if apiUser.Name == "" { + rows, err = p.db.Query(context.TODO(), q, parseResult.QueryValues...) + } else { + parseResult.QueryValues = append(parseResult.QueryValues, apiUser.Name) + rows, err = p.db.Query(context.TODO(), q, parseResult.QueryValues...) + } + if err != nil { + log.Debug("[Annotation Tasks] Couldn't get available annotation tasks: ", err.Error()) + raven.CaptureError(err, nil) + return annotationTasks, err + } + + defer rows.Close() + + for rows.Next() { + var annotationTask datastructures.AnnotationTask + err = rows.Scan(&annotationTask.Image.Id, &annotationTask.Image.Width, &annotationTask.Image.Height, + &annotationTask.Id, &annotationTask.Image.Unlocked, &annotationTask.Label.Accessor) + if err != nil { + log.Debug("[Annotation Tasks] Couldn't get available annotation tasks: ", err.Error()) + raven.CaptureError(err, nil) + return annotationTasks, err + } + + if annotationTask.Id == "" { + continue + } + + annotationTask.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, annotationTask.Image.Id, annotationTask.Image.Unlocked) + + annotationTasks = append(annotationTasks, annotationTask) + } + + return annotationTasks, nil } func (p *ImageMonkeyDatabase) GetRandomAnnotationForQuizRefinement() (datastructures.AnnotationRefinement, error) { - var bytes []byte - var annotationBytes []byte - var refinement datastructures.AnnotationRefinement - var annotations []json.RawMessage - rows, err := p.db.Query(`SELECT i.key, s.quiz_question_id, s.quiz_question, s.quiz_answers, s1.annotations, s.recommended_control::text, + var bytes []byte + var annotationBytes []byte + var refinement datastructures.AnnotationRefinement + var annotations []json.RawMessage + rows, err := p.db.Query(context.TODO(), + `SELECT i.key, s.quiz_question_id, s.quiz_question, s.quiz_answers, s1.annotations, s.recommended_control::text, s1.uuid, s.allow_unknown, s.allow_other, s.browse_by_example, s.multiselect FROM ( SELECT qq.question as quiz_question, qq.recommended_control as recommended_control, @@ -1969,67 +1973,67 @@ func (p *ImageMonkeyDatabase) GetRandomAnnotationForQuizRefinement() (datastruct ) ) LIMIT 1`) - if err != nil { - log.Debug("[Random Quiz question] Couldn't get random image quiz: ", err.Error()) - raven.CaptureError(err, nil) - return refinement, err - } - - defer rows.Close() - - if rows.Next() { - err = rows.Scan(&refinement.Image.Uuid, &refinement.Question.Uuid, - &refinement.Question.Question, &bytes, &annotationBytes, &refinement.Question.RecommendedControl, - &refinement.Annotation.Uuid, &refinement.Metainfo.AllowUnknown, &refinement.Metainfo.AllowOther, - &refinement.Metainfo.BrowseByExample, &refinement.Metainfo.MultiSelect) - - if err != nil { - log.Debug("[Random Quiz question] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return refinement, err - } - - err = json.Unmarshal(bytes, &refinement.Answers) - if err != nil { - log.Debug("[Random Quiz question] Couldn't unmarshal answers: ", err.Error()) - raven.CaptureError(err, nil) - return refinement, err - } - - err = json.Unmarshal(annotationBytes, &annotations) - if err != nil { - log.Debug("[Random Quiz question] Couldn't unmarshal annotations: ", err.Error()) - raven.CaptureError(err, nil) - return refinement, err - } - - if len(annotations) == 1 { - refinement.Annotation.Annotation = annotations[0] - } else if len(annotations) > 1 { - randomVal := commons.Random(0, (len(annotations) - 1)) - refinement.Annotation.Annotation = annotations[randomVal] - } - } - - return refinement, nil -} + if err != nil { + log.Debug("[Random Quiz question] Couldn't get random image quiz: ", err.Error()) + raven.CaptureError(err, nil) + return refinement, err + } + + defer rows.Close() + + if rows.Next() { + err = rows.Scan(&refinement.Image.Uuid, &refinement.Question.Uuid, + &refinement.Question.Question, &bytes, &annotationBytes, &refinement.Question.RecommendedControl, + &refinement.Annotation.Uuid, &refinement.Metainfo.AllowUnknown, &refinement.Metainfo.AllowOther, + &refinement.Metainfo.BrowseByExample, &refinement.Metainfo.MultiSelect) + if err != nil { + log.Debug("[Random Quiz question] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return refinement, err + } -func addOrUpdateRefinementsInTransaction(tx *sql.Tx, annotationUuid string, annotationDataId string, - annotationRefinementEntries []datastructures.AnnotationRefinementEntry, clientFingerprint string, - isSuggestion bool) error { - for _, item := range annotationRefinementEntries { + err = json.Unmarshal(bytes, &refinement.Answers) + if err != nil { + log.Debug("[Random Quiz question] Couldn't unmarshal answers: ", err.Error()) + raven.CaptureError(err, nil) + return refinement, err + } - _, err := uuid.FromString(item.LabelId) - if err != nil { - tx.Rollback() - log.Error("[Add or Update annotation refinement] Couldn't add/update refinements - invalid label id") - raven.CaptureError(err, nil) - return &InvalidLabelIdError{Description: "invalid label id"} - } + err = json.Unmarshal(annotationBytes, &annotations) + if err != nil { + log.Debug("[Random Quiz question] Couldn't unmarshal annotations: ", err.Error()) + raven.CaptureError(err, nil) + return refinement, err + } + + if len(annotations) == 1 { + refinement.Annotation.Annotation = annotations[0] + } else if len(annotations) > 1 { + randomVal := commons.Random(0, (len(annotations) - 1)) + refinement.Annotation.Annotation = annotations[randomVal] + } + } + + return refinement, nil +} + +func addOrUpdateRefinementsInTransaction(tx pgx.Tx, annotationUuid string, annotationDataId string, + annotationRefinementEntries []datastructures.AnnotationRefinementEntry, clientFingerprint string, + isSuggestion bool) error { + for _, item := range annotationRefinementEntries { + + _, err := uuid.FromString(item.LabelId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Add or Update annotation refinement] Couldn't add/update refinements - invalid label id") + raven.CaptureError(err, nil) + return &InvalidLabelIdError{Description: "invalid label id"} + } if isSuggestion { - _, err = tx.Exec(`INSERT INTO image_annotation_suggestion_refinement(annotation_suggestion_data_id, + _, err = tx.Exec(context.TODO(), + `INSERT INTO image_annotation_suggestion_refinement(annotation_suggestion_data_id, label_id, num_of_valid, fingerprint_of_last_modification) SELECT d.id, (SELECT l.id FROM label l WHERE l.uuid = $2), $3, $4 FROM image_annotation_suggestion a @@ -2038,10 +2042,11 @@ func addOrUpdateRefinementsInTransaction(tx *sql.Tx, annotationUuid string, anno ON CONFLICT (annotation_suggestion_data_id, label_id) DO UPDATE SET fingerprint_of_last_modification = $4, num_of_valid = image_annotation_suggestion_refinement.num_of_valid + 1 WHERE image_annotation_suggestion_refinement.annotation_suggestion_data_id = (SELECT d.id FROM annotation_suggestion_data d WHERE d.uuid = $1) - AND image_annotation_suggestion_refinement.label_id = (SELECT l.id FROM label l WHERE l.uuid = $2)`, - annotationDataId, item.LabelId, 1, clientFingerprint, annotationUuid) + AND image_annotation_suggestion_refinement.label_id = (SELECT l.id FROM label l WHERE l.uuid = $2)`, + annotationDataId, item.LabelId, 1, clientFingerprint, annotationUuid) } else { - _, err = tx.Exec(`INSERT INTO image_annotation_refinement(annotation_data_id, label_id, num_of_valid, fingerprint_of_last_modification) + _, err = tx.Exec(context.TODO(), + `INSERT INTO image_annotation_refinement(annotation_data_id, label_id, num_of_valid, fingerprint_of_last_modification) SELECT d.id, (SELECT l.id FROM label l WHERE l.uuid = $2), $3, $4 FROM image_annotation a JOIN annotation_data d ON d.image_annotation_id = a.id @@ -2049,23 +2054,24 @@ func addOrUpdateRefinementsInTransaction(tx *sql.Tx, annotationUuid string, anno ON CONFLICT (annotation_data_id, label_id) DO UPDATE SET fingerprint_of_last_modification = $4, num_of_valid = image_annotation_refinement.num_of_valid + 1 WHERE image_annotation_refinement.annotation_data_id = (SELECT d.id FROM annotation_data d WHERE d.uuid = $1) - AND image_annotation_refinement.label_id = (SELECT l.id FROM label l WHERE l.uuid = $2)`, - annotationDataId, item.LabelId, 1, clientFingerprint, annotationUuid) - } - - if err != nil { - tx.Rollback() - log.Error("[Add or Update annotation refinement] Couldn't update: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - } - return nil + AND image_annotation_refinement.label_id = (SELECT l.id FROM label l WHERE l.uuid = $2)`, + annotationDataId, item.LabelId, 1, clientFingerprint, annotationUuid) + } + + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Add or Update annotation refinement] Couldn't update: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + } + return nil } func (p *ImageMonkeyDatabase) AnnotationUuidIsASuggestion(annotationUuid string) (bool, error) { var isSuggestion bool = false - err := p.db.QueryRow(`SELECT is_suggestion FROM + err := p.db.QueryRow(context.TODO(), + `SELECT is_suggestion FROM ( SELECT count(*) as count, false as is_suggestion FROM image_annotation a @@ -2084,117 +2090,117 @@ func (p *ImageMonkeyDatabase) AnnotationUuidIsASuggestion(annotationUuid string) return isSuggestion, nil } -func (p *ImageMonkeyDatabase) AddOrUpdateRefinements(annotationUuid string, annotationDataId string, - annotationRefinementEntries []datastructures.AnnotationRefinementEntry, clientFingerprint string, - isSuggestion bool) error { - var err error - - tx, err := p.db.Begin() - if err != nil { - log.Error("[Add or Update annotation refinement] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - err = addOrUpdateRefinementsInTransaction(tx, annotationUuid, annotationDataId, annotationRefinementEntries, clientFingerprint, isSuggestion) - if err != nil { //transaction already rolled back, so we can return here - return err - } - - err = tx.Commit() - if err != nil { - log.Error("[Add or Update annotation refinement] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - return nil -} +func (p *ImageMonkeyDatabase) AddOrUpdateRefinements(annotationUuid string, annotationDataId string, + annotationRefinementEntries []datastructures.AnnotationRefinementEntry, clientFingerprint string, + isSuggestion bool) error { + var err error + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Error("[Add or Update annotation refinement] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } -func (p *ImageMonkeyDatabase) BatchAnnotationRefinement(annotationRefinementEntries []datastructures.BatchAnnotationRefinementEntry, - apiUser datastructures.APIUser) error { - var err error + err = addOrUpdateRefinementsInTransaction(tx, annotationUuid, annotationDataId, annotationRefinementEntries, clientFingerprint, isSuggestion) + if err != nil { //transaction already rolled back, so we can return here + return err + } - tx, err := p.db.Begin() - if err != nil { - log.Debug("[Add or Update annotation refinement] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + err = tx.Commit(context.TODO()) + if err != nil { + log.Error("[Add or Update annotation refinement] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + return nil +} + +func (p *ImageMonkeyDatabase) BatchAnnotationRefinement(annotationRefinementEntries []datastructures.BatchAnnotationRefinementEntry, + apiUser datastructures.APIUser) error { + var err error - for _, item := range annotationRefinementEntries { - _, err = tx.Exec(`INSERT INTO image_annotation_refinement(annotation_data_id, label_id, num_of_valid, fingerprint_of_last_modification) + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Debug("[Add or Update annotation refinement] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + for _, item := range annotationRefinementEntries { + _, err = tx.Exec(context.TODO(), + `INSERT INTO image_annotation_refinement(annotation_data_id, label_id, num_of_valid, fingerprint_of_last_modification) SELECT d.id, (SELECT l.id FROM label l WHERE l.uuid = $2), $3, $4 FROM annotation_data d WHERE d.uuid = $1 ON CONFLICT (annotation_data_id, label_id) DO UPDATE SET fingerprint_of_last_modification = $4, num_of_valid = image_annotation_refinement.num_of_valid + 1 - WHERE image_annotation_refinement.annotation_data_id = (SELECT d.id FROM annotation_data d WHERE d.uuid = $1)`, - item.AnnotationDataId, item.LabelId, 1, apiUser.ClientFingerprint) - - if err != nil { - tx.Rollback() - log.Debug("[Batch annotation refinement] Couldn't update: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - } - - err = tx.Commit() - if err != nil { - log.Debug("[Batch annotation refinement] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - return nil -} + WHERE image_annotation_refinement.annotation_data_id = (SELECT d.id FROM annotation_data d WHERE d.uuid = $1)`, + item.AnnotationDataId, item.LabelId, 1, apiUser.ClientFingerprint) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Batch annotation refinement] Couldn't update: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + } + + err = tx.Commit(context.TODO()) + if err != nil { + log.Debug("[Batch annotation refinement] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + return nil +} func (p *ImageMonkeyDatabase) GetImagesForAutoAnnotation(labels []string) ([]datastructures.AutoAnnotationImage, error) { - var autoAnnotationImages []datastructures.AutoAnnotationImage - rows, err := p.db.Query(`SELECT i.key, i.width, i.height, json_agg(l.name) FROM image i + var autoAnnotationImages []datastructures.AutoAnnotationImage + rows, err := p.db.Query(context.TODO(), + `SELECT i.key, i.width, i.height, json_agg(l.name) FROM image i JOIN image_validation v ON v.image_id = i.id JOIN label l on v.label_id = l.id WHERE i.id NOT IN ( SELECT image_id FROM image_annotation WHERE auto_generated = true ) AND l.parent_id is null AND i.unlocked = true AND l.name = ANY($1) - GROUP BY i.key, i.width, i.height`, - pq.Array(labels)) - if err != nil { - log.Debug("[Get images for auto annotation] Couldn't get: ", err.Error()) - raven.CaptureError(err, nil) - return autoAnnotationImages, err - } - - defer rows.Close() - - for rows.Next() { - var autoAnnotationImage datastructures.AutoAnnotationImage - var data []byte - err = rows.Scan(&autoAnnotationImage.Image.Id, &autoAnnotationImage.Image.Width, &autoAnnotationImage.Image.Height, &data) - if err != nil { - log.Debug("[Get images for auto annotation] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return autoAnnotationImages, err - } - - err = json.Unmarshal(data, &autoAnnotationImage.Labels) - if err != nil { - log.Debug("[Get images for auto annotation] Couldn't unmarshal: ", err.Error()) - raven.CaptureError(err, nil) - return autoAnnotationImages, err - } - - autoAnnotationImages = append(autoAnnotationImages, autoAnnotationImage) - } - return autoAnnotationImages, nil + GROUP BY i.key, i.width, i.height`, + labels) + if err != nil { + log.Debug("[Get images for auto annotation] Couldn't get: ", err.Error()) + raven.CaptureError(err, nil) + return autoAnnotationImages, err + } + + defer rows.Close() + + for rows.Next() { + var autoAnnotationImage datastructures.AutoAnnotationImage + var data []byte + err = rows.Scan(&autoAnnotationImage.Image.Id, &autoAnnotationImage.Image.Width, &autoAnnotationImage.Image.Height, &data) + if err != nil { + log.Debug("[Get images for auto annotation] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return autoAnnotationImages, err + } + + err = json.Unmarshal(data, &autoAnnotationImage.Labels) + if err != nil { + log.Debug("[Get images for auto annotation] Couldn't unmarshal: ", err.Error()) + raven.CaptureError(err, nil) + return autoAnnotationImages, err + } + + autoAnnotationImages = append(autoAnnotationImages, autoAnnotationImage) + } + return autoAnnotationImages, nil } func (p *ImageMonkeyDatabase) GetBoundingBoxesForImageLabel(imageId string, label string) ([]image.Rectangle, error) { - boundingBoxes := []image.Rectangle{} + boundingBoxes := []image.Rectangle{} - query := `WITH all_annotations AS ( + query := `WITH all_annotations AS ( SELECT an.image_id as image_id, d.id as annotation_data_id, d.annotation as annotation, t.name as annotation_type FROM image_annotation an JOIN annotation_data d ON d.image_annotation_id = an.id @@ -2259,25 +2265,25 @@ func (p *ImageMonkeyDatabase) GetBoundingBoxesForImageLabel(imageId string, labe ((geom->'coordinates'->>0)::jsonb->>2)::jsonb->0 as x1, ((geom->'coordinates'->>0)::jsonb->>2)::jsonb->1 as y1 FROM all_annotation_areas` - rows, err := p.db.Query(query, imageId, label) - if err != nil { - log.Error("[Get Bounding Boxes] Couldn't get bounding boxes for image label: ", err.Error()) - raven.CaptureError(err, nil) - return boundingBoxes, err - } - - defer rows.Close() - - for rows.Next() { - var x0, y0, x1, y1 int - err = rows.Scan(&x0, &y0, &x1, &y1) - if err != nil { - log.Error("[Get Bounding Boxes] Couldn't scan bounding boxes for image label: ", err.Error()) - raven.CaptureError(err, nil) - return boundingBoxes, err - } - boundingBoxes = append(boundingBoxes, image.Rect(x0, y0, x1, y1)) - } - - return boundingBoxes, nil + rows, err := p.db.Query(context.TODO(), query, imageId, label) + if err != nil { + log.Error("[Get Bounding Boxes] Couldn't get bounding boxes for image label: ", err.Error()) + raven.CaptureError(err, nil) + return boundingBoxes, err + } + + defer rows.Close() + + for rows.Next() { + var x0, y0, x1, y1 int + err = rows.Scan(&x0, &y0, &x1, &y1) + if err != nil { + log.Error("[Get Bounding Boxes] Couldn't scan bounding boxes for image label: ", err.Error()) + raven.CaptureError(err, nil) + return boundingBoxes, err + } + boundingBoxes = append(boundingBoxes, image.Rect(x0, y0, x1, y1)) + } + + return boundingBoxes, nil } diff --git a/src/database/image_collection.go b/src/database/image_collection.go index fb2bcc15..3d865ad3 100644 --- a/src/database/image_collection.go +++ b/src/database/image_collection.go @@ -1,20 +1,22 @@ package imagemonkeydb import ( - "database/sql" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgconn" commons "github.com/bbernhard/imagemonkey-core/commons" datastructures "github.com/bbernhard/imagemonkey-core/datastructures" "github.com/getsentry/raven-go" - "github.com/lib/pq" log "github.com/sirupsen/logrus" "fmt" "time" + "context" ) func (p *ImageMonkeyDatabase) GetImageCollections(apiUser datastructures.APIUser, apiBaseUrl string) ([]datastructures.ImageCollection, error) { imageCollections := []datastructures.ImageCollection{} - rows, err := p.db.Query(`SELECT u.name, u.description, COALESCE(q.num, 0), COALESCE(q1.image_key, ''), + rows, err := p.db.Query(context.TODO(), + `SELECT u.name, u.description, COALESCE(q.num, 0), COALESCE(q1.image_key, ''), COALESCE(q1.image_width, 0), COALESCE(q1.image_height, 0), COALESCE(q1.image_unlocked, false) FROM user_image_collection u JOIN account a ON a.id = u.account_id @@ -72,17 +74,18 @@ func (p *ImageMonkeyDatabase) GetImageCollections(apiUser datastructures.APIUser return imageCollections, nil } -func (p *ImageMonkeyDatabase) _addImageCollectionInTransaction(tx *sql.Tx, username string, name string, description string) error { - _, err := tx.Exec(`INSERT INTO user_image_collection(account_id, name, description) +func (p *ImageMonkeyDatabase) _addImageCollectionInTransaction(tx pgx.Tx, username string, name string, description string) error { + _, err := tx.Exec(context.TODO(), + `INSERT INTO user_image_collection(account_id, name, description) SELECT id, $2, $3 FROM account WHERE name = $1`, username, name, description) if err != nil { - if err, ok := err.(*pq.Error); ok { + if err, ok := err.(*pgconn.PgError); ok { if err.Code == "23505" { - tx.Rollback() + tx.Rollback(context.TODO()) return &DuplicateImageCollectionError{Description: "An Image Collection with that name already exists"} } } - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -90,7 +93,7 @@ func (p *ImageMonkeyDatabase) _addImageCollectionInTransaction(tx *sql.Tx, usern } func (p *ImageMonkeyDatabase) AddImageCollection(apiUser datastructures.APIUser, name string, description string) error { - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Error("[Image Collections] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -104,7 +107,7 @@ func (p *ImageMonkeyDatabase) AddImageCollection(apiUser datastructures.APIUser, return err } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Image Collections] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -114,7 +117,7 @@ func (p *ImageMonkeyDatabase) AddImageCollection(apiUser datastructures.APIUser, return nil } -func (p *ImageMonkeyDatabase) _addImageToImageCollectionInTransaction(tx *sql.Tx, username string, +func (p *ImageMonkeyDatabase) _addImageToImageCollectionInTransaction(tx pgx.Tx, username string, imageCollectionName string, imageId string, failIfAlreadyAssigned bool) error { currentUnixTimestamp := int64(time.Now().Unix()) @@ -132,18 +135,18 @@ func (p *ImageMonkeyDatabase) _addImageToImageCollectionInTransaction(tx *sql.Tx JOIN account a ON u.account_id = a.id WHERE u.name = $1 AND a.name = $2), (SELECT id FROM image WHERE key = $3), $4 %s`, q1) - _, err := tx.Exec(q, queryValues...) + _, err := tx.Exec(context.TODO(), q, queryValues...) if err != nil { - if err, ok := err.(*pq.Error); ok { + if err, ok := err.(*pgconn.PgError); ok { if err.Code == "23502" { - tx.Rollback() + tx.Rollback(context.TODO()) return &InvalidImageCollectionInputError{Description: "Invalid Image Collection Input"} } else if err.Code == "23505" { - tx.Rollback() + tx.Rollback(context.TODO()) return &DuplicateImageCollectionError{Description: "Image already assigned to Image Collection"} } } - tx.Rollback() + tx.Rollback(context.TODO()) return err } @@ -153,7 +156,7 @@ func (p *ImageMonkeyDatabase) _addImageToImageCollectionInTransaction(tx *sql.Tx func (p *ImageMonkeyDatabase) AddImageToImageCollection(apiUser datastructures.APIUser, imageCollectionName string, imageId string, failIfAlreadyAssigned bool) error { - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Error("[Add Image To Collection] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -167,7 +170,7 @@ func (p *ImageMonkeyDatabase) AddImageToImageCollection(apiUser datastructures.A return err } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Add Image to Collection] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) diff --git a/src/database/image_description.go b/src/database/image_description.go index c61b960a..5d171304 100644 --- a/src/database/image_description.go +++ b/src/database/image_description.go @@ -1,41 +1,41 @@ package imagemonkeydb import ( + "context" + "encoding/json" datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + languages "github.com/bbernhard/imagemonkey-core/languages" "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - "encoding/json" - "github.com/lib/pq" "github.com/gofrs/uuid" - languages "github.com/bbernhard/imagemonkey-core/languages" + log "github.com/sirupsen/logrus" + "github.com/jackc/pgtype" ) type UnlockImageDescriptionErrorType int const ( - UnlockImageDescriptionSuccess UnlockImageDescriptionErrorType = 1 << iota - UnlockImageDescriptionInternalError - UnlockImageDescriptionInvalidId + UnlockImageDescriptionSuccess UnlockImageDescriptionErrorType = 1 << iota + UnlockImageDescriptionInternalError + UnlockImageDescriptionInvalidId ) type LockImageDescriptionErrorType int const ( - LockImageDescriptionSuccess LockImageDescriptionErrorType = 1 << iota - LockImageDescriptionInternalError - LockImageDescriptionInvalidId + LockImageDescriptionSuccess LockImageDescriptionErrorType = 1 << iota + LockImageDescriptionInternalError + LockImageDescriptionInvalidId ) type AddImageDescriptionErrorType int const ( - AddImageDescriptionSuccess AddImageDescriptionErrorType = 1 << iota - AddImageDescriptionInternalError - AddImageDescriptionInvalidImageDescription - AddImageDescriptionInvalidLanguage + AddImageDescriptionSuccess AddImageDescriptionErrorType = 1 << iota + AddImageDescriptionInternalError + AddImageDescriptionInvalidImageDescription + AddImageDescriptionInvalidLanguage ) - func (p *ImageMonkeyDatabase) AddImageDescriptions(imageId string, descriptions []datastructures.ImageDescription) AddImageDescriptionErrorType { var imageDescriptions []string var langs []string @@ -51,90 +51,95 @@ func (p *ImageMonkeyDatabase) AddImageDescriptions(imageId string, descriptions langs = append(langs, val.Language) } - tx, err := p.db.Begin() - if err != nil { - log.Error("[Adding image description] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return AddImageDescriptionInternalError - } + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Error("[Adding image description] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return AddImageDescriptionInternalError + } - rows, err := tx.Query(`SELECT l.id FROM + rows, err := tx.Query(context.TODO(), + `SELECT l.id FROM ( SELECT lang, nr FROM unnest($1::text[]) WITH ORDINALITY As T (lang, nr) --ensure that result is correctly ordered after unnest ) q JOIN language l ON l.name = q.lang - ORDER BY nr`, pq.Array(langs)) - if err != nil { - tx.Rollback() - log.Error("[Adding image description] Couldn't get languages: ", err.Error()) - raven.CaptureError(err, nil) - return AddImageDescriptionInternalError - } - - defer rows.Close() - - var languageIds []int64 - for rows.Next() { - var languageId int64 - err = rows.Scan(&languageId) - if err != nil { - tx.Rollback() - log.Error("[Adding image description] Couldn't scan language: ", err.Error()) - raven.CaptureError(err, nil) - return AddImageDescriptionInternalError - } - - languageIds = append(languageIds, languageId) - } - - if len(languageIds) != len(imageDescriptions) { - tx.Rollback() - log.Error("[Adding image description] language ids mismatch: ", err.Error()) - raven.CaptureError(err, nil) - return AddImageDescriptionInternalError - } - - - _, err = tx.Exec(`INSERT INTO image_description(image_id, description, num_of_valid, num_of_invalid, state, processed_by, uuid, language_id) + ORDER BY nr`, langs) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Adding image description] Couldn't get languages: ", err.Error()) + raven.CaptureError(err, nil) + return AddImageDescriptionInternalError + } + + defer rows.Close() + + var languageIds []int64 + for rows.Next() { + var languageId int64 + err = rows.Scan(&languageId) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Adding image description] Couldn't scan language: ", err.Error()) + raven.CaptureError(err, nil) + return AddImageDescriptionInternalError + } + + languageIds = append(languageIds, languageId) + } + + if len(languageIds) != len(imageDescriptions) { + tx.Rollback(context.TODO()) + log.Error("[Adding image description] language ids mismatch: ", err.Error()) + raven.CaptureError(err, nil) + return AddImageDescriptionInternalError + } + + pgxLanguageIds := &pgtype.Int8Array{} + pgxLanguageIds.Set(languageIds) + + _, err = tx.Exec(context.TODO(), + `INSERT INTO image_description(image_id, description, num_of_valid, num_of_invalid, state, processed_by, uuid, language_id) SELECT (SELECT i.id FROM image i WHERE i.key = $1), - unnest($2::text[]), 0, 0, 'unknown', null, uuid_generate_v4(), unnest($3::integer[]) - ON CONFLICT(image_id, description) DO UPDATE SET num_of_valid = image_description.num_of_valid + 1`, - imageId, pq.Array(imageDescriptions), pq.Array(languageIds)) + unnest($2::text[]), 0, 0, 'unknown', null, uuid_generate_v4(), unnest($3::bigint[]) + ON CONFLICT(image_id, description) DO UPDATE SET num_of_valid = image_description.num_of_valid + 1`, + imageId, imageDescriptions, pgxLanguageIds) if err != nil { - tx.Rollback() - log.Error("[Adding image description] Couldn't add image description: ", err.Error()) - raven.CaptureError(err, nil) - return AddImageDescriptionInternalError + tx.Rollback(context.TODO()) + log.Error("[Adding image description] Couldn't add image description: ", err.Error()) + raven.CaptureError(err, nil) + return AddImageDescriptionInternalError } - err = tx.Commit() - if err != nil { - tx.Rollback() - log.Error("[Adding image description] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return AddImageDescriptionInternalError - } + err = tx.Commit(context.TODO()) + if err != nil { + tx.Rollback(context.TODO()) + log.Error("[Adding image description] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return AddImageDescriptionInternalError + } return AddImageDescriptionSuccess } -func (p *ImageMonkeyDatabase) UnlockImageDescription(apiUser datastructures.APIUser, imageId string, descriptionId string) (UnlockImageDescriptionErrorType) { - _, err := uuid.FromString(descriptionId) +func (p *ImageMonkeyDatabase) UnlockImageDescription(apiUser datastructures.APIUser, imageId string, descriptionId string) UnlockImageDescriptionErrorType { + _, err := uuid.FromString(descriptionId) if err != nil { return UnlockImageDescriptionInvalidId } - rows, err := p.db.Query(`UPDATE image_description AS d + rows, err := p.db.Query(context.TODO(), + `UPDATE image_description AS d SET state = 'unlocked', processed_by = (SELECT id FROM account WHERE name = $3) FROM image AS i WHERE i.id = d.image_id AND i.key = $1 AND uuid = $2 - RETURNING d.id`, - imageId, descriptionId, apiUser.Name) + RETURNING d.id`, + imageId, descriptionId, apiUser.Name) if err != nil { - log.Error("[Unlocking image description] Couldn't unlock image description: ", err.Error()) - raven.CaptureError(err, nil) - return UnlockImageDescriptionInternalError + log.Error("[Unlocking image description] Couldn't unlock image description: ", err.Error()) + raven.CaptureError(err, nil) + return UnlockImageDescriptionInternalError } defer rows.Close() @@ -146,23 +151,23 @@ func (p *ImageMonkeyDatabase) UnlockImageDescription(apiUser datastructures.APIU return UnlockImageDescriptionInvalidId } - -func (p *ImageMonkeyDatabase) LockImageDescription(apiUser datastructures.APIUser, imageId string, descriptionId string) (LockImageDescriptionErrorType) { - _, err := uuid.FromString(descriptionId) +func (p *ImageMonkeyDatabase) LockImageDescription(apiUser datastructures.APIUser, imageId string, descriptionId string) LockImageDescriptionErrorType { + _, err := uuid.FromString(descriptionId) if err != nil { return LockImageDescriptionInvalidId } - rows, err := p.db.Query(`UPDATE image_description AS d + rows, err := p.db.Query(context.TODO(), + `UPDATE image_description AS d SET state = 'locked', processed_by = (SELECT id FROM account WHERE name = $3) FROM image AS i WHERE i.id = d.image_id AND i.key = $1 AND uuid = $2 - RETURNING d.id`, - imageId, descriptionId, apiUser.Name) + RETURNING d.id`, + imageId, descriptionId, apiUser.Name) if err != nil { - log.Error("[Unlocking image description] Couldn't lock image description: ", err.Error()) - raven.CaptureError(err, nil) - return LockImageDescriptionInternalError + log.Error("[Unlocking image description] Couldn't lock image description: ", err.Error()) + raven.CaptureError(err, nil) + return LockImageDescriptionInternalError } defer rows.Close() @@ -175,9 +180,10 @@ func (p *ImageMonkeyDatabase) LockImageDescription(apiUser datastructures.APIUse } func (p *ImageMonkeyDatabase) GetUnprocessedImageDescriptions() ([]datastructures.DescriptionsPerImage, error) { - var res []datastructures.DescriptionsPerImage + var res []datastructures.DescriptionsPerImage - rows, err := p.db.Query(`SELECT i.key, json_agg(json_build_object('text', dsc.description, 'uuid', dsc.uuid, 'language', l.fullname)) + rows, err := p.db.Query(context.TODO(), + `SELECT i.key, json_agg(json_build_object('text', dsc.description, 'uuid', dsc.uuid, 'language', l.fullname)) FROM image_description dsc JOIN language l ON l.id = dsc.language_id JOIN image i ON i.id = dsc.image_id @@ -185,47 +191,44 @@ func (p *ImageMonkeyDatabase) GetUnprocessedImageDescriptions() ([]datastructure GROUP BY i.key`) if err != nil { - log.Error("[Get Locked image descriptions] Couldn't get locked image descriptions: ", err.Error()) - raven.CaptureError(err, nil) - return res, err + log.Error("[Get Locked image descriptions] Couldn't get locked image descriptions: ", err.Error()) + raven.CaptureError(err, nil) + return res, err } defer rows.Close() for rows.Next() { - var descriptionsPerImage datastructures.DescriptionsPerImage + var descriptionsPerImage datastructures.DescriptionsPerImage var descriptions []byte err = rows.Scan(&descriptionsPerImage.Image.Id, &descriptions) if err != nil { - log.Error("[Get Locked image descriptions] Couldn't get locked image descriptions: ", err.Error()) - raven.CaptureError(err, nil) - return res, err + log.Error("[Get Locked image descriptions] Couldn't get locked image descriptions: ", err.Error()) + raven.CaptureError(err, nil) + return res, err } err := json.Unmarshal(descriptions, &descriptionsPerImage.Image.Descriptions) - if err != nil { - log.Error("[Get Locked image descriptions] Couldn't unmarshal descriptions: ", err.Error()) - raven.CaptureError(err, nil) - return res, err - } + if err != nil { + log.Error("[Get Locked image descriptions] Couldn't unmarshal descriptions: ", err.Error()) + raven.CaptureError(err, nil) + return res, err + } - res = append(res, descriptionsPerImage) + res = append(res, descriptionsPerImage) } return res, nil } - func (p *ImageMonkeyDatabase) GetNumOfUnprocessedDescriptions() (int, error) { var num int num = 0 - err := p.db.QueryRow(`SELECT count(*) FROM image_description WHERE state = 'unknown'`).Scan(&num) + err := p.db.QueryRow(context.TODO(), `SELECT count(*) FROM image_description WHERE state = 'unknown'`).Scan(&num) if err != nil { return num, err } return num, nil } - - diff --git a/src/database/image_label.go b/src/database/image_label.go index 60140ac8..4be64dea 100644 --- a/src/database/image_label.go +++ b/src/database/image_label.go @@ -1,14 +1,15 @@ package imagemonkeydb import ( - "database/sql" + "context" "encoding/json" "fmt" commons "github.com/bbernhard/imagemonkey-core/commons" datastructures "github.com/bbernhard/imagemonkey-core/datastructures" parser "github.com/bbernhard/imagemonkey-core/parser/v2" "github.com/getsentry/raven-go" - "github.com/lib/pq" + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4" log "github.com/sirupsen/logrus" ) @@ -17,7 +18,7 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i var labelMeEntries []datastructures.LabelMeEntry image.Provider = "donation" - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[Get Image to Label] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -43,7 +44,7 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i )` } - var unlabeledRows *sql.Rows + var unlabeledRows pgx.Rows if imageId == "" { q := fmt.Sprintf(`WITH imgs AS ( SELECT i.key, i.unlocked, i.width, i.height @@ -63,13 +64,13 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i ) LIMIT 1`, includeOwnImageDonations) if username == "" { - unlabeledRows, err = tx.Query(q) + unlabeledRows, err = tx.Query(context.TODO(), q) } else { - unlabeledRows, err = tx.Query(q, username) + unlabeledRows, err = tx.Query(context.TODO(), q, username) } if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) raven.CaptureError(err, nil) log.Debug("[Get Image to Label] Couldn't get unlabeled image: ", err.Error()) return image, err @@ -145,23 +146,23 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i -- otherwise, the below logic won't work correctly `, q2, q1) - var rows *sql.Rows + var rows pgx.Rows if imageId == "" { if username == "" { - rows, err = tx.Query(q) + rows, err = tx.Query(context.TODO(), q) } else { - rows, err = tx.Query(q, username) + rows, err = tx.Query(context.TODO(), q, username) } } else { if username == "" { - rows, err = tx.Query(q, imageId) + rows, err = tx.Query(context.TODO(), q, imageId) } else { - rows, err = tx.Query(q, username, imageId) + rows, err = tx.Query(context.TODO(), q, username, imageId) } } if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) raven.CaptureError(err, nil) log.Debug("[Get Image to Label] Couldn't get image: ", err.Error()) return image, err @@ -187,7 +188,7 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i &imageDescriptions) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) raven.CaptureError(err, nil) log.Debug("[Get Image to Label] Couldn't scan labeled row: ", err.Error()) return image, err @@ -255,7 +256,7 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i } else { err = unlabeledRows.Scan(&image.Id, &image.Unlocked, &image.Width, &image.Height) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) raven.CaptureError(err, nil) log.Debug("[Get Image to Label] Couldn't scan row: ", err.Error()) return image, err @@ -265,7 +266,7 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i image.AllLabels = labelMeEntries - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { raven.CaptureError(err, nil) log.Debug("[Get Image to Label] Couldn't commit changes: ", err.Error()) @@ -277,7 +278,7 @@ func (p *ImageMonkeyDatabase) GetImageToLabel(imageId string, username string, i func (p *ImageMonkeyDatabase) AddLabelsToImage(apiUser datastructures.APIUser, labelMap map[string]datastructures.LabelMapEntry, metalabels *commons.MetaLabels, imageId string, labels []datastructures.LabelMeEntry) error { - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Error("[Adding image labels] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -300,7 +301,7 @@ func (p *ImageMonkeyDatabase) AddLabelsToImage(apiUser datastructures.APIUser, l } } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Adding image labels] Couldn't commit changes: ", err.Error()) raven.CaptureError(err, nil) @@ -309,7 +310,7 @@ func (p *ImageMonkeyDatabase) AddLabelsToImage(apiUser datastructures.APIUser, l return err } -func _addLabelsAndLabelSuggestionsToImageInTransaction(tx *sql.Tx, apiUser datastructures.APIUser, labelMap map[string]datastructures.LabelMapEntry, +func _addLabelsAndLabelSuggestionsToImageInTransaction(tx pgx.Tx, apiUser datastructures.APIUser, labelMap map[string]datastructures.LabelMapEntry, metalabels *commons.MetaLabels, imageId string, labels []datastructures.LabelMeEntry, numOfValid int, numOfNotAnnotatable int) ([]int64, error) { var insertedValidationIds []int64 @@ -323,7 +324,7 @@ func _addLabelsAndLabelSuggestionsToImageInTransaction(tx *sql.Tx, apiUser datas return insertedValidationIds, err //tx already rolled back in case of error, so we can just return here } } else { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("you need to be authenticated") return insertedValidationIds, &AuthenticationRequiredError{Description: "you need to be authenticated to perform this action"} } @@ -342,13 +343,13 @@ func _addLabelsAndLabelSuggestionsToImageInTransaction(tx *sql.Tx, apiUser datas return insertedValidationIds, nil } -func _addLabelSuggestionToImage(apiUser datastructures.APIUser, label string, imageId string, annotatable bool, tx *sql.Tx) error { +func _addLabelSuggestionToImage(apiUser datastructures.APIUser, label string, imageId string, annotatable bool, tx pgx.Tx) error { var labelSuggestionId int64 labelSuggestionId = -1 - rows, err := tx.Query("SELECT id FROM label_suggestion WHERE name = $1", label) + rows, err := tx.Query(context.TODO(), "SELECT id FROM label_suggestion WHERE name = $1", label) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding suggestion label] Couldn't get label: ", err.Error()) raven.CaptureError(err, nil) return err @@ -357,11 +358,12 @@ func _addLabelSuggestionToImage(apiUser datastructures.APIUser, label string, im if !rows.Next() { //label does not exist yet, insert it rows.Close() - err := tx.QueryRow(`INSERT INTO label_suggestion(name, proposed_by, uuid) + err := tx.QueryRow(context.TODO(), + `INSERT INTO label_suggestion(name, proposed_by, uuid) SELECT $1, id, uuid_generate_v4() FROM account a WHERE a.name = $2 ON CONFLICT (name) DO NOTHING RETURNING id`, label, apiUser.Name).Scan(&labelSuggestionId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding suggestion label] Couldn't add label: ", err.Error()) raven.CaptureError(err, nil) return err @@ -370,19 +372,20 @@ func _addLabelSuggestionToImage(apiUser datastructures.APIUser, label string, im err = rows.Scan(&labelSuggestionId) rows.Close() if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding suggestion label] Couldn't scan label: ", err.Error()) raven.CaptureError(err, nil) return err } } - _, err = tx.Exec(`INSERT INTO image_label_suggestion (fingerprint_of_last_modification, image_id, label_suggestion_id, annotatable, sys_period, uuid) + _, err = tx.Exec(context.TODO(), + `INSERT INTO image_label_suggestion (fingerprint_of_last_modification, image_id, label_suggestion_id, annotatable, sys_period, uuid) SELECT $1, id, $3, $4, '["now()",]'::tstzrange, uuid_generate_v4() FROM image WHERE key = $2 ON CONFLICT(image_id, label_suggestion_id) DO NOTHING`, apiUser.ClientFingerprint, imageId, labelSuggestionId, annotatable) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding image label suggestion] Couldn't add image label suggestion: ", err.Error()) raven.CaptureError(err, nil) return err @@ -392,12 +395,12 @@ func _addLabelSuggestionToImage(apiUser datastructures.APIUser, label string, im } func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, labels []datastructures.LabelMeEntry, - numOfValid int, numOfNotAnnotatable int, tx *sql.Tx) ([]int64, error) { + numOfValid int, numOfNotAnnotatable int, tx pgx.Tx) ([]int64, error) { var insertedIds []int64 for _, item := range labels { - rows, err := tx.Query(`SELECT i.id FROM image i WHERE i.key = $1`, imageId) + rows, err := tx.Query(context.TODO(), `SELECT i.id FROM image i WHERE i.key = $1`, imageId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding image labels] Couldn't get image ", err.Error()) raven.CaptureError(err, nil) return insertedIds, err @@ -409,7 +412,7 @@ func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, lab if rows.Next() { err = rows.Scan(&imageId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding image labels] Couldn't scan image image entry: ", err.Error()) raven.CaptureError(err, nil) return insertedIds, err @@ -420,16 +423,17 @@ func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, lab //add sublabels if len(item.Sublabels) > 0 { - rows, err = tx.Query(`INSERT INTO image_validation(image_id, num_of_valid, num_of_invalid, fingerprint_of_last_modification, label_id, uuid, num_of_not_annotatable) + rows, err = tx.Query(context.TODO(), + `INSERT INTO image_validation(image_id, num_of_valid, num_of_invalid, fingerprint_of_last_modification, label_id, uuid, num_of_not_annotatable) SELECT $1, $2, $3, $4, l.id, uuid_generate_v4(), $7 FROM label l LEFT JOIN label cl ON cl.id = l.parent_id WHERE (cl.name = $5 AND l.name = ANY($6)) ON CONFLICT DO NOTHING RETURNING id`, - imageId, numOfValid, 0, clientFingerprint, item.Label, pq.Array(sublabelsToStringlist(item.Sublabels)), numOfNotAnnotatable) + imageId, numOfValid, 0, clientFingerprint, item.Label, sublabelsToStringlist(item.Sublabels), numOfNotAnnotatable) if err != nil { - if err != sql.ErrNoRows { //handle no rows gracefully (can happen if label already exists) - pqErr := err.(*pq.Error) - if pqErr.Code.Name() != "unique_violation" { - tx.Rollback() + if err != pgx.ErrNoRows { //handle no rows gracefully (can happen if label already exists) + pgxErr := err.(*pgconn.PgError) + if pgxErr.Code != "unique_violation" { + tx.Rollback(context.TODO()) log.Debug("[Adding image labels] Couldn't insert image validation entries for sublabels: ", err.Error()) raven.CaptureError(err, nil) return insertedIds, err @@ -441,7 +445,7 @@ func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, lab err = rows.Scan(&insertedSublabelId) if err != nil { rows.Close() - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Adding image labels] Couldn't scan sublabels: ", err.Error()) raven.CaptureError(err, nil) return insertedIds, err @@ -454,7 +458,8 @@ func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, lab //add base label var insertedLabelId int64 - err = tx.QueryRow(`INSERT INTO image_validation(image_id, num_of_valid, num_of_invalid, fingerprint_of_last_modification, + err = tx.QueryRow(context.TODO(), + `INSERT INTO image_validation(image_id, num_of_valid, num_of_invalid, fingerprint_of_last_modification, label_id, uuid, num_of_not_annotatable) SELECT $1, $2, $3, $4, id, uuid_generate_v4(), $6 from label l WHERE id NOT IN ( @@ -464,10 +469,10 @@ func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, lab RETURNING id`, imageId, numOfValid, 0, clientFingerprint, item.Label, numOfNotAnnotatable).Scan(&insertedLabelId) if err != nil { - if err != sql.ErrNoRows { //handle no rows gracefully (can happen if label already exists) - pqErr := err.(*pq.Error) - if pqErr.Code.Name() != "unique_violation" { - tx.Rollback() + if err != pgx.ErrNoRows { //handle no rows gracefully (can happen if label already exists) + pgxErr := err.(*pgconn.PgError) + if pgxErr.Code != "unique_violation" { + tx.Rollback(context.TODO()) log.Debug("[Adding image labels] Couldn't insert image validation entry for label: ", err.Error()) raven.CaptureError(err, nil) return insertedIds, err @@ -484,7 +489,7 @@ func AddLabelsToImageInTransaction(clientFingerprint string, imageId string, lab func (p *ImageMonkeyDatabase) GetAllImageLabels() ([]string, error) { var labels []string - rows, err := p.db.Query(`SELECT l.name FROM label l`) + rows, err := p.db.Query(context.TODO(), `SELECT l.name FROM label l`) if err != nil { log.Debug("[Getting all image labels] Couldn't get image labels: ", err.Error()) raven.CaptureError(err, nil) @@ -641,12 +646,12 @@ func (p *ImageMonkeyDatabase) GetImagesLabels(apiUser datastructures.APIUser, pa %s`, includeOwnImageDonations, includeOwnImageDonations, q2, parseResult.Query, shuffleStr) - var rows *sql.Rows + var rows pgx.Rows if apiUser.Name != "" { parseResult.QueryValues = append(parseResult.QueryValues, apiUser.Name) } - rows, err := p.db.Query(q, parseResult.QueryValues...) + rows, err := p.db.Query(context.TODO(), q, parseResult.QueryValues...) if err != nil { log.Debug("[Get Image Labels] Couldn't get image labels: ", err.Error()) raven.CaptureError(err, nil) @@ -691,7 +696,8 @@ func (p *ImageMonkeyDatabase) GetImagesLabels(apiUser datastructures.APIUser, pa func (p *ImageMonkeyDatabase) GetTrendingLabels() ([]datastructures.TrendingLabel, error) { trendingLabels := []datastructures.TrendingLabel{} - rows, err := p.db.Query(`SELECT s.name, t.github_issue_id, t.closed, COALESCE(tb.state::text, ''), + rows, err := p.db.Query(context.TODO(), + `SELECT s.name, t.github_issue_id, t.closed, COALESCE(tb.state::text, ''), COALESCE(tb.job_url, ''), COALESCE(tb.label_type::text, ''), COALESCE(tb.branch_name, ''), COALESCE(tb.description, ''), COALESCE(tb.plural, ''), COALESCE(tb.rename_to, '') @@ -731,14 +737,15 @@ func (p *ImageMonkeyDatabase) AcceptTrendingLabel(name string, labelType string, status = "accepted" } - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Error("[Accept Trending Label] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) return err } - rows, err := tx.Query(`INSERT INTO trending_label_bot_task(trending_label_suggestion_id, state, try, label_type, description, plural, rename_to) + rows, err := tx.Query(context.TODO(), + `INSERT INTO trending_label_bot_task(trending_label_suggestion_id, state, try, label_type, description, plural, rename_to) SELECT l.id, $1, 1, $3, $4, $5, $6 FROM trending_label_suggestion l JOIN label_suggestion s ON s.id = l.label_suggestion_id @@ -747,7 +754,7 @@ func (p *ImageMonkeyDatabase) AcceptTrendingLabel(name string, labelType string, RETURNING id`, status, name, labelType, labelDescription, labelPlural, labelRenameTo) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Accept Trending Label] Couldn't accept trending label: ", err.Error()) raven.CaptureError(err, nil) return err @@ -763,7 +770,8 @@ func (p *ImageMonkeyDatabase) AcceptTrendingLabel(name string, labelType string, rows.Close() if !success { //already exists an entry - rows1, err := tx.Query(`UPDATE trending_label_bot_task + rows1, err := tx.Query(context.TODO(), + `UPDATE trending_label_bot_task SET state = CASE WHEN state = 'waiting for moderator approval' THEN $2 WHEN state = 'build-failed' THEN 'retry' @@ -780,7 +788,7 @@ func (p *ImageMonkeyDatabase) AcceptTrendingLabel(name string, labelType string, WHERE q.lid = trending_label_suggestion_id RETURNING id`, name, status) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Accept Trending Label] Couldn't accept trending label: ", err.Error()) raven.CaptureError(err, nil) return err @@ -795,7 +803,7 @@ func (p *ImageMonkeyDatabase) AcceptTrendingLabel(name string, labelType string, rows1.Close() } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Accept Trending Label] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) diff --git a/src/database/image_validation.go b/src/database/image_validation.go index fd158e57..eb6ab2cc 100644 --- a/src/database/image_validation.go +++ b/src/database/image_validation.go @@ -1,148 +1,148 @@ package imagemonkeydb import ( - "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" - parser "github.com/bbernhard/imagemonkey-core/parser/v2" - commons "github.com/bbernhard/imagemonkey-core/commons" - "database/sql" - "github.com/lib/pq" - "errors" - "fmt" + "context" + "errors" + "fmt" + commons "github.com/bbernhard/imagemonkey-core/commons" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + parser "github.com/bbernhard/imagemonkey-core/parser/v2" + "github.com/getsentry/raven-go" + "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" ) -func _addImageValidationSources(imageSourceId int64, imageValidationIds []int64, tx *sql.Tx) error { - for _, id := range imageValidationIds { - _, err := tx.Exec("INSERT INTO image_validation_source(image_source_id, image_validation_id) VALUES($1, $2)", imageSourceId, id) - if err != nil { - tx.Rollback() - log.Debug("[Add image validation source] Couldn't add image validation source: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - } - - return nil +func _addImageValidationSources(imageSourceId int64, imageValidationIds []int64, tx pgx.Tx) error { + for _, id := range imageValidationIds { + _, err := tx.Exec(context.TODO(), + "INSERT INTO image_validation_source(image_source_id, image_validation_id) VALUES($1, $2)", imageSourceId, id) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Add image validation source] Couldn't add image validation source: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + } + + return nil } -func (p *ImageMonkeyDatabase) ValidateImages(apiUser datastructures.APIUser, - imageValidationBatch datastructures.ImageValidationBatch, moderatorAction bool) error { - var validEntries []string - var invalidEntries []string - var updatedRowIds []int64 - - stepSize := 1 - if moderatorAction { - stepSize = 5 - } - - validations := imageValidationBatch.Validations - - for i := range validations { - if validations[i].Valid == "yes" { - validEntries = append(validEntries, validations[i].Uuid) - } else if validations[i].Valid == "no" { - invalidEntries = append(invalidEntries, validations[i].Uuid) - } - } - - - tx, err := p.db.Begin() - if err != nil { - log.Debug("[Batch Validating donated photos] Couldn't begin transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - if len(invalidEntries) > 0 { - rows, err := tx.Query(`UPDATE image_validation AS v +func (p *ImageMonkeyDatabase) ValidateImages(apiUser datastructures.APIUser, + imageValidationBatch datastructures.ImageValidationBatch, moderatorAction bool) error { + var validEntries []string + var invalidEntries []string + var updatedRowIds []int64 + + stepSize := 1 + if moderatorAction { + stepSize = 5 + } + + validations := imageValidationBatch.Validations + + for i := range validations { + if validations[i].Valid == "yes" { + validEntries = append(validEntries, validations[i].Uuid) + } else if validations[i].Valid == "no" { + invalidEntries = append(invalidEntries, validations[i].Uuid) + } + } + + tx, err := p.db.Begin(context.TODO()) + if err != nil { + log.Debug("[Batch Validating donated photos] Couldn't begin transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + if len(invalidEntries) > 0 { + rows, err := tx.Query(context.TODO(), + `UPDATE image_validation AS v SET num_of_invalid = num_of_invalid + $3, fingerprint_of_last_modification = $1 - WHERE uuid = ANY($2) RETURNING id`, - apiUser.ClientFingerprint, pq.Array(invalidEntries), stepSize) - if err != nil { - tx.Rollback() - log.Debug("[Batch Validating donated photos] Couldn't increase num_of_invalid: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - defer rows.Close() - - for rows.Next() { - var updatedRowId int64 - err = rows.Scan(&updatedRowId) - if err != nil { - tx.Rollback() - log.Debug("[Batch Validating donated photos] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - updatedRowIds = append(updatedRowIds, updatedRowId) - } - } - - if len(validEntries) > 0 { - rows1, err := tx.Query(`UPDATE image_validation AS v + WHERE uuid = ANY($2) RETURNING id`, + apiUser.ClientFingerprint, invalidEntries, stepSize) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Batch Validating donated photos] Couldn't increase num_of_invalid: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + defer rows.Close() + + for rows.Next() { + var updatedRowId int64 + err = rows.Scan(&updatedRowId) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Batch Validating donated photos] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + updatedRowIds = append(updatedRowIds, updatedRowId) + } + } + + if len(validEntries) > 0 { + rows1, err := tx.Query(context.TODO(), + `UPDATE image_validation AS v SET num_of_valid = num_of_valid + $3, fingerprint_of_last_modification = $1 - WHERE uuid = ANY($2) RETURNING id`, - apiUser.ClientFingerprint, pq.Array(validEntries), stepSize) - if err != nil { - tx.Rollback() - log.Debug("[Batch Validating donated photos] Couldn't increase num_of_valid: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - defer rows1.Close() - - for rows1.Next() { - var updatedRowId int64 - err = rows1.Scan(&updatedRowId) - if err != nil { - tx.Rollback() - log.Debug("[Batch Validating donated photos] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - updatedRowIds = append(updatedRowIds, updatedRowId) - } - } - - - if apiUser.Name != "" { - if len(updatedRowIds) == 0 { - tx.Rollback() - err := errors.New("nothing updated") - log.Debug("[Batch Validating donated photos] ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - - _, err = tx.Exec(`INSERT INTO user_image_validation(image_validation_id, account_id, timestamp) - SELECT unnest($1::integer[]), a.id, CURRENT_TIMESTAMP FROM account a WHERE a.name = $2`, pq.Array(updatedRowIds), apiUser.Name) - if err != nil { - tx.Rollback() - log.Debug("[Batch Validating donated photos] Couldn't add user image validation entry: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - } - - err = tx.Commit() - if err != nil { - log.Debug("[Batch Validating donated photos] Couldn't commit transaction: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - - - return nil -} + WHERE uuid = ANY($2) RETURNING id`, + apiUser.ClientFingerprint, validEntries, stepSize) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Batch Validating donated photos] Couldn't increase num_of_valid: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + defer rows1.Close() + + for rows1.Next() { + var updatedRowId int64 + err = rows1.Scan(&updatedRowId) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Batch Validating donated photos] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + updatedRowIds = append(updatedRowIds, updatedRowId) + } + } + if apiUser.Name != "" { + if len(updatedRowIds) == 0 { + tx.Rollback(context.TODO()) + err := errors.New("nothing updated") + log.Debug("[Batch Validating donated photos] ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + _, err = tx.Exec(context.TODO(), + `INSERT INTO user_image_validation(image_validation_id, account_id, timestamp) + SELECT unnest($1::bigint[]), a.id, CURRENT_TIMESTAMP FROM account a WHERE a.name = $2`, + updatedRowIds, apiUser.Name) + if err != nil { + tx.Rollback(context.TODO()) + log.Debug("[Batch Validating donated photos] Couldn't add user image validation entry: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + } + + err = tx.Commit(context.TODO()) + if err != nil { + log.Debug("[Batch Validating donated photos] Couldn't commit transaction: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + + return nil +} func (p *ImageMonkeyDatabase) GetImageToValidate(validationId string, username string) (datastructures.ValidationImage, error) { var image datastructures.ValidationImage @@ -151,11 +151,11 @@ func (p *ImageMonkeyDatabase) GetImageToValidate(validationId string, username s image.Label = "" image.Provider = "donation" - var queryParams []interface{} + var queryParams []interface{} - includeOwnImageDonations := "" - if username != "" { - includeOwnImageDonations = fmt.Sprintf(`OR ( + includeOwnImageDonations := "" + if username != "" { + includeOwnImageDonations = fmt.Sprintf(`OR ( EXISTS ( SELECT 1 @@ -169,58 +169,58 @@ func (p *ImageMonkeyDatabase) GetImageToValidate(validationId string, username s FROM image_quarantine q WHERE q.image_id = i.id ) - )`, len(queryParams) + 1) - queryParams = append(queryParams, username) - } - - orderRandomly := "ORDER BY RANDOM()" - - //either select a specific image with a given image id or try to select - //an image that's not already validated (as they have preference). - validationIdStr := "(v.num_of_valid = 0) AND (v.num_of_invalid = 0)" - if validationId != "" { - orderRandomly = "" - validationIdStr = fmt.Sprintf("v.uuid::text = $%d", len(queryParams) +1) - queryParams = append(queryParams, validationId) - } - - q := fmt.Sprintf(`SELECT i.key, l.name, COALESCE(pl.name, ''), v.num_of_valid, v.num_of_invalid, v.uuid, i.unlocked + )`, len(queryParams)+1) + queryParams = append(queryParams, username) + } + + orderRandomly := "ORDER BY RANDOM()" + + //either select a specific image with a given image id or try to select + //an image that's not already validated (as they have preference). + validationIdStr := "(v.num_of_valid = 0) AND (v.num_of_invalid = 0)" + if validationId != "" { + orderRandomly = "" + validationIdStr = fmt.Sprintf("v.uuid::text = $%d", len(queryParams)+1) + queryParams = append(queryParams, validationId) + } + + q := fmt.Sprintf(`SELECT i.key, l.name, COALESCE(pl.name, ''), v.num_of_valid, v.num_of_invalid, v.uuid, i.unlocked FROM image i JOIN image_provider p ON i.image_provider_id = p.id JOIN image_validation v ON v.image_id = i.id JOIN label l ON v.label_id = l.id LEFT JOIN label pl ON l.parent_id = pl.id WHERE ((i.unlocked = true %s) AND (p.name = 'donation') - AND %s) %s LIMIT 1`,includeOwnImageDonations, validationIdStr, orderRandomly) + AND %s) %s LIMIT 1`, includeOwnImageDonations, validationIdStr, orderRandomly) - var rows *sql.Rows - var err error + var rows pgx.Rows + var err error - rows, err = p.db.Query(q, queryParams...) + rows, err = p.db.Query(context.TODO(), q, queryParams...) - if err != nil { + if err != nil { log.Debug("[Fetch image] Couldn't fetch random image: ", err.Error()) raven.CaptureError(err, nil) return image, err } - defer rows.Close() - - var label1 string - var label2 string + defer rows.Close() + + var label1 string + var label2 string if !rows.Next() { - //if we provided a validation id, but we get no result, its an error. So return here - if validationId != "" { - return image, nil - } + //if we provided a validation id, but we get no result, its an error. So return here + if validationId != "" { + return image, nil + } - queryParams = nil - if username != "" { - queryParams = append(queryParams, username) - } + queryParams = nil + if username != "" { + queryParams = append(queryParams, username) + } - var otherRows *sql.Rows + var otherRows pgx.Rows - q1 := fmt.Sprintf(`SELECT i.key, l.name, COALESCE(pl.name, ''), v.num_of_valid, v.num_of_invalid, v.uuid, i.unlocked + q1 := fmt.Sprintf(`SELECT i.key, l.name, COALESCE(pl.name, ''), v.num_of_valid, v.num_of_invalid, v.uuid, i.unlocked FROM image i JOIN image_provider p ON i.image_provider_id = p.id JOIN image_validation v ON v.image_id = i.id @@ -236,83 +236,85 @@ func (p *ImageMonkeyDatabase) GetImageToValidate(validationId string, username s ) ) LIMIT 1`, includeOwnImageDonations, includeOwnImageDonations) - otherRows, err := p.db.Query(q1, queryParams...) - - if err != nil { - log.Debug("[Fetch random image] Couldn't fetch random image: ", err.Error()) - raven.CaptureError(err, nil) - return image, err - } - - defer otherRows.Close() - - if otherRows.Next() { - err = otherRows.Scan(&image.Id, &label1, &label2, &image.Validation.NumOfValid, - &image.Validation.NumOfInvalid, &image.Validation.Id, &image.Unlocked) - if err != nil { - log.Debug("[Fetch random image] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return image, err - } - } - } else{ - err = rows.Scan(&image.Id, &label1, &label2, &image.Validation.NumOfValid, - &image.Validation.NumOfInvalid, &image.Validation.Id, &image.Unlocked) - if err != nil { - log.Debug("[Fetch random image] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return image, err - } - } - - if label2 == "" { - image.Label = label1 - image.Sublabel = "" - } else { - image.Label = label2 - image.Sublabel = label1 - } + otherRows, err := p.db.Query(context.TODO(), q1, queryParams...) + + if err != nil { + log.Debug("[Fetch random image] Couldn't fetch random image: ", err.Error()) + raven.CaptureError(err, nil) + return image, err + } + + defer otherRows.Close() + + if otherRows.Next() { + err = otherRows.Scan(&image.Id, &label1, &label2, &image.Validation.NumOfValid, + &image.Validation.NumOfInvalid, &image.Validation.Id, &image.Unlocked) + if err != nil { + log.Debug("[Fetch random image] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return image, err + } + } + } else { + err = rows.Scan(&image.Id, &label1, &label2, &image.Validation.NumOfValid, + &image.Validation.NumOfInvalid, &image.Validation.Id, &image.Unlocked) + if err != nil { + log.Debug("[Fetch random image] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return image, err + } + } + + if label2 == "" { + image.Label = label1 + image.Sublabel = "" + } else { + image.Label = label2 + image.Sublabel = label1 + } return image, nil } func (p *ImageMonkeyDatabase) BlacklistForAnnotation(validationId string, apiUser datastructures.APIUser) error { - _, err := p.db.Exec(`INSERT INTO user_annotation_blacklist(image_validation_id, account_id) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO user_annotation_blacklist(image_validation_id, account_id) SELECT v.id, (SELECT a.id FROM account a WHERE a.name = $1) as account_id FROM image_validation v WHERE v.uuid = $2 ON CONFLICT DO NOTHING`, apiUser.Name, validationId) - if err != nil { - log.Debug("[Blacklist Annotation] Couldn't blacklist annotation: ", err.Error()) - raven.CaptureError(err, nil) - return err - } - return nil + if err != nil { + log.Debug("[Blacklist Annotation] Couldn't blacklist annotation: ", err.Error()) + raven.CaptureError(err, nil) + return err + } + return nil } func (p *ImageMonkeyDatabase) MarkValidationAsNotAnnotatable(validationId string) error { - _, err := p.db.Exec(`UPDATE image_validation SET num_of_not_annotatable = num_of_not_annotatable + 1 + _, err := p.db.Exec(context.TODO(), + `UPDATE image_validation SET num_of_not_annotatable = num_of_not_annotatable + 1 WHERE uuid = $1`, validationId) - if err != nil { - log.Debug("[Mark Validation as not annotatable] Couldn't mark validation as not-annotatable: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + if err != nil { + log.Debug("[Mark Validation as not annotatable] Couldn't mark validation as not-annotatable: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - return nil + return nil } -func (p *ImageMonkeyDatabase) GetImagesForValidation(apiUser datastructures.APIUser, parseResult parser.ParseResult, - orderRandomly bool, apiBaseUrl string) ([]datastructures.Validation, error) { - validations := []datastructures.Validation{} +func (p *ImageMonkeyDatabase) GetImagesForValidation(apiUser datastructures.APIUser, parseResult parser.ParseResult, + orderRandomly bool, apiBaseUrl string) ([]datastructures.Validation, error) { + validations := []datastructures.Validation{} - q1 := "" - if orderRandomly { - q1 = " ORDER BY RANDOM()" - } + q1 := "" + if orderRandomly { + q1 = " ORDER BY RANDOM()" + } - includeOwnImageDonations := "" - if apiUser.Name != "" { - includeOwnImageDonations = fmt.Sprintf(`OR ( + includeOwnImageDonations := "" + if apiUser.Name != "" { + includeOwnImageDonations = fmt.Sprintf(`OR ( EXISTS ( SELECT 1 @@ -326,10 +328,10 @@ func (p *ImageMonkeyDatabase) GetImagesForValidation(apiUser datastructures.APIU FROM image_quarantine q WHERE q.image_id = i.id ) - )`, len(parseResult.QueryValues) + 1) - } + )`, len(parseResult.QueryValues)+1) + } - q := fmt.Sprintf(`WITH + q := fmt.Sprintf(`WITH image_productive_labels AS ( SELECT i.id as image_id, i.key as image_key, i.width as image_width, i.height as image_height, i.unlocked as image_unlocked, array_agg(accessor)::text[] as accessors, COALESCE(c.annotated_percentage, 0) as annotated_percentage @@ -348,48 +350,48 @@ func (p *ImageMonkeyDatabase) GetImagesForValidation(apiUser datastructures.APIU WHERE (%s) AND %s %s`, includeOwnImageDonations, parseResult.Query, parseResult.Subquery, q1) - var rows *sql.Rows - var err error + var rows pgx.Rows + var err error - if apiUser.Name != "" { - parseResult.QueryValues = append(parseResult.QueryValues, apiUser.Name) - } + if apiUser.Name != "" { + parseResult.QueryValues = append(parseResult.QueryValues, apiUser.Name) + } + + rows, err = p.db.Query(context.TODO(), q, parseResult.QueryValues...) - rows, err = p.db.Query(q, parseResult.QueryValues...) - - if err != nil { - log.Error("[Get Validations] Couldn't get validations: ", err.Error()) - raven.CaptureError(err, nil) - return validations, err - } + if err != nil { + log.Error("[Get Validations] Couldn't get validations: ", err.Error()) + raven.CaptureError(err, nil) + return validations, err + } - defer rows.Close() + defer rows.Close() - for rows.Next() { - var validation datastructures.Validation - err = rows.Scan(&validation.Id, &validation.NumOfYes, &validation.NumOfNo, - &validation.Label.Name, &validation.Image.Id, &validation.Image.Width, - &validation.Image.Height, &validation.Image.Unlocked) - if err != nil { - log.Debug("[Get Validations] Couldn't scan rows: ", err.Error()) - raven.CaptureError(err, nil) - return validations, err - } + for rows.Next() { + var validation datastructures.Validation + err = rows.Scan(&validation.Id, &validation.NumOfYes, &validation.NumOfNo, + &validation.Label.Name, &validation.Image.Id, &validation.Image.Width, + &validation.Image.Height, &validation.Image.Unlocked) + if err != nil { + log.Debug("[Get Validations] Couldn't scan rows: ", err.Error()) + raven.CaptureError(err, nil) + return validations, err + } - validation.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, validation.Image.Id, validation.Image.Unlocked) + validation.Image.Url = commons.GetImageUrlFromImageId(apiBaseUrl, validation.Image.Id, validation.Image.Unlocked) - validations = append(validations, validation) - } + validations = append(validations, validation) + } - return validations, nil + return validations, nil } func (p *ImageMonkeyDatabase) GetUnannotatedValidations(apiUser datastructures.APIUser, imageId string) ([]datastructures.UnannotatedValidation, error) { - var unannotatedValidations []datastructures.UnannotatedValidation + var unannotatedValidations []datastructures.UnannotatedValidation - includeOwnImageDonations := "" - if apiUser.Name != "" { - includeOwnImageDonations = `OR ( + includeOwnImageDonations := "" + if apiUser.Name != "" { + includeOwnImageDonations = `OR ( EXISTS ( SELECT 1 @@ -404,9 +406,9 @@ func (p *ImageMonkeyDatabase) GetUnannotatedValidations(apiUser datastructures.A WHERE q.image_id = i.id ) )` - } + } - q := fmt.Sprintf(`SELECT v.uuid::text, l.name, COALESCE(pl.name, '') + q := fmt.Sprintf(`SELECT v.uuid::text, l.name, COALESCE(pl.name, '') FROM image_validation v JOIN label l ON v.label_id = l.id JOIN image i ON v.image_id = i.id @@ -415,35 +417,35 @@ func (p *ImageMonkeyDatabase) GetUnannotatedValidations(apiUser datastructures.A SELECT 1 FROM image_annotation a WHERE a.image_id = i.id AND a.label_id = l.id )`, includeOwnImageDonations) - var rows *sql.Rows - var err error - - if apiUser.Name == "" { - rows, err = p.db.Query(q, imageId) - } else { - rows, err = p.db.Query(q, imageId, apiUser.Name) - } - - if err != nil { - log.Debug("[Get unannotated validation ids] Couldn't get validation ids: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedValidations, err - } - - defer rows.Close() - - for rows.Next() { - var unannotatedValidation datastructures.UnannotatedValidation - err = rows.Scan(&unannotatedValidation.Validation.Id, &unannotatedValidation.Validation.Label, - &unannotatedValidation.Validation.Sublabel) - if err != nil { - log.Debug("[Get unannotated validation ids] Couldn't scan rows: ", err.Error()) - raven.CaptureError(err, nil) - return unannotatedValidations, err - } - - unannotatedValidations = append(unannotatedValidations, unannotatedValidation) - } - - return unannotatedValidations, nil + var rows pgx.Rows + var err error + + if apiUser.Name == "" { + rows, err = p.db.Query(context.TODO(), q, imageId) + } else { + rows, err = p.db.Query(context.TODO(), q, imageId, apiUser.Name) + } + + if err != nil { + log.Debug("[Get unannotated validation ids] Couldn't get validation ids: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedValidations, err + } + + defer rows.Close() + + for rows.Next() { + var unannotatedValidation datastructures.UnannotatedValidation + err = rows.Scan(&unannotatedValidation.Validation.Id, &unannotatedValidation.Validation.Label, + &unannotatedValidation.Validation.Sublabel) + if err != nil { + log.Debug("[Get unannotated validation ids] Couldn't scan rows: ", err.Error()) + raven.CaptureError(err, nil) + return unannotatedValidations, err + } + + unannotatedValidations = append(unannotatedValidations, unannotatedValidation) + } + + return unannotatedValidations, nil } diff --git a/src/database/label.go b/src/database/label.go index 152300fc..846eafff 100644 --- a/src/database/label.go +++ b/src/database/label.go @@ -1,109 +1,113 @@ package imagemonkeydb import ( - "github.com/getsentry/raven-go" - log "github.com/sirupsen/logrus" - "database/sql" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "context" "encoding/json" - "fmt" + "fmt" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "github.com/getsentry/raven-go" + "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" ) -func _getTotalLabelSuggestions(tx *sql.Tx) (int64, error) { - var numOfTotalLabelSuggestions int64 - numOfTotalLabelSuggestions = 0 +func _getTotalLabelSuggestions(tx pgx.Tx) (int64, error) { + var numOfTotalLabelSuggestions int64 + numOfTotalLabelSuggestions = 0 - rows, err := tx.Query(`SELECT count(*) FROM label_suggestion l`) - if err != nil { - return numOfTotalLabelSuggestions, nil - } + rows, err := tx.Query(context.TODO(), `SELECT count(*) FROM label_suggestion l`) + if err != nil { + return numOfTotalLabelSuggestions, nil + } - defer rows.Close() + defer rows.Close() - if rows.Next() { - err = rows.Scan(&numOfTotalLabelSuggestions) - if err != nil { - return numOfTotalLabelSuggestions, err - } - } + if rows.Next() { + err = rows.Scan(&numOfTotalLabelSuggestions) + if err != nil { + return numOfTotalLabelSuggestions, err + } + } - return numOfTotalLabelSuggestions, nil + return numOfTotalLabelSuggestions, nil } func (p *ImageMonkeyDatabase) GetMostPopularLabels(limit int32) ([]string, error) { - var labels []string + var labels []string - rows, err := p.db.Query(`SELECT l.name FROM image_validation v + rows, err := p.db.Query(context.TODO(), + `SELECT l.name FROM image_validation v JOIN label l ON v.label_id = l.id WHERE l.parent_id is NULL GROUP BY l.id ORDER BY count(l.id) DESC LIMIT $1`, limit) - if err != nil { - log.Debug("[Most Popular Labels] Couldn't fetch results: ", err.Error()) - raven.CaptureError(err, nil) - return labels, err - } - - defer rows.Close() - - for rows.Next() { - var label string - err = rows.Scan(&label) - if err != nil { - log.Debug("[Most Popular Labels] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return labels, err - } - - labels = append(labels, label) - } - - return labels, nil + if err != nil { + log.Debug("[Most Popular Labels] Couldn't fetch results: ", err.Error()) + raven.CaptureError(err, nil) + return labels, err + } + + defer rows.Close() + + for rows.Next() { + var label string + err = rows.Scan(&label) + if err != nil { + log.Debug("[Most Popular Labels] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return labels, err + } + + labels = append(labels, label) + } + + return labels, nil } func (p *ImageMonkeyDatabase) AddLabelSuggestion(suggestedLabel string) error { - _, err := p.db.Exec(`INSERT INTO label_suggestion(name, uuid) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO label_suggestion(name, uuid) SELECT $1, uuid_generate_v4() ON CONFLICT (name) DO NOTHING`, suggestedLabel) - if err != nil { - log.Debug("[Add label suggestion] Couldn't insert: ", err.Error()) - raven.CaptureError(err, nil) - return err - } + if err != nil { + log.Debug("[Add label suggestion] Couldn't insert: ", err.Error()) + raven.CaptureError(err, nil) + return err + } - return nil -} + return nil +} func (p *ImageMonkeyDatabase) GetLabelCategories() ([]string, error) { - var labels []string - rows, err := p.db.Query(`SELECT pl.name + var labels []string + rows, err := p.db.Query(context.TODO(), + `SELECT pl.name FROM label l JOIN label pl on pl.id = l.parent_id WHERE pl.label_type = 'refinement_category'`) - if err != nil { - log.Debug("[Get label categories] Couldn't get category: ", err.Error()) - raven.CaptureError(err, nil) - return labels, err - } - defer rows.Close() - - var label string - for rows.Next() { - err = rows.Scan(&label) - if err != nil { - log.Debug("[Get label categories] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return labels, err - } - - labels = append(labels, label) - } - - return labels, nil + if err != nil { + log.Debug("[Get label categories] Couldn't get category: ", err.Error()) + raven.CaptureError(err, nil) + return labels, err + } + defer rows.Close() + + var label string + for rows.Next() { + err = rows.Scan(&label) + if err != nil { + log.Debug("[Get label categories] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return labels, err + } + + labels = append(labels, label) + } + + return labels, nil } func (p *ImageMonkeyDatabase) GetLabelSuggestions(includeUnlocked bool) ([]string, error) { - labelSuggestions := []string{} + labelSuggestions := []string{} q1 := "" if !includeUnlocked { @@ -113,66 +117,66 @@ func (p *ImageMonkeyDatabase) GetLabelSuggestions(includeUnlocked bool) ([]strin q := fmt.Sprintf(`SELECT l.name FROM label_suggestion l %s`, q1) - rows, err := p.db.Query(q) - if err != nil { - log.Debug("[Get Label Suggestions] Couldn't get label suggestions: ", err.Error()) - raven.CaptureError(err, nil) - return labelSuggestions, err - } + rows, err := p.db.Query(context.TODO(), q) + if err != nil { + log.Debug("[Get Label Suggestions] Couldn't get label suggestions: ", err.Error()) + raven.CaptureError(err, nil) + return labelSuggestions, err + } - defer rows.Close() + defer rows.Close() - for rows.Next() { - var labelSuggestion string - err := rows.Scan(&labelSuggestion) - if err != nil { - log.Debug("[Get Label Suggestions] Couldn't scan label suggestions: ", err.Error()) - raven.CaptureError(err, nil) - return labelSuggestions, err - } + for rows.Next() { + var labelSuggestion string + err := rows.Scan(&labelSuggestion) + if err != nil { + log.Debug("[Get Label Suggestions] Couldn't scan label suggestions: ", err.Error()) + raven.CaptureError(err, nil) + return labelSuggestions, err + } - labelSuggestions = append(labelSuggestions, labelSuggestion) - } + labelSuggestions = append(labelSuggestions, labelSuggestion) + } - return labelSuggestions, nil + return labelSuggestions, nil } func (p *ImageMonkeyDatabase) GetLabelAccessors() ([]string, error) { - var labels []string - rows, err := p.db.Query(`SELECT accessor FROM label_accessor`) - if err != nil { - log.Debug("[Get label accessors] Couldn't get accessor: ", err.Error()) - raven.CaptureError(err, nil) - return labels, err - } - defer rows.Close() - - var label string - for rows.Next() { - err = rows.Scan(&label) - if err != nil { - log.Debug("[Get label accessors] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return labels, err - } - - labels = append(labels, label) - } - - return labels, nil + var labels []string + rows, err := p.db.Query(context.TODO(), `SELECT accessor FROM label_accessor`) + if err != nil { + log.Debug("[Get label accessors] Couldn't get accessor: ", err.Error()) + raven.CaptureError(err, nil) + return labels, err + } + defer rows.Close() + + var label string + for rows.Next() { + err = rows.Scan(&label) + if err != nil { + log.Debug("[Get label accessors] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return labels, err + } + + labels = append(labels, label) + } + + return labels, nil } func (p *ImageMonkeyDatabase) GetLabelAccessorDetails(labelType string) ([]datastructures.LabelAccessorDetail, error) { - var labelAccessorDetails []datastructures.LabelAccessorDetail - var queryValues []interface{} + var labelAccessorDetails []datastructures.LabelAccessorDetail + var queryValues []interface{} - q1 := "" - if labelType != "" { - q1 = "WHERE l.label_type = $1" - queryValues = append(queryValues, labelType) - } + q1 := "" + if labelType != "" { + q1 = "WHERE l.label_type = $1" + queryValues = append(queryValues, labelType) + } - query := fmt.Sprintf(`SELECT COALESCE(json_agg(json_build_object('accessor', acc.accessor, 'parent_accessor', pacc.accessor)) + query := fmt.Sprintf(`SELECT COALESCE(json_agg(json_build_object('accessor', acc.accessor, 'parent_accessor', pacc.accessor)) FILTER (WHERE l.id is not null), '[]'::json) FROM label_accessor acc JOIN label l ON l.id = acc.label_id @@ -180,38 +184,39 @@ func (p *ImageMonkeyDatabase) GetLabelAccessorDetails(labelType string) ([]datas LEFT JOIN label_accessor pacc ON pl.id = pacc.label_id %s`, q1) - rows, err := p.db.Query(query, queryValues...) - if err != nil { - log.Error("[Get detailed label accessors] Couldn't get accessors: ", err.Error()) - raven.CaptureError(err, nil) - return labelAccessorDetails, err - } - defer rows.Close() - - if rows.Next() { - var bytes []byte - err = rows.Scan(&bytes) - if err != nil { - log.Error("[Get detailed label accessors] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return labelAccessorDetails, err - } - - err = json.Unmarshal(bytes, &labelAccessorDetails) - if err != nil { - log.Error("[Get detailed label accessors] Couldn't unmarshal result: ", err.Error()) - raven.CaptureError(err, nil) - return labelAccessorDetails, err - } - } - - return labelAccessorDetails, nil + rows, err := p.db.Query(context.TODO(), query, queryValues...) + if err != nil { + log.Error("[Get detailed label accessors] Couldn't get accessors: ", err.Error()) + raven.CaptureError(err, nil) + return labelAccessorDetails, err + } + defer rows.Close() + + if rows.Next() { + var bytes []byte + err = rows.Scan(&bytes) + if err != nil { + log.Error("[Get detailed label accessors] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return labelAccessorDetails, err + } + + err = json.Unmarshal(bytes, &labelAccessorDetails) + if err != nil { + log.Error("[Get detailed label accessors] Couldn't unmarshal result: ", err.Error()) + raven.CaptureError(err, nil) + return labelAccessorDetails, err + } + } + + return labelAccessorDetails, nil } func (p *ImageMonkeyDatabase) GetLabelAccessorsMapping() (json.RawMessage, error) { - var res json.RawMessage + var res json.RawMessage - rows, err := p.db.Query(`SELECT json_object_agg(a.accessor, CASE + rows, err := p.db.Query(context.TODO(), + `SELECT json_object_agg(a.accessor, CASE WHEN pl.name is not null THEN (json_build_object('label', pl.name, 'sublabel', l.name)) ELSE @@ -221,21 +226,21 @@ func (p *ImageMonkeyDatabase) GetLabelAccessorsMapping() (json.RawMessage, error JOIN label l ON l.id = a.label_id LEFT JOIN label pl ON pl.id = l.parent_id WHERE l.label_type = 'normal' OR l.label_type = 'meta'`) - if err != nil { - log.Error("[Get Label Accessors Mapping] Couldn't get label accessors mapping: ", err.Error()) - raven.CaptureError(err, nil) - return res, err - } - - defer rows.Close() - - if rows.Next() { - err = rows.Scan(&res) - if err != nil { - log.Error("[Get Label Accessors Mapping] Couldn't scan row: ", err.Error()) - raven.CaptureError(err, nil) - return res, err - } - } - return res, nil + if err != nil { + log.Error("[Get Label Accessors Mapping] Couldn't get label accessors mapping: ", err.Error()) + raven.CaptureError(err, nil) + return res, err + } + + defer rows.Close() + + if rows.Next() { + err = rows.Scan(&res) + if err != nil { + log.Error("[Get Label Accessors Mapping] Couldn't scan row: ", err.Error()) + raven.CaptureError(err, nil) + return res, err + } + } + return res, nil } diff --git a/src/database/pg_stat.go b/src/database/pg_stat.go index c5923982..b0be255b 100644 --- a/src/database/pg_stat.go +++ b/src/database/pg_stat.go @@ -4,12 +4,14 @@ import ( datastructures "github.com/bbernhard/imagemonkey-core/datastructures" log "github.com/sirupsen/logrus" "github.com/getsentry/raven-go" + "context" ) func (p *ImageMonkeyDatabase) GetPgStatStatements() ([]datastructures.PgStatStatementResult, error) { res := []datastructures.PgStatStatementResult{} - rows, err := p.db.Query(`SELECT + rows, err := p.db.Query(context.TODO(), + `SELECT (total_time / 1000 / 60) as total, (total_time/calls) as avg, query diff --git a/src/database/statistics.go b/src/database/statistics.go index 0b62335f..dc3ace94 100644 --- a/src/database/statistics.go +++ b/src/database/statistics.go @@ -5,15 +5,16 @@ import ( "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" "errors" - "database/sql" + "github.com/jackc/pgx/v4" "fmt" "encoding/json" + "context" ) func (p *ImageMonkeyDatabase) GetNumOfDonatedImages() (int64, error) { var num int64 - err := p.db.QueryRow("SELECT count(*) FROM image").Scan(&num) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM image").Scan(&num) if err != nil { log.Debug("[Fetch images] Couldn't get num of available images: ", err.Error()) raven.CaptureError(err, nil) @@ -30,7 +31,8 @@ func (p *ImageMonkeyDatabase) GetImageDescriptionStatistics(period string) ([]da return imageDescriptionStatistics, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`WITH dates AS ( + rows, err := p.db.Query(context.TODO(), + `WITH dates AS ( SELECT * FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day') date ), @@ -79,7 +81,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationStatistics(period string) ([]datastru return annotationStatistics, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`WITH dates AS ( + rows, err := p.db.Query(context.TODO(), + `WITH dates AS ( SELECT * FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day') date ), @@ -128,7 +131,8 @@ func (p *ImageMonkeyDatabase) GetDonationsStatistics(period string) ([]datastruc return donationStatistics, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`WITH dates AS ( + rows, err := p.db.Query(context.TODO(), + `WITH dates AS ( SELECT * FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day') date ), @@ -174,7 +178,8 @@ func (p *ImageMonkeyDatabase) GetValidationStatistics(period string) ([]datastru return validationStatistics, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`WITH dates AS ( + rows, err := p.db.Query(context.TODO(), + `WITH dates AS ( SELECT * FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day') date ), @@ -226,7 +231,8 @@ func (p *ImageMonkeyDatabase) GetLabeledObjectsStatistics(period string) ([]data return labeledObjectsStatistics, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`WITH dates AS ( + rows, err := p.db.Query(context.TODO(), + `WITH dates AS ( SELECT * FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day') date ), @@ -282,7 +288,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationRefinementStatistics(period string) ( return annotationRefinementStatistics, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`WITH dates AS ( + rows, err := p.db.Query(context.TODO(), + `WITH dates AS ( SELECT * FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day') date ), @@ -327,7 +334,7 @@ func (p *ImageMonkeyDatabase) GetAnnotationRefinementStatistics(period string) ( func (p *ImageMonkeyDatabase) GetUserStatistics(username string) (datastructures.UserStatistics, error) { var userStatistics datastructures.UserStatistics - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[User Statistics] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -335,9 +342,9 @@ func (p *ImageMonkeyDatabase) GetUserStatistics(username string) (datastructures } userStatistics.Total.Annotations = 0 - err = tx.QueryRow("SELECT count(*) FROM image_annotation").Scan(&userStatistics.Total.Annotations) + err = tx.QueryRow(context.TODO(), "SELECT count(*) FROM image_annotation").Scan(&userStatistics.Total.Annotations) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[User Statistics] Couldn't get total annotations: ", err.Error()) raven.CaptureError(err, nil) return userStatistics, err @@ -345,10 +352,11 @@ func (p *ImageMonkeyDatabase) GetUserStatistics(username string) (datastructures userStatistics.User.Annotations = 0 - err = tx.QueryRow(`SELECT count(*) FROM user_image_annotation u + err = tx.QueryRow(context.TODO(), + `SELECT count(*) FROM user_image_annotation u JOIN account a on u.account_id = a.id WHERE a.name = $1`, username).Scan(&userStatistics.User.Annotations) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[User Statistics] Couldn't get user annotations: ", err.Error()) raven.CaptureError(err, nil) return userStatistics, err @@ -356,26 +364,28 @@ func (p *ImageMonkeyDatabase) GetUserStatistics(username string) (datastructures userStatistics.Total.Validations = 0 - err = tx.QueryRow("SELECT count(*) FROM image_validation").Scan(&userStatistics.Total.Validations) + err = tx.QueryRow(context.TODO(), + "SELECT count(*) FROM image_validation").Scan(&userStatistics.Total.Validations) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[User Statistics] Couldn't get total validations: ", err.Error()) raven.CaptureError(err, nil) return userStatistics, err } userStatistics.User.Validations = 0 - err = tx.QueryRow(`SELECT count(*) FROM user_image_validation u + err = tx.QueryRow(context.TODO(), + `SELECT count(*) FROM user_image_validation u JOIN account a on u.account_id = a.id WHERE a.name = $1`, username).Scan(&userStatistics.User.Validations) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[User Statistics] Couldn't get user validations: ", err.Error()) raven.CaptureError(err, nil) return userStatistics, err } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Debug("[User Statistics] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -392,14 +402,15 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics //use temporary map for faster lookup temp := make(map[string]datastructures.ValidationStat) - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Debug("[Explore] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) return statistics, err } - rows, err := tx.Query(`SELECT CASE WHEN pl.name is null THEN l.name ELSE l.name || '/' || pl.name END, count(l.name), + rows, err := tx.Query(context.TODO(), + `SELECT CASE WHEN pl.name is null THEN l.name ELSE l.name || '/' || pl.name END, count(l.name), CASE WHEN SUM(v.num_of_valid + v.num_of_invalid) = 0 THEN 0 ELSE (CAST (SUM(v.num_of_invalid) AS float)/(SUM(v.num_of_valid) + SUM(v.num_of_invalid))) @@ -410,7 +421,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics LEFT JOIN label pl on l.parent_id = pl.id GROUP BY l.name, pl.name ORDER BY count(l.name) DESC`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore data: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -421,7 +432,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics var validationStat datastructures.ValidationStat err = rows.Scan(&validationStat.Label, &validationStat.Count, &validationStat.ErrorRate, &validationStat.TotalValidations) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -446,9 +457,10 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics } //get donations grouped by country - donationsPerCountryRows, err := tx.Query(`SELECT country_code, count FROM donations_per_country ORDER BY count DESC`) + donationsPerCountryRows, err := tx.Query(context.TODO(), + `SELECT country_code, count FROM donations_per_country ORDER BY count DESC`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore data: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -459,7 +471,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics var donationsPerCountryStat datastructures.DonationsPerCountryStat err = donationsPerCountryRows.Scan(&donationsPerCountryStat.CountryCode, &donationsPerCountryStat.Count) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -470,9 +482,9 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics //get validations grouped by country - validationsPerCountryRows, err := tx.Query(`SELECT country_code, count FROM validations_per_country ORDER BY count DESC`) + validationsPerCountryRows, err := tx.Query(context.TODO(), `SELECT country_code, count FROM validations_per_country ORDER BY count DESC`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore data: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -483,7 +495,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics var validationsPerCountryStat datastructures.ValidationsPerCountryStat err = validationsPerCountryRows.Scan(&validationsPerCountryStat.CountryCode, &validationsPerCountryStat.Count) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -493,9 +505,9 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics } //get annotations grouped by country - annotationsPerCountryRows, err := tx.Query(`SELECT country_code, count FROM annotations_per_country ORDER BY count DESC`) + annotationsPerCountryRows, err := tx.Query(context.TODO(), `SELECT country_code, count FROM annotations_per_country ORDER BY count DESC`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore data: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -506,7 +518,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics var annotationsPerCountryStat datastructures.AnnotationsPerCountryStat err = annotationsPerCountryRows.Scan(&annotationsPerCountryStat.CountryCode, &annotationsPerCountryStat.Count) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -517,9 +529,10 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics //get annotation refinements grouped by country - annotationRefinementsPerCountryRows, err := tx.Query(`SELECT country_code, count FROM annotation_refinements_per_country ORDER BY count DESC`) + annotationRefinementsPerCountryRows, err := tx.Query(context.TODO(), + `SELECT country_code, count FROM annotation_refinements_per_country ORDER BY count DESC`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore data: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -530,7 +543,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics var annotationRefinementsPerCountryStat datastructures.AnnotationRefinementsPerCountryStat err = annotationRefinementsPerCountryRows.Scan(&annotationRefinementsPerCountryStat.CountryCode, &annotationRefinementsPerCountryStat.Count) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -541,9 +554,10 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics //get image descriptions grouped by country - imageDescriptionsPerCountryRows, err := tx.Query(`SELECT country_code, count FROM image_descriptions_per_country ORDER BY count DESC`) + imageDescriptionsPerCountryRows, err := tx.Query(context.TODO(), + `SELECT country_code, count FROM image_descriptions_per_country ORDER BY count DESC`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore data: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -554,7 +568,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics var imageDescriptionsPerCountryStat datastructures.ImageDescriptionsPerCountryStat err = imageDescriptionsPerCountryRows.Scan(&imageDescriptionsPerCountryStat.CountryCode, &imageDescriptionsPerCountryStat.Count) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -564,7 +578,8 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics } //get all unlabeled donations - err = tx.QueryRow(`SELECT count(i.id) from image i + err = tx.QueryRow(context.TODO(), + `SELECT count(i.id) from image i WHERE i.id NOT IN ( SELECT image_id FROM image_validation @@ -572,7 +587,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics SELECT image_id FROM image_label_suggestion )`).Scan(&statistics.NumOfUnlabeledDonations) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't scan data row: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -580,7 +595,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.AnnotationsPerApp, err = _exploreAnnotationsPerApp(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore annotations per app: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -588,7 +603,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.DonationsPerApp, err = _exploreDonationsPerApp(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore donations per app: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -596,7 +611,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.ValidationsPerApp, err = _exploreValidationsPerApp(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't explore validations per app: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -604,7 +619,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.NumOfDonations, err = _getTotalDonations(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't get total donations: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -612,7 +627,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.NumOfAnnotations, err = _getTotalAnnotations(tx, false) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't get total annotations: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -620,7 +635,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.NumOfValidations, err = _getTotalValidations(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't get total validations: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -628,7 +643,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.NumOfAnnotationRefinements, err = _getTotalAnnotationRefinements(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't get total annotation refinements: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -636,7 +651,7 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.NumOfLabelSuggestions, err = _getTotalLabelSuggestions(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't get total label suggestions: ", err.Error()) raven.CaptureError(err, nil) return statistics, err @@ -644,20 +659,20 @@ func (p *ImageMonkeyDatabase) Explore(words []string) (datastructures.Statistics statistics.NumOfLabels, err = _getTotalLabels(tx) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Debug("[Explore] Couldn't get total labels: ", err.Error()) raven.CaptureError(err, nil) return statistics, err } - return statistics, tx.Commit() + return statistics, tx.Commit(context.TODO()) } -func _getTotalDonations(tx *sql.Tx) (int64, error) { +func _getTotalDonations(tx pgx.Tx) (int64, error) { var numOfTotalDonations int64 numOfTotalDonations = 0 - rows, err := tx.Query(`SELECT count(*) FROM image i`) + rows, err := tx.Query(context.TODO(), `SELECT count(*) FROM image i`) if err != nil { return numOfTotalDonations, nil } @@ -674,11 +689,11 @@ func _getTotalDonations(tx *sql.Tx) (int64, error) { return numOfTotalDonations, nil } -func _getTotalLabels(tx *sql.Tx) (int64, error) { +func _getTotalLabels(tx pgx.Tx) (int64, error) { var numOfTotalLabels int64 numOfTotalLabels = 0 - rows, err := tx.Query(`SELECT count(*) FROM label l`) + rows, err := tx.Query(context.TODO(), `SELECT count(*) FROM label l`) if err != nil { return numOfTotalLabels, nil } @@ -695,7 +710,7 @@ func _getTotalLabels(tx *sql.Tx) (int64, error) { return numOfTotalLabels, nil } -func _getTotalAnnotations(tx *sql.Tx, includeAutoGeneratedAnnotations bool) (int64, error) { +func _getTotalAnnotations(tx pgx.Tx, includeAutoGeneratedAnnotations bool) (int64, error) { var numOfAnnotations int64 numOfAnnotations = 0 @@ -706,7 +721,7 @@ func _getTotalAnnotations(tx *sql.Tx, includeAutoGeneratedAnnotations bool) (int q := fmt.Sprintf(`SELECT count(*) FROM image_annotation a %s`, q1) - rows, err := tx.Query(q) + rows, err := tx.Query(context.TODO(), q) if err != nil { return numOfAnnotations, nil } @@ -723,11 +738,11 @@ func _getTotalAnnotations(tx *sql.Tx, includeAutoGeneratedAnnotations bool) (int return numOfAnnotations, nil } -func _getTotalValidations(tx *sql.Tx) (int64, error) { +func _getTotalValidations(tx pgx.Tx) (int64, error) { var numOfValidations int64 numOfValidations = 0 - rows, err := tx.Query(`SELECT count(*) FROM image_validation v`) + rows, err := tx.Query(context.TODO(), `SELECT count(*) FROM image_validation v`) if err != nil { return numOfValidations, nil } @@ -744,11 +759,11 @@ func _getTotalValidations(tx *sql.Tx) (int64, error) { return numOfValidations, nil } -func _getTotalAnnotationRefinements(tx *sql.Tx) (int64, error) { +func _getTotalAnnotationRefinements(tx pgx.Tx) (int64, error) { var numOfAnnotationRefinements int64 numOfAnnotationRefinements = 0 - rows, err := tx.Query(`SELECT count(*) FROM image_annotation_refinement r`) + rows, err := tx.Query(context.TODO(), `SELECT count(*) FROM image_annotation_refinement r`) if err != nil { return numOfAnnotationRefinements, nil } @@ -765,11 +780,11 @@ func _getTotalAnnotationRefinements(tx *sql.Tx) (int64, error) { return numOfAnnotationRefinements, nil } -func _exploreAnnotationsPerApp(tx *sql.Tx) ([]datastructures.AnnotationsPerAppStat, error) { +func _exploreAnnotationsPerApp(tx pgx.Tx) ([]datastructures.AnnotationsPerAppStat, error) { var annotationsPerApp []datastructures.AnnotationsPerAppStat //get annotations grouped by app - annotationsPerAppRows, err := tx.Query(`SELECT app_identifier, count FROM annotations_per_app ORDER BY count DESC`) + annotationsPerAppRows, err := tx.Query(context.TODO(), `SELECT app_identifier, count FROM annotations_per_app ORDER BY count DESC`) if err != nil { return annotationsPerApp, err } @@ -788,11 +803,11 @@ func _exploreAnnotationsPerApp(tx *sql.Tx) ([]datastructures.AnnotationsPerAppSt return annotationsPerApp, nil } -func _exploreDonationsPerApp(tx *sql.Tx) ([]datastructures.DonationsPerAppStat, error) { +func _exploreDonationsPerApp(tx pgx.Tx) ([]datastructures.DonationsPerAppStat, error) { var donationsPerApp []datastructures.DonationsPerAppStat //get donations grouped by app - donationsPerAppRows, err := tx.Query(`SELECT app_identifier, count FROM donations_per_app ORDER BY count DESC`) + donationsPerAppRows, err := tx.Query(context.TODO(), `SELECT app_identifier, count FROM donations_per_app ORDER BY count DESC`) if err != nil { return donationsPerApp, err } @@ -811,11 +826,11 @@ func _exploreDonationsPerApp(tx *sql.Tx) ([]datastructures.DonationsPerAppStat, return donationsPerApp, nil } -func _exploreValidationsPerApp(tx *sql.Tx) ([]datastructures.ValidationsPerAppStat, error) { +func _exploreValidationsPerApp(tx pgx.Tx) ([]datastructures.ValidationsPerAppStat, error) { var validationsPerApp []datastructures.ValidationsPerAppStat //get validations grouped by app - validationsPerAppRows, err := tx.Query(`SELECT app_identifier, count FROM validations_per_app ORDER BY count DESC`) + validationsPerAppRows, err := tx.Query(context.TODO(), `SELECT app_identifier, count FROM validations_per_app ORDER BY count DESC`) if err != nil { return validationsPerApp, err } @@ -836,7 +851,8 @@ func _exploreValidationsPerApp(tx *sql.Tx) ([]datastructures.ValidationsPerAppSt func (p *ImageMonkeyDatabase) UpdateContributionsPerCountry(contributionType string, countryCode string) error { if contributionType == "donation" { - _, err := p.db.Exec(`INSERT INTO donations_per_country (country_code, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO donations_per_country (country_code, count) VALUES ($1, $2) ON CONFLICT (country_code) DO UPDATE SET count = donations_per_country.count + 1`, countryCode, 1) if err != nil { @@ -844,7 +860,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerCountry(contributionType str return err } } else if contributionType == "validation" { - _, err := p.db.Exec(`INSERT INTO validations_per_country (country_code, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO validations_per_country (country_code, count) VALUES ($1, $2) ON CONFLICT (country_code) DO UPDATE SET count = validations_per_country.count + 1`, countryCode, 1) if err != nil { @@ -852,7 +869,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerCountry(contributionType str return err } } else if contributionType == "annotation" { - _, err := p.db.Exec(`INSERT INTO annotations_per_country (country_code, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO annotations_per_country (country_code, count) VALUES ($1, $2) ON CONFLICT (country_code) DO UPDATE SET count = annotations_per_country.count + 1`, countryCode, 1) if err != nil { @@ -860,7 +878,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerCountry(contributionType str return err } } else if contributionType == "annotation-refinement" { - _, err := p.db.Exec(`INSERT INTO annotation_refinements_per_country (country_code, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO annotation_refinements_per_country (country_code, count) VALUES ($1, $2) ON CONFLICT (country_code) DO UPDATE SET count = annotation_refinements_per_country.count + 1`, countryCode, 1) if err != nil { @@ -868,7 +887,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerCountry(contributionType str return err } } else if contributionType == "image-description" { - _, err := p.db.Exec(`INSERT INTO image_descriptions_per_country (country_code, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO image_descriptions_per_country (country_code, count) VALUES ($1, $2) ON CONFLICT (country_code) DO UPDATE SET count = image_descriptions_per_country.count + 1`, countryCode, 1) if err != nil { @@ -882,7 +902,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerCountry(contributionType str func (p *ImageMonkeyDatabase) UpdateContributionsPerApp(contributionType string, appIdentifier string) error { if contributionType == "donation" { - _, err := p.db.Exec(`INSERT INTO donations_per_app (app_identifier, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO donations_per_app (app_identifier, count) VALUES ($1, $2) ON CONFLICT (app_identifier) DO UPDATE SET count = donations_per_app.count + 1`, appIdentifier, 1) if err != nil { @@ -890,7 +911,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerApp(contributionType string, return err } } else if contributionType == "validation" { - _, err := p.db.Exec(`INSERT INTO validations_per_app (app_identifier, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO validations_per_app (app_identifier, count) VALUES ($1, $2) ON CONFLICT (app_identifier) DO UPDATE SET count = validations_per_app.count + 1`, appIdentifier, 1) if err != nil { @@ -898,7 +920,8 @@ func (p *ImageMonkeyDatabase) UpdateContributionsPerApp(contributionType string, return err } } else if contributionType == "annotation" { - _, err := p.db.Exec(`INSERT INTO annotations_per_app (app_identifier, count) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO annotations_per_app (app_identifier, count) VALUES ($1, $2) ON CONFLICT (app_identifier) DO UPDATE SET count = annotations_per_app.count + 1`, appIdentifier, 1) if err != nil { @@ -969,7 +992,7 @@ func (p *ImageMonkeyDatabase) GetAnnotatedStatistics(apiUser datastructures.APIU END DESC`, includeOwnImageDonations, includeOwnImageDonations, q1) - rows, err := p.db.Query(q, queryValues...) + rows, err := p.db.Query(context.TODO(), q, queryValues...) if err != nil { log.Debug("[Get Annotated Statistics] Couldn't get annotated statistics: ", err.Error()) raven.CaptureError(err, nil) @@ -1045,7 +1068,7 @@ func (p *ImageMonkeyDatabase) GetValidatedStatistics(apiUser datastructures.APIU END DESC`, includeOwnImageDonations, includeOwnImageDonations) - rows, err := p.db.Query(q, queryValues...) + rows, err := p.db.Query(context.TODO(), q, queryValues...) if err != nil { log.Debug("[Get Validated Statistics] Couldn't get validated statistics: ", err.Error()) raven.CaptureError(err, nil) @@ -1077,7 +1100,8 @@ func (p *ImageMonkeyDatabase) GetActivity(period string) ([]datastructures.Activ return activity, errors.New("Only last-month statistics are supported at the moment") } - rows, err := p.db.Query(`SELECT l.name, i.key, q.type, date(q.dt), i.width, i.height, + rows, err := p.db.Query(context.TODO(), + `SELECT l.name, i.key, q.type, date(q.dt)::text, i.width, i.height, (d.annotation || ('{"type":"' || t.name || '"}')::jsonb)::jsonb as annotation, q.activity_name FROM ( @@ -1221,7 +1245,7 @@ func (p *ImageMonkeyDatabase) GetActivity(period string) ([]datastructures.Activ func (p *ImageMonkeyDatabase) GetNumOfAnnotatedImages() (int64, error) { var num int64 - err := p.db.QueryRow("SELECT count(*) FROM image_annotation").Scan(&num) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM image_annotation").Scan(&num) if err != nil { log.Debug("[Fetch images] Couldn't get num of annotated images: ", err.Error()) raven.CaptureError(err, nil) @@ -1234,7 +1258,7 @@ func (p *ImageMonkeyDatabase) GetNumOfAnnotatedImages() (int64, error) { func (p *ImageMonkeyDatabase) GetNumOfValidatedImages() (int64, error) { var num int64 - err := p.db.QueryRow("SELECT count(*) FROM image_validation").Scan(&num) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM image_validation").Scan(&num) if err != nil { log.Debug("[Fetch images] Couldn't get num of validated images: ", err.Error()) raven.CaptureError(err, nil) @@ -1247,7 +1271,8 @@ func (p *ImageMonkeyDatabase) GetNumOfValidatedImages() (int64, error) { func (p *ImageMonkeyDatabase) GetValidationsCount(minProbability float64, minCount int) ([]datastructures.ValidationCount, error) { validationCounts := []datastructures.ValidationCount{} - rows, err := p.db.Query(`SELECT a.accessor, COUNT(*) + rows, err := p.db.Query(context.TODO(), + `SELECT a.accessor, COUNT(*) FROM ( SELECT v.id as validation_id, v.label_id as label_id @@ -1289,7 +1314,8 @@ func (p *ImageMonkeyDatabase) GetValidationsCount(minProbability float64, minCou func (p *ImageMonkeyDatabase) GetAnnotationsCount(minProbability float64, minCount int) ([]datastructures.AnnotationCount, error) { annotationCounts := []datastructures.AnnotationCount{} - rows, err := p.db.Query(`SELECT a.accessor, COUNT(*) + rows, err := p.db.Query(context.TODO(), + `SELECT a.accessor, COUNT(*) FROM ( SELECT a.id as annotation_id, a.label_id as label_id diff --git a/src/database/user.go b/src/database/user.go index b6c63598..382a6042 100644 --- a/src/database/user.go +++ b/src/database/user.go @@ -6,6 +6,7 @@ import ( "github.com/getsentry/raven-go" log "github.com/sirupsen/logrus" "time" + "context" ) func (p *ImageMonkeyDatabase) GetUserInfo(username string) (datastructures.UserInfo, error) { @@ -23,7 +24,8 @@ func (p *ImageMonkeyDatabase) GetUserInfo(username string) (datastructures.UserI userInfo.IsModerator = false userInfo.Permissions = nil - rows, err := p.db.Query(`SELECT a.name, COALESCE(a.profile_picture, ''), a.created, a.is_moderator, + rows, err := p.db.Query(context.TODO(), + `SELECT a.name, COALESCE(a.profile_picture, ''), a.created, a.is_moderator, COALESCE(p.can_remove_label, false) as remove_label_permission, COALESCE(p.can_unlock_image_description, false) as unlock_image_description, COALESCE(p.can_unlock_image, false) as unlock_image, @@ -70,7 +72,7 @@ func (p *ImageMonkeyDatabase) GetUserInfo(username string) (datastructures.UserI func (p *ImageMonkeyDatabase) UserExists(username string) (bool, error) { var numOfExistingUsers int32 - err := p.db.QueryRow("SELECT count(*) FROM account u WHERE u.name = $1", username).Scan(&numOfExistingUsers) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM account u WHERE u.name = $1", username).Scan(&numOfExistingUsers) if err != nil { log.Error("[User exists] Couldn't get num of existing users: ", err.Error()) raven.CaptureError(err, nil) @@ -85,7 +87,7 @@ func (p *ImageMonkeyDatabase) UserExists(username string) (bool, error) { func (p *ImageMonkeyDatabase) EmailExists(email string) (bool, error) { var numOfExistingUsers int32 - err := p.db.QueryRow("SELECT count(*) FROM account u WHERE u.email = $1", email).Scan(&numOfExistingUsers) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM account u WHERE u.email = $1", email).Scan(&numOfExistingUsers) if err != nil { log.Error("[Email exists] Couldn't get num of existing users: ", err.Error()) raven.CaptureError(err, nil) @@ -100,7 +102,7 @@ func (p *ImageMonkeyDatabase) EmailExists(email string) (bool, error) { func (p *ImageMonkeyDatabase) GetHashedPasswordForUser(username string) (string, error) { var hashedPassword string - err := p.db.QueryRow("SELECT hashed_password FROM account u WHERE u.name = $1", username).Scan(&hashedPassword) + err := p.db.QueryRow(context.TODO(), "SELECT hashed_password FROM account u WHERE u.name = $1", username).Scan(&hashedPassword) if err != nil { log.Error("[Hashed Password] Couldn't get hashed password for user: ", err.Error()) raven.CaptureError(err, nil) @@ -125,7 +127,7 @@ func (p *ImageMonkeyDatabase) CreateUser(username string, hashedPassword []byte, DefaultImageCollection{Name: MyOpenTasks, Description: "My open tasks"}, } - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Error("[Creating User] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -133,18 +135,19 @@ func (p *ImageMonkeyDatabase) CreateUser(username string, hashedPassword []byte, } insertedId = 0 - err = tx.QueryRow(`INSERT INTO account(name, hashed_password, email, created, is_moderator) + err = tx.QueryRow(context.TODO(), + `INSERT INTO account(name, hashed_password, email, created, is_moderator) VALUES($1, $2, $3, $4, $5) RETURNING id`, username, hashedPassword, email, created, false).Scan(&insertedId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Creating User] Couldn't create user: ", err.Error()) raven.CaptureError(err, nil) return err } if insertedId == 0 { - tx.Rollback() + tx.Rollback(context.TODO()) return errors.New("nothing inserted") } @@ -158,7 +161,7 @@ func (p *ImageMonkeyDatabase) CreateUser(username string, hashedPassword []byte, } } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Creating User] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) @@ -173,28 +176,30 @@ func (p *ImageMonkeyDatabase) ChangeProfilePicture(username string, uuid string) existingProfilePicture = "" - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { log.Error("[Change Profile Picture] Couldn't begin transaction: ", err.Error()) raven.CaptureError(err, nil) return existingProfilePicture, err } - err = tx.QueryRow(`SELECT COALESCE(a.profile_picture, '') FROM account a WHERE a.name = $1`, username).Scan(&existingProfilePicture) + err = tx.QueryRow(context.TODO(), + `SELECT COALESCE(a.profile_picture, '') FROM account a WHERE a.name = $1`, username).Scan(&existingProfilePicture) if err != nil { log.Error("[Change Profile Picture] Couldn't get existing profile picture: ", err.Error()) raven.CaptureError(err, nil) return existingProfilePicture, err } - _, err = tx.Exec(`UPDATE account SET profile_picture = $1 WHERE name = $2`, uuid, username) + _, err = tx.Exec(context.TODO(), + `UPDATE account SET profile_picture = $1 WHERE name = $2`, uuid, username) if err != nil { log.Error("[Change Profile Picture] Couldn't change profile picture: ", err.Error()) raven.CaptureError(err, nil) return existingProfilePicture, err } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Change Profile Picture] Couldn't commit transaction: ", err.Error()) raven.CaptureError(err, nil) diff --git a/src/datastructures/datastructures.go b/src/datastructures/datastructures.go index a7174d16..2621b6c2 100644 --- a/src/datastructures/datastructures.go +++ b/src/datastructures/datastructures.go @@ -630,8 +630,8 @@ type DescriptionsPerImage struct { type Validation struct { Id string `json:"uuid"` - NumOfYes string `json:"num_of_yes"` - NumOfNo string `json:"num_of_no"` + NumOfYes int `json:"num_of_yes"` + NumOfNo int `json:"num_of_no"` Image struct { Id string `json:"uuid"` Width int32 `json:"width"` @@ -676,7 +676,7 @@ type AnnotationCount struct { type ImageCollection struct { Name string `json:"name"` Description string `json:"description"` - Count string `json:"count"` + Count int `json:"count"` SampleImage struct { Id string `json:"uuid"` diff --git a/src/go.mod b/src/go.mod index 04c99340..a68a24ff 100644 --- a/src/go.mod +++ b/src/go.mod @@ -20,19 +20,20 @@ require ( github.com/google/go-jsonnet v0.13.0 github.com/google/go-querystring v1.0.0 // indirect github.com/h2non/filetype v1.0.8 + github.com/jackc/pgconn v1.1.0 + github.com/jackc/pgtype v1.0.2 + github.com/jackc/pgx/v4 v4.1.2 github.com/justinas/nosurf v0.0.0-20190416172904-05988550ea18 // indirect - github.com/lib/pq v1.1.1 github.com/oschwald/geoip2-golang v1.3.0 github.com/oschwald/maxminddb-golang v1.3.1 // indirect github.com/sirupsen/logrus v1.4.2 github.com/ugorji/go v1.1.5-pre // indirect gocv.io/x/gocv v0.20.0 golang.org/dl v0.0.0-20190507014322-219d744c5398 // indirect - golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 + golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 gopkg.in/go-playground/validator.v8 v8.18.2 // indirect gopkg.in/h2non/bimg.v1 v1.0.19 gopkg.in/resty.v1 v1.12.0 gopkg.in/src-d/go-git.v4 v4.12.0 - gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/src/go.sum b/src/go.sum index c5f3817d..9236c05b 100644 --- a/src/go.sum +++ b/src/go.sum @@ -14,6 +14,10 @@ github.com/bbernhard/imghash v0.0.0-20151018235733-6afea89d4c0e h1:CPK6hjxST5FmH github.com/bbernhard/imghash v0.0.0-20151018235733-6afea89d4c0e/go.mod h1:bW1S/4fBqZTcE9GNO2wF4I3/yv9imfo6r2S7mHRbP8w= github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc= github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -36,6 +40,7 @@ github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6 github.com/gin-gonic/gin v1.4.0 h1:3tMoCCfM7ppqsR0ptz/wi1impNpT7/9wQtMZ8lr1mCQ= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gliderlabs/ssh v0.1.3/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -53,6 +58,41 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14= github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.1.0 h1:10i6DMVJOSko/sD3FLpFKBHONzDGKkX8pbLyHC8B92o= +github.com/jackc/pgconn v1.1.0/go.mod h1:GgY/Lbj1VonNaVdNUHs9AwWom3yP2eymFQ1C8z9r/Lk= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0 h1:FApgMJ/GtaXfI0s8Lvd0kaLaRwMOhs4VH92pwkwQQvU= +github.com/jackc/pgproto3/v2 v2.0.0/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.0.2 h1:TVyes5WLzcWjLUQ5C7WUQOZ/+yd+v7bCfKRd7XMP6Mk= +github.com/jackc/pgtype v1.0.2/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgx v3.6.0+incompatible h1:bJeo4JdVbDAW8KB2m8XkFeo8CPipREoG37BwEoKGz+Q= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.1.2 h1:xZwqiD9cP6zF7oJ1NO2j9txtjpA7I+MdfP3h/TAT1Q8= +github.com/jackc/pgx/v4 v4.1.2/go.mod h1:0cQ5ee0A6fEsg29vZekucSFk5OcWy8sT4qkhuPXHuIE= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.0.0 h1:rbjAshlgKscNa7j0jAM0uNQflis5o2XUogPMVAwtcsM= +github.com/jackc/puddle v1.0.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -64,15 +104,24 @@ github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -89,19 +138,26 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.5-pre h1:jyJKFOSEbdOc2HODrf2qcCkYOdq7zzXqA9bhW5oV4fM= @@ -110,35 +166,64 @@ github.com/ugorji/go/codec v1.1.5-pre h1:5YV9PsFAN+ndcCtTM7s60no7nY7eTG3LPtxhSwu github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= gocv.io/x/gocv v0.20.0 h1:2q75zQ8Zel2tB69G6qrmf/E7EdvaCs90qvkHzdSBOAg= gocv.io/x/gocv v0.20.0/go.mod h1:vZETJRwLnl11muQ6iL3q4ju+0oJRrdmYdv5xJTH7WYA= golang.org/dl v0.0.0-20190507014322-219d744c5398 h1:O7c4+c8Xs0kSsx6YioGx+SZ5bn5RAsObRSkrTDh6NWM= golang.org/dl v0.0.0-20190507014322-219d744c5398/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -150,6 +235,7 @@ gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2G gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/h2non/bimg.v1 v1.0.19 h1:Li7mgBrIvCHvShB4nyCcGJ2Z2rWR/95kgI/U2+U2eYw= gopkg.in/h2non/bimg.v1 v1.0.19/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= diff --git a/src/labels_downloader.go b/src/labels_downloader.go index 85d4b30e..9301c938 100644 --- a/src/labels_downloader.go +++ b/src/labels_downloader.go @@ -6,15 +6,15 @@ import ( clients "github.com/bbernhard/imagemonkey-core/clients" "github.com/getsentry/raven-go" "github.com/gomodule/redigo/redis" - _ "github.com/lib/pq" + "github.com/jackc/pgx/v4" "time" - "database/sql" "flag" "os" "strconv" + "context" ) -var db *sql.DB +var db *pgx.Conn func createBackup(from string, to string) error { return os.Rename(from, to) @@ -37,7 +37,8 @@ type TrendingLabel struct { func getTrendingLabelsForDeployment() ([]TrendingLabel, error) { trendingLabels := []TrendingLabel{} - rows, err := db.Query(`SELECT b.id, s.name, b.rename_to + rows, err := db.Query(context.TODO(), + `SELECT b.id, s.name, b.rename_to FROM trending_label_suggestion t JOIN trending_label_bot_task b ON b.trending_label_suggestion_id = t.id JOIN label_suggestion s ON s.id = t.label_suggestion_id @@ -72,7 +73,8 @@ func restoreBackupDueToError(labelsDir string, backupPath string) { } func setTrendingLabelBotTaskStateProductive(id int64) error { - _, err := db.Exec(`UPDATE trending_label_bot_task + _, err := db.Exec(context.TODO(), + `UPDATE trending_label_bot_task SET state = 'productive' WHERE id = $1`, id) return err @@ -127,18 +129,18 @@ func main() { var err error //open database and make sure that we can ping it imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") - db, err = sql.Open("postgres", imageMonkeyDbConnectionString) + db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString) if err != nil { raven.CaptureError(err, nil) log.Fatal("Couldn't open database: ", err.Error()) } - err = db.Ping() + err = db.Ping(context.Background()) if err != nil { raven.CaptureError(err, nil) log.Fatal("Couldn't ping database: ", err.Error()) } - defer db.Close() + defer db.Close(context.Background()) labelsDownloader := clients.NewLabelsDownloader(*labelsRepositoryUrl, *downloadDir) @@ -286,7 +288,7 @@ func main() { } redisConn := redisPool.Get() - _, err = redisConn.Do("PUBLISH", "reloadlabels", "reloadlabels") + _, err = redisConn.Do("PUBLISH", "tasks", "reloadlabels") if err != nil { raven.CaptureError(err, nil) log.Error("Couldn't publish message: ", err.Error()) diff --git a/src/attach_labels_to_user_collection.go b/src/migrations/attach_labels_to_user_collection.go similarity index 100% rename from src/attach_labels_to_user_collection.go rename to src/migrations/attach_labels_to_user_collection.go diff --git a/src/statworker.go b/src/statworker.go index 3e202bb4..33f5d926 100644 --- a/src/statworker.go +++ b/src/statworker.go @@ -22,6 +22,7 @@ func main(){ redisMaxConnections := flag.Int("redis_max_connections", 10, "Max connections to Redis") singleshot := flag.Bool("singleshot", false, "Terminate after work is done") useSentry := flag.Bool("use_sentry", false, "Use Sentry for error logging") + maxNumOfDatabaseConnections := flag.Int("db_max_connections", 5, "Max. number of database connections") flag.Parse() @@ -29,7 +30,7 @@ func main(){ imageDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") imageMonkeyDatabase := imagemonkeydb.NewImageMonkeyDatabase() - err = imageMonkeyDatabase.Open(imageDbConnectionString) + err = imageMonkeyDatabase.Open(imageDbConnectionString, int32(*maxNumOfDatabaseConnections)) if err != nil { log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error()) } diff --git a/src/trendinglabelsworker.go b/src/trendinglabelsworker.go index 260aa8bc..fdb7c8bf 100644 --- a/src/trendinglabelsworker.go +++ b/src/trendinglabelsworker.go @@ -1,51 +1,50 @@ package main import ( - "time" - log "github.com/sirupsen/logrus" - "github.com/google/go-github/github" - "database/sql" - _ "github.com/lib/pq" - "github.com/getsentry/raven-go" - "flag" "context" - "golang.org/x/oauth2" + "flag" "fmt" - "github.com/lib/pq" - datastructures "github.com/bbernhard/imagemonkey-core/datastructures" - imagemonkeydb "github.com/bbernhard/imagemonkey-core/database" commons "github.com/bbernhard/imagemonkey-core/commons" + imagemonkeydb "github.com/bbernhard/imagemonkey-core/database" + datastructures "github.com/bbernhard/imagemonkey-core/datastructures" + "github.com/getsentry/raven-go" + "github.com/google/go-github/github" + "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" + "time" ) -var db *sql.DB +var db *pgx.Conn type TrendingLabel struct { - Name string `json:"name"` - Id string `json:"id"` - Count int32 `json:"count"` + Name string `json:"name"` + Id int `json:"id"` + Count int32 `json:"count"` GithubIssue struct { - Id int `json:"id"` - Exists bool `json:"exists"` - } `json:"github_issue"` + Id int `json:"id"` + Exists bool `json:"exists"` + } `json:"github_issue"` } func handleRecurringLabelSuggestions() error { type ResultEntry struct { - ImageId string - Annotatable bool - LabelMeEntry datastructures.LabelMeEntry + ImageId string + Annotatable bool + LabelMeEntry datastructures.LabelMeEntry ProductionLabelId int64 - LabelSuggestion string + LabelSuggestion string } - tx, err := db.Begin() - if err != nil { - log.Error("[Mark label suggestion as productive] Couldn't begin trensaction: ", err.Error()) - return err - } + tx, err := db.Begin(context.TODO()) + if err != nil { + log.Error("[Mark label suggestion as productive] Couldn't begin trensaction: ", err.Error()) + return err + } - rows, err := tx.Query(`SELECT s.id, i.key, ils.annotatable, l.name, COALESCE(pl.name, ''), t.productive_label_id, s.name + rows, err := tx.Query(context.TODO(), + `SELECT s.id, i.key, ils.annotatable, l.name, COALESCE(pl.name, ''), t.productive_label_id, s.name FROM label_suggestion s JOIN trending_label_suggestion t ON t.label_suggestion_id = s.id JOIN image_label_suggestion ils ON ils.label_suggestion_id = s.id @@ -54,7 +53,7 @@ func handleRecurringLabelSuggestions() error { LEFT JOIN label pl ON l.parent_id = pl.id WHERE t.github_issue_id is not null AND t.productive_label_id is not null`) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Mark label suggestions as productive] Couldn't get entries: ", err.Error()) raven.CaptureError(err, nil) return err @@ -68,23 +67,23 @@ func handleRecurringLabelSuggestions() error { var label1 string var label2 string var resultEntry ResultEntry - err = rows.Scan(&labelSuggestionId, &resultEntry.ImageId, &resultEntry.Annotatable, &label1, &label2, - &resultEntry.ProductionLabelId, &resultEntry.LabelSuggestion) + err = rows.Scan(&labelSuggestionId, &resultEntry.ImageId, &resultEntry.Annotatable, &label1, &label2, + &resultEntry.ProductionLabelId, &resultEntry.LabelSuggestion) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Mark label suggestions as productive] Couldn't scan row: ", err.Error()) raven.CaptureError(err, nil) return err } if label2 == "" { - resultEntry.LabelMeEntry.Label = label1 - } else { - resultEntry.LabelMeEntry.Label = label2 - resultEntry.LabelMeEntry.Sublabels = append(resultEntry.LabelMeEntry.Sublabels, - datastructures.Sublabel{Name: label1}) - } - results = append(results, resultEntry) + resultEntry.LabelMeEntry.Label = label1 + } else { + resultEntry.LabelMeEntry.Label = label2 + resultEntry.LabelMeEntry.Sublabels = append(resultEntry.LabelMeEntry.Sublabels, + datastructures.Sublabel{Name: label1}) + } + results = append(results, resultEntry) labelSuggestionIds = append(labelSuggestionIds, labelSuggestionId) } rows.Close() @@ -93,7 +92,7 @@ func handleRecurringLabelSuggestions() error { for _, elem := range results { labels := []datastructures.LabelMeEntry{} labels = append(labels, elem.LabelMeEntry) - + numOfNotAnnotatable := 0 if elem.Annotatable { numOfNotAnnotatable = 0 @@ -101,20 +100,20 @@ func handleRecurringLabelSuggestions() error { //if label is not annotatable, set num_of_not_annotatable to 10 numOfNotAnnotatable = 10 } - - _, err = imagemonkeydb.AddLabelsToImageInTransaction("", elem.ImageId, labels, 0, numOfNotAnnotatable, tx) + + _, err = imagemonkeydb.AddLabelsToImageInTransaction("", elem.ImageId, labels, 0, numOfNotAnnotatable, tx) if err != nil { //transaction already rolled back in AddLabelsToImageInTransaction() log.Error("[Mark label suggestions as productive] Couldn't add labels: ", err.Error()) raven.CaptureError(err, nil) return err } - + if len(elem.LabelMeEntry.Sublabels) == 0 { //if the label re-occurs and there are annotations too, migrate them also err = imagemonkeydb.MakeAnnotationsProductive(tx, elem.LabelSuggestion, elem.ProductionLabelId) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Mark label suggestions as productive] Couldn't make annotations productive", err.Error()) raven.CaptureError(err, nil) return err @@ -122,19 +121,19 @@ func handleRecurringLabelSuggestions() error { } } - //remove label suggestions - _, err := tx.Exec(`DELETE FROM image_label_suggestion s - WHERE s.label_suggestion_id = ANY($1)`, pq.Array(labelSuggestionIds)) + _, err := tx.Exec(context.TODO(), + `DELETE FROM image_label_suggestion s + WHERE s.label_suggestion_id = ANY($1)`, labelSuggestionIds) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) log.Error("[Mark label suggestions as productive] Couldn't delete label suggestions: ", err.Error()) raven.CaptureError(err, nil) return err } } - err = tx.Commit() + err = tx.Commit(context.TODO()) if err != nil { log.Error("[Mark label suggestions as productive] Couldn't commit transaction: ", err.Error()) return err @@ -145,18 +144,19 @@ func handleRecurringLabelSuggestions() error { func getNewTrendingLabels(trendingLabelTreshold int) ([]TrendingLabel, error) { var trendingLabels []TrendingLabel - rows, err := db.Query(`SELECT s.id, s.name, COUNT(t.id), COALESCE(github_issue_id, -1) FROM label_suggestion s + rows, err := db.Query(context.TODO(), + `SELECT s.id, s.name, COUNT(t.id), COALESCE(github_issue_id, -1) FROM label_suggestion s JOIN image_label_suggestion i ON i.label_suggestion_id = s.id LEFT JOIN trending_label_suggestion t ON t.label_suggestion_id = s.id GROUP BY s.name, s.id, num_of_last_sent, github_issue_id - HAVING COUNT(*) > (COALESCE(num_of_last_sent, 0) + $1)`, trendingLabelTreshold) + HAVING COUNT(*) > (COALESCE(num_of_last_sent, 0) + $1)`, trendingLabelTreshold) if err != nil { return trendingLabels, err } defer rows.Close() for rows.Next() { - var trendingLabel TrendingLabel + var trendingLabel TrendingLabel err := rows.Scan(&trendingLabel.Id, &trendingLabel.Name, &trendingLabel.Count, &trendingLabel.GithubIssue.Id) if err != nil { return trendingLabels, err @@ -188,8 +188,8 @@ func createGithubTicket(trendingLabel TrendingLabel, repository string, githubPr //create a new Issue issueRequest := &github.IssueRequest{ - Title: github.String(title), - Body: github.String(body), + Title: github.String(title), + Body: github.String(body), } issue, _, err := client.Issues.Create(ctx, githubProjectOwner, repository, issueRequest) @@ -202,8 +202,8 @@ func createGithubTicket(trendingLabel TrendingLabel, repository string, githubPr return trendingLabel, err } -func addCommentToGithubTicket(trendingLabel TrendingLabel, repository string, - githubProjectOwner string, githubApiToken string) error { +func addCommentToGithubTicket(trendingLabel TrendingLabel, repository string, + githubProjectOwner string, githubApiToken string) error { ctx := context.Background() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: githubApiToken}, @@ -212,11 +212,11 @@ func addCommentToGithubTicket(trendingLabel TrendingLabel, repository string, client := github.NewClient(tc) - body := fmt.Sprintf("New label count: %d" ,trendingLabel.Count) + body := fmt.Sprintf("New label count: %d", trendingLabel.Count) //create a new comment commentRequest := &github.IssueComment{ - Body: github.String(body), + Body: github.String(body), } _, _, err := client.Issues.CreateComment(ctx, githubProjectOwner, repository, trendingLabel.GithubIssue.Id, commentRequest) @@ -225,9 +225,10 @@ func addCommentToGithubTicket(trendingLabel TrendingLabel, repository string, } func updateSentTrendingLabelCount(trendingLabel TrendingLabel) error { - _, err := db.Exec(`INSERT INTO trending_label_suggestion(label_suggestion_id, num_of_last_sent, github_issue_id, closed) VALUES($1, $2, $3, $4) - ON CONFLICT(label_suggestion_id) DO UPDATE SET num_of_last_sent = $2, github_issue_id = $3`, - trendingLabel.Id, trendingLabel.Count, trendingLabel.GithubIssue.Id, false) + _, err := db.Exec(context.TODO(), + `INSERT INTO trending_label_suggestion(label_suggestion_id, num_of_last_sent, github_issue_id, closed) VALUES($1, $2, $3, $4) + ON CONFLICT(label_suggestion_id) DO UPDATE SET num_of_last_sent = $2, github_issue_id = $3`, + trendingLabel.Id, trendingLabel.Count, trendingLabel.GithubIssue.Id, false) return err } @@ -240,10 +241,9 @@ func main() { useGithub := flag.Bool("use_github", true, "Create Issue in Issues tracker") flag.Parse() - githubProjectOwner := "" githubApiToken := "" - + if *useGithub { githubProjectOwner = commons.MustGetEnv("GITHUB_PROJECT_OWNER") githubApiToken = commons.MustGetEnv("GITHUB_API_TOKEN") @@ -261,21 +261,21 @@ func main() { } log.Info("[Main] Starting up Trending Labels Worker...") - imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") + imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") var err error - db, err = sql.Open("postgres", imageMonkeyDbConnectionString) + db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString) if err != nil { raven.CaptureError(err, nil) log.Fatal(err) } - err = db.Ping() + err = db.Ping(context.Background()) if err != nil { raven.CaptureError(err, nil) log.Fatal("[Main] Couldn't ping database: ", err.Error()) } - defer db.Close() + defer db.Close(context.Background()) for { trendingLabels, err := getNewTrendingLabels(*trendingLabelsTreshold) @@ -315,7 +315,7 @@ func main() { if githubErr != nil { log.Error("[Main] Couldn't update trending label count for trending label: ", err.Error()) raven.CaptureError(err, nil) - } + } } if githubErr == nil { @@ -329,8 +329,8 @@ func main() { } } - //in case someone adds a trending label that was already made productive, we can - //transition the label suggestion automatically to productive. + //in case someone adds a trending label that was already made productive, we can + //transition the label suggestion automatically to productive. err = handleRecurringLabelSuggestions() if err != nil { log.Error("[Main] Couldn't mark trending labels as productive: ", err.Error()) @@ -341,5 +341,5 @@ func main() { } time.Sleep((time.Second * 120)) //sleep for 120 seconds - } + } } diff --git a/src/web.go b/src/web.go index 0b14c595..fdca398a 100644 --- a/src/web.go +++ b/src/web.go @@ -138,6 +138,7 @@ func main() { trendingLabelsRepositoryUrl := flag.String("trending_labels_repository_url", "https://github.com/bbernhard/imagemonkey-trending-labels-test", "Trending Labels Repository") redisAddress := flag.String("redis_address", ":6379", "Address to the Redis server") redisMaxConnections := flag.Int("redis_max_connections", 5, "Max connections to Redis") + maxNumOfDatabaseConnections := flag.Int("db_max_connections", 5, "Max. number of database connections") webAppIdentifier := "edd77e5fb6fc0775a00d2499b59b75d" browserExtensionAppIdentifier := "adf78e53bd6fc0875a00d2499c59b75" @@ -245,6 +246,20 @@ func main() { assetVersion = strconv.FormatInt(int64(time.Now().Unix()), 10) + + imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") + + imageMonkeyDatabase := imagemonkeydb.NewImageMonkeyDatabase() + err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString, int32(*maxNumOfDatabaseConnections)) + if err != nil { + log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error()) + } + defer imageMonkeyDatabase.Close() + + if *useSentry { + imageMonkeyDatabase.InitializeSentry(sentryDsn, sentryEnvironment) + } + //create redis pool redisPool := redis.NewPool(func() (redis.Conn, error) { c, err := redis.Dial("tcp", *redisAddress) @@ -263,8 +278,8 @@ func main() { psc := redis.PubSubConn{Conn: redisConn} defer psc.Close() - if err := psc.Subscribe(redis.Args{}.AddFlat([]string{"reloadlabels"})...); err != nil { - log.Fatal("Couldn't subscribe to topic 'reloadlabels': ", err.Error()) + if err := psc.Subscribe(redis.Args{}.AddFlat([]string{"tasks"})...); err != nil { + log.Fatal("Couldn't subscribe to topic 'tasks': ", err.Error()) } done := make(chan error, 1) @@ -276,24 +291,38 @@ func main() { done <- n return case redis.Message: - log.Info("[Main] Reloading labels") - err := labelRepository.Load() - if err != nil { - log.Error("Couldn't read label map: ", err.Error()) - raven.CaptureError(err, nil) - } - words = labelRepository.GetWords() - - err = metaLabels.Load() - if err != nil { - log.Error("Couldn't read metalabels map: ", err.Error()) - raven.CaptureError(err, nil) - } - - labelRefinementsMap, err = commons.GetLabelRefinementsMap(*labelRefinementsPath) - if err != nil { - log.Error("Couldn't read label refinements: ", err.Error()) - raven.CaptureError(err, nil) + if n.Channel == "tasks" { + if string(n.Data) == "reloadlabels" { + log.Info("[Main] Reloading labels") + err := labelRepository.Load() + if err != nil { + log.Error("Couldn't read label map: ", err.Error()) + raven.CaptureError(err, nil) + } + words = labelRepository.GetWords() + + err = metaLabels.Load() + if err != nil { + log.Error("Couldn't read metalabels map: ", err.Error()) + raven.CaptureError(err, nil) + } + + labelRefinementsMap, err = commons.GetLabelRefinementsMap(*labelRefinementsPath) + if err != nil { + log.Error("Couldn't read label refinements: ", err.Error()) + raven.CaptureError(err, nil) + } + } else if string(n.Data) == "reconnectdb" { + log.Info("Reconnecting to Database") + + //close existing db handle + reconnect + imageMonkeyDatabase.Close() + err = imageMonkeyDatabase.Open(imageMonkeyDbConnectionString, int32(*maxNumOfDatabaseConnections)) + if err != nil { + raven.CaptureError(err, nil) + log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error()) + } + } } case redis.Subscription: switch n.Count { @@ -306,19 +335,6 @@ func main() { } }() - imageDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") - - imageMonkeyDatabase := imagemonkeydb.NewImageMonkeyDatabase() - err = imageMonkeyDatabase.Open(imageDbConnectionString) - if err != nil { - log.Fatal("[Main] Couldn't ping ImageMonkey database: ", err.Error()) - } - defer imageMonkeyDatabase.Close() - - if *useSentry { - imageMonkeyDatabase.InitializeSentry(sentryDsn, sentryEnvironment) - } - jwtSecret := commons.MustGetEnv("JWT_SECRET") sessionCookieHandler := NewSessionCookieHandler(imageMonkeyDatabase, jwtSecret) diff --git a/tests/database.go b/tests/database.go index 3da9e1c8..baa1d8c9 100644 --- a/tests/database.go +++ b/tests/database.go @@ -1,7 +1,7 @@ package tests import ( - "database/sql" + "github.com/jackc/pgx/v4" "bytes" "os/exec" "fmt" @@ -9,6 +9,7 @@ import ( "math/rand" "time" "github.com/bbernhard/imagemonkey-core/commons" + "context" ) func random(min, max int) int { @@ -226,7 +227,7 @@ func installTruncateAllTablesFunction() error { type ImageMonkeyDatabase struct { - db *sql.DB + db *pgx.Conn } func NewImageMonkeyDatabase() *ImageMonkeyDatabase { @@ -237,12 +238,12 @@ func (p *ImageMonkeyDatabase) Open() error { imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") var err error - p.db, err = sql.Open("postgres", imageMonkeyDbConnectionString) + p.db, err = pgx.Connect(context.Background(), imageMonkeyDbConnectionString) if err != nil { return err } - err = p.db.Ping() + err = p.db.Ping(context.Background()) if err != nil { return err } @@ -253,27 +254,28 @@ func (p *ImageMonkeyDatabase) Open() error { func (p *ImageMonkeyDatabase) Initialize() error { //connect as user postgres, in order to drop + re-create database imagemonkey - localDb, err := sql.Open("postgres", "user=postgres sslmode=disable port="+ DB_PORT) + localDb, err := pgx.Connect(context.Background(), "user=postgres host=127.0.0.1 sslmode=disable port="+ DB_PORT) if err != nil { return err } - defer localDb.Close() + defer localDb.Close(context.Background()) //terminate any open database connections (we need to do this, before we can drop the database) - _, err = localDb.Exec(`SELECT pg_terminate_backend(pid) + _, err = localDb.Exec(context.TODO(), + `SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = 'imagemonkey'`) if err != nil { return err } - _, err = localDb.Exec("DROP DATABASE IF EXISTS imagemonkey") + _, err = localDb.Exec(context.TODO(), "DROP DATABASE IF EXISTS imagemonkey") if err != nil { return err } - _, err = localDb.Exec("CREATE DATABASE imagemonkey OWNER monkey") + _, err = localDb.Exec(context.TODO(), "CREATE DATABASE imagemonkey OWNER monkey") if err != nil { return err } @@ -332,7 +334,7 @@ func (p *ImageMonkeyDatabase) Initialize() error { } func (p *ImageMonkeyDatabase) ClearAll() error { - _, err := p.db.Exec(`SELECT truncate_tables('monkey')`) + _, err := p.db.Exec(context.TODO(), `SELECT truncate_tables('monkey')`) if err != nil { return err } @@ -356,12 +358,12 @@ func (p *ImageMonkeyDatabase) ClearAll() error { func (p *ImageMonkeyDatabase) TablesAreEmpty() (bool, error) { var empty bool - err := p.db.QueryRow("SELECT tables_empty('monkey')").Scan(&empty) + err := p.db.QueryRow(context.TODO(), "SELECT tables_empty('monkey')").Scan(&empty) return empty, err } func (p *ImageMonkeyDatabase) UnlockAllImages() error { - _, err := p.db.Exec(`UPDATE image SET unlocked = true`) + _, err := p.db.Exec(context.TODO(), `UPDATE image SET unlocked = true`) if err != nil { return err } @@ -370,12 +372,13 @@ func (p *ImageMonkeyDatabase) UnlockAllImages() error { } func (p *ImageMonkeyDatabase) GiveUserModeratorRights(name string) error { - _, err := p.db.Exec("UPDATE account SET is_moderator = true WHERE name = $1", name) + _, err := p.db.Exec(context.TODO(), "UPDATE account SET is_moderator = true WHERE name = $1", name) if err != nil { return err } - _, err = p.db.Exec(`INSERT INTO account_permission(account_id, can_remove_label, can_unlock_image_description, + _, err = p.db.Exec(context.TODO(), + `INSERT INTO account_permission(account_id, can_remove_label, can_unlock_image_description, can_monitor_system, can_accept_trending_label, can_access_pg_stat) SELECT a.id, true, true, true, true, true FROM account a WHERE a.name = $1`, name) if err != nil { @@ -386,7 +389,8 @@ func (p *ImageMonkeyDatabase) GiveUserModeratorRights(name string) error { } func (p *ImageMonkeyDatabase) GiveUserUnlockImagePermissions(name string) error { - _, err := p.db.Exec(`UPDATE account_permission + _, err := p.db.Exec(context.TODO(), + `UPDATE account_permission SET can_unlock_image = true FROM account a WHERE a.id = account_id AND a.name = $1`, name) @@ -395,7 +399,7 @@ func (p *ImageMonkeyDatabase) GiveUserUnlockImagePermissions(name string) error func (p *ImageMonkeyDatabase) GetNumberOfImages() (int32, error) { var numOfImages int32 - err := p.db.QueryRow("SELECT count(*) FROM image").Scan(&numOfImages) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM image").Scan(&numOfImages) if err != nil { return 0, err } @@ -405,7 +409,7 @@ func (p *ImageMonkeyDatabase) GetNumberOfImages() (int32, error) { func (p *ImageMonkeyDatabase) GetNumberOfLabels() (int32, error) { var numOfLabels int32 - err := p.db.QueryRow("SELECT count(*) FROM label").Scan(&numOfLabels) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM label").Scan(&numOfLabels) if err != nil { return 0, err } @@ -415,7 +419,7 @@ func (p *ImageMonkeyDatabase) GetNumberOfLabels() (int32, error) { func (p *ImageMonkeyDatabase) GetNumberOfUsers() (int32, error) { var numOfUsers int32 - err := p.db.QueryRow("SELECT count(*) FROM account").Scan(&numOfUsers) + err := p.db.QueryRow(context.TODO(), "SELECT count(*) FROM account").Scan(&numOfUsers) if err != nil { return 0, err } @@ -426,7 +430,7 @@ func (p *ImageMonkeyDatabase) GetNumberOfUsers() (int32, error) { func (p *ImageMonkeyDatabase) GetAllValidationIds() ([]string, error) { var validationIds []string - rows, err := p.db.Query("SELECT uuid FROM image_validation") + rows, err := p.db.Query(context.TODO(), "SELECT uuid FROM image_validation") if err != nil { return validationIds, err } @@ -449,7 +453,8 @@ func (p *ImageMonkeyDatabase) GetAllValidationIds() ([]string, error) { func (p *ImageMonkeyDatabase) GetAllValidationIdsForLabel(label string) ([]string, error) { var validationIds []string - rows, err := p.db.Query(`SELECT v.uuid FROM image_validation v + rows, err := p.db.Query(context.TODO(), + `SELECT v.uuid FROM image_validation v JOIN label l ON v.label_id = l.id WHERE l.name = $1 AND l.parent_id is null`, label) if err != nil { @@ -491,7 +496,8 @@ func (p *ImageMonkeyDatabase) GetRandomValidationId() (string, error) { func (p *ImageMonkeyDatabase) GetValidationCount(uuid string) (int32, int32, error) { var numOfYes int32 var numOfNo int32 - err := p.db.QueryRow(`SELECT num_of_valid, num_of_invalid + err := p.db.QueryRow(context.TODO(), + `SELECT num_of_valid, num_of_invalid FROM image_validation WHERE uuid = $1`, uuid).Scan(&numOfYes, &numOfNo) return numOfYes, numOfNo, err @@ -499,7 +505,8 @@ func (p *ImageMonkeyDatabase) GetValidationCount(uuid string) (int32, int32, err func (p *ImageMonkeyDatabase) GetAnnotationRevision(annotationId string) (int32, error) { var revision int32 - err := p.db.QueryRow(`SELECT revision + err := p.db.QueryRow(context.TODO(), + `SELECT revision FROM image_annotation WHERE uuid = $1`, annotationId).Scan(&revision) return revision, err @@ -507,7 +514,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationRevision(annotationId string) (int32, func (p *ImageMonkeyDatabase) GetAnnotationSuggestionRevision(annotationId string) (int32, error) { var revision int32 - err := p.db.QueryRow(`SELECT revision + err := p.db.QueryRow(context.TODO(), + `SELECT revision FROM image_annotation_suggestion WHERE uuid = $1`, annotationId).Scan(&revision) return revision, err @@ -516,7 +524,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationSuggestionRevision(annotationId strin func (p *ImageMonkeyDatabase) GetOldAnnotationDataIds(annotationId string, revision int32) ([]int64, error) { var ids []int64 - rows, err := p.db.Query(`SELECT d.id + rows, err := p.db.Query(context.TODO(), + `SELECT d.id FROM annotation_data d JOIN image_annotation_revision r ON d.image_annotation_revision_id = r.id JOIN image_annotation a ON r.image_annotation_id = a.id @@ -543,7 +552,8 @@ func (p *ImageMonkeyDatabase) GetOldAnnotationDataIds(annotationId string, revis func (p *ImageMonkeyDatabase) GetOldAnnotationSuggestionDataIds(annotationId string, revision int32) ([]int64, error) { var ids []int64 - rows, err := p.db.Query(`SELECT d.id + rows, err := p.db.Query(context.TODO(), + `SELECT d.id FROM annotation_suggestion_data d JOIN image_annotation_suggestion_revision r ON d.image_annotation_suggestion_revision_id = r.id JOIN image_annotation_suggestion a ON r.image_annotation_suggestion_id = a.id @@ -569,7 +579,8 @@ func (p *ImageMonkeyDatabase) GetOldAnnotationSuggestionDataIds(annotationId str func (p *ImageMonkeyDatabase) GetAnnotationDataIds(annotationId string) ([]int64, error) { var ids []int64 - rows, err := p.db.Query(`SELECT d.id + rows, err := p.db.Query(context.TODO(), + `SELECT d.id FROM annotation_data d JOIN image_annotation a ON d.image_annotation_id = a.id WHERE a.uuid = $1`, annotationId) @@ -594,7 +605,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationDataIds(annotationId string) ([]int64 func (p *ImageMonkeyDatabase) GetAnnotationSuggestionDataIds(annotationId string) ([]int64, error) { var ids []int64 - rows, err := p.db.Query(`SELECT d.id + rows, err := p.db.Query(context.TODO(), + `SELECT d.id FROM annotation_suggestion_data d JOIN image_annotation_suggestion a ON d.image_annotation_suggestion_id = a.id WHERE a.uuid = $1`, annotationId) @@ -620,7 +632,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationSuggestionDataIds(annotationId string func (p *ImageMonkeyDatabase) GetRandomImageForAnnotation() (AnnotationRow, error) { var annotationRow AnnotationRow - err := p.db.QueryRow(`SELECT i.key, v.uuid, l.name, COALESCE(pl.name, '') + err := p.db.QueryRow(context.TODO(), + `SELECT i.key, v.uuid, l.name, COALESCE(pl.name, '') FROM image i JOIN image_validation v ON v.image_id = i.id JOIN label l ON v.label_id = l.id @@ -636,7 +649,8 @@ func (p *ImageMonkeyDatabase) GetRandomImageForAnnotation() (AnnotationRow, erro func (p *ImageMonkeyDatabase) AnnotationUuidIsASuggestion(annotationUuid string) (bool, error) { var isSuggestion bool = false - err := p.db.QueryRow(`SELECT is_suggestion FROM + err := p.db.QueryRow(context.TODO(), + `SELECT is_suggestion FROM ( SELECT count(*) as count, false as is_suggestion FROM image_annotation a @@ -657,7 +671,8 @@ func (p *ImageMonkeyDatabase) AnnotationUuidIsASuggestion(annotationUuid string) func (p *ImageMonkeyDatabase) GetImageAnnotationSuggestionIdsForImage(imageId string) ([]string, error) { annotationUuids := []string{} - rows, err := p.db.Query(`SELECT a.uuid + rows, err := p.db.Query(context.TODO(), + `SELECT a.uuid FROM image_annotation_suggestion a JOIN image i ON a.image_id = i.id WHERE i.key = $1`, imageId) @@ -682,38 +697,39 @@ func (p *ImageMonkeyDatabase) GetImageAnnotationSuggestionIdsForImage(imageId st func (p *ImageMonkeyDatabase) GetRandomAnnotationId() (string, error) { var annotationId string - err := p.db.QueryRow(`SELECT a.uuid FROM image_annotation a ORDER BY random() LIMIT 1`).Scan(&annotationId) + err := p.db.QueryRow(context.TODO(), `SELECT a.uuid FROM image_annotation a ORDER BY random() LIMIT 1`).Scan(&annotationId) return annotationId, err } func (p *ImageMonkeyDatabase) GetLastAddedAnnotationDataId() (string, error) { var annotationDataId string - err := p.db.QueryRow(`SELECT d.uuid FROM annotation_data d ORDER BY d.id DESC LIMIT 1`).Scan(&annotationDataId) + err := p.db.QueryRow(context.TODO(), `SELECT d.uuid FROM annotation_data d ORDER BY d.id DESC LIMIT 1`).Scan(&annotationDataId) return annotationDataId, err } func (p *ImageMonkeyDatabase) GetLastAddedAnnotationId() (string, error) { var annotationId string - err := p.db.QueryRow(`SELECT a.uuid FROM image_annotation a ORDER BY a.id DESC LIMIT 1`).Scan(&annotationId) + err := p.db.QueryRow(context.TODO(), `SELECT a.uuid FROM image_annotation a ORDER BY a.id DESC LIMIT 1`).Scan(&annotationId) return annotationId, err } func (p *ImageMonkeyDatabase) GetRandomLabelId() (int64, error) { var labelId int64 - err := p.db.QueryRow(`SELECT l.id FROM label l ORDER BY random() LIMIT 1`).Scan(&labelId) + err := p.db.QueryRow(context.TODO(), `SELECT l.id FROM label l ORDER BY random() LIMIT 1`).Scan(&labelId) return labelId, err } func (p *ImageMonkeyDatabase) GetRandomLabelUuid() (string, error) { var labelUuid string - err := p.db.QueryRow(`SELECT l.uuid FROM label l ORDER BY random() LIMIT 1`).Scan(&labelUuid) + err := p.db.QueryRow(context.TODO(), `SELECT l.uuid FROM label l ORDER BY random() LIMIT 1`).Scan(&labelUuid) return labelUuid, err } func (p *ImageMonkeyDatabase) GetRandomAnnotationData() (string, string, error) { var annotationId string var annotationDataId string - err := p.db.QueryRow(`SELECT a.uuid, d.uuid + err := p.db.QueryRow(context.TODO(), + `SELECT a.uuid, d.uuid FROM image_annotation a JOIN annotation_data d ON d.image_annotation_id = a.id ORDER BY random() LIMIT 1`).Scan(&annotationId, &annotationDataId) @@ -723,7 +739,8 @@ func (p *ImageMonkeyDatabase) GetRandomAnnotationData() (string, string, error) func (p *ImageMonkeyDatabase) GetLastAddedAnnotationData() (string, string, error) { var annotationId string var annotationDataId string - err := p.db.QueryRow(`SELECT a.uuid, d.uuid + err := p.db.QueryRow(context.TODO(), + `SELECT a.uuid, d.uuid FROM image_annotation a JOIN annotation_data d ON d.image_annotation_id = a.id ORDER BY a.id DESC LIMIT 1`).Scan(&annotationId, &annotationDataId) @@ -732,7 +749,8 @@ func (p *ImageMonkeyDatabase) GetLastAddedAnnotationData() (string, string, erro func (p *ImageMonkeyDatabase) GetNumberOfImagesWithLabel(label string) (int32, error) { var num int32 - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_validation v JOIN label l ON v.label_id = l.id WHERE l.name = $1 AND l.parent_id is null`, label).Scan(&num) @@ -741,7 +759,8 @@ func (p *ImageMonkeyDatabase) GetNumberOfImagesWithLabel(label string) (int32, e func (p *ImageMonkeyDatabase) GetNumberOfImagesWithLabelUuid(labelUuid string) (int32, error) { var num int32 - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_validation v JOIN label l ON v.label_id = l.id WHERE l.uuid::text = $1`, labelUuid).Scan(&num) @@ -750,7 +769,8 @@ func (p *ImageMonkeyDatabase) GetNumberOfImagesWithLabelUuid(labelUuid string) ( func (p *ImageMonkeyDatabase) GetNumberOfImagesWithLabelSuggestions(label string) (int32, error) { var num int32 - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_label_suggestion ils JOIN label_suggestion l ON l.id = ils.label_suggestion_id WHERE l.name = $1 @@ -760,14 +780,16 @@ func (p *ImageMonkeyDatabase) GetNumberOfImagesWithLabelSuggestions(label string func (p *ImageMonkeyDatabase) GetNumberOfTrendingLabelSuggestions() (int32, error) { var num int32 - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM trending_label_suggestion`).Scan(&num) return num, err } func (p *ImageMonkeyDatabase) GetNumberOfImageHuntTasksForImageWithLabel(imageId string, label string) (int32, error) { var num int32 - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM imagehunt_task h JOIN image_validation v ON v.id = h.image_validation_id JOIN label l ON l.id = v.label_id @@ -778,7 +800,8 @@ func (p *ImageMonkeyDatabase) GetNumberOfImageHuntTasksForImageWithLabel(imageId func (p *ImageMonkeyDatabase) GetNumberOfImageUserEntriesForImageAndUser(imageId string, username string) (int32, error) { var num int32 - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image i JOIN user_image u ON u.image_id = i.id JOIN account a ON a.id = u.account_id @@ -789,7 +812,8 @@ func (p *ImageMonkeyDatabase) GetNumberOfImageUserEntriesForImageAndUser(imageId func (p *ImageMonkeyDatabase) GetProductiveLabelIdsForTrendingLabels() ([]int64, error) { productiveLabelIds := []int64{} - rows, err := p.db.Query(`SELECT t.productive_label_id FROM trending_label_suggestion t + rows, err := p.db.Query(context.TODO(), + `SELECT t.productive_label_id FROM trending_label_suggestion t WHERE t.productive_label_id is not null`) if err != nil { return productiveLabelIds, err @@ -825,14 +849,14 @@ func (p *ImageMonkeyDatabase) GetRandomLabelName(skipLabel string) (string, erro var label string - err := p.db.QueryRow(query, queryParams...).Scan(&label) + err := p.db.QueryRow(context.TODO(), query, queryParams...).Scan(&label) return label, err } func (p *ImageMonkeyDatabase) GetAllImageIds() ([]string, error) { var imageIds []string - rows, err := p.db.Query(`SELECT i.key FROM image i ORDER BY random()`) + rows, err := p.db.Query(context.TODO(), `SELECT i.key FROM image i ORDER BY random()`) if err != nil { return imageIds, err } @@ -854,19 +878,21 @@ func (p *ImageMonkeyDatabase) GetAllImageIds() ([]string, error) { func (p *ImageMonkeyDatabase) GetLatestDonatedImageId() (string,error) { var imageId string - err := p.db.QueryRow(`SELECT i.key FROM image i ORDER BY id DESC LIMIT 1`).Scan(&imageId) + err := p.db.QueryRow(context.TODO(), `SELECT i.key FROM image i ORDER BY id DESC LIMIT 1`).Scan(&imageId) return imageId, err } func (p *ImageMonkeyDatabase) PutImageInQuarantine(imageId string) error { - _, err := p.db.Exec(`INSERT INTO image_quarantine(image_id) + _, err := p.db.Exec(context.TODO(), + `INSERT INTO image_quarantine(image_id) SELECT id FROM image WHERE key = $1`, imageId) return err } func (p *ImageMonkeyDatabase) GetLabelUuidFromName(label string) (string, error) { var uuid string - err := p.db.QueryRow(`SELECT l.uuid + err := p.db.QueryRow(context.TODO(), + `SELECT l.uuid FROM label l WHERE l.name = $1 and l.parent_id is null`, label).Scan(&uuid) return uuid, err @@ -874,7 +900,8 @@ func (p *ImageMonkeyDatabase) GetLabelUuidFromName(label string) (string, error) func (p *ImageMonkeyDatabase) GetLabelIdFromName(label string) (int64, error) { var labelId int64 - err := p.db.QueryRow(`SELECT l.id + err := p.db.QueryRow(context.TODO(), + `SELECT l.id FROM label l WHERE l.name = $1 and l.parent_id is null`, label).Scan(&labelId) return labelId, err @@ -882,7 +909,8 @@ func (p *ImageMonkeyDatabase) GetLabelIdFromName(label string) (int64, error) { func (p *ImageMonkeyDatabase) GetLabelIdFromUuid(labelUuid string) (int64, error) { var labelId int64 - err := p.db.QueryRow(`SELECT l.id + err := p.db.QueryRow(context.TODO(), + `SELECT l.id FROM label l WHERE l.uuid::text = $1`, labelUuid).Scan(&labelId) return labelId, err @@ -890,7 +918,8 @@ func (p *ImageMonkeyDatabase) GetLabelIdFromUuid(labelUuid string) (int64, error func (p *ImageMonkeyDatabase) GetLabelNameFromId(id int64) (string, error) { var labelName string - err := p.db.QueryRow(`SELECT l.name + err := p.db.QueryRow(context.TODO(), + `SELECT l.name FROM label l WHERE l.id = $1`, id).Scan(&labelName) return labelName, err @@ -898,7 +927,8 @@ func (p *ImageMonkeyDatabase) GetLabelNameFromId(id int64) (string, error) { func (p *ImageMonkeyDatabase) GetLabelSuggestionNameFromId(id int64) (string, error) { var labelName string - err := p.db.QueryRow(`SELECT l.name + err := p.db.QueryRow(context.TODO(), + `SELECT l.name FROM label_suggestion l WHERE l.id = $1`, id).Scan(&labelName) return labelName, err @@ -906,7 +936,8 @@ func (p *ImageMonkeyDatabase) GetLabelSuggestionNameFromId(id int64) (string, er func (p *ImageMonkeyDatabase) GetNumOfSentOfTrendingLabel(tendingLabel string) (int, error) { var tendingLabelId int - err := p.db.QueryRow(`SELECT t.num_of_last_sent + err := p.db.QueryRow(context.TODO(), + `SELECT t.num_of_last_sent FROM trending_label_suggestion t JOIN label_suggestion l ON t.label_suggestion_id = l.id WHERE l.name = $1`, tendingLabel).Scan(&tendingLabelId) @@ -914,7 +945,8 @@ func (p *ImageMonkeyDatabase) GetNumOfSentOfTrendingLabel(tendingLabel string) ( } func (p *ImageMonkeyDatabase) SetValidationValid(validationId string, num int) error { - _, err := p.db.Exec(`UPDATE image_validation + _, err := p.db.Exec(context.TODO(), + `UPDATE image_validation SET num_of_valid = $2 WHERE uuid = $1`, validationId, num) return err @@ -922,14 +954,14 @@ func (p *ImageMonkeyDatabase) SetValidationValid(validationId string, num int) e func (p *ImageMonkeyDatabase) GetNumOfRefinements() (int, error) { var num int - err := p.db.QueryRow(`SELECT count(*) FROM image_annotation_refinement`).Scan(&num) + err := p.db.QueryRow(context.TODO(), `SELECT count(*) FROM image_annotation_refinement`).Scan(&num) return num, err } func (p *ImageMonkeyDatabase) GetAllAnnotationIds() ([]string, error) { var annotationIds []string - rows, err := p.db.Query("SELECT uuid FROM image_annotation") + rows, err := p.db.Query(context.TODO(), "SELECT uuid FROM image_annotation") if err != nil { return annotationIds, err } @@ -950,14 +982,16 @@ func (p *ImageMonkeyDatabase) GetAllAnnotationIds() ([]string, error) { } func (p *ImageMonkeyDatabase) SetAnnotationValid(annotationId string, num int) error { - _, err := p.db.Exec(`UPDATE image_annotation + _, err := p.db.Exec(context.TODO(), + `UPDATE image_annotation SET num_of_valid = $2 WHERE uuid = $1`, annotationId, num) return err } func (p *ImageMonkeyDatabase) GetImageAnnotationCoverageForImageId(imageId string) (int, error) { - rows, err := p.db.Query(`SELECT annotated_percentage + rows, err := p.db.Query(context.TODO(), + `SELECT annotated_percentage FROM image_annotation_coverage c JOIN image i ON i.id = c.image_id WHERE i.key = $1`, imageId) @@ -982,7 +1016,8 @@ func (p *ImageMonkeyDatabase) GetImageAnnotationCoverageForImageId(imageId strin func (p *ImageMonkeyDatabase) GetImageDescriptionForImageId(imageId string) ([]ImageDescriptionSummary, error) { var descriptionSummaries []ImageDescriptionSummary - rows, err := p.db.Query(`SELECT dsc.description, dsc.num_of_valid, dsc.uuid, dsc.state, l.name + rows, err := p.db.Query(context.TODO(), + `SELECT dsc.description, dsc.num_of_valid, dsc.uuid, dsc.state, l.name FROM image_description dsc JOIN language l ON l.id = dsc.language_id JOIN image i ON i.id = dsc.image_id @@ -1018,7 +1053,8 @@ func (p *ImageMonkeyDatabase) GetImageDescriptionForImageId(imageId string) ([]I func (p *ImageMonkeyDatabase) GetModeratorWhoProcessedImageDescription(imageId string, imageDescription string) (string, error) { - rows, err := p.db.Query(`SELECT a.name + rows, err := p.db.Query(context.TODO(), + `SELECT a.name FROM image_description dsc JOIN image i ON i.id = dsc.image_id JOIN account a ON a.id = dsc.processed_by @@ -1042,7 +1078,7 @@ func (p *ImageMonkeyDatabase) GetModeratorWhoProcessedImageDescription(imageId s } func (p *ImageMonkeyDatabase) IsImageUnlocked(imageId string) (bool, error) { - rows, err := p.db.Query(`SELECT unlocked FROM image i WHERE i.key = $1`, imageId) + rows, err := p.db.Query(context.TODO(), `SELECT unlocked FROM image i WHERE i.key = $1`, imageId) if err != nil { return false, err } @@ -1062,7 +1098,8 @@ func (p *ImageMonkeyDatabase) IsImageUnlocked(imageId string) (bool, error) { } func (p *ImageMonkeyDatabase) IsImageInQuarantine(imageId string) (bool, error) { - rows, err := p.db.Query(`SELECT CASE + rows, err := p.db.Query(context.TODO(), + `SELECT CASE WHEN COUNT(*) <> 0 THEN true ELSE false END as in_quarantine @@ -1089,7 +1126,8 @@ func (p *ImageMonkeyDatabase) IsImageInQuarantine(imageId string) (bool, error) } func (p *ImageMonkeyDatabase) DoLabelAccessorsBelongToMoreThanOneLabelId() (bool, error) { - rows, err := p.db.Query(`SELECT label_id + rows, err := p.db.Query(context.TODO(), + `SELECT label_id FROM label_accessor GROUP BY label_id HAVING COUNT(label_id) > 1`) @@ -1108,7 +1146,8 @@ func (p *ImageMonkeyDatabase) DoLabelAccessorsBelongToMoreThanOneLabelId() (bool func (p *ImageMonkeyDatabase) GetNumOfMetaLabelImageValidations() (int, error) { var num int - err := p.db.QueryRow(`SELECT count(*) FROM + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_validation v JOIN label l ON l.id = v.label_id WHERE l.label_type = 'meta'`).Scan(&num) @@ -1117,71 +1156,81 @@ func (p *ImageMonkeyDatabase) GetNumOfMetaLabelImageValidations() (int, error) { func (p *ImageMonkeyDatabase) GetNumOfDatesFromNowTilOneMonthAgo() (int, error) { var num int - err := p.db.QueryRow(`SELECT COUNT(*) + err := p.db.QueryRow(context.TODO(), + `SELECT COUNT(*) FROM generate_series((CURRENT_DATE - interval '1 month'), CURRENT_DATE, '1 day')`).Scan(&num) return num, err } func (p *ImageMonkeyDatabase) RemoveLabel(labelName string) error { - tx, err := p.db.Begin() + tx, err := p.db.Begin(context.TODO()) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM label_accessor a WHERE a.label_id IN (SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM label_accessor a WHERE a.label_id IN (SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM label_accessor a WHERE a.label_id IN (SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE pl.name = $1)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM label_accessor a WHERE a.label_id IN (SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE pl.name = $1)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM quiz_answer q WHERE q.label_id IN (SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM quiz_answer q WHERE q.label_id IN (SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM quiz_answer q WHERE q.label_id IN (SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE pl.name = $1)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM quiz_answer q WHERE q.label_id IN (SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE pl.name = $1)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM quiz_question q WHERE q.refines_label_id IN (SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM quiz_question q WHERE q.refines_label_id IN (SELECT l.id FROM label l WHERE l.name = $1 AND l.parent_id is null)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM quiz_question q WHERE q.refines_label_id IN (SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE pl.name = $1)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM quiz_question q WHERE q.refines_label_id IN (SELECT l.id FROM label l JOIN label pl ON pl.id = l.parent_id WHERE pl.name = $1)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM label l WHERE l.parent_id IN (SELECT id FROM label pl WHERE pl.name = $1)", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM label l WHERE l.parent_id IN (SELECT id FROM label pl WHERE pl.name = $1)", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - _, err = tx.Exec("DELETE FROM label l WHERE l.name = $1 AND l.parent_id is null", labelName) + _, err = tx.Exec(context.TODO(), + "DELETE FROM label l WHERE l.name = $1 AND l.parent_id is null", labelName) if err != nil { - tx.Rollback() + tx.Rollback(context.TODO()) return err } - err = tx.Commit() + err = tx.Commit(context.TODO()) return err } func (p *ImageMonkeyDatabase) GetNumOfNotAnnotatable(uuid string) (int, error) { - rows, err := p.db.Query("SELECT num_of_not_annotatable FROM image_validation WHERE uuid = $1", uuid) + rows, err := p.db.Query(context.TODO(), + "SELECT num_of_not_annotatable FROM image_validation WHERE uuid = $1", uuid) if err != nil { return 0, err } @@ -1198,7 +1247,8 @@ func (p *ImageMonkeyDatabase) GetNumOfNotAnnotatable(uuid string) (int, error) { } func (p *ImageMonkeyDatabase) GetTrendingLabelBotTaskState(labelSuggestion string) (string, error) { - rows, err := p.db.Query(`SELECT COALESCE(bt.state::text, '') + rows, err := p.db.Query(context.TODO(), + `SELECT COALESCE(bt.state::text, '') FROM trending_label_bot_task bt RIGHT JOIN trending_label_suggestion l ON l.id = bt.trending_label_suggestion_id RIGHT JOIN label_suggestion s ON s.id = l.label_suggestion_id @@ -1222,7 +1272,8 @@ func (p *ImageMonkeyDatabase) GetTrendingLabelBotTaskState(labelSuggestion strin } func (p *ImageMonkeyDatabase) SetTrendingLabelBotTaskState(labelSuggestion string, state string) error { - _, err := p.db.Exec(`UPDATE trending_label_bot_task + _, err := p.db.Exec(context.TODO(), + `UPDATE trending_label_bot_task SET state = $2 FROM ( SELECT l.id as lid @@ -1235,14 +1286,15 @@ func (p *ImageMonkeyDatabase) SetTrendingLabelBotTaskState(labelSuggestion strin } func (p *ImageMonkeyDatabase) Close() { - p.db.Close() + p.db.Close(context.TODO()) } func (p *ImageMonkeyDatabase) AddDummyTrendingLabelBotTask(trendingLabelName string, renameTo string, branchName string, labelType string, state string) (int64, error) { var trendingLabelBotTaskId int64 - rows, err := p.db.Query(`INSERT INTO trending_label_bot_task (trending_label_suggestion_id, branch_name, state, label_type, rename_to) + rows, err := p.db.Query(context.TODO(), + `INSERT INTO trending_label_bot_task (trending_label_suggestion_id, branch_name, state, label_type, rename_to) SELECT t.id, $1, $2 , $3, $4 FROM trending_label_suggestion t JOIN label_suggestion l ON t.label_suggestion_id = l.id @@ -1263,7 +1315,7 @@ func (p *ImageMonkeyDatabase) AddDummyTrendingLabelBotTask(trendingLabelName str type ImageAnnotationEntry struct { Uuid string - ImageId string + ImageId int64 NumOfValid int32 NumOfInvalid int32 FingerprintOfLastModification string @@ -1276,8 +1328,9 @@ type ImageAnnotationEntry struct { func (p *ImageMonkeyDatabase) GetImageAnnotationEntries() ([]ImageAnnotationEntry, error) { imageAnnotationEntries :=[]ImageAnnotationEntry{} - rows, err := p.db.Query(`SELECT uuid, image_id, num_of_valid, num_of_invalid, - COALESCE(fingerprint_of_last_modification, ''), sys_period, label_id, auto_generated, revision, id + rows, err := p.db.Query(context.TODO(), + `SELECT uuid, image_id, num_of_valid, num_of_invalid, + COALESCE(fingerprint_of_last_modification, ''), sys_period::text, label_id, auto_generated, revision, id FROM image_annotation ORDER BY uuid`) if err != nil { @@ -1302,8 +1355,9 @@ func (p *ImageMonkeyDatabase) GetImageAnnotationEntries() ([]ImageAnnotationEntr func (p *ImageMonkeyDatabase) GetImageAnnotationSuggestionEntries() ([]ImageAnnotationEntry, error) { imageAnnotationSuggestionEntries :=[]ImageAnnotationEntry{} - rows, err := p.db.Query(`SELECT uuid, image_id, num_of_valid, num_of_invalid, - COALESCE(fingerprint_of_last_modification, ''), sys_period, label_suggestion_id, auto_generated, revision, id + rows, err := p.db.Query(context.TODO(), + `SELECT uuid, image_id, num_of_valid, num_of_invalid, + COALESCE(fingerprint_of_last_modification, ''), sys_period::text, label_suggestion_id, auto_generated, revision, id FROM image_annotation_suggestion ORDER BY uuid`) if err != nil { @@ -1338,7 +1392,8 @@ type AnnotationDataEntry struct { func (p *ImageMonkeyDatabase) GetAnnotationDataEntries() ([]AnnotationDataEntry, error) { annotationDataEntries := []AnnotationDataEntry{} - rows, err := p.db.Query(`SELECT COALESCE(image_annotation_id, -1), annotation, + rows, err := p.db.Query(context.TODO(), + `SELECT COALESCE(image_annotation_id, -1), annotation, annotation_type_id, COALESCE(image_annotation_revision_id, 0), uuid FROM annotation_data ORDER BY uuid`) @@ -1363,7 +1418,8 @@ func (p *ImageMonkeyDatabase) GetAnnotationDataEntries() ([]AnnotationDataEntry, func (p *ImageMonkeyDatabase) GetAnnotationSuggestionDataEntries() ([]AnnotationDataEntry, error) { annotationSuggestionDataEntries := []AnnotationDataEntry{} - rows, err := p.db.Query(`SELECT COALESCE(image_annotation_suggestion_id, -1), annotation, + rows, err := p.db.Query(context.TODO(), + `SELECT COALESCE(image_annotation_suggestion_id, -1), annotation, annotation_type_id, COALESCE(image_annotation_suggestion_revision_id, 0), uuid FROM annotation_suggestion_data ORDER BY uuid`) @@ -1396,7 +1452,8 @@ type ImageAnnotationRevisionEntry struct { func (p *ImageMonkeyDatabase) GetImageAnnotationRevisionEntries() ([]ImageAnnotationRevisionEntry, error) { imageAnnotationRevisions := []ImageAnnotationRevisionEntry{} - rows, err := p.db.Query(`SELECT r.id, r.image_annotation_id, r.revision + rows, err := p.db.Query(context.TODO(), + `SELECT r.id, r.image_annotation_id, r.revision FROM image_annotation_revision r JOIN image_annotation a ON r.image_annotation_id = a.id ORDER BY uuid`) @@ -1422,7 +1479,8 @@ func (p *ImageMonkeyDatabase) GetImageAnnotationRevisionEntries() ([]ImageAnnota func (p *ImageMonkeyDatabase) GetImageAnnotationSuggestionRevisionEntries() ([]ImageAnnotationRevisionEntry, error) { imageAnnotationSuggestionRevisions := []ImageAnnotationRevisionEntry{} - rows, err := p.db.Query(`SELECT r.id, r.image_annotation_suggestion_id, r.revision + rows, err := p.db.Query(context.TODO(), + `SELECT r.id, r.image_annotation_suggestion_id, r.revision FROM image_annotation_suggestion_revision r JOIN image_annotation_suggestion a ON r.image_annotation_suggestion_id = a.id ORDER BY uuid`) @@ -1447,7 +1505,8 @@ func (p *ImageMonkeyDatabase) GetImageAnnotationSuggestionRevisionEntries() ([]I func (p *ImageMonkeyDatabase) GetNumberOfLabelSuggestionsForImage(imageId string) (int, error) { var num int - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_label_suggestion ils JOIN label_suggestion l ON l.id = ils.label_suggestion_id JOIN image i ON i.id = ils.image_id @@ -1462,7 +1521,8 @@ func (p *ImageMonkeyDatabase) GetNumberOfLabelSuggestionsForImage(imageId string func (p *ImageMonkeyDatabase) GetNumberOfLabelSuggestionsWithLabelForImage(imageId string, labelName string) (int, error) { var num int - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_label_suggestion ils JOIN label_suggestion l ON l.id = ils.label_suggestion_id JOIN image i ON i.id = ils.image_id @@ -1478,7 +1538,8 @@ func (p *ImageMonkeyDatabase) GetNumberOfLabelSuggestionsWithLabelForImage(image func (p *ImageMonkeyDatabase) GetLabelUuidsForImage(imageId string) ([]string, error) { labelUuids := []string{} - rows, err := p.db.Query(`SELECT l.uuid + rows, err := p.db.Query(context.TODO(), + `SELECT l.uuid FROM image_validation v JOIN label l ON v.label_id = l.id JOIN image i ON v.image_id = i.id @@ -1503,13 +1564,14 @@ func (p *ImageMonkeyDatabase) GetLabelUuidsForImage(imageId string) ([]string, e } func (p *ImageMonkeyDatabase) CloseAllTrendingLabelTasks() error { - _, err := p.db.Exec("UPDATE trending_label_suggestion SET closed = true") + _, err := p.db.Exec(context.TODO(), "UPDATE trending_label_suggestion SET closed = true") return err } func (p *ImageMonkeyDatabase) GetNumOfImagesInImageCollection(username string, imageCollectionName string) (int, error) { var num int - err := p.db.QueryRow(`SELECT count(*) + err := p.db.QueryRow(context.TODO(), + `SELECT count(*) FROM image_collection_image ici JOIN image i ON i.id = ici.image_id JOIN user_image_collection u ON u.id = ici.user_image_collection_id diff --git a/tests/go.mod b/tests/go.mod index 6e113379..f88af6df 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -4,7 +4,8 @@ go 1.12 require ( github.com/bbernhard/imagemonkey-core v0.0.0-00010101000000-000000000000 - github.com/lib/pq v1.1.1 + github.com/gomodule/redigo v2.0.0+incompatible + github.com/jackc/pgx/v4 v4.1.2 github.com/sirupsen/logrus v1.4.2 gopkg.in/resty.v1 v1.12.0 ) diff --git a/tests/go.sum b/tests/go.sum index 5b63a388..5d00832a 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -14,6 +14,10 @@ github.com/bbernhard/imghash v0.0.0-20151018235733-6afea89d4c0e h1:CPK6hjxST5FmH github.com/bbernhard/imghash v0.0.0-20151018235733-6afea89d4c0e/go.mod h1:bW1S/4fBqZTcE9GNO2wF4I3/yv9imfo6r2S7mHRbP8w= github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713 h1:UNOqI3EKhvbqV8f1Vm3NIwkrhq388sGCeAH2Op7w0rc= github.com/certifi/gocertifi v0.0.0-20190506164543-d2eda7129713/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -32,6 +36,7 @@ github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm github.com/gin-gonic/gin v1.3.0 h1:kCmZyPklC0gVdL728E6Aj20uYBJV93nj/TkwBTKhFbs= github.com/gin-gonic/gin v1.3.0/go.mod h1:7cKuhb5qV2ggCFctp2fJQ+ErvciLZrIeoOSOm6mUr7Y= github.com/gliderlabs/ssh v0.1.3/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= @@ -47,6 +52,39 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14= github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= +github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= +github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= +github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v0.0.0-20190420214824-7e0022ef6ba3/go.mod h1:jkELnwuX+w9qN5YIfX0fl88Ehu4XC3keFuOJJk9pcnA= +github.com/jackc/pgconn v0.0.0-20190824142844-760dd75542eb/go.mod h1:lLjNuW/+OfW9/pnVKPazfWOgNfH2aPem8YQ7ilXGvJE= +github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsUgOEh9hBm+xYTstcNHg7UPMVJqRfQxq4s= +github.com/jackc/pgconn v1.1.0 h1:10i6DMVJOSko/sD3FLpFKBHONzDGKkX8pbLyHC8B92o= +github.com/jackc/pgconn v1.1.0/go.mod h1:GgY/Lbj1VonNaVdNUHs9AwWom3yP2eymFQ1C8z9r/Lk= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= +github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= +github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgproto3/v2 v2.0.0 h1:FApgMJ/GtaXfI0s8Lvd0kaLaRwMOhs4VH92pwkwQQvU= +github.com/jackc/pgproto3/v2 v2.0.0/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= +github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= +github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= +github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= +github.com/jackc/pgtype v1.0.2 h1:TVyes5WLzcWjLUQ5C7WUQOZ/+yd+v7bCfKRd7XMP6Mk= +github.com/jackc/pgtype v1.0.2/go.mod h1:5m2OfMh1wTK7x+Fk952IDmI4nw3nPrvtQdM0ZT4WpC0= +github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= +github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= +github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= +github.com/jackc/pgx/v4 v4.1.2 h1:xZwqiD9cP6zF7oJ1NO2j9txtjpA7I+MdfP3h/TAT1Q8= +github.com/jackc/pgx/v4 v4.1.2/go.mod h1:0cQ5ee0A6fEsg29vZekucSFk5OcWy8sT4qkhuPXHuIE= +github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jackc/puddle v1.0.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -54,15 +92,25 @@ github.com/justinas/nosurf v0.0.0-20190416172904-05988550ea18/go.mod h1:Aucr5I5c github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e h1:RgQk53JHp/Cjunrr1WlsXSZpqXn+uREuHvUVcK82CV8= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8= @@ -75,56 +123,93 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= +github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= +github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/ugorji/go v1.1.5-pre h1:jyJKFOSEbdOc2HODrf2qcCkYOdq7zzXqA9bhW5oV4fM= github.com/ugorji/go v1.1.5-pre/go.mod h1:FwP/aQVg39TXzItUBMwnWp9T9gPQnXw4Poh4/oBQZ/0= github.com/ugorji/go/codec v1.1.5-pre h1:5YV9PsFAN+ndcCtTM7s60no7nY7eTG3LPtxhSwuxzCs= github.com/ugorji/go/codec v1.1.5-pre/go.mod h1:tULtS6Gy1AE1yCENaw4Vb//HLH5njI2tfCQDUqRd8fI= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= gocv.io/x/gocv v0.20.0/go.mod h1:vZETJRwLnl11muQ6iL3q4ju+0oJRrdmYdv5xJTH7WYA= golang.org/dl v0.0.0-20190507014322-219d744c5398/go.mod h1:IUMfjQLJQd4UTqG1Z90tenwKoCX93Gn3MAQJMOSBsDQ= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190422183909-d864b10871cd/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 h1:58fnuSXlxZmFdJyvtTFVmVhcMLU6v5fEb/ok4wyqtNU= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 h1:0hQKqeLdqlt5iIwVOBErRisrHJAN57yOiPRQItI20fU= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab h1:9RfW3ktsOZxgo9YNbBAjq1FWzc/igwEcUzZz8IXgSbk= golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2GVOI3xgiMrQ= gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/h2non/bimg.v1 v1.0.19/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/src-d/go-billy.v4 v4.3.0 h1:KtlZ4c1OWbIs4jCv5ZXrTqG8EQocr0g/d4DjNg70aek= diff --git a/tests/tests.go b/tests/tests.go index fc80db4a..bf838578 100644 --- a/tests/tests.go +++ b/tests/tests.go @@ -2,9 +2,9 @@ package tests import ( log "github.com/sirupsen/logrus" - _"github.com/lib/pq" "flag" commons "github.com/bbernhard/imagemonkey-core/commons" + "github.com/gomodule/redigo/redis" ) var db *ImageMonkeyDatabase @@ -66,4 +66,25 @@ func init() { if err != nil { log.Fatal("[Main] Couldn't open database: ", err.Error()) } + + + //create redis pool + redisPool := redis.NewPool(func() (redis.Conn, error) { + c, err := redis.Dial("tcp", REDIS_ADDRESS) + + if err != nil { + log.Fatal("[Main] Couldn't dial redis: ", err.Error()) + } + + return c, err + }, 1) + defer redisPool.Close() + + log.Info("Notify api/web service to reconnect to database (to avoid flaky testcases)") + redisConn := redisPool.Get() + _, err = redisConn.Do("PUBLISH", "tasks", "reconnectdb") + if err != nil { + log.Fatal("Couldn't publish message: ", err.Error()) + } + defer redisConn.Close() }