diff --git a/.gitignore b/.gitignore index fabac85c..41012504 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,7 @@ src/data_processor src/trendinglabelsworker src/populate_labels src/statworker +src/labels_downloader +src/labels_merger +src/make_labels_productive +src/bot diff --git a/env/docker/Dockerfile.blogsubscriptionworker b/env/docker/Dockerfile.blogsubscriptionworker index 495a624c..99f6fe18 100644 --- a/env/docker/Dockerfile.blogsubscriptionworker +++ b/env/docker/Dockerfile.blogsubscriptionworker @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster ENV REDIS_ADDRESS=:6379 ENV SENTRY_DSN= diff --git a/env/docker/Dockerfile.bot b/env/docker/Dockerfile.bot index 209ede6c..c1d815ab 100644 --- a/env/docker/Dockerfile.bot +++ b/env/docker/Dockerfile.bot @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster RUN mkdir -p /home/go/bin ENV GOPATH=/home/go diff --git a/env/docker/Dockerfile.buildbase b/env/docker/Dockerfile.buildbase index 12888e88..576f4f6b 100644 --- a/env/docker/Dockerfile.buildbase +++ b/env/docker/Dockerfile.buildbase @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster ENV GOPATH=/home/go ENV GOBIN=/home/go/bin diff --git a/env/docker/Dockerfile.dataprocessor b/env/docker/Dockerfile.dataprocessor index 8cca6540..1a3d8a8c 100644 --- a/env/docker/Dockerfile.dataprocessor +++ b/env/docker/Dockerfile.dataprocessor @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster ENV USE_SENTRY=true ARG SINGLESHOT=false diff --git a/env/docker/Dockerfile.postgres b/env/docker/Dockerfile.postgres index 1ef71ca1..907fbb0f 100644 --- a/env/docker/Dockerfile.postgres +++ b/env/docker/Dockerfile.postgres @@ -1,4 +1,4 @@ -FROM postgres:9.6.13 +FROM postgres:9.6.18 ARG TEMPORAL_TABLE_VERSION=1.2.0 ARG POSTGIS_VERSION=2.5 diff --git a/env/docker/Dockerfile.statworker b/env/docker/Dockerfile.statworker index 59f9dd7c..1bb9f7b9 100644 --- a/env/docker/Dockerfile.statworker +++ b/env/docker/Dockerfile.statworker @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster ENV SENTRY_DSN= ENV IMAGEMONKEY_DB_CONNECTION_STRING= diff --git a/env/docker/Dockerfile.testing b/env/docker/Dockerfile.testing index 95836ee1..2f7e023e 100644 --- a/env/docker/Dockerfile.testing +++ b/env/docker/Dockerfile.testing @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster # when changing the base image, change also the debian version here ARG DEBIAN_VERSION=buster diff --git a/env/docker/Dockerfile.trendinglabelsworker b/env/docker/Dockerfile.trendinglabelsworker index a5151b38..42145547 100644 --- a/env/docker/Dockerfile.trendinglabelsworker +++ b/env/docker/Dockerfile.trendinglabelsworker @@ -1,4 +1,4 @@ -FROM golang:1.12.14-buster +FROM golang:1.14.4-buster ENV USE_SENTRY=true ENV SINGLESHOT=false diff --git a/src/migrations/fix_duplicate_label_suggestions.go b/src/migrations/fix_duplicate_label_suggestions.go new file mode 100644 index 00000000..307e9bce --- /dev/null +++ b/src/migrations/fix_duplicate_label_suggestions.go @@ -0,0 +1,865 @@ +package main + +import ( + commons "github.com/bbernhard/imagemonkey-core/commons" + "github.com/jackc/pgx/v4" + log "github.com/sirupsen/logrus" + "context" + "flag" + "os" + "fmt" + "github.com/jackc/pgtype" + "golang.org/x/oauth2" + "github.com/google/go-github/github" +) + +var unrecoverableDeletedAnnotationData int = 0 +var unrecoverableDeletedAnnotations int = 0 +var githubIssueIds []int64 +var numOfDeleteLabelSuggestions int = 0 +var numOfDeletedImageLabelSuggestions int = 0 +var numOfDeletedImageAnnotationSuggestionHistoryEntries int = 0 +var numOfUpdatedImageAnnotationSuggestionEntries int = 0 +var numOfUpdatedImageLabelSuggestionEntries int = 0 +var numOfDeletedTrendingLabelSuggestionEntries int = 0 +var numOfDeletedImageLabelSuggestionEntries int = 0 +var numOfDeletedAnnotationSuggestionDataEntries int = 0 +var numOfDeletedUserImageAnnotationSuggestionEntries int = 0 +var numOfDeletedImageAnnotationSuggestionEntries int = 0 + +var typeOfScans []string = []string{"enable_indexscan", "enable_indexonlyscan"} + +func removeElemFromSlice(s []int64, r int64) []int64 { + for i, v := range s { + if v == r { + return append(s[:i], s[i+1:]...) + } + } + return s +} + +func removeImageAnnotationSuggestionHistoryEntry(tx pgx.Tx, imageAnnotationSuggestionId int64) error { + var num int = 0 + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM image_annotation_suggestion_history WHERE id = $1`, imageAnnotationSuggestionId).Scan(&num) + + if num > 0 { + _, err = tx.Exec(context.TODO(), `DELETE FROM image_annotation_suggestion_history WHERE id = $1`, imageAnnotationSuggestionId) + numOfDeletedImageAnnotationSuggestionHistoryEntries += 1 + } + return err +} + +func disableTriggers(tx pgx.Tx) error { + _, err := tx.Exec(context.TODO(), `ALTER TABLE image_annotation_suggestion DISABLE TRIGGER image_annotation_suggestion_versioning_trigger`) + return err +} + +func enableTriggers(tx pgx.Tx) error { + _, err := tx.Exec(context.TODO(), `ALTER TABLE image_annotation_suggestion ENABLE TRIGGER image_annotation_suggestion_versioning_trigger`) + return err +} + +func persistDuplicateLabels(duplicateLabels []string) error { + f, err := os.Create("/tmp/duplicate_labels.txt") + if err != nil { + return err + } + + s := "" + for _, duplicateLabel := range duplicateLabels { + s += duplicateLabel + "\n" + } + + _, err = f.WriteString(s) + if err != nil { + return err + } + + return nil +} + +func closeGithubIssue(githubIssueId int, repository string, githubProjectOwner string, githubApiToken string) error { + ctx := context.Background() + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: githubApiToken}, + ) + tc := oauth2.NewClient(ctx, ts) + + client := github.NewClient(tc) + + body := "Label is a duplicate." + + //create a new comment + commentRequest := &github.IssueComment{ + Body: github.String(body), + } + + //we do not care whether we can successfully close the github issue..if it doesn't work, one can always close it + //manually. + _, _, err := client.Issues.CreateComment(ctx, githubProjectOwner, repository, githubIssueId, commentRequest) + if err == nil { //if comment was successfully created, close issue + issueRequest := &github.IssueRequest{ + State: github.String("closed"), + } + + _, _, err = client.Issues.Edit(ctx, githubProjectOwner, repository, githubIssueId, issueRequest) + return err + } else { + return err + } + + return nil +} + +//TODO: add unique constraint to label_suggestion table + +//TODO: verify num of image labels + annotations before and after + +func getNumOfImageAnnotationSuggestions(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM image_annotation_suggestion`).Scan(&num) + return num, err +} + +func getNumOfImageAnnotationSuggestionHistoryEntries(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM image_annotation_suggestion_history`).Scan(&num) + return num, err +} + +func getNumOfImages(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM image`).Scan(&num) + return num, err +} + +func getNumOfLabelSuggestions(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) from label_suggestion`).Scan(&num) + return num, err +} + +func getNumOfAnnotationSuggestionData(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM annotation_suggestion_data`).Scan(&num) + return num, err +} + +func getNumOfImageLabelSuggestions(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM image_label_suggestion`).Scan(&num) + return num, err +} + +func getNumOfProductiveLabelSuggestions(tx pgx.Tx) (int, error) { + var num int + err := tx.QueryRow(context.TODO(), `SELECT count(*) FROM trending_label_suggestion WHERE closed = true`).Scan(&num) + return num, err +} + +func unifyTrendingLabelSuggestions(tx pgx.Tx, sourceIds *pgtype.Int8Array) error { + rows, err := tx.Query(context.TODO(), `SELECT id, github_issue_id + FROM trending_label_suggestion + WHERE label_suggestion_id = ANY($1)`, sourceIds) + if err != nil { + return err + } + + defer rows.Close() + + trendingLabelSuggestionIds := []int64{} + for rows.Next() { + var trendingLabelSuggestionId int64 + var githubIssueId int64 + err := rows.Scan(&trendingLabelSuggestionId, &githubIssueId) + if err != nil { + return err + } + + trendingLabelSuggestionIds = append(trendingLabelSuggestionIds, trendingLabelSuggestionId) + githubIssueIds = append(githubIssueIds, githubIssueId) + } + + for _, trendingLabelSuggestionId := range trendingLabelSuggestionIds { + _, err = tx.Exec(context.TODO(), `DELETE FROM trending_label_suggestion + WHERE id = $1`, trendingLabelSuggestionId) + numOfDeletedTrendingLabelSuggestionEntries += 1 + if err != nil { + return err + } + } + + return nil +} + +func unifyImagesWithNoDuplicateLabels(tx pgx.Tx, imageIdsThatHaveNoDuplicateLabels []int64, sourceIds *pgtype.Int8Array, target int64) error { + //unify images with no duplicate labels + rows, err := tx.Query(context.TODO(), `SELECT id FROM image_label_suggestion i + WHERE label_suggestion_id = ANY($1) AND image_id = ANY($2)`, sourceIds, imageIdsThatHaveNoDuplicateLabels) + if err != nil { + return err + } + defer rows.Close() + + imageLabelSuggestionIds := []int64{} + for rows.Next() { + var imageLabelSuggestionId int64 + err := rows.Scan(&imageLabelSuggestionId) + if err != nil { + return err + } + imageLabelSuggestionIds = append(imageLabelSuggestionIds, imageLabelSuggestionId) + } + + rows.Close() + + for _, imageLabelSuggestionId := range imageLabelSuggestionIds { + _, err := tx.Exec(context.TODO(), `UPDATE image_label_suggestion SET label_suggestion_id = $1 WHERE id = $2`, target, imageLabelSuggestionId) + numOfUpdatedImageLabelSuggestionEntries+= 1 + if err != nil { + return err + } + } + + return nil +} + +func unifyImagesWithDuplicateLabels(tx pgx.Tx, imageIdsWithDuplicateLabels []int64, sourceIds *pgtype.Int8Array, allIds *pgtype.Int8Array, target int64) error { + for _, imageIdWithDuplicateLabels := range imageIdsWithDuplicateLabels { + rows, err := tx.Query(context.TODO(), + ` SELECT id + FROM image_label_suggestion + WHERE image_id = $1 AND label_suggestion_id = ANY($2)`, imageIdWithDuplicateLabels, allIds) + if err != nil { + return err + } + + imageLabelSuggestionIds := []int64{} + for rows.Next() { + var imageLabelSuggestionId int64 + err = rows.Scan(&imageLabelSuggestionId) + if err != nil { + return err + } + imageLabelSuggestionIds = append(imageLabelSuggestionIds, imageLabelSuggestionId) + } + rows.Close() + + //delate all occurences, except one + for i := 0; i < len(imageLabelSuggestionIds)-1; i++ { + numOfDeletedImageLabelSuggestions += 1 + _, err = tx.Exec(context.TODO(), `DELETE FROM image_label_suggestion WHERE id = $1`, imageLabelSuggestionIds[i]) + numOfDeletedImageLabelSuggestionEntries += 1 + if err != nil { + return err + } + } + + _, err = tx.Exec(context.TODO(), + `UPDATE image_label_suggestion + SET label_suggestion_id = $1 + WHERE id = $2`, target, imageLabelSuggestionIds[len(imageLabelSuggestionIds)-1]) + numOfUpdatedImageLabelSuggestionEntries += 1 + if err != nil { + return err + } + } + + return nil +} + +func unifyImageLabelSuggestions(tx pgx.Tx, sourceIds *pgtype.Int8Array, allIds *pgtype.Int8Array, target int64) error { + rows, err := tx.Query(context.TODO(), `SELECT a.image_id, count(*) FROM image_label_suggestion a + WHERE label_suggestion_id = ANY($1) GROUP BY image_id`, allIds) + if err != nil { + return err + } + defer rows.Close() + + imageIdsThatHaveNoDuplicateLabels := []int64{} + imageIdsWithDuplicateLabels := []int64{} + for rows.Next() { + var count int + var imageId int64 + err := rows.Scan(&imageId, &count) + if err != nil { + return err + } + + if count > 1 { + imageIdsWithDuplicateLabels = append(imageIdsWithDuplicateLabels, imageId) + } else { + imageIdsThatHaveNoDuplicateLabels = append(imageIdsThatHaveNoDuplicateLabels, imageId) + } + } + + rows.Close() + + err = unifyImagesWithNoDuplicateLabels(tx, imageIdsThatHaveNoDuplicateLabels, sourceIds, target) + if err != nil { + return err + } + + err = unifyImagesWithDuplicateLabels(tx, imageIdsWithDuplicateLabels, sourceIds, allIds, target) + if err != nil { + return err + } + + return nil +} + +func unifyImageAnnotationsWithNoDuplicateLabels(tx pgx.Tx, imageIdsThatHaveNoDuplicateLabels []int64, sourceIds *pgtype.Int8Array, target int64) error { + rows, err := tx.Query(context.TODO(), `SELECT id + FROM image_annotation_suggestion a + WHERE label_suggestion_id = ANY($1) + AND a.image_id = ANY($2)`, sourceIds, imageIdsThatHaveNoDuplicateLabels) + if err != nil { + return err + } + defer rows.Close() + + imageAnnotationLabelSuggestionIds := []int64{} + for rows.Next() { + var imageAnnotationLabelSuggestionId int64 + err := rows.Scan(&imageAnnotationLabelSuggestionId) + if err != nil { + return err + } + imageAnnotationLabelSuggestionIds = append(imageAnnotationLabelSuggestionIds, imageAnnotationLabelSuggestionId) + } + + rows.Close() + + for _, imageAnnotationLabelSuggestionId := range imageAnnotationLabelSuggestionIds { + _, err = tx.Exec(context.TODO(), `UPDATE image_annotation_suggestion + SET label_suggestion_id = $1 WHERE id = $2`, target, imageAnnotationLabelSuggestionId) + numOfUpdatedImageAnnotationSuggestionEntries += 1 + if err != nil { + return err + } + } + + return nil +} + +func unifyImageAnnotationsWithDuplicateLabels(tx pgx.Tx, imageIdsWithDuplicateLabels []int64, allIds *pgtype.Int8Array, target int64) error { + for _, imageIdWithDuplicateLabels := range imageIdsWithDuplicateLabels { + rows, err := tx.Query(context.TODO(), + ` SELECT id + FROM image_annotation_suggestion + WHERE image_id = $1 AND label_suggestion_id = ANY($2)`, imageIdWithDuplicateLabels, allIds) + if err != nil { + return err + } + + imageAnnotationSuggestionIds := []int64{} + for rows.Next() { + var imageAnnotationSuggestionId int64 + err = rows.Scan(&imageAnnotationSuggestionId) + if err != nil { + return err + } + imageAnnotationSuggestionIds = append(imageAnnotationSuggestionIds, imageAnnotationSuggestionId) + } + rows.Close() + for _, imageAnnotationSuggestionId := range imageAnnotationSuggestionIds { + unrecoverableDeletedAnnotations += 1 + var unrecoverableDeletedAnnotationDataTemp int + + err = tx.QueryRow(context.TODO(), + `SELECT count(*) FROM annotation_suggestion_data WHERE image_annotation_suggestion_id = $1`, + imageAnnotationSuggestionId).Scan(&unrecoverableDeletedAnnotationDataTemp) + if err != nil { + return err + } + + unrecoverableDeletedAnnotationData += unrecoverableDeletedAnnotationDataTemp + + _, err = tx.Exec(context.TODO(), `DELETE FROM annotation_suggestion_data + WHERE image_annotation_suggestion_id = $1`, imageAnnotationSuggestionId) + numOfDeletedAnnotationSuggestionDataEntries += 1 + if err != nil { + return err + } + + _, err = tx.Exec(context.TODO(), `DELETE FROM user_image_annotation_suggestion + WHERE image_annotation_suggestion_id = $1`, imageAnnotationSuggestionId) + numOfDeletedUserImageAnnotationSuggestionEntries += 1 + if err != nil { + return err + } + + _, err = tx.Exec(context.TODO(), `DELETE FROM image_annotation_suggestion WHERE id = $1`, imageAnnotationSuggestionId) + numOfDeletedImageAnnotationSuggestionEntries += 1 + if err != nil { + return err + } + + err = removeImageAnnotationSuggestionHistoryEntry(tx, imageAnnotationSuggestionId) + if err != nil { + return err + } + } + } + + return nil +} + +func unifyImageAnnotationSuggestions(tx pgx.Tx, sourceIds *pgtype.Int8Array, allIds *pgtype.Int8Array, target int64) error { + rows, err := tx.Query(context.TODO(), `SELECT a.image_id, count(*) FROM image_annotation_suggestion a + WHERE label_suggestion_id = ANY($1) GROUP BY image_id`, allIds) + if err != nil { + return err + } + defer rows.Close() + + imageIdsThatHaveNoDuplicateLabels := []int64{} + imageIdsWithDuplicateLabels := []int64{} + for rows.Next() { + var count int + var imageId int64 + err := rows.Scan(&imageId, &count) + if err != nil { + return err + } + + if count > 1 { + imageIdsWithDuplicateLabels = append(imageIdsWithDuplicateLabels, imageId) + } else { + imageIdsThatHaveNoDuplicateLabels = append(imageIdsThatHaveNoDuplicateLabels, imageId) + } + } + + rows.Close() + + + err = unifyImageAnnotationsWithNoDuplicateLabels(tx, imageIdsThatHaveNoDuplicateLabels, sourceIds, target) + if err != nil { + return err + } + + err = unifyImageAnnotationsWithDuplicateLabels(tx, imageIdsWithDuplicateLabels, allIds, target) + if err != nil { + return err + } + + return nil +} + +func unifyDuplicateLabelSuggestions(tx pgx.Tx, source []int64, target int64) error { + sourceIds := &pgtype.Int8Array{} + sourceIds.Set(source) + + temp := []int64{} + temp = append(temp, source...) + temp = append(temp, target) + allIds := &pgtype.Int8Array{} + allIds.Set(temp) + + err := unifyImageLabelSuggestions(tx, sourceIds, allIds, target) + if err != nil { + return err + } + + err = unifyImageAnnotationSuggestions(tx, sourceIds, allIds, target) + if err != nil { + return err + } + + err = unifyTrendingLabelSuggestions(tx, sourceIds) + if err != nil { + return err + } + + numOfDeleteLabelSuggestions += len(source) + _, err = tx.Exec(context.TODO(), `DELETE FROM label_suggestion WHERE id = ANY($1)`, sourceIds) + if err != nil { + return err + } + + return nil +} + +func getDuplicateLabelSuggestions(tx pgx.Tx) ([]string, error) { + duplicateLabelSuggestions := []string{} + rows, err := tx.Query(context.TODO(), `SELECT name FROM label_suggestion GROUP BY name HAVING COUNT(name) > 1`) + if err != nil { + return duplicateLabelSuggestions, err + } + defer rows.Close() + + for rows.Next() { + var duplicateLabelSuggestion string + err := rows.Scan(&duplicateLabelSuggestion) + if err != nil { + return duplicateLabelSuggestions, err + } + + duplicateLabelSuggestions = append(duplicateLabelSuggestions, duplicateLabelSuggestion) + } + + return duplicateLabelSuggestions, nil +} + +func getLabelSuggestionIdsForLabelSuggestion(tx pgx.Tx, labelSuggestion string) ([]int64, error) { + duplicateLabelSuggestionIds := []int64{} + rows, err := tx.Query(context.TODO(), `SELECT id FROM label_suggestion WHERE name = $1`, labelSuggestion) + if err != nil { + return duplicateLabelSuggestionIds, err + } + defer rows.Close() + + for rows.Next() { + var duplicateLabelSuggestionId int64 + err := rows.Scan(&duplicateLabelSuggestionId) + if err != nil { + return duplicateLabelSuggestionIds, err + } + + duplicateLabelSuggestionIds = append(duplicateLabelSuggestionIds, duplicateLabelSuggestionId) + } + + return duplicateLabelSuggestionIds, nil +} + +func isLabelSuggestionIdProductive(tx pgx.Tx, labelSuggestionId int64) (bool, error) { + rows, err := tx.Query(context.TODO(), + `SELECT closed FROM trending_label_suggestion WHERE label_suggestion_id = $1`, labelSuggestionId) + if err != nil { + return false, err + } + defer rows.Close() + + if rows.Next() { + var productive bool + err := rows.Scan(&productive) + return productive, err + } + return false, nil +} + +func dropUniqueConstraint(tx pgx.Tx) error { + _, err := tx.Exec(context.TODO(), `DROP INDEX label_suggestion_name_unique`) + return err +} + +func disableIndexScans(tx pgx.Tx) error { + for _, typeOfScan := range typeOfScans { + q := fmt.Sprintf(`SET %s = OFF`, typeOfScan) + _, err := tx.Exec(context.TODO(), q) + if err != nil { + return err + } + } + return nil +} + +func enableIndexScans(tx pgx.Tx) error { + for _, typeOfScan := range typeOfScans { + q := fmt.Sprintf(`SET %s = ON`, typeOfScan) + _, err := tx.Exec(context.TODO(), q) + if err != nil { + return err + } + } + return nil +} + +func main() { + dryRun := flag.Bool("dryrun", true, "Do a dry run") + closeIssue := flag.Bool("close-github-issue", false, "Close github issue") + githubRepository := flag.String("repository", "", "Github repository") + + flag.Parse() + + githubProjectOwner := "" + githubApiToken := "" + if *closeIssue { + githubApiToken = commons.MustGetEnv("GITHUB_API_TOKEN") + githubProjectOwner = commons.MustGetEnv("GITHUB_PROJECT_OWNER") + + if *githubRepository == "" { + log.Fatal("Please provide a github repository!") + } + } + + imageMonkeyDbConnectionString := commons.MustGetEnv("IMAGEMONKEY_DB_CONNECTION_STRING") + db, err := pgx.Connect(context.TODO(), imageMonkeyDbConnectionString) + if err != nil { + log.Fatal("Couldn't begin transaction: ", err.Error()) + } + + tx, err := db.Begin(context.TODO()) + if err != nil { + log.Fatal("Couldn't begin transaction: ", err.Error()) + } + + err = dropUniqueConstraint(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't drop unique constraint: ", err.Error()) + } + + err = disableIndexScans(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't disable index scans: ", err.Error()) + } + + err = disableTriggers(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't disable triggers: ", err.Error()) + } + + numOfImagesBefore, err := getNumOfImages(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of images: ", err.Error()) + } + + numOfImageAnnotationSuggestionsBefore, err := getNumOfImageAnnotationSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of image annotation suggestions: ", err.Error()) + } + + numOfLabelSuggestionsBefore, err := getNumOfLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of label suggestions: ", err.Error()) + } + + numOfAnnotationSuggestionDataBefore, err := getNumOfAnnotationSuggestionData(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of annotation suggestion data: ", err.Error()) + } + + numOfProductiveLabelSuggestionsBefore, err := getNumOfProductiveLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of productive label suggestions: ", err.Error()) + } + + numOfImageLabelSuggestionsBefore, err := getNumOfImageLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of image label suggestions: ", err.Error()) + } + + numOfImageAnnotationSuggestionHistoryEntriesBefore, err := getNumOfImageAnnotationSuggestionHistoryEntries(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of image annotation suggestion history entries: ", err.Error()) + } + + duplicateLabelSuggestions, err := getDuplicateLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get duplicate label suggestions: ", err.Error()) + } + + log.Info("Found ", len(duplicateLabelSuggestions), " duplicate labels") + + for _, duplicateLabelSuggestion := range duplicateLabelSuggestions { + labelSuggestionIds, err := getLabelSuggestionIdsForLabelSuggestion(tx, duplicateLabelSuggestion) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get label suggestion ids: ", err.Error()) + } + + + var productiveLabelSuggestionId int64 = -1 + for _, labelSuggestionId := range labelSuggestionIds { + isProductive, err := isLabelSuggestionIdProductive(tx, labelSuggestionId) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get label suggestion id info: ", err.Error()) + } + + if isProductive { + if productiveLabelSuggestionId != -1 { + tx.Rollback(context.TODO()) + log.Fatal("more than one productive label!") + } + productiveLabelSuggestionId = labelSuggestionId; + } + } + + if productiveLabelSuggestionId != -1 { + err = unifyDuplicateLabelSuggestions(tx, removeElemFromSlice(labelSuggestionIds, productiveLabelSuggestionId), productiveLabelSuggestionId) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't unify: ", err.Error()) + } + } else { + err = unifyDuplicateLabelSuggestions(tx, labelSuggestionIds[1:], labelSuggestionIds[0]) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't unify: ", err.Error()) + } + } + } + + //just a sanity check if everything is clean now + duplicateLabelSuggestionsAfterUnification, err := getDuplicateLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("[Verify] Couldn't get duplicate label suggestions: ", err.Error()) + } + if len(duplicateLabelSuggestionsAfterUnification) > 0 { + tx.Rollback(context.TODO()) + log.Fatal("Verification failed. There are still duplicates! (", len(duplicateLabelSuggestionsAfterUnification), ")") + } + + numOfImagesAfter, err := getNumOfImages(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of images: ", err.Error()) + } + + if numOfImagesBefore != numOfImagesAfter { + tx.Rollback(context.TODO()) + log.Fatal("Num of images doesn't match!") + } + + numOfImageAnnotationSuggestionsAfter, err := getNumOfImageAnnotationSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of image annotation suggestions: ", err.Error()) + } + + if numOfImageAnnotationSuggestionsBefore != (numOfImageAnnotationSuggestionsAfter + unrecoverableDeletedAnnotations) { + tx.Rollback(context.TODO()) + log.Fatal("Num of image annotation suggestions do not match!") + } + + numOfAnnotationSuggestionDataAfter, err := getNumOfAnnotationSuggestionData(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of annotation suggestion data: ", err.Error()) + } + + if numOfAnnotationSuggestionDataBefore != (numOfAnnotationSuggestionDataAfter + unrecoverableDeletedAnnotationData) { + tx.Rollback(context.TODO()) + log.Fatal("fail: ") + } + + numOfLabelSuggestionsAfter, err := getNumOfLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of label suggestions: ", err.Error()) + } + + if numOfLabelSuggestionsBefore != (numOfLabelSuggestionsAfter + numOfDeleteLabelSuggestions) { + tx.Rollback(context.TODO()) + log.Fatal("Num of label suggestions do not match!") + } + + numOfProductiveLabelSuggestionsAfter, err := getNumOfProductiveLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of productive label suggestions: ", err.Error()) + } + + if numOfProductiveLabelSuggestionsBefore != numOfProductiveLabelSuggestionsAfter { + tx.Rollback(context.TODO()) + log.Fatal("Num of production label suggestions do not match!") + } + + numOfImageLabelSuggestionsAfter, err := getNumOfImageLabelSuggestions(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of image label suggestions") + } + + if numOfImageLabelSuggestionsBefore != (numOfImageLabelSuggestionsAfter + numOfDeletedImageLabelSuggestions) { + log.Fatal("Num of image label suggestions do not match!") + } + + numOfImageAnnotationSuggestionHistoryEntriesAfter, err := getNumOfImageAnnotationSuggestionHistoryEntries(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't get num of image annotation suggestion history entries: ", err.Error()) + } + + if numOfImageAnnotationSuggestionHistoryEntriesBefore != + (numOfImageAnnotationSuggestionHistoryEntriesAfter + numOfDeletedImageAnnotationSuggestionHistoryEntries) { + tx.Rollback(context.TODO()) + log.Fatal("Num of image annotation suggestion history entries do not match!") + } + + log.Info("Verification successful") + log.Info("") + log.Info("Statistics:") + log.Info("Unrecoverable deleted annotations: ", unrecoverableDeletedAnnotations) + log.Info("Unrecoverable deleted annotation data: ", unrecoverableDeletedAnnotationData) + log.Info("Num of deleted label suggestions: ", numOfDeleteLabelSuggestions) + log.Info("Num of deleted image label suggestions: ", numOfDeletedImageLabelSuggestions) + log.Info("Num of deleted image annotation suggestion history entries: ", numOfDeletedImageAnnotationSuggestionHistoryEntries) + log.Info("Num of updated image annotation suggestion entries: ", numOfUpdatedImageAnnotationSuggestionEntries) + log.Info("Num of updated image label suggestion entries: ", numOfUpdatedImageLabelSuggestionEntries) + log.Info("Num of deleted trending label suggestion entries: ", numOfDeletedTrendingLabelSuggestionEntries) + log.Info("Num of deleted image label suggestion entries: ", numOfDeletedImageLabelSuggestionEntries) + log.Info("Num of deleted image annotation suggestion data entries: ", numOfDeletedAnnotationSuggestionDataEntries) + log.Info("Num of deleted user image annotation suggestion entries: ", numOfDeletedUserImageAnnotationSuggestionEntries) + log.Info("Num of deleted image annotation suggestion entries: ", numOfDeletedImageAnnotationSuggestionEntries) + log.Info("-----------------------------------") + log.Info("") + + log.Info("The following ", len(githubIssueIds) , " github issues can be closed:") + for _, githubIssueId := range githubIssueIds { + log.Info(githubIssueId) + } + + err = enableTriggers(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't disable triggers: ", err.Error()) + } + + err = enableIndexScans(tx) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't enable index scans: ", err.Error()) + } + + err = persistDuplicateLabels(duplicateLabelSuggestions) + if err != nil { + tx.Rollback(context.TODO()) + log.Fatal("Couldn't persist duplicate labels: ", err.Error()) + } + + if *dryRun { + log.Info("Just a dry run..rolling back transaction") + err := tx.Rollback(context.TODO()) + if err != nil { + log.Fatal("Couldn't rollback transaction: ", err.Error()) + } + } else { + err := tx.Commit(context.TODO()) + if err != nil { + log.Fatal("Couldn't commit transaction: ", err.Error()) + } + + + unclosedGithubIssues := []int64{} + if *closeIssue { + for _, githubIssueId := range githubIssueIds { + err := closeGithubIssue(int(githubIssueId), *githubRepository, githubProjectOwner, githubApiToken) + if err != nil { + unclosedGithubIssues = append(unclosedGithubIssues, githubIssueId) + } + } + } + + log.Error("The following github issues couldn't be closed: ") + for _, unclosedGithubIssue := range unclosedGithubIssues { + log.Error(unclosedGithubIssue) + } + + log.Info("done") + } + +} diff --git a/src/parser/v2/bin/antlr4_go.sh b/src/parser/v2/bin/antlr4_go.sh new file mode 100644 index 00000000..55d0bd5a --- /dev/null +++ b/src/parser/v2/bin/antlr4_go.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export CLASSPATH=antlr-4.7.1-complete.jar:$CLASSPATH + +java org.antlr.v4.Tool -package imagemonkeyquerylang -Dlanguage=Go ../grammar/ImagemonkeyQueryLang.g4 + +cp ../grammar/imagemonkeyquerylang_base_listener.go ../ +cp ../grammar/imagemonkeyquerylang_lexer.go ../ +cp ../grammar/imagemonkeyquerylang_listener.go ../ +cp ../grammar/imagemonkeyquerylang_parser.go ../ diff --git a/src/parser/v2/grammar/ImagemonkeyQueryLang.g4 b/src/parser/v2/grammar/ImagemonkeyQueryLang.g4 index 49abe971..ff3a9a3a 100644 --- a/src/parser/v2/grammar/ImagemonkeyQueryLang.g4 +++ b/src/parser/v2/grammar/ImagemonkeyQueryLang.g4 @@ -39,6 +39,8 @@ fragment UPPERLOWERCASE : [a-zA-Z]; fragment UPPERLOWERCASEWS : [a-zA-Z ]; fragment UUIDBLOCK : [A-Za-z0-9] [A-Za-z0-9] [A-Za-z0-9] [A-Za-z0-9]; fragment WS : ' '; +fragment UNDERSCORE : '_'; +fragment SLASH : '/'; fragment DESC : [Dd] [Ee] [Ss] [Cc]; fragment ASC : [Aa] [Ss] [Cc]; SEP : '!'; @@ -54,7 +56,7 @@ ORDER_BY : [Oo] [Rr] [Dd] [Ee] [Rr] WS+ [Bb] [Yy]; ORDER_BY_VALIDATION_DESC : ORDER_BY WS+ 'validation' WS+ DESC; ORDER_BY_VALIDATION_ASC : ORDER_BY WS+ 'validation' WS+ ASC; ORDER_BY_VALIDATION : ORDER_BY WS+ 'validation'; -LABEL : UPPERLOWERCASE | (UPPERLOWERCASE (WS | UPPERLOWERCASE)* UPPERLOWERCASE); +LABEL : UPPERLOWERCASE | (UPPERLOWERCASE (WS | UPPERLOWERCASE | UNDERSCORE | SLASH)* UPPERLOWERCASE); UUID : UUIDBLOCK UUIDBLOCK '-' UUIDBLOCK '-' UUIDBLOCK '-' UUIDBLOCK '-' UUIDBLOCK UUIDBLOCK UUIDBLOCK; VAL : [0-9]+; AND : '&'; @@ -63,4 +65,4 @@ NOT : '~'; LPAR : '('; RPAR : ')'; -SKIPPED_TOKENS : [ \t\r\n]+ -> skip ; // skip spaces, tabs, newlines \ No newline at end of file +SKIPPED_TOKENS : [ \t\r\n]+ -> skip ; // skip spaces, tabs, newlines diff --git a/src/parser/v2/imagemonkeyquerylang_lexer.go b/src/parser/v2/imagemonkeyquerylang_lexer.go index 9bd921d6..3e8b28c7 100644 --- a/src/parser/v2/imagemonkeyquerylang_lexer.go +++ b/src/parser/v2/imagemonkeyquerylang_lexer.go @@ -14,145 +14,149 @@ var _ = fmt.Printf var _ = unicode.IsLetter var serializedLexerAtn = []uint16{ - 3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 23, 305, + 3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 2, 23, 315, 8, 1, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 4, 9, 9, 9, 4, 10, 9, 10, 4, 11, 9, 11, 4, 12, 9, 12, 4, 13, 9, 13, 4, 14, 9, 14, 4, 15, 9, 15, 4, 16, 9, 16, 4, 17, 9, 17, 4, 18, 9, 18, 4, 19, 9, 19, 4, 20, 9, 20, 4, 21, 9, 21, 4, 22, 9, 22, 4, 23, 9, 23, 4, 24, 9, 24, 4, 25, 9, 25, 4, 26, 9, 26, 4, 27, 9, 27, 4, 28, 9, - 28, 4, 29, 9, 29, 4, 30, 9, 30, 3, 2, 3, 2, 3, 3, 3, 3, 3, 4, 3, 4, 3, - 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 7, 3, 7, 3, 8, 3, 8, 3, 8, 3, - 8, 3, 8, 3, 9, 3, 9, 3, 9, 3, 9, 3, 10, 3, 10, 3, 11, 3, 11, 3, 11, 3, - 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, - 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 12, 3, 12, 3, - 12, 3, 12, 3, 12, 3, 12, 3, 12, 3, 12, 3, 12, 3, 12, 3, 13, 3, 13, 3, 13, - 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, - 14, 3, 14, 3, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, 3, 16, - 5, 16, 144, 10, 16, 3, 17, 6, 17, 147, 10, 17, 13, 17, 14, 17, 148, 3, - 17, 3, 17, 6, 17, 153, 10, 17, 13, 17, 14, 17, 154, 5, 17, 157, 10, 17, - 3, 17, 7, 17, 160, 10, 17, 12, 17, 14, 17, 163, 11, 17, 3, 17, 3, 17, 7, - 17, 167, 10, 17, 12, 17, 14, 17, 170, 11, 17, 3, 17, 3, 17, 6, 17, 174, - 10, 17, 13, 17, 14, 17, 175, 3, 17, 3, 17, 3, 18, 3, 18, 3, 18, 3, 18, - 3, 18, 3, 18, 6, 18, 186, 10, 18, 13, 18, 14, 18, 187, 3, 18, 3, 18, 3, - 18, 3, 19, 3, 19, 6, 19, 195, 10, 19, 13, 19, 14, 19, 196, 3, 19, 3, 19, - 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 3, 19, 6, - 19, 211, 10, 19, 13, 19, 14, 19, 212, 3, 19, 3, 19, 3, 20, 3, 20, 6, 20, - 219, 10, 20, 13, 20, 14, 20, 220, 3, 20, 3, 20, 3, 20, 3, 20, 3, 20, 3, - 20, 3, 20, 3, 20, 3, 20, 3, 20, 3, 20, 3, 20, 6, 20, 235, 10, 20, 13, 20, - 14, 20, 236, 3, 20, 3, 20, 3, 21, 3, 21, 6, 21, 243, 10, 21, 13, 21, 14, - 21, 244, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, - 3, 21, 3, 21, 3, 22, 3, 22, 3, 22, 3, 22, 7, 22, 262, 10, 22, 12, 22, 14, - 22, 265, 11, 22, 3, 22, 3, 22, 5, 22, 269, 10, 22, 3, 23, 3, 23, 3, 23, - 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, - 24, 6, 24, 285, 10, 24, 13, 24, 14, 24, 286, 3, 25, 3, 25, 3, 26, 3, 26, - 3, 27, 3, 27, 3, 28, 3, 28, 3, 29, 3, 29, 3, 30, 6, 30, 300, 10, 30, 13, - 30, 14, 30, 301, 3, 30, 3, 30, 2, 2, 31, 3, 2, 5, 2, 7, 2, 9, 2, 11, 2, - 13, 2, 15, 2, 17, 2, 19, 3, 21, 4, 23, 5, 25, 6, 27, 7, 29, 8, 31, 9, 33, - 10, 35, 11, 37, 12, 39, 13, 41, 14, 43, 15, 45, 16, 47, 17, 49, 18, 51, - 19, 53, 20, 55, 21, 57, 22, 59, 23, 3, 2, 19, 3, 2, 99, 124, 3, 2, 67, - 92, 4, 2, 67, 92, 99, 124, 5, 2, 34, 34, 67, 92, 99, 124, 5, 2, 50, 59, - 67, 92, 99, 124, 4, 2, 70, 70, 102, 102, 4, 2, 71, 71, 103, 103, 4, 2, - 85, 85, 117, 117, 4, 2, 69, 69, 101, 101, 4, 2, 67, 67, 99, 99, 4, 2, 62, - 62, 64, 64, 4, 2, 81, 81, 113, 113, 4, 2, 84, 84, 116, 116, 4, 2, 68, 68, - 100, 100, 4, 2, 91, 91, 123, 123, 3, 2, 50, 59, 5, 2, 11, 12, 15, 15, 34, - 34, 2, 316, 2, 19, 3, 2, 2, 2, 2, 21, 3, 2, 2, 2, 2, 23, 3, 2, 2, 2, 2, - 25, 3, 2, 2, 2, 2, 27, 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, - 2, 33, 3, 2, 2, 2, 2, 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, - 2, 2, 41, 3, 2, 2, 2, 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, - 2, 2, 2, 49, 3, 2, 2, 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, - 2, 2, 2, 2, 57, 3, 2, 2, 2, 2, 59, 3, 2, 2, 2, 3, 61, 3, 2, 2, 2, 5, 63, - 3, 2, 2, 2, 7, 65, 3, 2, 2, 2, 9, 67, 3, 2, 2, 2, 11, 69, 3, 2, 2, 2, 13, - 74, 3, 2, 2, 2, 15, 76, 3, 2, 2, 2, 17, 81, 3, 2, 2, 2, 19, 85, 3, 2, 2, - 2, 21, 87, 3, 2, 2, 2, 23, 107, 3, 2, 2, 2, 25, 119, 3, 2, 2, 2, 27, 132, - 3, 2, 2, 2, 29, 134, 3, 2, 2, 2, 31, 143, 3, 2, 2, 2, 33, 146, 3, 2, 2, - 2, 35, 179, 3, 2, 2, 2, 37, 192, 3, 2, 2, 2, 39, 216, 3, 2, 2, 2, 41, 240, - 3, 2, 2, 2, 43, 268, 3, 2, 2, 2, 45, 270, 3, 2, 2, 2, 47, 284, 3, 2, 2, - 2, 49, 288, 3, 2, 2, 2, 51, 290, 3, 2, 2, 2, 53, 292, 3, 2, 2, 2, 55, 294, - 3, 2, 2, 2, 57, 296, 3, 2, 2, 2, 59, 299, 3, 2, 2, 2, 61, 62, 9, 2, 2, - 2, 62, 4, 3, 2, 2, 2, 63, 64, 9, 3, 2, 2, 64, 6, 3, 2, 2, 2, 65, 66, 9, - 4, 2, 2, 66, 8, 3, 2, 2, 2, 67, 68, 9, 5, 2, 2, 68, 10, 3, 2, 2, 2, 69, - 70, 9, 6, 2, 2, 70, 71, 9, 6, 2, 2, 71, 72, 9, 6, 2, 2, 72, 73, 9, 6, 2, - 2, 73, 12, 3, 2, 2, 2, 74, 75, 7, 34, 2, 2, 75, 14, 3, 2, 2, 2, 76, 77, - 9, 7, 2, 2, 77, 78, 9, 8, 2, 2, 78, 79, 9, 9, 2, 2, 79, 80, 9, 10, 2, 2, - 80, 16, 3, 2, 2, 2, 81, 82, 9, 11, 2, 2, 82, 83, 9, 9, 2, 2, 83, 84, 9, - 10, 2, 2, 84, 18, 3, 2, 2, 2, 85, 86, 7, 35, 2, 2, 86, 20, 3, 2, 2, 2, - 87, 88, 7, 99, 2, 2, 88, 89, 7, 112, 2, 2, 89, 90, 7, 112, 2, 2, 90, 91, - 7, 113, 2, 2, 91, 92, 7, 118, 2, 2, 92, 93, 7, 99, 2, 2, 93, 94, 7, 118, - 2, 2, 94, 95, 7, 107, 2, 2, 95, 96, 7, 113, 2, 2, 96, 97, 7, 112, 2, 2, - 97, 98, 7, 48, 2, 2, 98, 99, 7, 101, 2, 2, 99, 100, 7, 113, 2, 2, 100, - 101, 7, 120, 2, 2, 101, 102, 7, 103, 2, 2, 102, 103, 7, 116, 2, 2, 103, - 104, 7, 99, 2, 2, 104, 105, 7, 105, 2, 2, 105, 106, 7, 103, 2, 2, 106, - 22, 3, 2, 2, 2, 107, 108, 7, 107, 2, 2, 108, 109, 7, 111, 2, 2, 109, 110, - 7, 99, 2, 2, 110, 111, 7, 105, 2, 2, 111, 112, 7, 103, 2, 2, 112, 113, - 7, 48, 2, 2, 113, 114, 7, 121, 2, 2, 114, 115, 7, 107, 2, 2, 115, 116, - 7, 102, 2, 2, 116, 117, 7, 118, 2, 2, 117, 118, 7, 106, 2, 2, 118, 24, - 3, 2, 2, 2, 119, 120, 7, 107, 2, 2, 120, 121, 7, 111, 2, 2, 121, 122, 7, - 99, 2, 2, 122, 123, 7, 105, 2, 2, 123, 124, 7, 103, 2, 2, 124, 125, 7, - 48, 2, 2, 125, 126, 7, 106, 2, 2, 126, 127, 7, 103, 2, 2, 127, 128, 7, - 107, 2, 2, 128, 129, 7, 105, 2, 2, 129, 130, 7, 106, 2, 2, 130, 131, 7, - 118, 2, 2, 131, 26, 3, 2, 2, 2, 132, 133, 7, 39, 2, 2, 133, 28, 3, 2, 2, - 2, 134, 135, 7, 114, 2, 2, 135, 136, 7, 122, 2, 2, 136, 30, 3, 2, 2, 2, - 137, 144, 9, 12, 2, 2, 138, 139, 7, 64, 2, 2, 139, 144, 7, 63, 2, 2, 140, - 144, 7, 63, 2, 2, 141, 142, 7, 62, 2, 2, 142, 144, 7, 63, 2, 2, 143, 137, - 3, 2, 2, 2, 143, 138, 3, 2, 2, 2, 143, 140, 3, 2, 2, 2, 143, 141, 3, 2, - 2, 2, 144, 32, 3, 2, 2, 2, 145, 147, 5, 7, 4, 2, 146, 145, 3, 2, 2, 2, - 147, 148, 3, 2, 2, 2, 148, 146, 3, 2, 2, 2, 148, 149, 3, 2, 2, 2, 149, - 156, 3, 2, 2, 2, 150, 152, 7, 48, 2, 2, 151, 153, 5, 7, 4, 2, 152, 151, - 3, 2, 2, 2, 153, 154, 3, 2, 2, 2, 154, 152, 3, 2, 2, 2, 154, 155, 3, 2, - 2, 2, 155, 157, 3, 2, 2, 2, 156, 150, 3, 2, 2, 2, 156, 157, 3, 2, 2, 2, - 157, 161, 3, 2, 2, 2, 158, 160, 5, 13, 7, 2, 159, 158, 3, 2, 2, 2, 160, - 163, 3, 2, 2, 2, 161, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 164, - 3, 2, 2, 2, 163, 161, 3, 2, 2, 2, 164, 168, 7, 63, 2, 2, 165, 167, 5, 13, - 7, 2, 166, 165, 3, 2, 2, 2, 167, 170, 3, 2, 2, 2, 168, 166, 3, 2, 2, 2, - 168, 169, 3, 2, 2, 2, 169, 171, 3, 2, 2, 2, 170, 168, 3, 2, 2, 2, 171, - 173, 7, 41, 2, 2, 172, 174, 5, 9, 5, 2, 173, 172, 3, 2, 2, 2, 174, 175, - 3, 2, 2, 2, 175, 173, 3, 2, 2, 2, 175, 176, 3, 2, 2, 2, 176, 177, 3, 2, - 2, 2, 177, 178, 7, 41, 2, 2, 178, 34, 3, 2, 2, 2, 179, 180, 9, 13, 2, 2, - 180, 181, 9, 14, 2, 2, 181, 182, 9, 7, 2, 2, 182, 183, 9, 8, 2, 2, 183, - 185, 9, 14, 2, 2, 184, 186, 5, 13, 7, 2, 185, 184, 3, 2, 2, 2, 186, 187, - 3, 2, 2, 2, 187, 185, 3, 2, 2, 2, 187, 188, 3, 2, 2, 2, 188, 189, 3, 2, - 2, 2, 189, 190, 9, 15, 2, 2, 190, 191, 9, 16, 2, 2, 191, 36, 3, 2, 2, 2, - 192, 194, 5, 35, 18, 2, 193, 195, 5, 13, 7, 2, 194, 193, 3, 2, 2, 2, 195, - 196, 3, 2, 2, 2, 196, 194, 3, 2, 2, 2, 196, 197, 3, 2, 2, 2, 197, 198, - 3, 2, 2, 2, 198, 199, 7, 120, 2, 2, 199, 200, 7, 99, 2, 2, 200, 201, 7, - 110, 2, 2, 201, 202, 7, 107, 2, 2, 202, 203, 7, 102, 2, 2, 203, 204, 7, - 99, 2, 2, 204, 205, 7, 118, 2, 2, 205, 206, 7, 107, 2, 2, 206, 207, 7, - 113, 2, 2, 207, 208, 7, 112, 2, 2, 208, 210, 3, 2, 2, 2, 209, 211, 5, 13, - 7, 2, 210, 209, 3, 2, 2, 2, 211, 212, 3, 2, 2, 2, 212, 210, 3, 2, 2, 2, - 212, 213, 3, 2, 2, 2, 213, 214, 3, 2, 2, 2, 214, 215, 5, 15, 8, 2, 215, - 38, 3, 2, 2, 2, 216, 218, 5, 35, 18, 2, 217, 219, 5, 13, 7, 2, 218, 217, - 3, 2, 2, 2, 219, 220, 3, 2, 2, 2, 220, 218, 3, 2, 2, 2, 220, 221, 3, 2, - 2, 2, 221, 222, 3, 2, 2, 2, 222, 223, 7, 120, 2, 2, 223, 224, 7, 99, 2, - 2, 224, 225, 7, 110, 2, 2, 225, 226, 7, 107, 2, 2, 226, 227, 7, 102, 2, - 2, 227, 228, 7, 99, 2, 2, 228, 229, 7, 118, 2, 2, 229, 230, 7, 107, 2, - 2, 230, 231, 7, 113, 2, 2, 231, 232, 7, 112, 2, 2, 232, 234, 3, 2, 2, 2, - 233, 235, 5, 13, 7, 2, 234, 233, 3, 2, 2, 2, 235, 236, 3, 2, 2, 2, 236, - 234, 3, 2, 2, 2, 236, 237, 3, 2, 2, 2, 237, 238, 3, 2, 2, 2, 238, 239, - 5, 17, 9, 2, 239, 40, 3, 2, 2, 2, 240, 242, 5, 35, 18, 2, 241, 243, 5, - 13, 7, 2, 242, 241, 3, 2, 2, 2, 243, 244, 3, 2, 2, 2, 244, 242, 3, 2, 2, - 2, 244, 245, 3, 2, 2, 2, 245, 246, 3, 2, 2, 2, 246, 247, 7, 120, 2, 2, - 247, 248, 7, 99, 2, 2, 248, 249, 7, 110, 2, 2, 249, 250, 7, 107, 2, 2, - 250, 251, 7, 102, 2, 2, 251, 252, 7, 99, 2, 2, 252, 253, 7, 118, 2, 2, - 253, 254, 7, 107, 2, 2, 254, 255, 7, 113, 2, 2, 255, 256, 7, 112, 2, 2, - 256, 42, 3, 2, 2, 2, 257, 269, 5, 7, 4, 2, 258, 263, 5, 7, 4, 2, 259, 262, - 5, 13, 7, 2, 260, 262, 5, 7, 4, 2, 261, 259, 3, 2, 2, 2, 261, 260, 3, 2, - 2, 2, 262, 265, 3, 2, 2, 2, 263, 261, 3, 2, 2, 2, 263, 264, 3, 2, 2, 2, - 264, 266, 3, 2, 2, 2, 265, 263, 3, 2, 2, 2, 266, 267, 5, 7, 4, 2, 267, - 269, 3, 2, 2, 2, 268, 257, 3, 2, 2, 2, 268, 258, 3, 2, 2, 2, 269, 44, 3, - 2, 2, 2, 270, 271, 5, 11, 6, 2, 271, 272, 5, 11, 6, 2, 272, 273, 7, 47, - 2, 2, 273, 274, 5, 11, 6, 2, 274, 275, 7, 47, 2, 2, 275, 276, 5, 11, 6, - 2, 276, 277, 7, 47, 2, 2, 277, 278, 5, 11, 6, 2, 278, 279, 7, 47, 2, 2, - 279, 280, 5, 11, 6, 2, 280, 281, 5, 11, 6, 2, 281, 282, 5, 11, 6, 2, 282, - 46, 3, 2, 2, 2, 283, 285, 9, 17, 2, 2, 284, 283, 3, 2, 2, 2, 285, 286, - 3, 2, 2, 2, 286, 284, 3, 2, 2, 2, 286, 287, 3, 2, 2, 2, 287, 48, 3, 2, - 2, 2, 288, 289, 7, 40, 2, 2, 289, 50, 3, 2, 2, 2, 290, 291, 7, 126, 2, - 2, 291, 52, 3, 2, 2, 2, 292, 293, 7, 128, 2, 2, 293, 54, 3, 2, 2, 2, 294, - 295, 7, 42, 2, 2, 295, 56, 3, 2, 2, 2, 296, 297, 7, 43, 2, 2, 297, 58, - 3, 2, 2, 2, 298, 300, 9, 18, 2, 2, 299, 298, 3, 2, 2, 2, 300, 301, 3, 2, - 2, 2, 301, 299, 3, 2, 2, 2, 301, 302, 3, 2, 2, 2, 302, 303, 3, 2, 2, 2, - 303, 304, 8, 30, 2, 2, 304, 60, 3, 2, 2, 2, 21, 2, 143, 148, 154, 156, - 161, 168, 175, 187, 196, 212, 220, 236, 244, 261, 263, 268, 286, 301, 3, - 8, 2, 2, + 28, 4, 29, 9, 29, 4, 30, 9, 30, 4, 31, 9, 31, 4, 32, 9, 32, 3, 2, 3, 2, + 3, 3, 3, 3, 3, 4, 3, 4, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 3, 7, + 3, 7, 3, 8, 3, 8, 3, 9, 3, 9, 3, 10, 3, 10, 3, 10, 3, 10, 3, 10, 3, 11, + 3, 11, 3, 11, 3, 11, 3, 12, 3, 12, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, + 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, 3, 13, + 3, 13, 3, 13, 3, 13, 3, 13, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, + 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 14, 3, 15, 3, 15, 3, 15, 3, 15, 3, 15, + 3, 15, 3, 15, 3, 15, 3, 15, 3, 15, 3, 15, 3, 15, 3, 15, 3, 16, 3, 16, 3, + 17, 3, 17, 3, 17, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 3, 18, 5, 18, 152, + 10, 18, 3, 19, 6, 19, 155, 10, 19, 13, 19, 14, 19, 156, 3, 19, 3, 19, 6, + 19, 161, 10, 19, 13, 19, 14, 19, 162, 5, 19, 165, 10, 19, 3, 19, 7, 19, + 168, 10, 19, 12, 19, 14, 19, 171, 11, 19, 3, 19, 3, 19, 7, 19, 175, 10, + 19, 12, 19, 14, 19, 178, 11, 19, 3, 19, 3, 19, 6, 19, 182, 10, 19, 13, + 19, 14, 19, 183, 3, 19, 3, 19, 3, 20, 3, 20, 3, 20, 3, 20, 3, 20, 3, 20, + 6, 20, 194, 10, 20, 13, 20, 14, 20, 195, 3, 20, 3, 20, 3, 20, 3, 21, 3, + 21, 6, 21, 203, 10, 21, 13, 21, 14, 21, 204, 3, 21, 3, 21, 3, 21, 3, 21, + 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 3, 21, 6, 21, 219, 10, + 21, 13, 21, 14, 21, 220, 3, 21, 3, 21, 3, 22, 3, 22, 6, 22, 227, 10, 22, + 13, 22, 14, 22, 228, 3, 22, 3, 22, 3, 22, 3, 22, 3, 22, 3, 22, 3, 22, 3, + 22, 3, 22, 3, 22, 3, 22, 3, 22, 6, 22, 243, 10, 22, 13, 22, 14, 22, 244, + 3, 22, 3, 22, 3, 23, 3, 23, 6, 23, 251, 10, 23, 13, 23, 14, 23, 252, 3, + 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, 3, 23, + 3, 24, 3, 24, 3, 24, 3, 24, 3, 24, 3, 24, 7, 24, 272, 10, 24, 12, 24, 14, + 24, 275, 11, 24, 3, 24, 3, 24, 5, 24, 279, 10, 24, 3, 25, 3, 25, 3, 25, + 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, 25, 3, + 26, 6, 26, 295, 10, 26, 13, 26, 14, 26, 296, 3, 27, 3, 27, 3, 28, 3, 28, + 3, 29, 3, 29, 3, 30, 3, 30, 3, 31, 3, 31, 3, 32, 6, 32, 310, 10, 32, 13, + 32, 14, 32, 311, 3, 32, 3, 32, 2, 2, 33, 3, 2, 5, 2, 7, 2, 9, 2, 11, 2, + 13, 2, 15, 2, 17, 2, 19, 2, 21, 2, 23, 3, 25, 4, 27, 5, 29, 6, 31, 7, 33, + 8, 35, 9, 37, 10, 39, 11, 41, 12, 43, 13, 45, 14, 47, 15, 49, 16, 51, 17, + 53, 18, 55, 19, 57, 20, 59, 21, 61, 22, 63, 23, 3, 2, 19, 3, 2, 99, 124, + 3, 2, 67, 92, 4, 2, 67, 92, 99, 124, 5, 2, 34, 34, 67, 92, 99, 124, 5, + 2, 50, 59, 67, 92, 99, 124, 4, 2, 70, 70, 102, 102, 4, 2, 71, 71, 103, + 103, 4, 2, 85, 85, 117, 117, 4, 2, 69, 69, 101, 101, 4, 2, 67, 67, 99, + 99, 4, 2, 62, 62, 64, 64, 4, 2, 81, 81, 113, 113, 4, 2, 84, 84, 116, 116, + 4, 2, 68, 68, 100, 100, 4, 2, 91, 91, 123, 123, 3, 2, 50, 59, 5, 2, 11, + 12, 15, 15, 34, 34, 2, 326, 2, 23, 3, 2, 2, 2, 2, 25, 3, 2, 2, 2, 2, 27, + 3, 2, 2, 2, 2, 29, 3, 2, 2, 2, 2, 31, 3, 2, 2, 2, 2, 33, 3, 2, 2, 2, 2, + 35, 3, 2, 2, 2, 2, 37, 3, 2, 2, 2, 2, 39, 3, 2, 2, 2, 2, 41, 3, 2, 2, 2, + 2, 43, 3, 2, 2, 2, 2, 45, 3, 2, 2, 2, 2, 47, 3, 2, 2, 2, 2, 49, 3, 2, 2, + 2, 2, 51, 3, 2, 2, 2, 2, 53, 3, 2, 2, 2, 2, 55, 3, 2, 2, 2, 2, 57, 3, 2, + 2, 2, 2, 59, 3, 2, 2, 2, 2, 61, 3, 2, 2, 2, 2, 63, 3, 2, 2, 2, 3, 65, 3, + 2, 2, 2, 5, 67, 3, 2, 2, 2, 7, 69, 3, 2, 2, 2, 9, 71, 3, 2, 2, 2, 11, 73, + 3, 2, 2, 2, 13, 78, 3, 2, 2, 2, 15, 80, 3, 2, 2, 2, 17, 82, 3, 2, 2, 2, + 19, 84, 3, 2, 2, 2, 21, 89, 3, 2, 2, 2, 23, 93, 3, 2, 2, 2, 25, 95, 3, + 2, 2, 2, 27, 115, 3, 2, 2, 2, 29, 127, 3, 2, 2, 2, 31, 140, 3, 2, 2, 2, + 33, 142, 3, 2, 2, 2, 35, 151, 3, 2, 2, 2, 37, 154, 3, 2, 2, 2, 39, 187, + 3, 2, 2, 2, 41, 200, 3, 2, 2, 2, 43, 224, 3, 2, 2, 2, 45, 248, 3, 2, 2, + 2, 47, 278, 3, 2, 2, 2, 49, 280, 3, 2, 2, 2, 51, 294, 3, 2, 2, 2, 53, 298, + 3, 2, 2, 2, 55, 300, 3, 2, 2, 2, 57, 302, 3, 2, 2, 2, 59, 304, 3, 2, 2, + 2, 61, 306, 3, 2, 2, 2, 63, 309, 3, 2, 2, 2, 65, 66, 9, 2, 2, 2, 66, 4, + 3, 2, 2, 2, 67, 68, 9, 3, 2, 2, 68, 6, 3, 2, 2, 2, 69, 70, 9, 4, 2, 2, + 70, 8, 3, 2, 2, 2, 71, 72, 9, 5, 2, 2, 72, 10, 3, 2, 2, 2, 73, 74, 9, 6, + 2, 2, 74, 75, 9, 6, 2, 2, 75, 76, 9, 6, 2, 2, 76, 77, 9, 6, 2, 2, 77, 12, + 3, 2, 2, 2, 78, 79, 7, 34, 2, 2, 79, 14, 3, 2, 2, 2, 80, 81, 7, 97, 2, + 2, 81, 16, 3, 2, 2, 2, 82, 83, 7, 49, 2, 2, 83, 18, 3, 2, 2, 2, 84, 85, + 9, 7, 2, 2, 85, 86, 9, 8, 2, 2, 86, 87, 9, 9, 2, 2, 87, 88, 9, 10, 2, 2, + 88, 20, 3, 2, 2, 2, 89, 90, 9, 11, 2, 2, 90, 91, 9, 9, 2, 2, 91, 92, 9, + 10, 2, 2, 92, 22, 3, 2, 2, 2, 93, 94, 7, 35, 2, 2, 94, 24, 3, 2, 2, 2, + 95, 96, 7, 99, 2, 2, 96, 97, 7, 112, 2, 2, 97, 98, 7, 112, 2, 2, 98, 99, + 7, 113, 2, 2, 99, 100, 7, 118, 2, 2, 100, 101, 7, 99, 2, 2, 101, 102, 7, + 118, 2, 2, 102, 103, 7, 107, 2, 2, 103, 104, 7, 113, 2, 2, 104, 105, 7, + 112, 2, 2, 105, 106, 7, 48, 2, 2, 106, 107, 7, 101, 2, 2, 107, 108, 7, + 113, 2, 2, 108, 109, 7, 120, 2, 2, 109, 110, 7, 103, 2, 2, 110, 111, 7, + 116, 2, 2, 111, 112, 7, 99, 2, 2, 112, 113, 7, 105, 2, 2, 113, 114, 7, + 103, 2, 2, 114, 26, 3, 2, 2, 2, 115, 116, 7, 107, 2, 2, 116, 117, 7, 111, + 2, 2, 117, 118, 7, 99, 2, 2, 118, 119, 7, 105, 2, 2, 119, 120, 7, 103, + 2, 2, 120, 121, 7, 48, 2, 2, 121, 122, 7, 121, 2, 2, 122, 123, 7, 107, + 2, 2, 123, 124, 7, 102, 2, 2, 124, 125, 7, 118, 2, 2, 125, 126, 7, 106, + 2, 2, 126, 28, 3, 2, 2, 2, 127, 128, 7, 107, 2, 2, 128, 129, 7, 111, 2, + 2, 129, 130, 7, 99, 2, 2, 130, 131, 7, 105, 2, 2, 131, 132, 7, 103, 2, + 2, 132, 133, 7, 48, 2, 2, 133, 134, 7, 106, 2, 2, 134, 135, 7, 103, 2, + 2, 135, 136, 7, 107, 2, 2, 136, 137, 7, 105, 2, 2, 137, 138, 7, 106, 2, + 2, 138, 139, 7, 118, 2, 2, 139, 30, 3, 2, 2, 2, 140, 141, 7, 39, 2, 2, + 141, 32, 3, 2, 2, 2, 142, 143, 7, 114, 2, 2, 143, 144, 7, 122, 2, 2, 144, + 34, 3, 2, 2, 2, 145, 152, 9, 12, 2, 2, 146, 147, 7, 64, 2, 2, 147, 152, + 7, 63, 2, 2, 148, 152, 7, 63, 2, 2, 149, 150, 7, 62, 2, 2, 150, 152, 7, + 63, 2, 2, 151, 145, 3, 2, 2, 2, 151, 146, 3, 2, 2, 2, 151, 148, 3, 2, 2, + 2, 151, 149, 3, 2, 2, 2, 152, 36, 3, 2, 2, 2, 153, 155, 5, 7, 4, 2, 154, + 153, 3, 2, 2, 2, 155, 156, 3, 2, 2, 2, 156, 154, 3, 2, 2, 2, 156, 157, + 3, 2, 2, 2, 157, 164, 3, 2, 2, 2, 158, 160, 7, 48, 2, 2, 159, 161, 5, 7, + 4, 2, 160, 159, 3, 2, 2, 2, 161, 162, 3, 2, 2, 2, 162, 160, 3, 2, 2, 2, + 162, 163, 3, 2, 2, 2, 163, 165, 3, 2, 2, 2, 164, 158, 3, 2, 2, 2, 164, + 165, 3, 2, 2, 2, 165, 169, 3, 2, 2, 2, 166, 168, 5, 13, 7, 2, 167, 166, + 3, 2, 2, 2, 168, 171, 3, 2, 2, 2, 169, 167, 3, 2, 2, 2, 169, 170, 3, 2, + 2, 2, 170, 172, 3, 2, 2, 2, 171, 169, 3, 2, 2, 2, 172, 176, 7, 63, 2, 2, + 173, 175, 5, 13, 7, 2, 174, 173, 3, 2, 2, 2, 175, 178, 3, 2, 2, 2, 176, + 174, 3, 2, 2, 2, 176, 177, 3, 2, 2, 2, 177, 179, 3, 2, 2, 2, 178, 176, + 3, 2, 2, 2, 179, 181, 7, 41, 2, 2, 180, 182, 5, 9, 5, 2, 181, 180, 3, 2, + 2, 2, 182, 183, 3, 2, 2, 2, 183, 181, 3, 2, 2, 2, 183, 184, 3, 2, 2, 2, + 184, 185, 3, 2, 2, 2, 185, 186, 7, 41, 2, 2, 186, 38, 3, 2, 2, 2, 187, + 188, 9, 13, 2, 2, 188, 189, 9, 14, 2, 2, 189, 190, 9, 7, 2, 2, 190, 191, + 9, 8, 2, 2, 191, 193, 9, 14, 2, 2, 192, 194, 5, 13, 7, 2, 193, 192, 3, + 2, 2, 2, 194, 195, 3, 2, 2, 2, 195, 193, 3, 2, 2, 2, 195, 196, 3, 2, 2, + 2, 196, 197, 3, 2, 2, 2, 197, 198, 9, 15, 2, 2, 198, 199, 9, 16, 2, 2, + 199, 40, 3, 2, 2, 2, 200, 202, 5, 39, 20, 2, 201, 203, 5, 13, 7, 2, 202, + 201, 3, 2, 2, 2, 203, 204, 3, 2, 2, 2, 204, 202, 3, 2, 2, 2, 204, 205, + 3, 2, 2, 2, 205, 206, 3, 2, 2, 2, 206, 207, 7, 120, 2, 2, 207, 208, 7, + 99, 2, 2, 208, 209, 7, 110, 2, 2, 209, 210, 7, 107, 2, 2, 210, 211, 7, + 102, 2, 2, 211, 212, 7, 99, 2, 2, 212, 213, 7, 118, 2, 2, 213, 214, 7, + 107, 2, 2, 214, 215, 7, 113, 2, 2, 215, 216, 7, 112, 2, 2, 216, 218, 3, + 2, 2, 2, 217, 219, 5, 13, 7, 2, 218, 217, 3, 2, 2, 2, 219, 220, 3, 2, 2, + 2, 220, 218, 3, 2, 2, 2, 220, 221, 3, 2, 2, 2, 221, 222, 3, 2, 2, 2, 222, + 223, 5, 19, 10, 2, 223, 42, 3, 2, 2, 2, 224, 226, 5, 39, 20, 2, 225, 227, + 5, 13, 7, 2, 226, 225, 3, 2, 2, 2, 227, 228, 3, 2, 2, 2, 228, 226, 3, 2, + 2, 2, 228, 229, 3, 2, 2, 2, 229, 230, 3, 2, 2, 2, 230, 231, 7, 120, 2, + 2, 231, 232, 7, 99, 2, 2, 232, 233, 7, 110, 2, 2, 233, 234, 7, 107, 2, + 2, 234, 235, 7, 102, 2, 2, 235, 236, 7, 99, 2, 2, 236, 237, 7, 118, 2, + 2, 237, 238, 7, 107, 2, 2, 238, 239, 7, 113, 2, 2, 239, 240, 7, 112, 2, + 2, 240, 242, 3, 2, 2, 2, 241, 243, 5, 13, 7, 2, 242, 241, 3, 2, 2, 2, 243, + 244, 3, 2, 2, 2, 244, 242, 3, 2, 2, 2, 244, 245, 3, 2, 2, 2, 245, 246, + 3, 2, 2, 2, 246, 247, 5, 21, 11, 2, 247, 44, 3, 2, 2, 2, 248, 250, 5, 39, + 20, 2, 249, 251, 5, 13, 7, 2, 250, 249, 3, 2, 2, 2, 251, 252, 3, 2, 2, + 2, 252, 250, 3, 2, 2, 2, 252, 253, 3, 2, 2, 2, 253, 254, 3, 2, 2, 2, 254, + 255, 7, 120, 2, 2, 255, 256, 7, 99, 2, 2, 256, 257, 7, 110, 2, 2, 257, + 258, 7, 107, 2, 2, 258, 259, 7, 102, 2, 2, 259, 260, 7, 99, 2, 2, 260, + 261, 7, 118, 2, 2, 261, 262, 7, 107, 2, 2, 262, 263, 7, 113, 2, 2, 263, + 264, 7, 112, 2, 2, 264, 46, 3, 2, 2, 2, 265, 279, 5, 7, 4, 2, 266, 273, + 5, 7, 4, 2, 267, 272, 5, 13, 7, 2, 268, 272, 5, 7, 4, 2, 269, 272, 5, 15, + 8, 2, 270, 272, 5, 17, 9, 2, 271, 267, 3, 2, 2, 2, 271, 268, 3, 2, 2, 2, + 271, 269, 3, 2, 2, 2, 271, 270, 3, 2, 2, 2, 272, 275, 3, 2, 2, 2, 273, + 271, 3, 2, 2, 2, 273, 274, 3, 2, 2, 2, 274, 276, 3, 2, 2, 2, 275, 273, + 3, 2, 2, 2, 276, 277, 5, 7, 4, 2, 277, 279, 3, 2, 2, 2, 278, 265, 3, 2, + 2, 2, 278, 266, 3, 2, 2, 2, 279, 48, 3, 2, 2, 2, 280, 281, 5, 11, 6, 2, + 281, 282, 5, 11, 6, 2, 282, 283, 7, 47, 2, 2, 283, 284, 5, 11, 6, 2, 284, + 285, 7, 47, 2, 2, 285, 286, 5, 11, 6, 2, 286, 287, 7, 47, 2, 2, 287, 288, + 5, 11, 6, 2, 288, 289, 7, 47, 2, 2, 289, 290, 5, 11, 6, 2, 290, 291, 5, + 11, 6, 2, 291, 292, 5, 11, 6, 2, 292, 50, 3, 2, 2, 2, 293, 295, 9, 17, + 2, 2, 294, 293, 3, 2, 2, 2, 295, 296, 3, 2, 2, 2, 296, 294, 3, 2, 2, 2, + 296, 297, 3, 2, 2, 2, 297, 52, 3, 2, 2, 2, 298, 299, 7, 40, 2, 2, 299, + 54, 3, 2, 2, 2, 300, 301, 7, 126, 2, 2, 301, 56, 3, 2, 2, 2, 302, 303, + 7, 128, 2, 2, 303, 58, 3, 2, 2, 2, 304, 305, 7, 42, 2, 2, 305, 60, 3, 2, + 2, 2, 306, 307, 7, 43, 2, 2, 307, 62, 3, 2, 2, 2, 308, 310, 9, 18, 2, 2, + 309, 308, 3, 2, 2, 2, 310, 311, 3, 2, 2, 2, 311, 309, 3, 2, 2, 2, 311, + 312, 3, 2, 2, 2, 312, 313, 3, 2, 2, 2, 313, 314, 8, 32, 2, 2, 314, 64, + 3, 2, 2, 2, 21, 2, 151, 156, 162, 164, 169, 176, 183, 195, 204, 220, 228, + 244, 252, 271, 273, 278, 296, 311, 3, 8, 2, 2, } var lexerDeserializer = antlr.NewATNDeserializer(nil) @@ -181,10 +185,11 @@ var lexerSymbolicNames = []string{ var lexerRuleNames = []string{ "LOWERCASE", "UPPERCASE", "UPPERLOWERCASE", "UPPERLOWERCASEWS", "UUIDBLOCK", - "WS", "DESC", "ASC", "SEP", "ANNOTATION_COVERAGE_PREFIX", "IMAGE_WIDTH_PREFIX", - "IMAGE_HEIGHT_PREFIX", "PERCENT", "PIXEL", "OPERATOR", "ASSIGNMENT", "ORDER_BY", - "ORDER_BY_VALIDATION_DESC", "ORDER_BY_VALIDATION_ASC", "ORDER_BY_VALIDATION", - "LABEL", "UUID", "VAL", "AND", "OR", "NOT", "LPAR", "RPAR", "SKIPPED_TOKENS", + "WS", "UNDERSCORE", "SLASH", "DESC", "ASC", "SEP", "ANNOTATION_COVERAGE_PREFIX", + "IMAGE_WIDTH_PREFIX", "IMAGE_HEIGHT_PREFIX", "PERCENT", "PIXEL", "OPERATOR", + "ASSIGNMENT", "ORDER_BY", "ORDER_BY_VALIDATION_DESC", "ORDER_BY_VALIDATION_ASC", + "ORDER_BY_VALIDATION", "LABEL", "UUID", "VAL", "AND", "OR", "NOT", "LPAR", + "RPAR", "SKIPPED_TOKENS", } type ImagemonkeyQueryLangLexer struct { diff --git a/src/parser/v2/parser_test.go b/src/parser/v2/parser_test.go index e7bb3037..5bd944ea 100644 --- a/src/parser/v2/parser_test.go +++ b/src/parser/v2/parser_test.go @@ -613,3 +613,27 @@ func TestQueryImageUnlabeledAndLabelShouldSucceed(t *testing.T) { equals(t, parseResult.Query, "q.accessors @> ARRAY[$1]::text[] OR is_unlabeled = $2") equals(t, parseResult.QueryValues, []interface{}{"apple", "true"}) } + +func TestQueryImageWithUnderscoreInLabelName(t *testing.T) { + queryParser := NewQueryParser("blue_fish") + parseResult, err := queryParser.Parse() + ok(t, err) + equals(t, parseResult.Query, "q.accessors @> ARRAY[$1]::text[]") + equals(t, parseResult.QueryValues, []interface{}{"blue_fish"}) +} + +func TestQueryImageWithSlashInLabelName(t *testing.T) { + queryParser := NewQueryParser("blue/fish") + parseResult, err := queryParser.Parse() + ok(t, err) + equals(t, parseResult.Query, "q.accessors @> ARRAY[$1]::text[]") + equals(t, parseResult.QueryValues, []interface{}{"blue/fish"}) +} + +func TestQueryImageWithSlashAndUnderscoreInLabelName(t *testing.T) { + queryParser := NewQueryParser("blue/red_fish") + parseResult, err := queryParser.Parse() + ok(t, err) + equals(t, parseResult.Query, "q.accessors @> ARRAY[$1]::text[]") + equals(t, parseResult.QueryValues, []interface{}{"blue/red_fish"}) +} diff --git a/tests/go.sum b/tests/go.sum index 5d00832a..9507fd4a 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -52,6 +52,7 @@ github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASu github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/h2non/filetype v1.0.8 h1:le8gpf+FQA0/DlDABbtisA1KiTS0Xi+YSC/E8yY3Y14= github.com/h2non/filetype v1.0.8/go.mod h1:isekKqOuhMj+s/7r3rIeTErIRy4Rub5uBWHfvMusLMU= +github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0 h1:DUwgMQuuPnS0rhMXenUtZpqZqrR/30NWY+qQvTpSvEs= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= @@ -65,6 +66,7 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= diff --git a/tests/helper.go b/tests/helper.go index c7c8808b..df60bda3 100644 --- a/tests/helper.go +++ b/tests/helper.go @@ -78,6 +78,8 @@ func notEquals(tb testing.TB, exp, act interface{}) { } func setupTestCase(t *testing.T) func(t *testing.T) { + setupInfrastructure() + t.Log("setup test case..clearing all database tables") err := db.ClearAll() diff --git a/tests/tests.go b/tests/tests.go index bf838578..a596546d 100644 --- a/tests/tests.go +++ b/tests/tests.go @@ -18,11 +18,11 @@ var X_CLIENT_SECRET string var DB_PORT string = "5432" var REDIS_ADDRESS string = ":6379" -func init() { - unverifiedDonationsDir := flag.String("unverified_donations_dir", "../unverified_donations/", "Path to unverified donations directory") - donationsDir := flag.String("donations_dir", "../donations/", "Path to donations directory") - flag.Parse() +var unverifiedDonationsDir = flag.String("unverified_donations_dir", "../unverified_donations/", "Path to unverified donations directory") +var donationsDir = flag.String("donations_dir", "../donations/", "Path to donations directory") + +func setupInfrastructure() { UNVERIFIED_DONATIONS_DIR = *unverifiedDonationsDir DONATIONS_DIR = *donationsDir