feat: generated column for tags

query/models.go

@@ -10,7 +10,9 @@ import (
 	"github.com/flanksource/duty/models"
 	"github.com/flanksource/duty/types"
 	"github.com/google/uuid"
+	"github.com/lib/pq"
 	"github.com/pkg/errors"
+	"github.com/samber/lo"
 	"github.com/timberio/go-datemath"
 	"gorm.io/gorm"
 
@@ -304,20 +306,30 @@ func (qm QueryModel) Apply(ctx context.Context, q types.QueryField, tx *gorm.DB)
 }
 
 func FilterJSONColumnValues(ctx context.Context, tx *gorm.DB, column string, op types.QueryOperator, path string, val string) *gorm.DB {
-	var subQueryCondition string
-	switch op {
-	case types.Neq:
-		subQueryCondition = "NOT EXISTS"
-	default:
-		subQueryCondition = "EXISTS"
+	if !slices.Contains([]types.QueryOperator{types.Eq, types.Neq}, op) {
+		op = types.Eq
 	}
 
 	values := strings.Split(val, ",")
-	tx = tx.Where(fmt.Sprintf(`%s (
+
+	switch column {
+	case "tags":
+		// `tags` column uses the more performant tags_values column
+		switch op {
+		case types.Neq:
+			tx = tx.Where("NOT tags_values ? ?", gorm.Expr("?|"), pq.StringArray(values))
+		default:
+			tx = tx.Where("tags_values ? ?", gorm.Expr("?|"), pq.StringArray(values))
+		}
+
+	default:
+		subQueryCondition := lo.Ternary(op == types.Neq, "NOT EXISTS", "EXISTS")
+		tx = tx.Where(fmt.Sprintf(`%s (
 			SELECT 1 
 			FROM jsonb_each_text(%s) 
 			WHERE value IN ?
 		)`, subQueryCondition, column), values)
+	}
 
 	return tx
 }

schema/config.hcl

@@ -308,6 +308,18 @@ table "config_items" {
     type    = jsonb
     comment = "contains a list of tags"
   }
+  column "tags_values" {
+    null    = true
+    type    = jsonb
+    comment = "a generated from tags column to get a better search on tag values"
+    as {
+      # without the ::jsonpath type casting
+      # we get this error from Atlas: failed to compute diff: failed to diff realms: 
+      # changing the generation expression for a column "tags_values" is not supported
+      expr = "(jsonb_path_query_array(tags, '$[*].*'::jsonpath))"
+      type = STORED
+    }
+  }
   column "properties" {
     null = true
     type = jsonb
@@ -400,6 +412,10 @@ table "config_items" {
     columns = [column.tags]
     type    = GIN
   }
+  index "idx_config_items_tags_values" {
+    columns = [column.tags_values]
+    type    = GIN
+  }
   index "idx_config_items_name" {
     columns = [column.agent_id, column.name, column.type, column.config_class]
   }