Merge pull request #39 from arenadata/1.29.4-sync

Sync 1.29.4 changes
arenadata · Sep 28, 2023 · 146799f · 146799f
2 parents 1be2727 + 91ca545
commit 146799f
Show file tree

Hide file tree

Showing 31 changed files with 806 additions and 1,322 deletions.
diff --git a/backup/queries_externals.go b/backup/queries_externals.go
@@ -86,7 +86,7 @@ func GetExternalTableDefinitions(connectionPool *dbconn.DBConn) map[uint32]Exter
 		coalesce(e.rejectlimit, 0) AS rejectlimit,
 		coalesce(e.rejectlimittype, '') AS rejectlimittype,
 		e.logerrors,
-		coalesce('log_errors=p' = any(ft.ftoptions), false) AS logerrpersist,
+		coalesce('log_errors=persistently' = any(ft.ftoptions), false) AS logerrpersist,
 		pg_encoding_to_char(e.encoding) AS encoding,
 		e.writable
 	FROM pg_exttable e

diff --git a/backup/queries_postdata.go b/backup/queries_postdata.go
@@ -250,40 +250,43 @@ func GetIndexes(connectionPool *dbconn.DBConn) []IndexDefinition {
 
 func GetRenameExchangedPartitionQuery(connection *dbconn.DBConn) string {
 	// In the case of exchanged partition tables, restoring index constraints with system-generated
-	// will cause a name collision in GPDB7+.  Rename those constraints to match their new owning
-	// tables.  In GPDB6 and below this renaming was done automatically by server code.
+	// will cause a name collision in GPDB7+. Rename those constraints to match their new owning
+	// tables. In GPDB6 and below this renaming was done automatically by server code.
 	cteClause := ""
 	if connectionPool.Version.Before("7") {
-		cteClause = `SELECT DISTINCT cl.relname
-            FROM pg_class cl
-                INNER JOIN pg_partitions pts
-					ON cl.relname = pts.partitiontablename
-					AND cl.relname != pts.tablename
-            WHERE cl.relkind IN ('r', 'f')`
+		cteClause = fmt.Sprintf(`
+            SELECT c.relname
+            FROM pg_class c
+                INNER JOIN pg_partition_rule p
+                    ON c.oid = p.parchildrelid
+                INNER JOIN pg_namespace n ON c.relnamespace = n.oid
+            WHERE %s`, relationAndSchemaFilterClause())
 	} else {
-		cteClause = `SELECT DISTINCT cl.relname
-             FROM pg_class cl
-             WHERE
-                cl.relkind IN ('r', 'f')
-                AND cl.relispartition = true
-                AND cl.relhassubclass = false`
+		cteClause = fmt.Sprintf(`
+            SELECT DISTINCT c.relname
+            FROM pg_class c
+                INNER JOIN pg_namespace n ON c.relnamespace = n.oid
+            WHERE %s
+                AND c.relkind IN ('r', 'f')
+                AND c.relispartition = true
+                AND c.relhassubclass = false`, relationAndSchemaFilterClause())
 	}
 	query := fmt.Sprintf(`
         WITH table_cte AS (%s)
         SELECT
             ic.relname AS origname,
-            rc.relname || SUBSTRING(ic.relname, LENGTH(ch.relname)+1, LENGTH(ch.relname)) AS newname
+            c.relname || SUBSTRING(ic.relname, LENGTH(ch.relname)+1, LENGTH(ch.relname)) AS newname
         FROM
             pg_index i
-            JOIN pg_class ic ON i.indexrelid = ic.oid
-            JOIN pg_class rc
-                ON i.indrelid = rc.oid
-                AND rc.relname != SUBSTRING(ic.relname, 1, LENGTH(rc.relname))
-            JOIN pg_namespace n ON rc.relnamespace = n.oid
+            INNER JOIN pg_class ic ON i.indexrelid = ic.oid
+            INNER JOIN pg_class c
+                ON i.indrelid = c.oid
+                AND c.relname != SUBSTRING(ic.relname, 1, LENGTH(c.relname))
+            INNER JOIN pg_namespace n ON c.relnamespace = n.oid
             INNER JOIN table_cte ch
                 ON SUBSTRING(ic.relname, 1, LENGTH(ch.relname)) = ch.relname
-                AND rc.relname != ch.relname
-        WHERE %s;`, cteClause, SchemaFilterClause("n"))
+                AND c.relname != ch.relname
+        WHERE %s;`, cteClause, relationAndSchemaFilterClause())
 	return query
 }
 

diff --git a/backup/queries_shared.go b/backup/queries_shared.go
@@ -128,7 +128,7 @@ func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) []
 	}
 	// This query is adapted from the queries underlying \d in psql.
 	tableQuery := ""
-	if connectionPool.Version.Before("7") {
+	if connectionPool.Version.Before("6") {
 		tableQuery = fmt.Sprintf(`
 		SELECT con.oid,
 			quote_ident(n.nspname) AS schema,
@@ -153,6 +153,31 @@ func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) []
 			AND conrelid NOT IN (SELECT parchildrelid FROM pg_partition_rule)
 			AND (conrelid, conname) NOT IN (SELECT i.inhrelid, con.conname FROM pg_inherits i JOIN pg_constraint con ON i.inhrelid = con.conrelid JOIN pg_constraint p ON i.inhparent = p.conrelid WHERE con.conname = p.conname)
 		GROUP BY con.oid, conname, contype, c.relname, n.nspname, %s pt.parrelid`, conIsLocal, "%s", ExtensionFilterClause("c"), conIsLocal)
+	} else if connectionPool.Version.Is("6") {
+		tableQuery = fmt.Sprintf(`
+		SELECT con.oid,
+			quote_ident(n.nspname) AS schema,
+			quote_ident(conname) AS name,
+			contype,
+			%s
+			pg_get_constraintdef(con.oid, TRUE) AS def,
+			quote_ident(n.nspname) || '.' || quote_ident(c.relname) AS owningobject,
+			'f' AS isdomainconstraint,
+			CASE
+				WHEN pt.parrelid IS NULL THEN 'f'
+				ELSE 't'
+			END AS ispartitionparent
+		FROM pg_constraint con
+			LEFT JOIN pg_class c ON con.conrelid = c.oid
+			LEFT JOIN pg_partition pt ON con.conrelid = pt.parrelid
+			JOIN pg_namespace n ON n.oid = con.connamespace
+		WHERE %s
+			AND %s
+			AND c.relname IS NOT NULL
+			AND contype != 't'
+			AND conrelid NOT IN (SELECT parchildrelid FROM pg_partition_rule)
+			AND coninhcount = 0
+		GROUP BY con.oid, conname, contype, c.relname, n.nspname, %s pt.parrelid`, conIsLocal, "%s", ExtensionFilterClause("c"), conIsLocal)
 	} else {
 		tableQuery = fmt.Sprintf(`
 		SELECT con.oid,
@@ -176,7 +201,7 @@ func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) []
 			AND c.relname IS NOT NULL
 			AND contype != 't'
 			AND (c.relispartition IS FALSE OR conislocal IS TRUE)
-			AND (conrelid, conname) NOT IN (SELECT i.inhrelid, con.conname FROM pg_inherits i JOIN pg_constraint con ON i.inhrelid = con.conrelid JOIN pg_constraint p ON i.inhparent = p.conrelid WHERE con.conname = p.conname)
+			AND coninhcount = 0
 		GROUP BY con.oid, conname, contype, c.relname, n.nspname, con.conislocal, pt.partrelid`, "%s", ExtensionFilterClause("c"))
 	}
 

diff --git a/ci/cleanup/plugin-cleanup.yml b/ci/cleanup/plugin-cleanup.yml
@@ -44,7 +44,7 @@ resources:
   source:
     bucket: ((dp/prod/gcs-ci-bucket))
     json_key: ((concourse-gcs-resources-service-account-key))
-    regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64((dp/dev/rc-build-type-gcs)).tar.gz
+    regexp: server/published/gpdb6/server-rc-(.*)-rhel8_x86_64.tar.gz
 
 - name: gpbackup
   type: git

diff --git a/ci/gen_pipeline.py b/ci/gen_pipeline.py
@@ -69,7 +69,8 @@ def create_pipeline(args):
         'timestamp': datetime.datetime.now(),
         'pipeline_name': args.pipeline_name,
         'nightly_trigger': args.nightly_trigger,
-        'is_prod': args.is_prod
+        'is_prod': args.is_prod,
+        'should_alert': args.should_alert
     }
 
     pipeline_yml = render_template(args.template_filename, context)
@@ -94,9 +95,9 @@ def print_output_message(args):
     if not args.is_prod:
         if git_branch == "main":
             print("\n[WARNING] You are generating a dev pipeline pointed to the main branch!\n")
-        cmd = """fly -t dp set-pipeline  -p dev:%s_%s \
+        cmd = """fly -t dp set-pipeline -p %s \
 -c %s/%s-dev-generated.yml \
--v gpbackup-git-branch=%s""" % (args.pipeline_name, git_branch, curr_dir, args.pipeline_name, git_branch)
+-v gpbackup-git-branch=%s""" % (git_branch, curr_dir, args.pipeline_name, git_branch)
         print("To set this pipeline on dev, run: \n%s" % (cmd))
         join = raw_input('Would you like to run the pipeline now? [yN]: ')
         if join.lower() == 'yes' or join.lower() == 'y':
@@ -109,11 +110,13 @@ def print_output_message(args):
     if args.is_prod:
         if git_branch != "main":
             print("\n[WARNING] You are generating a prod pipeline, but are not on the main branch!\n")
-        cmd1 = "fly -t gpdb-prod set-pipeline -p %s \
--c %s/%s-generated.yml" % (args.pipeline_name, curr_dir, args.pipeline_name)
+        cmd1 = "fly -t dp set-pipeline -p %s \
+-c %s/%s-generated.yml \
+-v gpbackup-git-branch=%s" % (args.pipeline_name, curr_dir, args.pipeline_name, git_branch)
         args.pipeline_name = "gpbackup"
-        cmd2 = "fly -t gpdb-prod set-pipeline -p %s \
--c %s/%s-generated.yml" % (args.pipeline_name, curr_dir, args.pipeline_name)
+        cmd2 = "fly -t dp set-pipeline -p %s \
+-c %s/%s-generated.yml \
+-v gpbackup-git-branch=%s" % (args.pipeline_name, curr_dir, args.pipeline_name, git_branch)
         print("To set these pipelines (gpbackup / gpbackup-release) on prod, run: \n%s\n%s" % (cmd2, cmd1))
 
 def main():
@@ -132,6 +135,15 @@ def main():
         help='Name of template to use, in templates/'
     )
 
+    parser.add_argument(
+        '-na',
+        '--no-alerts',
+        action='store_false',
+        dest='should_alert',
+        default=True,
+        help='Disable the Slack alert for failed jobs'
+    )
+
     parser.add_argument(
         '-nt',
         '--nightly-trigger',