feat(whpg19): support whpg19

Makefile

@@ -66,6 +66,7 @@ unit_all_gpdb_versions : $(GINKGO)
 		TEST_GPDB_VERSION=5.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
 		TEST_GPDB_VERSION=6.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1
 		TEST_GPDB_VERSION=7.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB main
+		TEST_GPDB_VERSION=19.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # WHPG19
 
 integration : build_test $(GINKGO)
 	ginkgo $(GINKGO_FLAGS) integration 2>&1

backup/data_test.go

@@ -34,6 +34,35 @@ var _ = Describe("backup/data tests", func() {
 			atts := backup.ConstructTableAttributesList(columnDefs)
 			Expect(atts).To(Equal(""))
 		})
+		It("skips data for a table whose every column is generated", func() {
+			// Such a table cannot be expressed as a COPY column list -- an
+			// empty list is a syntax error, and no list at all covers the
+			// generated column and fails the restore -- so it carries no data.
+			allGenerated := backup.Table{
+				Relation: backup.Relation{Oid: 1, Schema: "public", Name: "vgen_only"},
+				TableDefinition: backup.TableDefinition{ColumnDefs: []backup.ColumnDefinition{
+					{Name: "j", AttGenerated: "VIRTUAL"},
+				}},
+			}
+			Expect(allGenerated.SkipDataBackup()).To(BeTrue())
+		})
+		It("does not skip data for a table with no columns at all", func() {
+			noColumns := backup.Table{
+				Relation:        backup.Relation{Oid: 1, Schema: "public", Name: "nocols"},
+				TableDefinition: backup.TableDefinition{ColumnDefs: []backup.ColumnDefinition{}},
+			}
+			Expect(noColumns.SkipDataBackup()).To(BeFalse())
+		})
+		It("excludes generated columns, stored and virtual alike", func() {
+			// A virtual generated column (WHPG19+) has no stored value to copy.
+			columnDefs := []backup.ColumnDefinition{
+				{Name: "a"},
+				{Name: "b", AttGenerated: "STORED"},
+				{Name: "c", AttGenerated: "VIRTUAL"},
+			}
+			atts := backup.ConstructTableAttributesList(columnDefs)
+			Expect(atts).To(Equal("(a)"))
+		})
 	})
 	Describe("AddTableDataEntriesToTOC", func() {
 		var (

backup/metadata_globals.go

@@ -35,6 +35,37 @@ func PrintCreateDatabaseStatement(metadataFile *utils.FileWithByteCount, tocfile
 	if db.Encoding != "" && (db.Encoding != defaultDB.Encoding) {
 		metadataFile.MustPrintf(" ENCODING '%s'", db.Encoding)
 	}
+	// PG15+ (WHPG19): reproduce a non-default locale provider along with the
+	// locale it reads, which for icu and builtin is datlocale rather than
+	// datcollate/datctype.  Emitted whenever either differs from the default
+	// database, since a matching provider can still carry a different locale.
+	if db.LocProvider != "" && (db.LocProvider != defaultDB.LocProvider ||
+		db.Locale != defaultDB.Locale || db.IcuRules != defaultDB.IcuRules) {
+		switch db.LocProvider {
+		case "c":
+			// libc keeps its locale in datcollate/datctype, which the
+			// LC_COLLATE/LC_CTYPE clauses below carry; datlocale is asserted
+			// NULL for this provider, so there is nothing else to reproduce.
+			metadataFile.MustPrintf(" LOCALE_PROVIDER libc")
+		case "i":
+			metadataFile.MustPrintf(" LOCALE_PROVIDER icu")
+			if db.Locale != "" {
+				metadataFile.MustPrintf(" ICU_LOCALE '%s'", utils.EscapeSingleQuotes(db.Locale))
+			}
+			if db.IcuRules != "" {
+				// ICU tailoring uses the apostrophe as its own quoting
+				// character, so these genuinely do contain single quotes.
+				metadataFile.MustPrintf(" ICU_RULES '%s'", utils.EscapeSingleQuotes(db.IcuRules))
+			}
+		case "b":
+			metadataFile.MustPrintf(" LOCALE_PROVIDER builtin")
+			if db.Locale != "" {
+				metadataFile.MustPrintf(" BUILTIN_LOCALE '%s'", utils.EscapeSingleQuotes(db.Locale))
+			}
+		default:
+			gplog.Warn("Database %s has unrecognized locale provider '%s'; it will be restored with the default provider.", db.Name, db.LocProvider)
+		}
+	}
 	if db.Collate != "" && (db.Collate != defaultDB.Collate) {
 		metadataFile.MustPrintf(" LC_COLLATE '%s'", db.Collate)
 	}
@@ -398,8 +429,23 @@ func PrintRoleMembershipStatements(metadataFile *utils.FileWithByteCount, objToc
 	for _, roleMember := range roleMembers {
 		start := metadataFile.ByteCount
 		metadataFile.MustPrintf("\nGRANT %s TO %s", roleMember.Role, roleMember.Member)
+		// Built as a list because PG16+ (WHPG19) can carry INHERIT and SET
+		// alongside ADMIN OPTION under a single WITH. On older majors only the
+		// admin option is ever populated, so this still reads as it always did.
+		// Mirrors the option buffer in pg_dumpall's dumpRoleMembership():
+		// INHERIT is always spelled out, SET only when it is false.
+		options := make([]string, 0)
 		if roleMember.IsAdmin {
-			metadataFile.MustPrintf(" WITH ADMIN OPTION")
+			options = append(options, "ADMIN OPTION")
+		}
+		if roleMember.InheritOption != "" {
+			options = append(options, fmt.Sprintf("INHERIT %s", roleMember.InheritOption))
+		}
+		if roleMember.SetOption == "FALSE" {
+			options = append(options, "SET FALSE")
+		}
+		if len(options) > 0 {
+			metadataFile.MustPrintf(" WITH %s", strings.Join(options, ", "))
 		}
 		if roleMember.Grantor != "" {
 			metadataFile.MustPrintf(" GRANTED BY %s", roleMember.Grantor)

backup/metadata_globals_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/warehouse-pg/common-go-libs/testhelper"
 
 	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
 )
 
 var _ = Describe("backup/metadata_globals tests", func() {
@@ -63,6 +64,43 @@ GRANT TEMPORARY,CONNECT ON DATABASE testdb TO testrole;`,
 			backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap)
 			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 TABLESPACE test_tablespace ENCODING 'UTF8' LC_COLLATE 'en_US.utf-8' LC_CTYPE 'en_US.utf-8';`)
 		})
+		It("prints the locale provider and locale for an ICU database", func() {
+			db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default",
+				LocProvider: "i", Locale: "en-US", IcuRules: "&a < g"}
+			emptyMetadataMap := backup.MetadataMap{}
+			backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap)
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' ICU_RULES '&a < g';`)
+		})
+		It("prints the locale provider and locale for a builtin database", func() {
+			db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default",
+				LocProvider: "b", Locale: "C.UTF-8"}
+			emptyMetadataMap := backup.MetadataMap{}
+			backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap)
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER builtin BUILTIN_LOCALE 'C.UTF-8';`)
+		})
+		It("does not print the locale provider if it matches the default database", func() {
+			defaultDB := backup.Database{LocProvider: "c"}
+			db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default", LocProvider: "c"}
+			emptyMetadataMap := backup.MetadataMap{}
+			backup.PrintCreateDatabaseStatement(backupfile, tocfile, defaultDB, db, emptyMetadataMap)
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0;`)
+		})
+		It("prints the ICU rules when only they differ from the default database", func() {
+			defaultDB := backup.Database{LocProvider: "i", Locale: "en-US"}
+			db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default",
+				LocProvider: "i", Locale: "en-US", IcuRules: "&a < g"}
+			emptyMetadataMap := backup.MetadataMap{}
+			backup.PrintCreateDatabaseStatement(backupfile, tocfile, defaultDB, db, emptyMetadataMap)
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' ICU_RULES '&a < g';`)
+		})
+		It("escapes single quotes in the ICU rules", func() {
+			// ICU tailoring uses the apostrophe as its own quoting character.
+			db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default",
+				LocProvider: "i", Locale: "en-US", IcuRules: "&a < 'x'"}
+			emptyMetadataMap := backup.MetadataMap{}
+			backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap)
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' ICU_RULES '&a < ''x''';`)
+		})
 		It("does not print encoding information if it is the same as defaults", func() {
 			defaultDB := backup.Database{Oid: 0, Name: "", Tablespace: "", Encoding: "UTF8", Collate: "en_US.utf-8", CType: "en_US.utf-8"}
 			db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "test_tablespace", Encoding: "UTF8", Collate: "en_US.utf-8", CType: "en_US.utf-8"}
@@ -371,6 +409,63 @@ ALTER ROLE "testRole2" WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICAT
 			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, expectedStatements...)
 		})
 	})
+	Describe("OrderRoleMembersForRestore", func() {
+		// PG16+ requires the role named by GRANTED BY to already hold ADMIN
+		// OPTION on the role being granted, so a grant attributed to a
+		// non-superuser grantor has to follow that grantor's own admin grant.
+		It("moves a grant behind the admin grant its grantor depends on", func() {
+			members := []backup.RoleMember{
+				{Role: "usergroup", Member: "testuser", Grantor: "testrole", IsAdmin: false},
+				{Role: "usergroup", Member: "testrole", Grantor: "gpadmin", IsAdmin: true, GrantorIsBootstrapSuper: true},
+			}
+			ordered := backup.OrderRoleMembersForRestore(members)
+			Expect(ordered).To(HaveLen(2))
+			Expect(ordered[0].Member).To(Equal("testrole"))
+			Expect(ordered[1].Member).To(Equal("testuser"))
+		})
+		It("still defers a grant whose grantor is a superuser but not the bootstrap one", func() {
+			// check_role_grantor() exempts BOOTSTRAP_SUPERUSERID alone, and the
+			// select_best_admin() it otherwise defers to ignores super-userness,
+			// so testrole being SUPERUSER does not let this grant go first.
+			members := []backup.RoleMember{
+				{Role: "usergroup", Member: "testuser", Grantor: "testrole", GrantorIsBootstrapSuper: false},
+				{Role: "usergroup", Member: "testrole", Grantor: "gpadmin", IsAdmin: true, GrantorIsBootstrapSuper: true},
+			}
+			ordered := backup.OrderRoleMembersForRestore(members)
+			Expect(ordered[0].Member).To(Equal("testrole"))
+			Expect(ordered[1].Member).To(Equal("testuser"))
+		})
+		It("leaves an already-replayable order alone", func() {
+			members := []backup.RoleMember{
+				{Role: "usergroup", Member: "alice", Grantor: "gpadmin", GrantorIsBootstrapSuper: true},
+				{Role: "usergroup", Member: "bob", Grantor: "gpadmin", GrantorIsBootstrapSuper: true},
+			}
+			ordered := backup.OrderRoleMembersForRestore(members)
+			Expect(ordered[0].Member).To(Equal("alice"))
+			Expect(ordered[1].Member).To(Equal("bob"))
+		})
+		It("only orders within a role, keeping the roles in catalog order", func() {
+			members := []backup.RoleMember{
+				{Role: "groupone", Member: "alice", Grantor: "gpadmin", GrantorIsBootstrapSuper: true},
+				{Role: "grouptwo", Member: "carol", Grantor: "dave", IsAdmin: false},
+				{Role: "grouptwo", Member: "dave", Grantor: "gpadmin", IsAdmin: true, GrantorIsBootstrapSuper: true},
+			}
+			ordered := backup.OrderRoleMembersForRestore(members)
+			Expect(ordered[0].Role).To(Equal("groupone"))
+			Expect(ordered[1].Member).To(Equal("dave"))
+			Expect(ordered[2].Member).To(Equal("carol"))
+		})
+		It("emits a grant whose grantor never becomes available rather than dropping it", func() {
+			// A grantor that is not a member of the role at all: nothing can
+			// make it replayable, so it still has to reach the metadata file.
+			members := []backup.RoleMember{
+				{Role: "usergroup", Member: "testuser", Grantor: "someowner", IsAdmin: false},
+			}
+			ordered := backup.OrderRoleMembersForRestore(members)
+			Expect(ordered).To(HaveLen(1))
+			Expect(ordered[0].Member).To(Equal("testuser"))
+		})
+	})
 	Describe("PrintRoleMembershipStatements", func() {
 		roleWith := backup.RoleMember{Role: "group", Member: "rolewith", Grantor: "grantor", IsAdmin: true}
 		roleWithout := backup.RoleMember{Role: "group", Member: "rolewithout", Grantor: "grantor", IsAdmin: false}
@@ -384,6 +479,18 @@ ALTER ROLE "testRole2" WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICAT
 			backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{roleWith})
 			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `GRANT group TO rolewith WITH ADMIN OPTION GRANTED BY grantor;`)
 		})
+		It("prints the PG16+ grant options alongside ADMIN OPTION", func() {
+			member := backup.RoleMember{Role: "group", Member: "rolewith", Grantor: "grantor",
+				IsAdmin: true, InheritOption: "FALSE", SetOption: "FALSE"}
+			backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{member})
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `GRANT group TO rolewith WITH ADMIN OPTION, INHERIT FALSE, SET FALSE GRANTED BY grantor;`)
+		})
+		It("omits SET when it is true but still spells out INHERIT", func() {
+			member := backup.RoleMember{Role: "group", Member: "rolewith", Grantor: "",
+				IsAdmin: false, InheritOption: "TRUE", SetOption: "TRUE"}
+			backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{member})
+			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `GRANT group TO rolewith WITH INHERIT TRUE;`)
+		})
 		It("prints multiple roles", func() {
 			backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{roleWith, roleWithout})
 			testutils.AssertBufferContents(tocfile.GlobalEntries, buffer,

backup/predata_acl.go

@@ -52,6 +52,9 @@ type ACL struct {
 	TemporaryWithGrant  bool
 	Connect             bool
 	ConnectWithGrant    bool
+	// PG17+ (WHPG19), ACL character 'm'.
+	Maintain          bool
+	MaintainWithGrant bool
 }
 
 type MetadataMap map[UniqueID]ObjectMetadata
@@ -235,6 +238,8 @@ func ParseACL(aclStr string) *ACL {
 				acl.Temporary = true
 			case 'c':
 				acl.Connect = true
+			case 'm':
+				acl.Maintain = true
 			case '*':
 				switch lastChar {
 				case 'a':
@@ -273,6 +278,9 @@ func ParseACL(aclStr string) *ACL {
 				case 'c':
 					acl.Connect = false
 					acl.ConnectWithGrant = true
+				case 'm':
+					acl.Maintain = false
+					acl.MaintainWithGrant = true
 				}
 			}
 			lastChar = char
@@ -376,6 +384,13 @@ func createPrivilegeStrings(acl ACL, objectType string) (string, string) {
 		hasAllPrivileges = acl.Select && acl.Insert && acl.Update && acl.Delete && acl.Truncate && acl.References && acl.Trigger
 		hasAllPrivilegesWithGrant = acl.SelectWithGrant && acl.InsertWithGrant && acl.UpdateWithGrant && acl.DeleteWithGrant &&
 			acl.TruncateWithGrant && acl.ReferencesWithGrant && acl.TriggerWithGrant
+		// PG17+ added MAINTAIN to ACL_ALL_RIGHTS_RELATION. Without this a
+		// grantee holding only the older seven would still be written as GRANT
+		// ALL, and would come back holding MAINTAIN as well.
+		if connectionPool.Version.AtLeast("19") {
+			hasAllPrivileges = hasAllPrivileges && acl.Maintain
+			hasAllPrivilegesWithGrant = hasAllPrivilegesWithGrant && acl.MaintainWithGrant
+		}
 	case toc.OBJ_TABLESPACE:
 		hasAllPrivileges = acl.Create
 		hasAllPrivilegesWithGrant = acl.CreateWithGrant
@@ -408,6 +423,11 @@ func createPrivilegeStrings(acl ACL, objectType string) (string, string) {
 		if acl.Trigger {
 			privList = append(privList, "TRIGGER")
 		}
+		// Gated for the same reason as the ALL check above: MAINTAIN does not
+		// exist before PG17, so nothing should mention it on an older major.
+		if acl.Maintain && connectionPool.Version.AtLeast("19") {
+			privList = append(privList, "MAINTAIN")
+		}
 		/*
 		 * We skip checking whether acl.Execute is set here because only Functions have Execute,
 		 * and functions only have Execute, so Execute == hasAllPrivileges for Functions.
@@ -451,6 +471,9 @@ func createPrivilegeStrings(acl ACL, objectType string) (string, string) {
 		if acl.TriggerWithGrant {
 			privWithGrantList = append(privWithGrantList, "TRIGGER")
 		}
+		if acl.MaintainWithGrant && connectionPool.Version.AtLeast("19") {
+			privWithGrantList = append(privWithGrantList, "MAINTAIN")
+		}
 		// The comment above regarding Execute applies to ExecuteWithGrant as well.
 		if acl.UsageWithGrant {
 			privWithGrantList = append(privWithGrantList, "USAGE")

backup/predata_acl_test.go

@@ -24,10 +24,12 @@ var _ = Describe("backup/predata_acl tests", func() {
 		hasAllPrivileges := testutils.DefaultACLForType("anothertestrole", toc.OBJ_TABLE)
 		hasMostPrivileges := testutils.DefaultACLForType("testrole", toc.OBJ_TABLE)
 		hasMostPrivileges.Trigger = false
+		hasMostPrivileges.Maintain = false
 		hasSinglePrivilege := backup.ACL{Grantee: "", Trigger: true}
 		hasAllPrivilegesWithGrant := testutils.DefaultACLForTypeWithGrant("anothertestrole", toc.OBJ_TABLE)
 		hasMostPrivilegesWithGrant := testutils.DefaultACLForTypeWithGrant("testrole", toc.OBJ_TABLE)
 		hasMostPrivilegesWithGrant.TriggerWithGrant = false
+		hasMostPrivilegesWithGrant.MaintainWithGrant = false
 		hasSinglePrivilegeWithGrant := backup.ACL{Grantee: "", TriggerWithGrant: true}
 		privileges := []backup.ACL{hasAllPrivileges, hasMostPrivileges, hasSinglePrivilege}
 		privilegesWithGrant := []backup.ACL{hasAllPrivilegesWithGrant, hasMostPrivilegesWithGrant, hasSinglePrivilegeWithGrant}
@@ -430,9 +432,22 @@ ALTER DEFAULT PRIVILEGES FOR ROLE testrole GRANT USAGE ON TABLES TO somerole WIT
 			result := backup.ParseACL(aclStr)
 			structmatcher.ExpectStructsToMatch(&expected, result)
 		})
+		It("parses the PG17 MAINTAIN privilege", func() {
+			result := backup.ParseACL("testrole=m/gpadmin")
+			Expect(result.Maintain).To(BeTrue())
+			Expect(result.MaintainWithGrant).To(BeFalse())
+		})
+		It("parses MAINTAIN with grant option", func() {
+			result := backup.ParseACL("testrole=m*/gpadmin")
+			Expect(result.Maintain).To(BeFalse())
+			Expect(result.MaintainWithGrant).To(BeTrue())
+		})
 		It("parses an ACL string containing a role with multiple privileges", func() {
 			aclStr := "testrole=arwdDxt/gpadmin"
 			expected := testutils.DefaultACLForType("testrole", toc.OBJ_TABLE)
+			// The string above grants the seven privileges a relation had
+			// before PG17, so it never carries MAINTAIN whatever the version.
+			expected.Maintain = false
 			result := backup.ParseACL(aclStr)
 			structmatcher.ExpectStructsToMatch(&expected, result)
 		})

backup/predata_functions.go

@@ -18,14 +18,30 @@ func PrintCreateFunctionStatement(metadataFile *utils.FileWithByteCount, objToc
 	start := metadataFile.ByteCount
 	funcFQN := utils.MakeFQN(funcDef.Schema, funcDef.Name)
 
+	// A SQL-standard body (WHPG19+) is not introduced by AS, and has to follow
+	// the modifiers rather than precede LANGUAGE, so the statement is laid out
+	// differently in that case.  This mirrors dumpFunc() in pg_dump.c.
+	hasSqlBody := funcDef.SqlBody != ""
+	asClause := " AS"
+	if hasSqlBody {
+		asClause = ""
+	}
+
 	if connectionPool.Version.AtLeast("7") && funcDef.Kind == "p" {
-		metadataFile.MustPrintf("\n\nCREATE PROCEDURE %s(%s) AS", funcFQN, funcDef.Arguments.String)
+		metadataFile.MustPrintf("\n\nCREATE PROCEDURE %s(%s)%s", funcFQN, funcDef.Arguments.String, asClause)
 	} else {
-		metadataFile.MustPrintf("\n\nCREATE FUNCTION %s(%s) RETURNS %s AS", funcFQN, funcDef.Arguments.String, funcDef.ResultType.String)
+		metadataFile.MustPrintf("\n\nCREATE FUNCTION %s(%s) RETURNS %s%s", funcFQN, funcDef.Arguments.String, funcDef.ResultType.String, asClause)
+	}
+	if hasSqlBody {
+		metadataFile.MustPrintln()
+	} else {
+		PrintFunctionBodyOrPath(metadataFile, funcDef)
 	}
-	PrintFunctionBodyOrPath(metadataFile, funcDef)
 	metadataFile.MustPrintf("LANGUAGE %s", funcDef.Language)
 	PrintFunctionModifiers(metadataFile, funcDef)
+	if hasSqlBody {
+		metadataFile.MustPrintf("\n%s", funcDef.SqlBody)
+	}
 	metadataFile.MustPrintln(";")
 
 	section, entry := funcDef.GetMetadataEntry()