diff --git a/Makefile b/Makefile index 7494815b..ca8cd1a9 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ unit_all_gpdb_versions : $(GINKGO) TEST_GPDB_VERSION=5.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 TEST_GPDB_VERSION=6.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 TEST_GPDB_VERSION=7.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # GPDB main + TEST_GPDB_VERSION=19.999.0 ginkgo $(GINKGO_FLAGS) $(SUBDIRS_HAS_UNIT) 2>&1 # WHPG19 integration : build_test $(GINKGO) ginkgo $(GINKGO_FLAGS) integration 2>&1 diff --git a/backup/data_test.go b/backup/data_test.go index 10403526..e6d60d36 100644 --- a/backup/data_test.go +++ b/backup/data_test.go @@ -34,6 +34,35 @@ var _ = Describe("backup/data tests", func() { atts := backup.ConstructTableAttributesList(columnDefs) Expect(atts).To(Equal("")) }) + It("skips data for a table whose every column is generated", func() { + // Such a table cannot be expressed as a COPY column list -- an + // empty list is a syntax error, and no list at all covers the + // generated column and fails the restore -- so it carries no data. + allGenerated := backup.Table{ + Relation: backup.Relation{Oid: 1, Schema: "public", Name: "vgen_only"}, + TableDefinition: backup.TableDefinition{ColumnDefs: []backup.ColumnDefinition{ + {Name: "j", AttGenerated: "VIRTUAL"}, + }}, + } + Expect(allGenerated.SkipDataBackup()).To(BeTrue()) + }) + It("does not skip data for a table with no columns at all", func() { + noColumns := backup.Table{ + Relation: backup.Relation{Oid: 1, Schema: "public", Name: "nocols"}, + TableDefinition: backup.TableDefinition{ColumnDefs: []backup.ColumnDefinition{}}, + } + Expect(noColumns.SkipDataBackup()).To(BeFalse()) + }) + It("excludes generated columns, stored and virtual alike", func() { + // A virtual generated column (WHPG19+) has no stored value to copy. + columnDefs := []backup.ColumnDefinition{ + {Name: "a"}, + {Name: "b", AttGenerated: "STORED"}, + {Name: "c", AttGenerated: "VIRTUAL"}, + } + atts := backup.ConstructTableAttributesList(columnDefs) + Expect(atts).To(Equal("(a)")) + }) }) Describe("AddTableDataEntriesToTOC", func() { var ( diff --git a/backup/metadata_globals.go b/backup/metadata_globals.go index 6d6063f3..fdf72e89 100644 --- a/backup/metadata_globals.go +++ b/backup/metadata_globals.go @@ -35,6 +35,37 @@ func PrintCreateDatabaseStatement(metadataFile *utils.FileWithByteCount, tocfile if db.Encoding != "" && (db.Encoding != defaultDB.Encoding) { metadataFile.MustPrintf(" ENCODING '%s'", db.Encoding) } + // PG15+ (WHPG19): reproduce a non-default locale provider along with the + // locale it reads, which for icu and builtin is datlocale rather than + // datcollate/datctype. Emitted whenever either differs from the default + // database, since a matching provider can still carry a different locale. + if db.LocProvider != "" && (db.LocProvider != defaultDB.LocProvider || + db.Locale != defaultDB.Locale || db.IcuRules != defaultDB.IcuRules) { + switch db.LocProvider { + case "c": + // libc keeps its locale in datcollate/datctype, which the + // LC_COLLATE/LC_CTYPE clauses below carry; datlocale is asserted + // NULL for this provider, so there is nothing else to reproduce. + metadataFile.MustPrintf(" LOCALE_PROVIDER libc") + case "i": + metadataFile.MustPrintf(" LOCALE_PROVIDER icu") + if db.Locale != "" { + metadataFile.MustPrintf(" ICU_LOCALE '%s'", utils.EscapeSingleQuotes(db.Locale)) + } + if db.IcuRules != "" { + // ICU tailoring uses the apostrophe as its own quoting + // character, so these genuinely do contain single quotes. + metadataFile.MustPrintf(" ICU_RULES '%s'", utils.EscapeSingleQuotes(db.IcuRules)) + } + case "b": + metadataFile.MustPrintf(" LOCALE_PROVIDER builtin") + if db.Locale != "" { + metadataFile.MustPrintf(" BUILTIN_LOCALE '%s'", utils.EscapeSingleQuotes(db.Locale)) + } + default: + gplog.Warn("Database %s has unrecognized locale provider '%s'; it will be restored with the default provider.", db.Name, db.LocProvider) + } + } if db.Collate != "" && (db.Collate != defaultDB.Collate) { metadataFile.MustPrintf(" LC_COLLATE '%s'", db.Collate) } @@ -398,8 +429,23 @@ func PrintRoleMembershipStatements(metadataFile *utils.FileWithByteCount, objToc for _, roleMember := range roleMembers { start := metadataFile.ByteCount metadataFile.MustPrintf("\nGRANT %s TO %s", roleMember.Role, roleMember.Member) + // Built as a list because PG16+ (WHPG19) can carry INHERIT and SET + // alongside ADMIN OPTION under a single WITH. On older majors only the + // admin option is ever populated, so this still reads as it always did. + // Mirrors the option buffer in pg_dumpall's dumpRoleMembership(): + // INHERIT is always spelled out, SET only when it is false. + options := make([]string, 0) if roleMember.IsAdmin { - metadataFile.MustPrintf(" WITH ADMIN OPTION") + options = append(options, "ADMIN OPTION") + } + if roleMember.InheritOption != "" { + options = append(options, fmt.Sprintf("INHERIT %s", roleMember.InheritOption)) + } + if roleMember.SetOption == "FALSE" { + options = append(options, "SET FALSE") + } + if len(options) > 0 { + metadataFile.MustPrintf(" WITH %s", strings.Join(options, ", ")) } if roleMember.Grantor != "" { metadataFile.MustPrintf(" GRANTED BY %s", roleMember.Grantor) diff --git a/backup/metadata_globals_test.go b/backup/metadata_globals_test.go index efdbd692..26395216 100644 --- a/backup/metadata_globals_test.go +++ b/backup/metadata_globals_test.go @@ -9,6 +9,7 @@ import ( "github.com/warehouse-pg/common-go-libs/testhelper" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var _ = Describe("backup/metadata_globals tests", func() { @@ -63,6 +64,43 @@ GRANT TEMPORARY,CONNECT ON DATABASE testdb TO testrole;`, backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap) testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 TABLESPACE test_tablespace ENCODING 'UTF8' LC_COLLATE 'en_US.utf-8' LC_CTYPE 'en_US.utf-8';`) }) + It("prints the locale provider and locale for an ICU database", func() { + db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default", + LocProvider: "i", Locale: "en-US", IcuRules: "&a < g"} + emptyMetadataMap := backup.MetadataMap{} + backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' ICU_RULES '&a < g';`) + }) + It("prints the locale provider and locale for a builtin database", func() { + db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default", + LocProvider: "b", Locale: "C.UTF-8"} + emptyMetadataMap := backup.MetadataMap{} + backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER builtin BUILTIN_LOCALE 'C.UTF-8';`) + }) + It("does not print the locale provider if it matches the default database", func() { + defaultDB := backup.Database{LocProvider: "c"} + db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default", LocProvider: "c"} + emptyMetadataMap := backup.MetadataMap{} + backup.PrintCreateDatabaseStatement(backupfile, tocfile, defaultDB, db, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0;`) + }) + It("prints the ICU rules when only they differ from the default database", func() { + defaultDB := backup.Database{LocProvider: "i", Locale: "en-US"} + db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default", + LocProvider: "i", Locale: "en-US", IcuRules: "&a < g"} + emptyMetadataMap := backup.MetadataMap{} + backup.PrintCreateDatabaseStatement(backupfile, tocfile, defaultDB, db, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' ICU_RULES '&a < g';`) + }) + It("escapes single quotes in the ICU rules", func() { + // ICU tailoring uses the apostrophe as its own quoting character. + db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "pg_default", + LocProvider: "i", Locale: "en-US", IcuRules: "&a < 'x'"} + emptyMetadataMap := backup.MetadataMap{} + backup.PrintCreateDatabaseStatement(backupfile, tocfile, emptyDB, db, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `CREATE DATABASE testdb TEMPLATE template0 LOCALE_PROVIDER icu ICU_LOCALE 'en-US' ICU_RULES '&a < ''x''';`) + }) It("does not print encoding information if it is the same as defaults", func() { defaultDB := backup.Database{Oid: 0, Name: "", Tablespace: "", Encoding: "UTF8", Collate: "en_US.utf-8", CType: "en_US.utf-8"} db := backup.Database{Oid: 1, Name: "testdb", Tablespace: "test_tablespace", Encoding: "UTF8", Collate: "en_US.utf-8", CType: "en_US.utf-8"} @@ -371,6 +409,63 @@ ALTER ROLE "testRole2" WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICAT testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, expectedStatements...) }) }) + Describe("OrderRoleMembersForRestore", func() { + // PG16+ requires the role named by GRANTED BY to already hold ADMIN + // OPTION on the role being granted, so a grant attributed to a + // non-superuser grantor has to follow that grantor's own admin grant. + It("moves a grant behind the admin grant its grantor depends on", func() { + members := []backup.RoleMember{ + {Role: "usergroup", Member: "testuser", Grantor: "testrole", IsAdmin: false}, + {Role: "usergroup", Member: "testrole", Grantor: "gpadmin", IsAdmin: true, GrantorIsBootstrapSuper: true}, + } + ordered := backup.OrderRoleMembersForRestore(members) + Expect(ordered).To(HaveLen(2)) + Expect(ordered[0].Member).To(Equal("testrole")) + Expect(ordered[1].Member).To(Equal("testuser")) + }) + It("still defers a grant whose grantor is a superuser but not the bootstrap one", func() { + // check_role_grantor() exempts BOOTSTRAP_SUPERUSERID alone, and the + // select_best_admin() it otherwise defers to ignores super-userness, + // so testrole being SUPERUSER does not let this grant go first. + members := []backup.RoleMember{ + {Role: "usergroup", Member: "testuser", Grantor: "testrole", GrantorIsBootstrapSuper: false}, + {Role: "usergroup", Member: "testrole", Grantor: "gpadmin", IsAdmin: true, GrantorIsBootstrapSuper: true}, + } + ordered := backup.OrderRoleMembersForRestore(members) + Expect(ordered[0].Member).To(Equal("testrole")) + Expect(ordered[1].Member).To(Equal("testuser")) + }) + It("leaves an already-replayable order alone", func() { + members := []backup.RoleMember{ + {Role: "usergroup", Member: "alice", Grantor: "gpadmin", GrantorIsBootstrapSuper: true}, + {Role: "usergroup", Member: "bob", Grantor: "gpadmin", GrantorIsBootstrapSuper: true}, + } + ordered := backup.OrderRoleMembersForRestore(members) + Expect(ordered[0].Member).To(Equal("alice")) + Expect(ordered[1].Member).To(Equal("bob")) + }) + It("only orders within a role, keeping the roles in catalog order", func() { + members := []backup.RoleMember{ + {Role: "groupone", Member: "alice", Grantor: "gpadmin", GrantorIsBootstrapSuper: true}, + {Role: "grouptwo", Member: "carol", Grantor: "dave", IsAdmin: false}, + {Role: "grouptwo", Member: "dave", Grantor: "gpadmin", IsAdmin: true, GrantorIsBootstrapSuper: true}, + } + ordered := backup.OrderRoleMembersForRestore(members) + Expect(ordered[0].Role).To(Equal("groupone")) + Expect(ordered[1].Member).To(Equal("dave")) + Expect(ordered[2].Member).To(Equal("carol")) + }) + It("emits a grant whose grantor never becomes available rather than dropping it", func() { + // A grantor that is not a member of the role at all: nothing can + // make it replayable, so it still has to reach the metadata file. + members := []backup.RoleMember{ + {Role: "usergroup", Member: "testuser", Grantor: "someowner", IsAdmin: false}, + } + ordered := backup.OrderRoleMembersForRestore(members) + Expect(ordered).To(HaveLen(1)) + Expect(ordered[0].Member).To(Equal("testuser")) + }) + }) Describe("PrintRoleMembershipStatements", func() { roleWith := backup.RoleMember{Role: "group", Member: "rolewith", Grantor: "grantor", IsAdmin: true} roleWithout := backup.RoleMember{Role: "group", Member: "rolewithout", Grantor: "grantor", IsAdmin: false} @@ -384,6 +479,18 @@ ALTER ROLE "testRole2" WITH SUPERUSER INHERIT CREATEROLE CREATEDB LOGIN REPLICAT backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{roleWith}) testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `GRANT group TO rolewith WITH ADMIN OPTION GRANTED BY grantor;`) }) + It("prints the PG16+ grant options alongside ADMIN OPTION", func() { + member := backup.RoleMember{Role: "group", Member: "rolewith", Grantor: "grantor", + IsAdmin: true, InheritOption: "FALSE", SetOption: "FALSE"} + backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{member}) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `GRANT group TO rolewith WITH ADMIN OPTION, INHERIT FALSE, SET FALSE GRANTED BY grantor;`) + }) + It("omits SET when it is true but still spells out INHERIT", func() { + member := backup.RoleMember{Role: "group", Member: "rolewith", Grantor: "", + IsAdmin: false, InheritOption: "TRUE", SetOption: "TRUE"} + backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{member}) + testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, `GRANT group TO rolewith WITH INHERIT TRUE;`) + }) It("prints multiple roles", func() { backup.PrintRoleMembershipStatements(backupfile, tocfile, []backup.RoleMember{roleWith, roleWithout}) testutils.AssertBufferContents(tocfile.GlobalEntries, buffer, diff --git a/backup/predata_acl.go b/backup/predata_acl.go index 22d79d39..2f37e4ec 100644 --- a/backup/predata_acl.go +++ b/backup/predata_acl.go @@ -52,6 +52,9 @@ type ACL struct { TemporaryWithGrant bool Connect bool ConnectWithGrant bool + // PG17+ (WHPG19), ACL character 'm'. + Maintain bool + MaintainWithGrant bool } type MetadataMap map[UniqueID]ObjectMetadata @@ -235,6 +238,8 @@ func ParseACL(aclStr string) *ACL { acl.Temporary = true case 'c': acl.Connect = true + case 'm': + acl.Maintain = true case '*': switch lastChar { case 'a': @@ -273,6 +278,9 @@ func ParseACL(aclStr string) *ACL { case 'c': acl.Connect = false acl.ConnectWithGrant = true + case 'm': + acl.Maintain = false + acl.MaintainWithGrant = true } } lastChar = char @@ -376,6 +384,13 @@ func createPrivilegeStrings(acl ACL, objectType string) (string, string) { hasAllPrivileges = acl.Select && acl.Insert && acl.Update && acl.Delete && acl.Truncate && acl.References && acl.Trigger hasAllPrivilegesWithGrant = acl.SelectWithGrant && acl.InsertWithGrant && acl.UpdateWithGrant && acl.DeleteWithGrant && acl.TruncateWithGrant && acl.ReferencesWithGrant && acl.TriggerWithGrant + // PG17+ added MAINTAIN to ACL_ALL_RIGHTS_RELATION. Without this a + // grantee holding only the older seven would still be written as GRANT + // ALL, and would come back holding MAINTAIN as well. + if connectionPool.Version.AtLeast("19") { + hasAllPrivileges = hasAllPrivileges && acl.Maintain + hasAllPrivilegesWithGrant = hasAllPrivilegesWithGrant && acl.MaintainWithGrant + } case toc.OBJ_TABLESPACE: hasAllPrivileges = acl.Create hasAllPrivilegesWithGrant = acl.CreateWithGrant @@ -408,6 +423,11 @@ func createPrivilegeStrings(acl ACL, objectType string) (string, string) { if acl.Trigger { privList = append(privList, "TRIGGER") } + // Gated for the same reason as the ALL check above: MAINTAIN does not + // exist before PG17, so nothing should mention it on an older major. + if acl.Maintain && connectionPool.Version.AtLeast("19") { + privList = append(privList, "MAINTAIN") + } /* * We skip checking whether acl.Execute is set here because only Functions have Execute, * and functions only have Execute, so Execute == hasAllPrivileges for Functions. @@ -451,6 +471,9 @@ func createPrivilegeStrings(acl ACL, objectType string) (string, string) { if acl.TriggerWithGrant { privWithGrantList = append(privWithGrantList, "TRIGGER") } + if acl.MaintainWithGrant && connectionPool.Version.AtLeast("19") { + privWithGrantList = append(privWithGrantList, "MAINTAIN") + } // The comment above regarding Execute applies to ExecuteWithGrant as well. if acl.UsageWithGrant { privWithGrantList = append(privWithGrantList, "USAGE") diff --git a/backup/predata_acl_test.go b/backup/predata_acl_test.go index bc950249..13210687 100644 --- a/backup/predata_acl_test.go +++ b/backup/predata_acl_test.go @@ -24,10 +24,12 @@ var _ = Describe("backup/predata_acl tests", func() { hasAllPrivileges := testutils.DefaultACLForType("anothertestrole", toc.OBJ_TABLE) hasMostPrivileges := testutils.DefaultACLForType("testrole", toc.OBJ_TABLE) hasMostPrivileges.Trigger = false + hasMostPrivileges.Maintain = false hasSinglePrivilege := backup.ACL{Grantee: "", Trigger: true} hasAllPrivilegesWithGrant := testutils.DefaultACLForTypeWithGrant("anothertestrole", toc.OBJ_TABLE) hasMostPrivilegesWithGrant := testutils.DefaultACLForTypeWithGrant("testrole", toc.OBJ_TABLE) hasMostPrivilegesWithGrant.TriggerWithGrant = false + hasMostPrivilegesWithGrant.MaintainWithGrant = false hasSinglePrivilegeWithGrant := backup.ACL{Grantee: "", TriggerWithGrant: true} privileges := []backup.ACL{hasAllPrivileges, hasMostPrivileges, hasSinglePrivilege} privilegesWithGrant := []backup.ACL{hasAllPrivilegesWithGrant, hasMostPrivilegesWithGrant, hasSinglePrivilegeWithGrant} @@ -430,9 +432,22 @@ ALTER DEFAULT PRIVILEGES FOR ROLE testrole GRANT USAGE ON TABLES TO somerole WIT result := backup.ParseACL(aclStr) structmatcher.ExpectStructsToMatch(&expected, result) }) + It("parses the PG17 MAINTAIN privilege", func() { + result := backup.ParseACL("testrole=m/gpadmin") + Expect(result.Maintain).To(BeTrue()) + Expect(result.MaintainWithGrant).To(BeFalse()) + }) + It("parses MAINTAIN with grant option", func() { + result := backup.ParseACL("testrole=m*/gpadmin") + Expect(result.Maintain).To(BeFalse()) + Expect(result.MaintainWithGrant).To(BeTrue()) + }) It("parses an ACL string containing a role with multiple privileges", func() { aclStr := "testrole=arwdDxt/gpadmin" expected := testutils.DefaultACLForType("testrole", toc.OBJ_TABLE) + // The string above grants the seven privileges a relation had + // before PG17, so it never carries MAINTAIN whatever the version. + expected.Maintain = false result := backup.ParseACL(aclStr) structmatcher.ExpectStructsToMatch(&expected, result) }) diff --git a/backup/predata_functions.go b/backup/predata_functions.go index f01748e3..d75457ba 100644 --- a/backup/predata_functions.go +++ b/backup/predata_functions.go @@ -18,14 +18,30 @@ func PrintCreateFunctionStatement(metadataFile *utils.FileWithByteCount, objToc start := metadataFile.ByteCount funcFQN := utils.MakeFQN(funcDef.Schema, funcDef.Name) + // A SQL-standard body (WHPG19+) is not introduced by AS, and has to follow + // the modifiers rather than precede LANGUAGE, so the statement is laid out + // differently in that case. This mirrors dumpFunc() in pg_dump.c. + hasSqlBody := funcDef.SqlBody != "" + asClause := " AS" + if hasSqlBody { + asClause = "" + } + if connectionPool.Version.AtLeast("7") && funcDef.Kind == "p" { - metadataFile.MustPrintf("\n\nCREATE PROCEDURE %s(%s) AS", funcFQN, funcDef.Arguments.String) + metadataFile.MustPrintf("\n\nCREATE PROCEDURE %s(%s)%s", funcFQN, funcDef.Arguments.String, asClause) } else { - metadataFile.MustPrintf("\n\nCREATE FUNCTION %s(%s) RETURNS %s AS", funcFQN, funcDef.Arguments.String, funcDef.ResultType.String) + metadataFile.MustPrintf("\n\nCREATE FUNCTION %s(%s) RETURNS %s%s", funcFQN, funcDef.Arguments.String, funcDef.ResultType.String, asClause) + } + if hasSqlBody { + metadataFile.MustPrintln() + } else { + PrintFunctionBodyOrPath(metadataFile, funcDef) } - PrintFunctionBodyOrPath(metadataFile, funcDef) metadataFile.MustPrintf("LANGUAGE %s", funcDef.Language) PrintFunctionModifiers(metadataFile, funcDef) + if hasSqlBody { + metadataFile.MustPrintf("\n%s", funcDef.SqlBody) + } metadataFile.MustPrintln(";") section, entry := funcDef.GetMetadataEntry() diff --git a/backup/predata_functions_test.go b/backup/predata_functions_test.go index 90e94559..51835f2d 100644 --- a/backup/predata_functions_test.go +++ b/backup/predata_functions_test.go @@ -45,6 +45,19 @@ var _ = Describe("backup/predata_functions tests", func() { testutils.AssertBufferContents(tocfile.PredataEntries, buffer, fmt.Sprintf(`CREATE FUNCTION public.func_name(integer, integer) RETURNS integer AS $$add_two_ints$$ LANGUAGE internal%s;`, DEFAULT_PARALLEL)) + }) + It("prints a SQL-standard function body after the modifiers, with no AS clause", func() { + // WHPG19+: the body comes from prosqlbody, and prosrc is empty. + funcDef.Language = "sql" + funcDef.FunctionBody = "" + funcDef.Cost = float32(100) + funcDef.SqlBody = "BEGIN ATOMIC\n SELECT $1 + $2;\nEND" + backup.PrintCreateFunctionStatement(backupfile, tocfile, funcDef, funcMetadata) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, fmt.Sprintf(`CREATE FUNCTION public.func_name(integer, integer) RETURNS integer +LANGUAGE sql%s +BEGIN ATOMIC + SELECT $1 + $2; +END;`, DEFAULT_PARALLEL)) }) It("prints a function definition for a function that returns a set", func() { funcDef.ReturnsSet = true diff --git a/backup/predata_relations.go b/backup/predata_relations.go index c7ed1c7d..42b37a7e 100644 --- a/backup/predata_relations.go +++ b/backup/predata_relations.go @@ -251,6 +251,11 @@ func printAlterColumnStatements(metadataFile *utils.FileWithByteCount, table Tab if column.Options != "" { metadataFile.MustPrintf("\nALTER TABLE ONLY %s ALTER COLUMN %s SET (%s);", table.FQN(), column.Name, column.Options) } + // PG14+ (WHPG19). Emitted as its own ALTER, the way pg_dump does it, + // rather than inline in the column definition. + if column.Compression != "" { + metadataFile.MustPrintf("\nALTER TABLE ONLY %s ALTER COLUMN %s SET COMPRESSION %s;", table.FQN(), column.Name, column.Compression) + } } } diff --git a/backup/predata_relations_tables_test.go b/backup/predata_relations_tables_test.go index c4dd84d0..76be533d 100644 --- a/backup/predata_relations_tables_test.go +++ b/backup/predata_relations_tables_test.go @@ -61,6 +61,7 @@ ENCODING 'UTF-8';`) colWithCollation := backup.ColumnDefinition{Oid: 0, Num: 1, Name: "c", Type: "character (8)", StatTarget: -1, Collation: "public.some_coll"} rowTwoGenerated := backup.ColumnDefinition{Oid: 0, Num: 2, Name: "j", HasDefault: true, Type: "integer", StatTarget: -1, DefaultVal: "(i * 2)", AttGenerated: "STORED"} rowTwoGeneratedInherited := backup.ColumnDefinition{Oid: 0, Num: 2, Name: "j", HasDefault: true, Type: "integer", StatTarget: -1, DefaultVal: "(i * 2)", AttGenerated: "STORED", IsInherited: true} + rowTwoGeneratedVirtual := backup.ColumnDefinition{Oid: 0, Num: 2, Name: "j", HasDefault: true, Type: "integer", StatTarget: -1, DefaultVal: "(i * 2)", AttGenerated: "VIRTUAL"} Context("No special table attributes", func() { It("prints a CREATE TABLE OF type block with one attribute", func() { @@ -182,6 +183,17 @@ ALTER TABLE ONLY public.tablename ALTER COLUMN i SET STORAGE PLAIN;`) ) DISTRIBUTED RANDOMLY; ALTER TABLE ONLY public.tablename ALTER COLUMN i SET (n_distinct=1);`) + }) + It("prints a CREATE TABLE block followed by an ALTER COLUMN ... SET COMPRESSION statement", func() { + // PG14+ (WHPG19): an explicit TOAST compression method. + colCompressed := backup.ColumnDefinition{Oid: 0, Num: 1, Name: "i", Type: "text", StatTarget: -1, Compression: "lz4"} + testTable.ColumnDefs = []backup.ColumnDefinition{colCompressed} + backup.PrintRegularTableCreateStatement(backupfile, tocfile, testTable) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TABLE public.tablename ( + i text +) DISTRIBUTED RANDOMLY; + +ALTER TABLE ONLY public.tablename ALTER COLUMN i SET COMPRESSION lz4;`) }) It("prints a CREATE TABLE block with one line with regular attribute and a line with generated attribute", func() { col := []backup.ColumnDefinition{rowOne, rowTwoGenerated} @@ -190,6 +202,16 @@ ALTER TABLE ONLY public.tablename ALTER COLUMN i SET (n_distinct=1);`) testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TABLE public.tablename ( i integer, j integer GENERATED ALWAYS AS (i * 2) STORED +) DISTRIBUTED RANDOMLY;`) + }) + It("prints a CREATE TABLE block with a virtual generated attribute", func() { + // WHPG19+ (PG18) adds virtual generated columns. + col := []backup.ColumnDefinition{rowOne, rowTwoGeneratedVirtual} + testTable.ColumnDefs = col + backup.PrintRegularTableCreateStatement(backupfile, tocfile, testTable) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TABLE public.tablename ( + i integer, + j integer GENERATED ALWAYS AS (i * 2) VIRTUAL ) DISTRIBUTED RANDOMLY;`) }) It("prints a CREATE TABLE block that omits the generated attribute if it inherits from another table", func() { diff --git a/backup/predata_types.go b/backup/predata_types.go index 3e27bb17..c0cceea6 100644 --- a/backup/predata_types.go +++ b/backup/predata_types.go @@ -117,6 +117,9 @@ func PrintCreateBaseTypeStatement(metadataFile *utils.FileWithByteCount, objToc if base.Element != "" { metadataFile.MustPrintf(",\n\tELEMENT = %s", base.Element) } + if base.Subscript != "" { + metadataFile.MustPrintf(",\n\tSUBSCRIPT = %s", base.Subscript) + } if base.Delimiter != "" { metadataFile.MustPrintf(",\n\tDELIMITER = '%s'", base.Delimiter) } @@ -187,6 +190,9 @@ func PrintCreateRangeTypeStatement(metadataFile *utils.FileWithByteCount, objToc start := metadataFile.ByteCount metadataFile.MustPrintf("\n\nCREATE TYPE %s AS RANGE (\n\tSUBTYPE = %s", rangeType.FQN(), rangeType.SubType) + if rangeType.MultiRangeName != "" { + metadataFile.MustPrintf(",\n\tMULTIRANGE_TYPE_NAME = %s", rangeType.MultiRangeName) + } if rangeType.SubTypeOpClass != "" { metadataFile.MustPrintf(",\n\tSUBTYPE_OPCLASS = %s", rangeType.SubTypeOpClass) } @@ -210,7 +216,14 @@ func PrintCreateRangeTypeStatement(metadataFile *utils.FileWithByteCount, objToc func PrintCreateCollationStatements(metadataFile *utils.FileWithByteCount, objToc *toc.TOC, collations []Collation, collationMetadata MetadataMap) { for _, collation := range collations { start := metadataFile.ByteCount - metadataFile.MustPrintf("\nCREATE COLLATION %s (LC_COLLATE = '%s', LC_CTYPE = '%s'", collation.FQN(), collation.Collate, collation.Ctype) + // A non-libc collation on PG17+ (WHPG19) has no LC_COLLATE/LC_CTYPE to + // reproduce; its single locale comes from colllocale instead, which is + // spelled LOCALE in CREATE COLLATION. + if collation.Locale != "" { + metadataFile.MustPrintf("\nCREATE COLLATION %s (LOCALE = '%s'", collation.FQN(), utils.EscapeSingleQuotes(collation.Locale)) + } else { + metadataFile.MustPrintf("\nCREATE COLLATION %s (LC_COLLATE = '%s', LC_CTYPE = '%s'", collation.FQN(), collation.Collate, collation.Ctype) + } if collation.Provider != "" { providerOption := "" switch collation.Provider { @@ -220,11 +233,18 @@ func PrintCreateCollationStatements(metadataFile *utils.FileWithByteCount, objTo providerOption = "icu" case "d": providerOption = "default" + case "b": // PG17+ (WHPG19) + providerOption = "builtin" default: - gplog.Fatal(errors.Errorf("Unexpected collation provider: expected 'c|i|d' got '%s'\n", collation.Provider), "") + gplog.Fatal(errors.Errorf("Unexpected collation provider: expected 'c|i|d|b' got '%s'\n", collation.Provider), "") } metadataFile.MustPrintf(", PROVIDER = '%s'", providerOption) } + if collation.IcuRules != "" { + // ICU tailoring uses the apostrophe as its own quoting character, + // so these genuinely do contain single quotes. + metadataFile.MustPrintf(", RULES = '%s'", utils.EscapeSingleQuotes(collation.IcuRules)) + } if collation.IsDeterministic == "f" { metadataFile.MustPrintf(", DETERMINISTIC = 'false'") } diff --git a/backup/predata_types_test.go b/backup/predata_types_test.go index 12645ce0..208fa268 100644 --- a/backup/predata_types_test.go +++ b/backup/predata_types_test.go @@ -250,6 +250,15 @@ ALTER TYPE public.base_type testutils.ExpectEntry(tocfile.PredataEntries, 0, "public", "", "rangetype", toc.OBJ_TYPE) testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TYPE public.rangetype AS RANGE ( SUBTYPE = test_subtype_schema.test_subtype +);`) + }) + It("prints a range type with a multirange type name", func() { + rangeType := basicRangeType + rangeType.MultiRangeName = "public.my_multirange" + backup.PrintCreateRangeTypeStatement(backupfile, tocfile, rangeType, emptyMetadata) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE TYPE public.rangetype AS RANGE ( + SUBTYPE = test_subtype_schema.test_subtype, + MULTIRANGE_TYPE_NAME = public.my_multirange );`) }) It("prints a complex range type", func() { @@ -290,5 +299,24 @@ ALTER TYPE public.base_type "ALTER COLLATION schema1.collation1 OWNER TO testrole;"} testutils.AssertBufferContents(tocfile.PredataEntries, buffer, expectedStatements...) }) + It("prints an ICU collation statement with a locale and rules", func() { + collation := backup.Collation{Oid: 1, Name: "collation1", Schema: "schema1", + Provider: "i", Locale: "en-US", IcuRules: "&a < g"} + backup.PrintCreateCollationStatements(backupfile, tocfile, []backup.Collation{collation}, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE COLLATION schema1.collation1 (LOCALE = 'en-US', PROVIDER = 'icu', RULES = '&a < g');`) + }) + It("escapes single quotes in the ICU rules", func() { + // ICU tailoring uses the apostrophe as its own quoting character. + collation := backup.Collation{Oid: 1, Name: "collation1", Schema: "schema1", + Provider: "i", Locale: "en-US", IcuRules: "&a < 'x'"} + backup.PrintCreateCollationStatements(backupfile, tocfile, []backup.Collation{collation}, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE COLLATION schema1.collation1 (LOCALE = 'en-US', PROVIDER = 'icu', RULES = '&a < ''x''');`) + }) + It("prints a builtin collation statement with a locale", func() { + collation := backup.Collation{Oid: 1, Name: "collation1", Schema: "schema1", + Provider: "b", Locale: "C.UTF-8"} + backup.PrintCreateCollationStatements(backupfile, tocfile, []backup.Collation{collation}, emptyMetadataMap) + testutils.AssertBufferContents(tocfile.PredataEntries, buffer, `CREATE COLLATION schema1.collation1 (LOCALE = 'C.UTF-8', PROVIDER = 'builtin');`) + }) }) }) diff --git a/backup/queries_functions.go b/backup/queries_functions.go index 602e63a1..a3399fb3 100644 --- a/backup/queries_functions.go +++ b/backup/queries_functions.go @@ -42,6 +42,7 @@ type Function struct { ExecLocation string `db:"proexeclocation"` Parallel string `db:"proparallel"` // GPDB 7+ TransformTypes string // GPDB 7+ + SqlBody string // WHPG19+ } func (f Function) GetMetadataEntry() (string, toc.MetadataEntry) { @@ -79,6 +80,17 @@ func GetFunctions(connectionPool *dbconn.DBConn) []Function { AND objid = p.oid AND deptype = 'i')` } + // PG14+ (WHPG19) lets a SQL function carry a pre-parsed, SQL-standard body + // (BEGIN ATOMIC ... END, or RETURN expr) in pg_proc.prosqlbody. For those + // functions the server stores prosrc as the empty string -- see the + // sql_body branch of interpret_AS_clause() -- so backing up prosrc alone + // yields a function with no body at all. Deparse the real body the way + // pg_dump does, and prefer it over prosrc wherever it is present. + sqlBodyAtt := "" + if connectionPool.Version.AtLeast("19") { + sqlBodyAtt = "coalesce(pg_catalog.pg_get_function_sqlbody(p.oid), '') AS sqlbody," + } + locationAtts := "" if connectionPool.Version.Before("6") { locationAtts = "'a' AS proexeclocation," @@ -124,6 +136,7 @@ func GetFunctions(connectionPool *dbconn.DBConn) []Function { quote_ident(nspname) AS schema, quote_ident(p.proname) AS name, proretset, + %s coalesce(prosrc, '') AS functionbody, coalesce(probin, '') AS binarypath, pg_catalog.pg_get_function_arguments(p.oid) AS arguments, @@ -154,7 +167,7 @@ func GetFunctions(connectionPool *dbconn.DBConn) []Function { WHERE %s AND prokind <> 'a' AND %s%s - ORDER BY nspname, proname, identargs`, locationAtts, + ORDER BY nspname, proname, identargs`, sqlBodyAtt, locationAtts, SchemaFilterClause("n"), ExtensionFilterClause("p"), excludeImplicitFunctionsClause) @@ -576,6 +589,20 @@ func GetCasts(connectionPool *dbconn.DBConn) []Cast { } else { methodStr = "CASE WHEN c.castfunc = 0 THEN 'b' ELSE 'f' END AS castmethod," } + // "CREATE TYPE ... AS RANGE" on PG14+ (WHPG19) also creates the + // range->multirange cast. It is not independently creatable via CREATE + // CAST, and restoring it explicitly errors, because at that point in + // predata restore its endpoint type is still just a shell. Exclude it + // exactly as pg_dump's getCasts() does, and only where it can exist -- + // pg_range.rngmultitypid is itself PG14+. + multirangeCastClause := "" + if connectionPool.Version.AtLeast("19") { + multirangeCastClause = `AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_range r + WHERE c.castsource = r.rngtypid AND c.casttarget = r.rngmultitypid + )` + } + query := fmt.Sprintf(` SELECT c.oid, @@ -596,9 +623,10 @@ func GetCasts(connectionPool *dbconn.DBConn) []Cast { LEFT JOIN pg_namespace n ON p.pronamespace = n.oid WHERE ((%s) OR (%s) OR (%s)) AND %s + %s ORDER BY 1, 2`, methodStr, SchemaFilterClause("sn"), SchemaFilterClause("tn"), - SchemaFilterClause("n"), ExtensionFilterClause("c")) + SchemaFilterClause("n"), ExtensionFilterClause("c"), multirangeCastClause) casts := make([]Cast, 0) err := connectionPool.Select(&casts, query) @@ -839,7 +867,8 @@ func GetForeignDataWrappers(connectionPool *dbconn.DBConn) []ForeignDataWrapper SELECT pg_catalog.quote_ident(option_name) || ' ' || pg_catalog.quote_literal(option_value) FROM pg_options_to_table(fdwoptions) ORDER BY option_name), ', ') AS options FROM pg_foreign_data_wrapper - WHERE oid >= %d AND %s`, FIRST_NORMAL_OBJECT_ID, ExtensionFilterClause("")) + WHERE oid >= %d AND %s + ORDER BY fdwname`, FIRST_NORMAL_OBJECT_ID, ExtensionFilterClause("")) err := connectionPool.Select(&results, query) gplog.FatalOnError(err) diff --git a/backup/queries_globals.go b/backup/queries_globals.go index 85a0253f..15d32335 100644 --- a/backup/queries_globals.go +++ b/backup/queries_globals.go @@ -39,12 +39,15 @@ func GetSessionGUCs(connectionPool *dbconn.DBConn) SessionGUCs { } type Database struct { - Oid uint32 - Name string - Tablespace string - Collate string - CType string - Encoding string + Oid uint32 + Name string + Tablespace string + Collate string + CType string + Encoding string + LocProvider string // WHPG19+ + Locale string // WHPG19+ + IcuRules string // WHPG19+ } func (db Database) GetMetadataEntry() (string, toc.MetadataEntry) { @@ -72,6 +75,14 @@ func GetDefaultDatabaseEncodingInfo(connectionPool *dbconn.DBConn) Database { if connectionPool.Version.AtLeast("6") { lcQuery = "datcollate AS collate, datctype AS ctype," } + // template0's own provider and locale, so that PrintCreateDatabaseStatement + // only spells them out for a database that actually diverges from it. + if connectionPool.Version.AtLeast("19") { + lcQuery += ` + datlocprovider AS locprovider, + coalesce(datlocale, '') AS locale, + coalesce(daticurules, '') AS icurules,` + } query := fmt.Sprintf(` SELECT datname AS name, @@ -91,6 +102,17 @@ func GetDatabaseInfo(connectionPool *dbconn.DBConn) Database { if connectionPool.Version.AtLeast("6") { lcQuery = "datcollate AS collate, datctype AS ctype," } + // PG15+ (WHPG19) lets a database pick a locale provider other than libc, + // in which case the locale that actually governs it lives in datlocale + // (plus daticurules for ICU tailoring) rather than in datcollate/datctype. + // Those two stay populated regardless, so without the provider the database + // silently comes back as libc with a different collation. + if connectionPool.Version.AtLeast("19") { + lcQuery += ` + datlocprovider AS locprovider, + coalesce(datlocale, '') AS locale, + coalesce(daticurules, '') AS icurules,` + } query := fmt.Sprintf(` SELECT d.oid, quote_ident(d.datname) AS name, @@ -505,6 +527,12 @@ type RoleMember struct { Member string Grantor string IsAdmin bool + // WHPG19+. Empty on older majors, where the grant carries no such option. + InheritOption string + SetOption string + // WHPG19+. Whether the grantor is the bootstrap superuser, the only role + // whose grants can be replayed before its own membership on the role. + GrantorIsBootstrapSuper bool } func (rm RoleMember) GetMetadataEntry() (string, toc.MetadataEntry) { @@ -528,23 +556,99 @@ func GetRoleMembers(connectionPool *dbconn.DBConn) []RoleMember { whereClause = `` } + // PG16+ (WHPG19) records two more per-grant options alongside admin_option. + // Without them a GRANT ... WITH INHERIT FALSE / SET FALSE comes back with + // the defaults, quietly widening what the member can do. + grantOptionAtts := "" + if connectionPool.Version.AtLeast("19") { + grantOptionAtts = `CASE WHEN pga.inherit_option THEN 'TRUE' ELSE 'FALSE' END AS inheritoption, + CASE WHEN pga.set_option THEN 'TRUE' ELSE 'FALSE' END AS setoption, + (pga.grantor = 10::oid) AS grantorisbootstrapsuper,` + } + query := fmt.Sprintf(` SELECT quote_ident(pg_get_userbyid(pga.roleid)) AS role, quote_ident(pg_get_userbyid(pga.member)) AS member, CASE WHEN pg_get_userbyid(pga.grantor) like 'unknown (OID='||pga.grantor::regclass||')' THEN '' ELSE quote_ident(pg_get_userbyid(pga.grantor)) END AS grantor, + %s admin_option AS isadmin FROM pg_auth_members pga % s - ORDER BY roleid, member`, whereClause) + ORDER BY roleid, member`, grantOptionAtts, whereClause) results := make([]RoleMember, 0) err := connectionPool.Select(&results, query) gplog.FatalOnError(err) + if connectionPool.Version.AtLeast("19") { + return OrderRoleMembersForRestore(results) + } return results } +// OrderRoleMembersForRestore puts role grants into an order the restore can +// actually replay. +// +// PG16+ requires the role named by GRANTED BY to hold ADMIN OPTION on the role +// being granted at the moment the grant is replayed, so such a grant must +// follow that grantor's own ADMIN OPTION grant. The catalog order the query +// asks for -- roleid, member -- says nothing about that: the two rows share a +// roleid, so it falls to the members' OIDs, and a member that happens to +// predate its grantor restores first and fails. +// +// Only the bootstrap superuser is exempt, and being a superuser is not enough: +// check_role_grantor() skips the check for BOOTSTRAP_SUPERUSERID alone, and the +// select_best_admin() it defers to otherwise is documented as ignoring +// super-userness. pg_dumpall compares the recorded grantor against +// BOOTSTRAP_SUPERUSERID for the same reason. +// +// Same shape as pg_dumpall's dumpRoleMembership(): repeatedly emit whatever has +// become replayable, and if a pass makes no progress emit the remainder in +// catalog order rather than dropping grants on the floor. +func OrderRoleMembersForRestore(roleMembers []RoleMember) []RoleMember { + // A grant is only constrained relative to other grants of the same role. + membersByRole := make(map[string][]RoleMember) + roleOrder := make([]string, 0) + for _, member := range roleMembers { + if _, seen := membersByRole[member.Role]; !seen { + roleOrder = append(roleOrder, member.Role) + } + membersByRole[member.Role] = append(membersByRole[member.Role], member) + } + + ordered := make([]RoleMember, 0, len(roleMembers)) + for _, role := range roleOrder { + remaining := membersByRole[role] + // Roles known to hold ADMIN OPTION on this role by the time we get here. + canGrant := make(map[string]bool) + for len(remaining) > 0 { + deferred := make([]RoleMember, 0, len(remaining)) + progressed := false + for _, member := range remaining { + if member.Grantor == "" || member.GrantorIsBootstrapSuper || canGrant[member.Grantor] { + ordered = append(ordered, member) + if member.IsAdmin { + canGrant[member.Member] = true + } + progressed = true + } else { + deferred = append(deferred, member) + } + } + if !progressed { + // Nothing left can be justified from this role's own + // membership -- a grantor that is not a member of it, say. + // Emit as-is and let the restore report it. + ordered = append(ordered, deferred...) + break + } + remaining = deferred + } + } + return ordered +} + type Tablespace struct { Oid uint32 Tablespace string diff --git a/backup/queries_operators.go b/backup/queries_operators.go index c56ab4ce..6b0428d4 100644 --- a/backup/queries_operators.go +++ b/backup/queries_operators.go @@ -7,6 +7,7 @@ package backup import ( "fmt" + "sort" "github.com/greenplum-db/gpbackup/toc" "github.com/greenplum-db/gpbackup/utils" @@ -239,13 +240,82 @@ func GetOperatorClassOperators(connectionPool *dbconn.DBConn) map[uint32][]Opera err := connectionPool.Select(&results, query) gplog.FatalOnError(err) + // PG14+ (WHPG19) records a GiST/GIN/SP-GiST opclass's OPERATOR members as + // depending on the operator FAMILY even when they were declared inline in + // CREATE OPERATOR CLASS -- see the comment above the operator loop in + // gistvalidate(), reached through the amadjustmembers callback added in + // PG14 (9f9682783, reachable from REL_14_0 but not REL_13_0). The + // pg_depend query above therefore returns nothing for them, and since + // gpbackup only ever emits a bare CREATE OPERATOR FAMILY and never dumps + // family members, those operators would be lost outright. + // Recover them from the family, and merge rather than replace: btree and + // hash still depend on the opclass (nbtvalidate/hashvalidate), and that + // attribution is exact where it exists. + if connectionPool.Version.AtLeast("19") { + familyResults := make([]OperatorClassOperator, 0) + err = connectionPool.Select(&familyResults, familyMemberQuery(` + amopstrategy AS strategynumber, + amopopr::pg_catalog.regoperator AS operator, + coalesce(quote_ident(sort_ns.nspname) || '.' || quote_ident(sort_opf.opfname), '') AS orderbyfamily`, + `JOIN pg_catalog.pg_amop ao ON ao.amopfamily = cls.opcfamily + AND ao.amoplefttype = cls.opcintype AND ao.amoprighttype = cls.opcintype + LEFT JOIN pg_catalog.pg_opfamily sort_opf ON sort_opf.oid = ao.amopsortfamily + LEFT JOIN pg_catalog.pg_namespace sort_ns ON sort_ns.oid = sort_opf.opfnamespace`)) + gplog.FatalOnError(err) + results = append(results, familyResults...) + } + operators := make(map[uint32][]OperatorClassOperator) + seen := make(map[OperatorClassOperator]bool) for _, result := range results { + // A class whose members pg_depend already attributed exactly will match + // the family lookup as well; keep the first sighting of each member. + if seen[result] { + continue + } + seen[result] = true operators[result.ClassOid] = append(operators[result.ClassOid], result) } + for classOid := range operators { + sort.SliceStable(operators[classOid], func(i, j int) bool { + return operators[classOid][i].StrategyNumber < operators[classOid][j].StrategyNumber + }) + } return operators } +// familyMemberQuery builds the WHPG19 fallback that attributes an operator +// family's members to the one opclass that family belongs to. +// +// It deliberately covers only families holding a single opclass -- the shape +// CREATE OPERATOR CLASS produces when no FAMILY is named. In a family shared by +// several classes there is nothing in the catalog that says which class a +// family-level member was declared with, and guessing would fold members that +// belong to a sibling class (or that were added by ALTER OPERATOR FAMILY) into +// this one: on restore those come back as class members, and if the family +// already held a matching member the restore fails outright with "operator +// already exists in operator family". Losing them, as gpbackup already does +// today for family members generally, is the safer direction. +// +// The schema and extension filters matter for more than tidiness: built-in +// catalog members are pinned and have no pg_depend rows, so the query they +// supplement never returned them. Without the filters every backup would fetch +// the entire catalog's members only for GetOperatorClasses to discard them. +func familyMemberQuery(selectList string, joins string) string { + return fmt.Sprintf(` + SELECT cls.oid AS classoid, + %s + FROM pg_catalog.pg_opclass cls + JOIN pg_catalog.pg_namespace cls_ns ON cls_ns.oid = cls.opcnamespace + %s + WHERE %s + AND %s + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_opclass other + WHERE other.opcfamily = cls.opcfamily AND other.oid <> cls.oid)`, + selectList, joins, SchemaFilterClause("cls_ns"), ExtensionFilterClause("cls")) +} + type OperatorClassFunction struct { ClassOid uint32 SupportNumber int @@ -271,9 +341,37 @@ func GetOperatorClassFunctions(connectionPool *dbconn.DBConn) map[uint32][]Opera err := connectionPool.Select(&results, query) gplog.FatalOnError(err) + // Unlike the OPERATOR members, a GiST opclass's *required* support + // functions keep a hard dependency on the opclass, so the query above + // still finds those on WHPG19; only the optional ones (compress, distance, + // options, ...) are forced onto the family. Merge those in the same way, + // rather than replacing a query that is exact for everything else. + if connectionPool.Version.AtLeast("19") { + familyResults := make([]OperatorClassFunction, 0) + err = connectionPool.Select(&familyResults, familyMemberQuery(` + amprocnum AS supportnumber, + amproclefttype::regtype, + amprocrighttype::regtype, + amproc::regprocedure::text AS functionname`, + `JOIN pg_catalog.pg_amproc ap ON ap.amprocfamily = cls.opcfamily + AND ap.amproclefttype = cls.opcintype AND ap.amprocrighttype = cls.opcintype`)) + gplog.FatalOnError(err) + results = append(results, familyResults...) + } + functions := make(map[uint32][]OperatorClassFunction) + seen := make(map[OperatorClassFunction]bool) for _, result := range results { + if seen[result] { + continue + } + seen[result] = true functions[result.ClassOid] = append(functions[result.ClassOid], result) } + for classOid := range functions { + sort.SliceStable(functions[classOid], func(i, j int) bool { + return functions[classOid][i].SupportNumber < functions[classOid][j].SupportNumber + }) + } return functions } diff --git a/backup/queries_postdata.go b/backup/queries_postdata.go index ed148726..c336b563 100644 --- a/backup/queries_postdata.go +++ b/backup/queries_postdata.go @@ -438,6 +438,18 @@ func GetTriggers(connectionPool *dbconn.DBConn) []TriggerDefinition { constraintClause := "NOT tgisinternal" if connectionPool.Version.Before("6") { constraintClause = "tgisconstraint = 'f'" + } else if connectionPool.Version.AtLeast("19") { + // A row trigger on a partitioned table is cloned onto every child, and + // PG13+ records the clone with tgisinternal = isInternal (trigger.c + // stores the flag verbatim) plus a tgparentid link -- so for a user + // trigger the clones are NOT internal and this filter lets them + // through. Each child then gets its own CREATE TRIGGER, while the + // parent's statement already recreates the clones, and the restore + // fails with "trigger ... already exists". Match on the parent link + // instead, as pg_dump's getTriggers() does. tgparentid does not exist + // before PG13, hence the version gate rather than an unconditional + // clause. + constraintClause = "NOT tgisinternal AND t.tgparentid = 0" } query := fmt.Sprintf(` SELECT t.oid AS oid, diff --git a/backup/queries_shared.go b/backup/queries_shared.go index 8165a8c4..bd8d7a34 100644 --- a/backup/queries_shared.go +++ b/backup/queries_shared.go @@ -173,6 +173,10 @@ func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) [] AND coninhcount = 0 GROUP BY con.oid, conname, contype, c.relname, n.nspname, %s pt.parrelid`, conIsLocal, "%s", ExtensionFilterClause("c"), conIsLocal) } else { + // PG18+ (WHPG19) represents column NOT NULL as pg_constraint rows + // (contype 'n'). Those are already captured in the column definitions + // (attnotnull), so exclude them here like pg_dump does; 'n' does not + // exist as a contype before PG17 so the filter is harmless on GPDB7. tableQuery = fmt.Sprintf(` SELECT con.oid, quote_ident(n.nspname) AS schema, @@ -193,7 +197,7 @@ func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) [] WHERE %s AND %s AND c.relname IS NOT NULL - AND contype != 't' + AND contype NOT IN ('t', 'n') AND (c.relispartition IS FALSE OR conislocal IS TRUE) AND coninhcount = 0 GROUP BY con.oid, conname, contype, c.relname, n.nspname, con.conislocal, pt.partrelid`, "%s", ExtensionFilterClause("c")) @@ -215,6 +219,7 @@ func GetConstraints(connectionPool *dbconn.DBConn, includeTables ...Relation) [] WHERE %s AND %s AND t.typname IS NOT NULL + AND contype != 'n' GROUP BY con.oid, conname, contype, n.nspname, %s t.typname ORDER BY name`, conIsLocal, SchemaFilterClause("n"), ExtensionFilterClause("con"), conIsLocal) diff --git a/backup/queries_table_defs.go b/backup/queries_table_defs.go index 51bdbbac..e206e906 100644 --- a/backup/queries_table_defs.go +++ b/backup/queries_table_defs.go @@ -24,7 +24,26 @@ type Table struct { func (t Table) SkipDataBackup() bool { def := t.TableDefinition - return def.IsExternal || (def.ForeignDef != ForeignTableDefinition{}) + return def.IsExternal || (def.ForeignDef != ForeignTableDefinition{}) || t.hasOnlyGeneratedColumns() +} + +// A table whose every column is generated holds no data to copy, and that +// cannot be expressed as a COPY column list: an empty list is a syntax error, +// and omitting the list makes COPY cover the generated column, which fails the +// restore with "cannot be inserted into". PG18's virtual generated columns make +// such a table easy to construct, so skip its data rather than write out a COPY +// that cannot work. A table with no columns at all is left alone -- COPY +// without a list is correct there. +func (t Table) hasOnlyGeneratedColumns() bool { + if len(t.ColumnDefs) == 0 { + return false + } + for _, column := range t.ColumnDefs { + if column.AttGenerated == "" { + return false + } + } + return true } func (t Table) GetMetadataEntry() (string, toc.MetadataEntry) { @@ -226,6 +245,7 @@ type ColumnDefinition struct { SecurityLabelProvider string SecurityLabel string AttGenerated string + Compression string // WHPG19+ IsInherited bool } @@ -238,6 +258,12 @@ var storageTypeCodes = map[string]string{ var attGeneratedCodes = map[string]string{ "s": "STORED", + // PG18+ (WHPG19) adds virtual generated columns. Mapping the code matters + // twice over: the keyword is needed to reproduce the column, and an + // unmapped code reads as "not generated", which would both emit a plain + // DEFAULT and put the column into the COPY list -- see + // ConstructTableAttributesList -- even though it has no stored value. + "v": "VIRTUAL", } func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefinition { @@ -325,6 +351,15 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi ORDER BY a.attrelid, a.attnum`, relationAndSchemaFilterClause()) // In GPDB7+ we do not want to exclude child partitions, they function as separate tables. + // PG14+ (WHPG19) lets a column pick its TOAST compression method. + // attcompression is '\0' when unset, 'p' for pglz and 'l' for lz4; a column + // left at the default keeps '\0' and needs nothing emitted, but an explicit + // choice has to be reproduced or an lz4 column silently comes back as pglz. + compressionAtt := "'' AS compression," + if connectionPool.Version.AtLeast("19") { + compressionAtt = `CASE a.attcompression WHEN 'p' THEN 'pglz' WHEN 'l' THEN 'lz4' ELSE '' END AS compression,` + } + // Cannot use unnest() in CASE statements anymore in GPDB 7+ so convert // it to a LEFT JOIN LATERAL. We do not use LEFT JOIN LATERAL for GPDB 6 // because the CASE unnest() logic is more performant. @@ -336,7 +371,8 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi a.atthasdef, pg_catalog.format_type(t.oid,a.atttypmod) AS type, coalesce(pg_catalog.array_to_string(e.attoptions, ','), '') AS encoding, - a.attstattarget, + coalesce(a.attstattarget, -1) AS attstattarget, + %s CASE WHEN a.attstorage != t.typstorage THEN a.attstorage ELSE '' END AS storagetype, coalesce('('||pg_catalog.pg_get_expr(ad.adbin, ad.adrelid)||')', '') AS defaultval, coalesce(d.description, '') AS comment, @@ -368,7 +404,7 @@ func GetColumnDefinitions(connectionPool *dbconn.DBConn) map[uint32][]ColumnDefi AND c.reltype <> 0 AND a.attnum > 0::pg_catalog.int2 AND a.attisdropped = 'f' - ORDER BY a.attrelid, a.attnum`, relationAndSchemaFilterClause()) + ORDER BY a.attrelid, a.attnum`, compressionAtt, relationAndSchemaFilterClause()) query := `` if connectionPool.Version.Before("6") { diff --git a/backup/queries_types.go b/backup/queries_types.go index b2e0e742..c8553b47 100644 --- a/backup/queries_types.go +++ b/backup/queries_types.go @@ -42,6 +42,7 @@ type BaseType struct { Storage string DefaultVal string Element string + Subscript string Category string Preferred bool Delimiter string @@ -96,6 +97,14 @@ func GetBaseTypes(connectionPool *dbconn.DBConn) []BaseType { AND ut.oid IS NULL AND %s`, SchemaFilterClause("n"), ExtensionFilterClause("t")) + // PG14+ (WHPG19) types may declare a subscript function; without backing it + // up, a type's ELEMENT cannot be restored (PG14+ rejects ELEMENT without + // SUBSCRIPT). The column does not exist before PG14, so only select it there. + subscriptCol := "" + if connectionPool.Version.AtLeast("19") { + subscriptCol = `CASE WHEN t.typsubscript = '-'::regproc THEN '' ELSE t.typsubscript::regproc::text END AS subscript,` + } + atLeast6Query := fmt.Sprintf(` SELECT t.oid, quote_ident(n.nspname) AS schema, @@ -112,6 +121,7 @@ func GetBaseTypes(connectionPool *dbconn.DBConn) []BaseType { t.typstorage AS storage, coalesce(t.typdefault, '') AS defaultval, CASE WHEN t.typelem != 0::regproc THEN pg_catalog.format_type(t.typelem, NULL) ELSE '' END AS element, + %s t.typcategory AS category, t.typispreferred AS preferred, t.typdelim AS delimiter, @@ -128,7 +138,7 @@ func GetBaseTypes(connectionPool *dbconn.DBConn) []BaseType { WHERE %s AND t.typtype = 'b' AND ut.oid IS NULL - AND %s`, SchemaFilterClause("n"), ExtensionFilterClause("t")) + AND %s`, subscriptCol, SchemaFilterClause("n"), ExtensionFilterClause("t")) results := make([]BaseType, 0) var err error @@ -387,6 +397,7 @@ type RangeType struct { SubTypeOpClass string Canonical string SubTypeDiff string + MultiRangeName string // WHPG19+ } func (t RangeType) GetMetadataEntry() (string, toc.MetadataEntry) { @@ -403,8 +414,30 @@ func (t RangeType) FQN() string { func GetRangeTypes(connectionPool *dbconn.DBConn) []RangeType { gplog.Verbose("Retrieving range types") + // PG14+ (WHPG19) gives every range type a companion multirange type, whose + // name CREATE TYPE ... AS RANGE can set with MULTIRANGE_TYPE_NAME. Without + // it a custom name restores as the generated default, and anything that + // referred to the custom name -- a column, a function signature -- fails. + // Spell the name out unconditionally, as pg_dump's dumpRangeType() does, + // rather than reimplementing makeMultirangeTypeName()'s truncation rules to + // decide whether it is the default. + // + // Built from pg_type/pg_namespace rather than format_type(): that only + // schema-qualifies a type the current search_path cannot see, so it would + // depend on SetSessionGUCs() having set search_path to pg_catalog -- true + // today, but nothing here says so. DefineRange() resolves an unqualified + // MULTIRANGE_TYPE_NAME against the *restore* session's search_path rather + // than the range type's schema, so a bare name is not merely untidy. + multiRangeNameAtt := "" + multiRangeNameJoin := "" + if connectionPool.Version.AtLeast("19") { + multiRangeNameAtt = "quote_ident(mn.nspname) || '.' || quote_ident(mt.typname) AS multirangename," + multiRangeNameJoin = `JOIN pg_type mt ON mt.oid = r.rngmultitypid + JOIN pg_namespace mn ON mn.oid = mt.typnamespace` + } query := fmt.Sprintf(` SELECT t.oid, + %s quote_ident(n.nspname) AS schema, quote_ident(t.typname) AS name, format_type(st.oid, st.typtypmod) AS subtype, @@ -428,13 +461,14 @@ func GetRangeTypes(connectionPool *dbconn.DBConn) []RangeType { JOIN pg_type t ON t.oid = r.rngtypid JOIN pg_namespace n ON t.typnamespace = n.oid JOIN pg_type st ON st.oid = r.rngsubtype + %s LEFT JOIN pg_collation c ON c.oid = r.rngcollation LEFT JOIN pg_namespace nc ON nc.oid = c.collnamespace LEFT JOIN pg_opclass opc ON opc.oid = r.rngsubopc LEFT JOIN pg_namespace nopc ON nopc.oid = opc.opcnamespace WHERE %s AND t.typtype = 'r' - AND %s`, SchemaFilterClause("n"), ExtensionFilterClause("t")) + AND %s`, multiRangeNameAtt, multiRangeNameJoin, SchemaFilterClause("n"), ExtensionFilterClause("t")) results := make([]RangeType, 0) err := connectionPool.Select(&results, query) @@ -487,6 +521,8 @@ type Collation struct { Ctype string Provider string IsDeterministic string + Locale string // WHPG19+ + IcuRules string // WHPG19+ } func (c Collation) GetMetadataEntry() (string, toc.MetadataEntry) { @@ -533,9 +569,29 @@ func GetCollations(connectionPool *dbconn.DBConn) []Collation { JOIN pg_namespace n ON c.collnamespace = n.oid WHERE %s`, SchemaFilterClause("n")) + // PG17+ (WHPG19) keeps a non-libc collation's locale in colllocale and + // leaves collcollate/collctype NULL, so the query above would fail outright + // on an ICU or builtin collation rather than merely lose the locale. ICU + // collations may also carry custom tailoring in collicurules. + atLeast19Query := fmt.Sprintf(` + SELECT c.oid, + quote_ident(n.nspname) AS schema, + quote_ident(c.collname) AS name, + coalesce(c.collcollate, '') AS collate, + coalesce(c.collctype, '') AS ctype, + coalesce(c.colllocale, '') AS locale, + coalesce(c.collicurules, '') AS icurules, + c.collprovider as provider, + c.collisdeterministic as IsDeterministic + FROM pg_collation c + JOIN pg_namespace n ON c.collnamespace = n.oid + WHERE %s`, SchemaFilterClause("n")) + query := "" if connectionPool.Version.Before("7") { query = before7Query + } else if connectionPool.Version.AtLeast("19") { + query = atLeast19Query } else { query = atLeast7Query } diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 4ee21844..67d1cdf2 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -1823,9 +1823,12 @@ var _ = Describe("backup and restore end to end tests", func() { Expect(tableCopyCount).To(Equal(strconv.Itoa(1))) }) It("does not retrieve trigger constraints with the rest of the constraints", func() { - if backupConn.Version.Is("7") { + if testutils.IsGPDB7OrLater(backupConn) { // TODO: Remove this once support is added - Skip("Triggers on statements not yet supported in GPDB7, per src/backend/parser/gram.y:39460,39488") + // WHPG19 hits the identical gram.y restriction (confirmed + // live: "ERROR: Triggers for statements are not yet + // supported", same gram.y/base_yyparse site) as GPDB7. + Skip("Triggers on statements not yet supported in GPDB7/WHPG19, per src/backend/parser/gram.y:39460,39488") } testutils.SkipIfBefore6(backupConn) testhelper.AssertQueryRuns(backupConn, @@ -2070,8 +2073,16 @@ LANGUAGE plpgsql NO SQL;`) Expect(contents).To(ContainSubstring("CREATE VIEW public.key_dependent_view AS \nSELECT\n\tNULL::integer AS key,\n\tNULL::character varying(20) COLLATE pg_catalog.\"C\" AS data;")) Expect(contents).To(ContainSubstring("CREATE VIEW public.key_dependent_view_no_cols AS \nSELECT;")) Expect(contents).To(ContainSubstring("ALTER TABLE ONLY public.view_base_table ADD CONSTRAINT view_base_table_pkey PRIMARY KEY (key);")) - Expect(contents).To(ContainSubstring("CREATE OR REPLACE VIEW public.key_dependent_view AS SELECT view_base_table.key,\n (view_base_table.data COLLATE \"C\") AS data\n FROM public.view_base_table\n GROUP BY view_base_table.key;")) - Expect(contents).To(ContainSubstring("CREATE OR REPLACE VIEW public.key_dependent_view_no_cols AS SELECT\n FROM public.view_base_table\n GROUP BY view_base_table.key\n HAVING (length((view_base_table.data)::text) > 0);")) + if backupConn.Version.AtLeast("19") { + // PG19's view deparser omits redundant table/column + // qualification for single-relation queries, but still + // schema-qualifies the relation itself. + Expect(contents).To(ContainSubstring("CREATE OR REPLACE VIEW public.key_dependent_view AS SELECT key,\n (data COLLATE \"C\") AS data\n FROM public.view_base_table\n GROUP BY key;")) + Expect(contents).To(ContainSubstring("CREATE OR REPLACE VIEW public.key_dependent_view_no_cols AS SELECT\n FROM public.view_base_table\n GROUP BY key\n HAVING (length((data)::text) > 0);")) + } else { + Expect(contents).To(ContainSubstring("CREATE OR REPLACE VIEW public.key_dependent_view AS SELECT view_base_table.key,\n (view_base_table.data COLLATE \"C\") AS data\n FROM public.view_base_table\n GROUP BY view_base_table.key;")) + Expect(contents).To(ContainSubstring("CREATE OR REPLACE VIEW public.key_dependent_view_no_cols AS SELECT\n FROM public.view_base_table\n GROUP BY view_base_table.key\n HAVING (length((view_base_table.data)::text) > 0);")) + } gprestore(gprestorePath, restoreHelperPath, timestamp, "--redirect-db", "restoredb", diff --git a/end_to_end/plugin_test.go b/end_to_end/plugin_test.go index 4944f1bd..9328c4e6 100644 --- a/end_to_end/plugin_test.go +++ b/end_to_end/plugin_test.go @@ -77,6 +77,17 @@ func forceMetadataFileDownloadFromPlugin(conn *dbconn.DBConn, timestamp string) } } +// gpdb4ObjectsFile picks the shared GPDB4 fixture, or the WHPG19 variant of it. +// PG19 dropped postfix operators, the legacy operator-class RECHECK keyword and +// the MULE_INTERNAL encoding, so three statements in the shared file do not +// parse there; resources/gpdb4_objects_whpg19.sql carries the substitutions. +func gpdb4ObjectsFile(conn *dbconn.DBConn) string { + if conn.Version.AtLeast("19") { + return "resources/gpdb4_objects_whpg19.sql" + } + return "resources/gpdb4_objects.sql" +} + var _ = Describe("End to End plugin tests", func() { BeforeEach(func() { end_to_end_setup() @@ -172,11 +183,20 @@ var _ = Describe("End to End plugin tests", func() { if backupConn.Version.AtLeast("7") { plpythonDropStatement = "DROP PROCEDURAL LANGUAGE IF EXISTS plpython3u;" } + if backupConn.Version.AtLeast("19") { + // WHPG19 ships plpython3u as a real extension (control file + // and all), so CREATE LANGUAGE plpython3u auto-converts to + // CREATE EXTENSION plpython3u; plain DROP LANGUAGE then fails + // ("cannot drop language ... because extension ... requires + // it"). GPDB6/7 don't package it as an extension, so DROP + // LANGUAGE works fine there. + plpythonDropStatement = "DROP EXTENSION IF EXISTS plpython3u CASCADE;" + } testhelper.AssertQueryRuns(backupConn, plpythonDropStatement) defer testhelper.AssertQueryRuns(backupConn, plpythonDropStatement) defer testhelper.AssertQueryRuns(restoreConn, plpythonDropStatement) - testutils.ExecuteSQLFile(backupConn, "resources/gpdb4_objects.sql") + testutils.ExecuteSQLFile(backupConn, gpdb4ObjectsFile(backupConn)) if backupConn.Version.Before("7") { testutils.ExecuteSQLFile(backupConn, "resources/gpdb4_compatible_objects_before_gpdb7.sql") } else { @@ -237,7 +257,7 @@ var _ = Describe("End to End plugin tests", func() { "CREATE ROLE testrole SUPERUSER") defer testhelper.AssertQueryRuns(backupConn, "DROP ROLE testrole") - testutils.ExecuteSQLFile(backupConn, "resources/gpdb4_objects.sql") + testutils.ExecuteSQLFile(backupConn, gpdb4ObjectsFile(backupConn)) if backupConn.Version.AtLeast("5") { testutils.ExecuteSQLFile(backupConn, "resources/gpdb5_objects.sql") } diff --git a/end_to_end/resources/gpdb4_objects_whpg19.sql b/end_to_end/resources/gpdb4_objects_whpg19.sql new file mode 100644 index 00000000..89114e6c --- /dev/null +++ b/end_to_end/resources/gpdb4_objects_whpg19.sql @@ -0,0 +1,430 @@ +-- WHPG19 variant of gpdb4_objects.sql. +-- +-- This is a fork of that file with exactly three substitutions, each marked +-- with a comment below: the postfix operator, the operator-class RECHECK +-- keyword, and the MULE_INTERNAL conversion -- none of which PG19 accepts. +-- +-- Because it is a fork, an object added to gpdb4_objects.sql is NOT exercised +-- on WHPG19 until it is added here too, and nothing fails to say so. Mirror +-- every change to that file into this one until the two are restructured into +-- a shared base plus per-version deltas. + +SET statement_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SET check_function_bodies = false; +SET client_min_messages = warning; + +SET default_with_oids = false; + + +CREATE SCHEMA schema2; + + +SET search_path = public, pg_catalog; + + +CREATE TYPE base_type; + + + +CREATE FUNCTION base_fn_in(cstring) RETURNS base_type + AS $$boolin$$ + LANGUAGE internal NO SQL; + + + + +CREATE FUNCTION base_fn_out(base_type) RETURNS cstring + AS $$boolout$$ + LANGUAGE internal NO SQL; + + +CREATE TYPE base_type ( + INTERNALLENGTH = variable, + INPUT = base_fn_in, + OUTPUT = base_fn_out, + ALIGNMENT = int4, + STORAGE = plain +); + + +CREATE TYPE composite_type AS ( + name integer, + name1 integer, + name2 text +); + + +CREATE FUNCTION casttoint(text) RETURNS integer + AS $_$ +SELECT cast($1 as integer); +$_$ + LANGUAGE sql IMMUTABLE STRICT CONTAINS SQL; + + +CREATE FUNCTION dup(integer, OUT f1 integer, OUT f2 text) RETURNS record + AS $_$ +SELECT $1, CAST($1 AS text) || ' is text' +$_$ + LANGUAGE sql CONTAINS SQL; + + +CREATE FUNCTION mypre_accum(numeric, numeric) RETURNS numeric + AS $_$ +select $1 + $2 +$_$ + LANGUAGE sql IMMUTABLE STRICT CONTAINS SQL; + + +CREATE FUNCTION mysfunc_accum(numeric, numeric, numeric) RETURNS numeric + AS $_$ +select $1 + $2 + $3 +$_$ + LANGUAGE sql IMMUTABLE STRICT CONTAINS SQL; + + +CREATE FUNCTION plusone(x text) RETURNS text + AS $$ +BEGIN + RETURN x || 'x'; +END; +$$ + LANGUAGE plpgsql NO SQL; + + +CREATE AGGREGATE agg_prefunc(numeric, numeric) ( + SFUNC = mysfunc_accum, + STYPE = numeric, + INITCOND = '0', + PREFUNC = mypre_accum +); + + +CREATE AGGREGATE agg_test(integer) ( + SFUNC = int4xor, + STYPE = integer, + INITCOND = '0' +); + + +-- PG19 dropped support for postfix operators ("Postfix operators are not +-- supported" / "operator right argument type must be specified"), so this +-- operator is defined as a prefix operator (RIGHTARG, built-in `factorial`) +-- instead of postfix (LEFTARG, `numeric_fac`) -- same pattern already used +-- in integration/predata_operators_queries_test.go for WHPG19. +CREATE OPERATOR #### ( + PROCEDURE = factorial, + RIGHTARG = bigint +); + + +-- PG19 dropped the legacy RECHECK keyword on operator class OPERATOR entries +-- (syntax error, not just a no-op as in older GPDB/Postgres). +CREATE OPERATOR CLASS testclass + FOR TYPE int USING gist AS + OPERATOR 1 = , + OPERATOR 2 < , + FUNCTION 1 abs(integer), + FUNCTION 2 int4out(integer); + +CREATE OPERATOR CLASS range_class + FOR TYPE text USING btree AS + STORAGE text; + +SET default_tablespace = ''; + + +CREATE TABLE bar ( + i integer NOT NULL, + j text, + k smallint NOT NULL, + l character varying(20) +) DISTRIBUTED BY (i); + + + + +COPY bar (i, j, k, l) FROM stdin; +\. + + +CREATE TABLE foo ( + k text, + i integer, + j text +) DISTRIBUTED RANDOMLY; + + +COPY foo (k, i, j) FROM stdin; +\. + + +CREATE TABLE foo2 ( + k text, + l character varying(20) +) +INHERITS (foo) DISTRIBUTED RANDOMLY; + + +COPY foo2 (k, i, j, l) FROM stdin; +\. + + +SET search_path = schema2, pg_catalog; + + +CREATE TABLE foo3 ( + m double precision +) +INHERITS (public.foo2) DISTRIBUTED RANDOMLY; + + + +SET search_path = public, pg_catalog; + + +CREATE TABLE foo4 ( + n integer +) +INHERITS (schema2.foo3) DISTRIBUTED RANDOMLY; + + +COPY foo4 (k, i, j, l, m, n) FROM stdin; +\. + + + +CREATE TABLE gpcrondump_history ( + rec_date timestamp without time zone, + start_time character(8), + end_time character(8), + options text, + dump_key character varying(20), + dump_exit_status smallint, + script_exit_status smallint, + exit_text character varying(10) +) DISTRIBUTED BY (rec_date); + + +COPY gpcrondump_history (rec_date, start_time, end_time, options, dump_key, dump_exit_status, script_exit_status, exit_text) FROM stdin; +\. + + +SET search_path = schema2, pg_catalog; + + +COPY foo3 (k, i, j, l, m) FROM stdin; +\. + + + +CREATE TABLE noatts ( +) DISTRIBUTED RANDOMLY; + + + + +COPY noatts FROM stdin; +\. + + +SET search_path = public, pg_catalog; + + +CREATE TABLE pk_table ( + a integer NOT NULL +) DISTRIBUTED BY (a); + + + + +COPY pk_table (a) FROM stdin; +\. + + + +CREATE TABLE reference_table ( + a integer, + b integer +) DISTRIBUTED BY (a); + + + + +COPY reference_table (a, b) FROM stdin; +\. + + +SET search_path = schema2, pg_catalog; + + +CREATE TABLE prime ( + i integer NOT NULL, + j integer +) DISTRIBUTED BY (i); + +COPY prime (i, j) FROM stdin; +\. + + +SET search_path = public, pg_catalog; + +CREATE WRITABLE EXTERNAL TABLE my_sales_ext ( + id integer, + year integer, + qtr integer, + day integer, + region text +) LOCATION ( + 'gpfdist://gpdb_test:8080/sales_2010' +) +FORMAT 'csv' (delimiter E',' null E'' escape E'"' quote E'"') +ENCODING 'UTF8' DISTRIBUTED BY (id); + + +CREATE EXTERNAL TABLE sales_1_prt_yr_1_external_partition__ ( + id integer, + year integer, + qtr integer, + day integer, + region text +) LOCATION ( + 'gpfdist://gpdb_test:8080/sales_2010' +) +FORMAT 'csv' (delimiter E',' null E'' escape E'"' quote E'"') +ENCODING 'UTF8'; + + +SET search_path = public, pg_catalog; + + +CREATE TABLE rule_table1 ( + i integer +) DISTRIBUTED BY (i); + + + + +COPY rule_table1 (i) FROM stdin; +\. + + +SET search_path = public, pg_catalog; + + +CREATE TABLE trigger_table1 ( + i integer +) DISTRIBUTED BY (i); + + + + +COPY trigger_table1 (i) FROM stdin; +\. + + + +CREATE TABLE uniq ( + i integer +) DISTRIBUTED BY (i); + + +COPY uniq (i) FROM stdin; +\. + + +SET search_path = schema2, pg_catalog; + + +CREATE TABLE with_multiple_check ( + a integer, + b character varying(40), + CONSTRAINT con1 CHECK (((a > 99) AND ((b)::text <> ''::text))) +) DISTRIBUTED BY (a); + + +COPY with_multiple_check (a, b) FROM stdin; +\. + +CREATE TABLE many_partitions (id int, year int) +DISTRIBUTED BY (id) +PARTITION BY RANGE (year) +( START (2000) END (2300) EVERY (1), + DEFAULT PARTITION extra ); + + +SET search_path = public, pg_catalog; + + +-- PG19 removed the MULE_INTERNAL encoding; use a conversion pair that still +-- exists there instead (same substitution already used in +-- integration/predata_functions_create_test.go's PrintCreateConversionStatements +-- test for WHPG19). +CREATE CONVERSION testconv FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8; + + + +CREATE VIEW test_view AS + SELECT pk_table.a FROM pk_table; + + + + +CREATE VIEW view_view AS + SELECT test_view.a FROM test_view; + + + +SET search_path = schema2, pg_catalog; + + +CREATE SEQUENCE seq_one + START WITH 3 + INCREMENT BY 1 + NO MAXVALUE + NO MINVALUE + CACHE 1; + + +ALTER SEQUENCE seq_one OWNED BY prime.j; + + +SELECT pg_catalog.setval('seq_one', 3, false); + + +SET search_path = public, pg_catalog; + + +ALTER TABLE ONLY pk_table + ADD CONSTRAINT pk_table_pkey PRIMARY KEY (a); + + + +ALTER TABLE ONLY uniq + ADD CONSTRAINT uniq_i_key UNIQUE (i); + + +SET search_path = schema2, pg_catalog; + + +ALTER TABLE ONLY prime + ADD CONSTRAINT prime_pkey PRIMARY KEY (i); + + +SET search_path = public, pg_catalog; + + +CREATE INDEX simple_table_idx1 ON foo4 USING btree (n); + + + +CREATE RULE double_insert AS ON INSERT TO rule_table1 DO INSERT INTO rule_table1 VALUES (1); + + +ALTER TABLE ONLY reference_table + ADD CONSTRAINT reference_table_b_fkey FOREIGN KEY (b) REFERENCES pk_table(a); + + +REVOKE ALL ON SCHEMA public FROM PUBLIC; +GRANT ALL ON SCHEMA public TO PUBLIC; diff --git a/integration/dependency_queries_test.go b/integration/dependency_queries_test.go index 83cfcb73..77cf25dc 100644 --- a/integration/dependency_queries_test.go +++ b/integration/dependency_queries_test.go @@ -31,6 +31,7 @@ var _ = Describe("backup integration tests", func() { Expect(deps[barEntry]).To(HaveKey(fooEntry)) }) It("constructs dependencies correctly for a table dependent on a protocol", func() { + testutils.SkipIfNoS3Protocol(connectionPool) testhelper.AssertQueryRuns(connectionPool, `CREATE FUNCTION public.read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE c STABLE NO SQL;`) diff --git a/integration/integration_suite_test.go b/integration/integration_suite_test.go index f2837c64..b1a15c79 100644 --- a/integration/integration_suite_test.go +++ b/integration/integration_suite_test.go @@ -67,6 +67,9 @@ var _ = BeforeSuite(func() { _, _ = connectionPool.Exec("CREATE ROLE anothertestrole SUPERUSER") backup.InitializeMetadataParams(connectionPool) backup.SetConnection(connectionPool) + // The ACL fixtures in testutils are compared against what this server hands + // back, and the full privilege set for a relation differs by major version. + testutils.SetTestVersion(connectionPool) segConfig := cluster.MustGetSegmentConfiguration(connectionPool) testCluster = cluster.NewCluster(segConfig) testhelper.AssertQueryRuns(connectionPool, "SET ROLE testrole") diff --git a/integration/metadata_globals_create_test.go b/integration/metadata_globals_create_test.go index 6eef35bd..876b9c84 100644 --- a/integration/metadata_globals_create_test.go +++ b/integration/metadata_globals_create_test.go @@ -410,6 +410,12 @@ var _ = Describe("backup integration create statement tests", func() { BeforeEach(func() { testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE usergroup`) testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE testuser`) + if connectionPool.Version.AtLeast("19") { + // PG16+: replaying GRANT ... GRANTED BY testrole requires + // testrole to actually hold ADMIN OPTION on the role (as it + // would after restoring its own admin membership first). + testhelper.AssertQueryRuns(connectionPool, `GRANT usergroup TO testrole WITH ADMIN OPTION`) + } }) AfterEach(func() { defer testhelper.AssertQueryRuns(connectionPool, `DROP ROLE usergroup`) @@ -425,7 +431,7 @@ var _ = Describe("backup integration create statement tests", func() { resultRoleMembers := backup.GetRoleMembers(connectionPool) Expect(resultRoleMembers).To(HaveLen(numRoleMembers + 1)) for _, roleMember := range resultRoleMembers { - if roleMember.Role == "usergroup" { + if roleMember.Role == "usergroup" && roleMember.Member == "testuser" { structmatcher.ExpectStructsToMatch(&expectedRoleMember, &roleMember) return } @@ -442,7 +448,7 @@ var _ = Describe("backup integration create statement tests", func() { resultRoleMembers := backup.GetRoleMembers(connectionPool) Expect(resultRoleMembers).To(HaveLen(numRoleMembers + 1)) for _, roleMember := range resultRoleMembers { - if roleMember.Role == "usergroup" { + if roleMember.Role == "usergroup" && roleMember.Member == "testuser" { structmatcher.ExpectStructsToMatch(&expectedRoleMember, &roleMember) return } diff --git a/integration/metadata_globals_queries_test.go b/integration/metadata_globals_queries_test.go index fce7fffd..b1ce05b7 100644 --- a/integration/metadata_globals_queries_test.go +++ b/integration/metadata_globals_queries_test.go @@ -344,7 +344,15 @@ CREATEEXTTABLE (protocol='gphdfs', type='writable')` for _, role := range results { if role.Name == "role1" { - structmatcher.ExpectStructsToMatchExcluding(&expectedRole, role, "TimeConstraints.Oid") + if connectionPool.Version.AtLeast("19") { + // WHPG19 defaults password_encryption to scram-sha-256, + // which salts the hash, so it can't be compared for + // exact equality across runs. + Expect(role.Password).To(HavePrefix("SCRAM-SHA-256$")) + structmatcher.ExpectStructsToMatchExcluding(&expectedRole, role, "TimeConstraints.Oid", "Password") + } else { + structmatcher.ExpectStructsToMatchExcluding(&expectedRole, role, "TimeConstraints.Oid") + } return } } @@ -499,19 +507,32 @@ CREATEEXTTABLE (protocol='gphdfs', type='writable')` BeforeEach(func() { testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE usergroup`) testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE testuser`) + if connectionPool.Version.AtLeast("19") { + // PG16+: a grantor recorded in pg_auth_members must actually + // hold ADMIN OPTION on the role; superuser grants without it + // are recorded as the bootstrap superuser instead of the + // current role, so give testrole ADMIN OPTION up front. + testhelper.AssertQueryRuns(connectionPool, `GRANT usergroup TO testrole WITH ADMIN OPTION`) + } }) AfterEach(func() { defer testhelper.AssertQueryRuns(connectionPool, `DROP ROLE usergroup`) defer testhelper.AssertQueryRuns(connectionPool, `DROP ROLE testuser`) }) It("returns a role without ADMIN OPTION", func() { - testhelper.AssertQueryRuns(connectionPool, "GRANT usergroup TO testuser") + if connectionPool.Version.AtLeast("19") { + // PG16+ records the bootstrap superuser as grantor for grants + // made by a superuser, unless the grantor is named explicitly. + testhelper.AssertQueryRuns(connectionPool, "GRANT usergroup TO testuser GRANTED BY testrole") + } else { + testhelper.AssertQueryRuns(connectionPool, "GRANT usergroup TO testuser") + } expectedRoleMember := backup.RoleMember{Role: "usergroup", Member: "testuser", Grantor: "testrole", IsAdmin: false} roleMembers := backup.GetRoleMembers(connectionPool) for _, roleMember := range roleMembers { - if roleMember.Role == "usergroup" { + if roleMember.Role == "usergroup" && roleMember.Member == "testuser" { structmatcher.ExpectStructsToMatch(&expectedRoleMember, &roleMember) return } @@ -525,7 +546,7 @@ CREATEEXTTABLE (protocol='gphdfs', type='writable')` roleMembers := backup.GetRoleMembers(connectionPool) for _, roleMember := range roleMembers { - if roleMember.Role == "usergroup" { + if roleMember.Role == "usergroup" && roleMember.Member == "testuser" { structmatcher.ExpectStructsToMatch(&expectedRoleMember, &roleMember) return } @@ -541,13 +562,21 @@ CREATEEXTTABLE (protocol='gphdfs', type='writable')` defer testhelper.AssertQueryRuns(connectionPool, `DROP ROLE "1usergroup"`) testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE "1testuser"`) defer testhelper.AssertQueryRuns(connectionPool, `DROP ROLE "1testuser"`) - testhelper.AssertQueryRuns(connectionPool, `GRANT "1usergroup" TO "1testuser"`) + if connectionPool.Version.AtLeast("19") { + // See BeforeEach: the grantor must hold ADMIN OPTION on PG16+, + // and must be named explicitly to be recorded (superuser grants + // otherwise record the bootstrap superuser). + testhelper.AssertQueryRuns(connectionPool, `GRANT "1usergroup" TO "1testrole" WITH ADMIN OPTION`) + testhelper.AssertQueryRuns(connectionPool, `GRANT "1usergroup" TO "1testuser" GRANTED BY "1testrole"`) + } else { + testhelper.AssertQueryRuns(connectionPool, `GRANT "1usergroup" TO "1testuser"`) + } expectedRoleMember := backup.RoleMember{Role: `"1usergroup"`, Member: `"1testuser"`, Grantor: `"1testrole"`, IsAdmin: false} roleMembers := backup.GetRoleMembers(connectionPool) for _, roleMember := range roleMembers { - if roleMember.Role == `"1usergroup"` { + if roleMember.Role == `"1usergroup"` && roleMember.Member == `"1testuser"` { structmatcher.ExpectStructsToMatch(&expectedRoleMember, &roleMember) return } @@ -555,6 +584,13 @@ CREATEEXTTABLE (protocol='gphdfs', type='writable')` Fail(`Role "1testuser" is not a member of role "1usergroup"`) }) It("handles dropped granter", func() { + if connectionPool.Version.AtLeast("19") { + // PG16+ records a dependency on the grantor of a role grant, + // so a role that granted a still-active membership can no + // longer be dropped — the stale-grantor state this exercises + // cannot exist. + Skip("PG16+ forbids dropping a role recorded as grantor of an active role grant") + } testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE testdropgranter_role`) defer testhelper.AssertQueryRuns(connectionPool, `DROP ROLE testdropgranter_role`) testhelper.AssertQueryRuns(connectionPool, `CREATE ROLE testdropgranter_member`) @@ -572,13 +608,18 @@ CREATEEXTTABLE (protocol='gphdfs', type='writable')` testhelper.AssertQueryRuns(connectionPool, "CREATE OR REPLACE FUNCTION pg_catalog.text(oid) RETURNS text STRICT IMMUTABLE LANGUAGE SQL AS 'SELECT textin(oidout($1));';") testhelper.AssertQueryRuns(connectionPool, "CREATE CAST (oid AS text) WITH FUNCTION pg_catalog.text(oid) AS IMPLICIT;") defer testhelper.AssertQueryRuns(connectionPool, "DROP FUNCTION pg_catalog.text(oid) CASCADE;") - testhelper.AssertQueryRuns(connectionPool, "GRANT usergroup TO testuser") + if connectionPool.Version.AtLeast("19") { + // See above: name the grantor explicitly so it is recorded. + testhelper.AssertQueryRuns(connectionPool, "GRANT usergroup TO testuser GRANTED BY testrole") + } else { + testhelper.AssertQueryRuns(connectionPool, "GRANT usergroup TO testuser") + } expectedRoleMember := backup.RoleMember{Role: "usergroup", Member: "testuser", Grantor: "testrole", IsAdmin: false} roleMembers := backup.GetRoleMembers(connectionPool) for _, roleMember := range roleMembers { - if roleMember.Role == "usergroup" { + if roleMember.Role == "usergroup" && roleMember.Member == "testuser" { structmatcher.ExpectStructsToMatch(&expectedRoleMember, &roleMember) return } diff --git a/integration/postdata_create_test.go b/integration/postdata_create_test.go index a795f6d9..4ef30ef1 100644 --- a/integration/postdata_create_test.go +++ b/integration/postdata_create_test.go @@ -422,7 +422,16 @@ AS $$ BEGIN RAISE EXCEPTION 'exception'; END; $$;`) resultViews := backup.GetAllViews(connectionPool) Expect(resultViews).To(HaveLen(1)) - structmatcher.ExpectStructsToMatchExcluding(&view1, &resultViews[0], "Oid", "ColumnDefs.Oid") + expectedView := view1 + if connectionPool.Version.AtLeast("19") { + // PG19's view deparser omits redundant table/column + // qualification for single-relation queries. + expectedView.Definition = sql.NullString{ + String: " SELECT key,\n (data COLLATE \"C\") AS data\n FROM public.view_base_table\n GROUP BY key;", + Valid: true, + } + } + structmatcher.ExpectStructsToMatchExcluding(&expectedView, &resultViews[0], "Oid", "ColumnDefs.Oid") }) }) }) diff --git a/integration/predata_acl_queries_test.go b/integration/predata_acl_queries_test.go index 14e66c8e..9ed938eb 100644 --- a/integration/predata_acl_queries_test.go +++ b/integration/predata_acl_queries_test.go @@ -129,6 +129,7 @@ var _ = Describe("backup integration tests", func() { structmatcher.ExpectStructsToMatchExcluding(&expectedMetadata, &resultMetadata, "Oid") }) It("returns a slice of default metadata for a procedural language", func() { + testutils.SkipIfLanguagesAreExtensionManaged(connectionPool) plpythonString := "plpythonu" if connectionPool.Version.AtLeast("7") { plpythonString = "plpython3u" @@ -337,6 +338,7 @@ LANGUAGE SQL`) structmatcher.ExpectStructsToMatchExcluding(&expectedMetadata, &resultMetadata, "Oid") }) It("returns a slice of default metadata for an external protocol", func() { + testutils.SkipIfNoS3Protocol(connectionPool) testhelper.AssertQueryRuns(connectionPool, `CREATE OR REPLACE FUNCTION public.read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;`) defer testhelper.AssertQueryRuns(connectionPool, "DROP FUNCTION public.read_from_s3()") testhelper.AssertQueryRuns(connectionPool, `CREATE TRUSTED PROTOCOL s3_read (readfunc = public.read_from_s3);`) @@ -369,10 +371,16 @@ LANGUAGE SQL`) structmatcher.ExpectStructsToMatchExcluding(&expectedMetadata, &resultMetadata, "Oid") }) It("returns a slice of default metadata for an operator", func() { - testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.#### (LEFTARG = bigint, PROCEDURE = numeric_fac)") - defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.#### (bigint, NONE)") - - testhelper.AssertQueryRuns(connectionPool, "COMMENT ON OPERATOR public.#### (bigint, NONE) IS 'This is an operator comment.'") + if connectionPool.Version.AtLeast("19") { + // PG14+ removed postfix (left-arg-only) operators; use a prefix operator instead. + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.#### (RIGHTARG = bigint, PROCEDURE = factorial)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.#### (NONE, bigint)") + testhelper.AssertQueryRuns(connectionPool, "COMMENT ON OPERATOR public.#### (NONE, bigint) IS 'This is an operator comment.'") + } else { + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.#### (LEFTARG = bigint, PROCEDURE = numeric_fac)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.#### (bigint, NONE)") + testhelper.AssertQueryRuns(connectionPool, "COMMENT ON OPERATOR public.#### (bigint, NONE) IS 'This is an operator comment.'") + } resultMetadataMap := backup.GetMetadataForObjectType(connectionPool, backup.TYPE_OPERATOR) @@ -688,13 +696,22 @@ LANGUAGE SQL`) structmatcher.ExpectStructsToMatchExcluding(&expectedMetadata, &resultMetadata, "Oid") }) It("returns a slice of default metadata for an operator in a specific schema", func() { - testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.#### (LEFTARG = bigint, PROCEDURE = numeric_fac)") - defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.#### (bigint, NONE)") testhelper.AssertQueryRuns(connectionPool, "CREATE SCHEMA testschema") defer testhelper.AssertQueryRuns(connectionPool, "DROP SCHEMA testschema") - testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR testschema.#### (LEFTARG = bigint, PROCEDURE = numeric_fac)") - defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR testschema.#### (bigint, NONE)") - testhelper.AssertQueryRuns(connectionPool, "COMMENT ON OPERATOR testschema.#### (bigint, NONE) IS 'This is an operator comment.'") + if connectionPool.Version.AtLeast("19") { + // PG14+ removed postfix (left-arg-only) operators; use prefix operators instead. + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.#### (RIGHTARG = bigint, PROCEDURE = factorial)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.#### (NONE, bigint)") + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR testschema.#### (RIGHTARG = bigint, PROCEDURE = factorial)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR testschema.#### (NONE, bigint)") + testhelper.AssertQueryRuns(connectionPool, "COMMENT ON OPERATOR testschema.#### (NONE, bigint) IS 'This is an operator comment.'") + } else { + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.#### (LEFTARG = bigint, PROCEDURE = numeric_fac)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.#### (bigint, NONE)") + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR testschema.#### (LEFTARG = bigint, PROCEDURE = numeric_fac)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR testschema.#### (bigint, NONE)") + testhelper.AssertQueryRuns(connectionPool, "COMMENT ON OPERATOR testschema.#### (bigint, NONE) IS 'This is an operator comment.'") + } _ = backupCmdFlags.Set(options.INCLUDE_SCHEMA, "testschema") resultMetadataMap := backup.GetMetadataForObjectType(connectionPool, backup.TYPE_OPERATOR) diff --git a/integration/predata_externals_create_test.go b/integration/predata_externals_create_test.go index 0c6cd603..b9c0282d 100644 --- a/integration/predata_externals_create_test.go +++ b/integration/predata_externals_create_test.go @@ -170,6 +170,7 @@ var _ = Describe("backup integration create statement tests", func() { } It("creates a trusted protocol with a read function, privileges, and an owner", func() { + testutils.SkipIfNoS3Protocol(connectionPool) protoMetadata := backup.ObjectMetadata{Privileges: []backup.ACL{{Grantee: "testrole", Select: true, Insert: true}}, Owner: "testrole"} backup.PrintCreateExternalProtocolStatement(backupfile, tocfile, protocolReadOnly, funcInfoMap, protoMetadata) @@ -186,6 +187,7 @@ var _ = Describe("backup integration create statement tests", func() { structmatcher.ExpectStructsToMatchExcluding(&protocolReadOnly, &resultExternalProtocols[0], "Oid", "ReadFunction", "FuncMap") }) It("creates a protocol with a write function", func() { + testutils.SkipIfNoS3Protocol(connectionPool) backup.PrintCreateExternalProtocolStatement(backupfile, tocfile, protocolWriteOnly, funcInfoMap, emptyMetadata) testhelper.AssertQueryRuns(connectionPool, "CREATE OR REPLACE FUNCTION public.write_to_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_export' LANGUAGE C STABLE;") @@ -200,6 +202,7 @@ var _ = Describe("backup integration create statement tests", func() { structmatcher.ExpectStructsToMatchExcluding(&protocolWriteOnly, &resultExternalProtocols[0], "Oid", "WriteFunction", "FuncMap") }) It("creates a protocol with a read and write function", func() { + testutils.SkipIfNoS3Protocol(connectionPool) backup.PrintCreateExternalProtocolStatement(backupfile, tocfile, protocolReadWrite, funcInfoMap, emptyMetadata) testhelper.AssertQueryRuns(connectionPool, "CREATE OR REPLACE FUNCTION public.read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;") diff --git a/integration/predata_externals_queries_test.go b/integration/predata_externals_queries_test.go index cf287538..8ed08cea 100644 --- a/integration/predata_externals_queries_test.go +++ b/integration/predata_externals_queries_test.go @@ -146,6 +146,7 @@ LOG ERRORS PERSISTENTLY SEGMENT REJECT LIMIT 10 PERCENT`) }) Describe("GetExternalProtocols", func() { It("returns a slice for a protocol", func() { + testutils.SkipIfNoS3Protocol(connectionPool) testhelper.AssertQueryRuns(connectionPool, "CREATE OR REPLACE FUNCTION public.write_to_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_export' LANGUAGE C STABLE;") defer testhelper.AssertQueryRuns(connectionPool, "DROP FUNCTION public.write_to_s3()") testhelper.AssertQueryRuns(connectionPool, "CREATE OR REPLACE FUNCTION public.read_from_s3() RETURNS integer AS '$libdir/gps3ext.so', 's3_import' LANGUAGE C STABLE;") diff --git a/integration/predata_functions_create_test.go b/integration/predata_functions_create_test.go index 2477232e..b69ca3c4 100644 --- a/integration/predata_functions_create_test.go +++ b/integration/predata_functions_create_test.go @@ -528,6 +528,7 @@ var _ = Describe("backup integration create statement tests", func() { }) Describe("PrintCreateLanguageStatements", func() { It("creates procedural languages", func() { + testutils.SkipIfLanguagesAreExtensionManaged(connectionPool) plpythonString := "plpython" if connectionPool.Version.AtLeast("7") { plpythonString = "plpython3" @@ -575,6 +576,8 @@ var _ = Describe("backup integration create statement tests", func() { Expect(resultExtensions).To(HaveLen(1)) } else { // gp_toolkit is installed by default as an extension in GPDB7+ + // (WHPG19 too: plpython3u ships available but isn't + // preinstalled/created by default, so it doesn't add to this count). Expect(resultExtensions).To(HaveLen(2)) } plperlMetadata := resultMetadataMap[plperlExtension.GetUniqueID()] @@ -623,8 +626,14 @@ var _ = Describe("backup integration create statement tests", func() { }) Describe("PrintCreateConversionStatements", func() { It("creates conversions", func() { - convOne := backup.Conversion{Oid: 1, Schema: "public", Name: "conv_one", ForEncoding: "LATIN1", ToEncoding: "MULE_INTERNAL", ConversionFunction: "pg_catalog.latin1_to_mic", IsDefault: false} - convTwo := backup.Conversion{Oid: 0, Schema: "public", Name: "conv_two", ForEncoding: "LATIN1", ToEncoding: "MULE_INTERNAL", ConversionFunction: "pg_catalog.latin1_to_mic", IsDefault: true} + // PG19 removed the MULE_INTERNAL encoding; use a pair that still + // exists there while keeping the historical fixture elsewhere. + toEncoding, convFunc := "MULE_INTERNAL", "pg_catalog.latin1_to_mic" + if connectionPool.Version.AtLeast("19") { + toEncoding, convFunc = "UTF8", "pg_catalog.iso8859_1_to_utf8" + } + convOne := backup.Conversion{Oid: 1, Schema: "public", Name: "conv_one", ForEncoding: "LATIN1", ToEncoding: toEncoding, ConversionFunction: convFunc, IsDefault: false} + convTwo := backup.Conversion{Oid: 0, Schema: "public", Name: "conv_two", ForEncoding: "LATIN1", ToEncoding: toEncoding, ConversionFunction: convFunc, IsDefault: true} conversions := []backup.Conversion{convOne, convTwo} convMetadataMap := testutils.DefaultMetadataMap(toc.OBJ_CONVERSION, false, true, true, false) convMetadata := convMetadataMap[convOne.GetUniqueID()] @@ -667,8 +676,22 @@ var _ = Describe("backup integration create statement tests", func() { resultWrappers := backup.GetForeignDataWrappers(connectionPool) Expect(resultWrappers).To(HaveLen(2)) - structmatcher.ExpectStructsToMatchExcluding(&foreignDataWrapperValidator, &resultWrappers[0], "Oid", "Validator") - structmatcher.ExpectStructsToMatchExcluding(&foreignDataWrapperOptions, &resultWrappers[1], "Oid", "Validator") + // The underlying query has no ORDER BY, so pg_foreign_data_wrapper's + // physical scan order (and thus slice order) isn't guaranteed; + // match by name instead of by fixed index. + var resultValidator, resultOptions *backup.ForeignDataWrapper + for i := range resultWrappers { + switch resultWrappers[i].Name { + case "foreigndata1": + resultValidator = &resultWrappers[i] + case "foreigndata2": + resultOptions = &resultWrappers[i] + } + } + Expect(resultValidator).ToNot(BeNil()) + Expect(resultOptions).ToNot(BeNil()) + structmatcher.ExpectStructsToMatchExcluding(&foreignDataWrapperValidator, resultValidator, "Oid", "Validator") + structmatcher.ExpectStructsToMatchExcluding(&foreignDataWrapperOptions, resultOptions, "Oid", "Validator") }) }) Describe("PrintCreateServerStatement", func() { diff --git a/integration/predata_functions_queries_test.go b/integration/predata_functions_queries_test.go index 909be365..ddb259b4 100644 --- a/integration/predata_functions_queries_test.go +++ b/integration/predata_functions_queries_test.go @@ -426,6 +426,14 @@ INSERT INTO public.tbl VALUES (b); firstProcedure.DataAccess = "" secondProcedure.DataAccess = "" } + if connectionPool.Version.AtLeast("19") { + // PG19's pg_get_function_(identity_)arguments now spells out + // the "IN" direction for procedure arguments explicitly. + firstProcedure.Arguments = sql.NullString{String: "IN a integer, IN b integer", Valid: true} + firstProcedure.IdentArgs = sql.NullString{String: "IN a integer, IN b integer", Valid: true} + secondProcedure.Arguments = sql.NullString{String: "IN a integer, IN b integer", Valid: true} + secondProcedure.IdentArgs = sql.NullString{String: "IN a integer, IN b integer", Valid: true} + } Expect(results).To(HaveLen(2)) structmatcher.ExpectStructsToMatchExcluding(&results[0], &firstProcedure, "Oid") @@ -806,6 +814,8 @@ LANGUAGE SQL`) Expect(results).To(HaveLen(1)) } else { // gp_toolkit is installed by default as an extension in GPDB7+ + // (WHPG19 too: plpython3u ships available but isn't + // preinstalled/created by default, so it doesn't add to this count). Expect(results).To(HaveLen(2)) } @@ -821,6 +831,7 @@ LANGUAGE SQL`) }) Describe("GetProceduralLanguages", func() { It("returns a slice of procedural languages", func() { + testutils.SkipIfLanguagesAreExtensionManaged(connectionPool) plpythonString := "plpython" if connectionPool.Version.AtLeast("7") { plpythonString = "plpython3" @@ -847,11 +858,22 @@ LANGUAGE SQL`) }) }) Describe("GetConversions", func() { + // PG19 removed the MULE_INTERNAL encoding; use a conversion pair that + // still exists there while keeping the historical fixture elsewhere. + // Resolved in BeforeEach because the connection is only established + // once the suite runs, not when the spec tree is built. + var toEncoding, convFunc string + BeforeEach(func() { + toEncoding, convFunc = "MULE_INTERNAL", "latin1_to_mic" + if connectionPool.Version.AtLeast("19") { + toEncoding, convFunc = "UTF8", "iso8859_1_to_utf8" + } + }) It("returns a slice of conversions", func() { - testhelper.AssertQueryRuns(connectionPool, "CREATE CONVERSION public.testconv FOR 'LATIN1' TO 'MULE_INTERNAL' FROM latin1_to_mic") + testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf("CREATE CONVERSION public.testconv FOR 'LATIN1' TO '%s' FROM %s", toEncoding, convFunc)) defer testhelper.AssertQueryRuns(connectionPool, "DROP CONVERSION public.testconv") - expectedConversion := backup.Conversion{Oid: 0, Schema: "public", Name: "testconv", ForEncoding: "LATIN1", ToEncoding: "MULE_INTERNAL", ConversionFunction: "pg_catalog.latin1_to_mic", IsDefault: false} + expectedConversion := backup.Conversion{Oid: 0, Schema: "public", Name: "testconv", ForEncoding: "LATIN1", ToEncoding: toEncoding, ConversionFunction: "pg_catalog." + convFunc, IsDefault: false} resultConversions := backup.GetConversions(connectionPool) @@ -859,14 +881,14 @@ LANGUAGE SQL`) structmatcher.ExpectStructsToMatchExcluding(&expectedConversion, &resultConversions[0], "Oid") }) It("returns a slice of conversions in a specific schema", func() { - testhelper.AssertQueryRuns(connectionPool, "CREATE CONVERSION public.testconv FOR 'LATIN1' TO 'MULE_INTERNAL' FROM latin1_to_mic") + testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf("CREATE CONVERSION public.testconv FOR 'LATIN1' TO '%s' FROM %s", toEncoding, convFunc)) defer testhelper.AssertQueryRuns(connectionPool, "DROP CONVERSION public.testconv") testhelper.AssertQueryRuns(connectionPool, "CREATE SCHEMA testschema") defer testhelper.AssertQueryRuns(connectionPool, "DROP SCHEMA testschema") - testhelper.AssertQueryRuns(connectionPool, "CREATE CONVERSION testschema.testconv FOR 'LATIN1' TO 'MULE_INTERNAL' FROM latin1_to_mic") + testhelper.AssertQueryRuns(connectionPool, fmt.Sprintf("CREATE CONVERSION testschema.testconv FOR 'LATIN1' TO '%s' FROM %s", toEncoding, convFunc)) defer testhelper.AssertQueryRuns(connectionPool, "DROP CONVERSION testschema.testconv") - expectedConversion := backup.Conversion{Oid: 0, Schema: "testschema", Name: "testconv", ForEncoding: "LATIN1", ToEncoding: "MULE_INTERNAL", ConversionFunction: "pg_catalog.latin1_to_mic", IsDefault: false} + expectedConversion := backup.Conversion{Oid: 0, Schema: "testschema", Name: "testconv", ForEncoding: "LATIN1", ToEncoding: toEncoding, ConversionFunction: "pg_catalog." + convFunc, IsDefault: false} _ = backupCmdFlags.Set(options.INCLUDE_SCHEMA, "testschema") resultConversions := backup.GetConversions(connectionPool) diff --git a/integration/predata_operators_queries_test.go b/integration/predata_operators_queries_test.go index 831458b1..b7e949df 100644 --- a/integration/predata_operators_queries_test.go +++ b/integration/predata_operators_queries_test.go @@ -14,10 +14,18 @@ import ( var _ = Describe("backup integration tests", func() { Describe("GetOperators", func() { It("returns a slice of operators", func() { - testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.## (LEFTARG = bigint, PROCEDURE = numeric_fac)") - defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.## (bigint, NONE)") - expectedOperator := backup.Operator{Oid: 0, Schema: "public", Name: "##", Procedure: "numeric_fac", LeftArgType: "bigint", RightArgType: "-", CommutatorOp: "0", NegatorOp: "0", RestrictFunction: "-", JoinFunction: "-", CanHash: false, CanMerge: false} + if connectionPool.Version.AtLeast("19") { + // PG14+ removed postfix (left-arg-only) operators; use a prefix operator instead. + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.## (RIGHTARG = bigint, PROCEDURE = factorial)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.## (NONE, bigint)") + expectedOperator.Procedure = "factorial" + expectedOperator.LeftArgType = "-" + expectedOperator.RightArgType = "bigint" + } else { + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.## (LEFTARG = bigint, PROCEDURE = numeric_fac)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.## (bigint, NONE)") + } results := backup.GetOperators(connectionPool) @@ -55,15 +63,25 @@ var _ = Describe("backup integration tests", func() { }) It("returns a slice of operators from a specific schema", func() { - testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.## (LEFTARG = bigint, PROCEDURE = numeric_fac)") - defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.## (bigint, NONE)") testhelper.AssertQueryRuns(connectionPool, "CREATE SCHEMA testschema") defer testhelper.AssertQueryRuns(connectionPool, "DROP SCHEMA testschema") - testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR testschema.## (LEFTARG = bigint, PROCEDURE = numeric_fac)") - defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR testschema.## (bigint, NONE)") - _ = backupCmdFlags.Set(options.INCLUDE_SCHEMA, "testschema") - expectedOperator := backup.Operator{Oid: 0, Schema: "testschema", Name: "##", Procedure: "numeric_fac", LeftArgType: "bigint", RightArgType: "-", CommutatorOp: "0", NegatorOp: "0", RestrictFunction: "-", JoinFunction: "-", CanHash: false, CanMerge: false} + if connectionPool.Version.AtLeast("19") { + // PG14+ removed postfix (left-arg-only) operators; use prefix operators instead. + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.## (RIGHTARG = bigint, PROCEDURE = factorial)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.## (NONE, bigint)") + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR testschema.## (RIGHTARG = bigint, PROCEDURE = factorial)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR testschema.## (NONE, bigint)") + expectedOperator.Procedure = "factorial" + expectedOperator.LeftArgType = "-" + expectedOperator.RightArgType = "bigint" + } else { + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR public.## (LEFTARG = bigint, PROCEDURE = numeric_fac)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR public.## (bigint, NONE)") + testhelper.AssertQueryRuns(connectionPool, "CREATE OPERATOR testschema.## (LEFTARG = bigint, PROCEDURE = numeric_fac)") + defer testhelper.AssertQueryRuns(connectionPool, "DROP OPERATOR testschema.## (bigint, NONE)") + } + _ = backupCmdFlags.Set(options.INCLUDE_SCHEMA, "testschema") results := backup.GetOperators(connectionPool) diff --git a/integration/predata_table_defs_queries_test.go b/integration/predata_table_defs_queries_test.go index 5c7f3e26..09741700 100644 --- a/integration/predata_table_defs_queries_test.go +++ b/integration/predata_table_defs_queries_test.go @@ -763,7 +763,8 @@ FORMAT 'TEXT'`) oid := testutils.OidFromObjectName(connectionPool, "public", "ext_table", backup.TYPE_RELATION) result := backup.GetForeignTableDefinitions(connectionPool) - expectedResult := backup.ForeignTableDefinition{Oid: oid, Options: "delimiter '\t', encoding 'UTF8', escape E'\\\\', execute_on 'ALL_SEGMENTS', format 'text', is_writable 'false', location_uris 'gpfdist://tmp:8080/myfile.txt', log_errors 'disable', \"null\" E'\\\\N'", Server: "gp_exttable_server"} + options := "delimiter '\t', encoding 'UTF8', escape E'\\\\', execute_on 'ALL_SEGMENTS', format 'text', is_writable 'false', location_uris 'gpfdist://tmp:8080/myfile.txt', log_errors 'disable', \"null\" E'\\\\N'" + expectedResult := backup.ForeignTableDefinition{Oid: oid, Options: options, Server: "gp_exttable_server"} Expect(result).To(HaveLen(4)) Expect(result[oid]).To(Equal(expectedResult)) }) diff --git a/integration/predata_types_create_test.go b/integration/predata_types_create_test.go index d5a35570..db000a9f 100644 --- a/integration/predata_types_create_test.go +++ b/integration/predata_types_create_test.go @@ -114,6 +114,10 @@ var _ = Describe("backup integration create statement tests", func() { baseType.Preferred = true baseType.Collatable = true } + if connectionPool.Version.AtLeast("19") { + // PG14+ rejects CREATE TYPE with ELEMENT but no SUBSCRIPT. + baseType.Subscript = "raw_array_subscript_handler" + } metadata := testutils.DefaultMetadata(toc.OBJ_TYPE, false, true, true, includeSecurityLabels) backup.PrintCreateBaseTypeStatement(backupfile, tocfile, baseType, metadata) @@ -183,6 +187,11 @@ var _ = Describe("backup integration create statement tests", func() { resultTypes := backup.GetRangeTypes(connectionPool) Expect(len(resultTypes)).To(Equal(1)) + if connectionPool.Version.AtLeast("19") { + // The statement above names no multirange, so PG derives + // one: the range name with "range" replaced by "multirange". + rangeType.MultiRangeName = "public.textmultirange" + } structmatcher.ExpectStructsToMatchExcluding(&rangeType, &resultTypes[0], "Oid") }) It("creates a range type in a specific schema with a subtype diff function", func() { @@ -206,6 +215,9 @@ var _ = Describe("backup integration create statement tests", func() { resultTypes := backup.GetRangeTypes(connectionPool) Expect(len(resultTypes)).To(Equal(1)) + if connectionPool.Version.AtLeast("19") { + rangeType.MultiRangeName = "testschema.timemultirange" + } structmatcher.ExpectStructsToMatchExcluding(&rangeType, &resultTypes[0], "Oid") }) }) diff --git a/integration/predata_types_queries_test.go b/integration/predata_types_queries_test.go index ada521ed..91b5293e 100644 --- a/integration/predata_types_queries_test.go +++ b/integration/predata_types_queries_test.go @@ -114,6 +114,9 @@ var _ = Describe("backup integration tests", func() { testhelper.AssertQueryRuns(connectionPool, "CREATE FUNCTION public.base_fn_out(public.base_type) RETURNS cstring AS 'boolout' LANGUAGE internal") if connectionPool.Version.Before("6") { testhelper.AssertQueryRuns(connectionPool, "CREATE TYPE public.base_type(INPUT=public.base_fn_in, OUTPUT=public.base_fn_out, INTERNALLENGTH=8, PASSEDBYVALUE, ALIGNMENT=double, STORAGE=plain, DEFAULT=0, ELEMENT=integer, DELIMITER=';')") + } else if connectionPool.Version.AtLeast("19") { + // PG14+ requires a SUBSCRIPT function to declare ELEMENT. + testhelper.AssertQueryRuns(connectionPool, "CREATE TYPE public.base_type(INPUT=public.base_fn_in, OUTPUT=public.base_fn_out, INTERNALLENGTH=8, PASSEDBYVALUE, ALIGNMENT=double, STORAGE=plain, DEFAULT=0, ELEMENT=integer, DELIMITER=';', SUBSCRIPT=raw_array_subscript_handler, CATEGORY='N', PREFERRED=true, COLLATABLE=true)") } else { testhelper.AssertQueryRuns(connectionPool, "CREATE TYPE public.base_type(INPUT=public.base_fn_in, OUTPUT=public.base_fn_out, INTERNALLENGTH=8, PASSEDBYVALUE, ALIGNMENT=double, STORAGE=plain, DEFAULT=0, ELEMENT=integer, DELIMITER=';', CATEGORY='N', PREFERRED=true, COLLATABLE=true)") } @@ -128,6 +131,9 @@ var _ = Describe("backup integration tests", func() { baseTypeCustom.Category = "N" baseTypeCustom.Preferred = true baseTypeCustom.Collatable = true + if connectionPool.Version.AtLeast("19") { + baseTypeCustom.Subscript = "raw_array_subscript_handler" + } structmatcher.ExpectStructsToMatchExcluding(&baseTypeCustom, &results[0], "Oid") } }) @@ -142,7 +148,13 @@ var _ = Describe("backup integration tests", func() { defer testhelper.AssertQueryRuns(connectionPool, "DROP TYPE public.base_array_type CASCADE") testhelper.AssertQueryRuns(connectionPool, "CREATE FUNCTION public.base_array_fn_in(cstring) RETURNS public.base_array_type AS 'boolin' LANGUAGE internal") testhelper.AssertQueryRuns(connectionPool, "CREATE FUNCTION public.base_array_fn_out(public.base_array_type) RETURNS cstring AS 'boolout' LANGUAGE internal") - testhelper.AssertQueryRuns(connectionPool, "CREATE TYPE public.base_array_type(INPUT=public.base_array_fn_in, OUTPUT=public.base_array_fn_out, ELEMENT=text)") + if connectionPool.Version.AtLeast("19") { + // PG14+ requires a SUBSCRIPT function to declare ELEMENT. + testhelper.AssertQueryRuns(connectionPool, "CREATE TYPE public.base_array_type(INPUT=public.base_array_fn_in, OUTPUT=public.base_array_fn_out, ELEMENT=text, SUBSCRIPT=raw_array_subscript_handler)") + arrayType.Subscript = "raw_array_subscript_handler" + } else { + testhelper.AssertQueryRuns(connectionPool, "CREATE TYPE public.base_array_type(INPUT=public.base_array_fn_in, OUTPUT=public.base_array_fn_out, ELEMENT=text)") + } results := backup.GetBaseTypes(connectionPool) @@ -277,6 +289,11 @@ var _ = Describe("backup integration tests", func() { Collation: "public.some_coll", SubTypeOpClass: "pg_catalog.text_ops", } + if connectionPool.Version.AtLeast("19") { + // PG14+ always names the companion multirange; the default is + // the range name with "range" replaced by "multirange". + expectedRangeType.MultiRangeName = "public.textmultirange" + } structmatcher.ExpectStructsToMatchExcluding(&expectedRangeType, &results[0], "Oid") }) It("returns a slice of a range type in a specific schema with a subtype diff function", func() { @@ -301,6 +318,11 @@ var _ = Describe("backup integration tests", func() { SubTypeOpClass: "pg_catalog.time_ops", SubTypeDiff: "testschema.time_subtype_diff", } + if connectionPool.Version.AtLeast("19") { + // Schema-qualified from the range type's own schema, not from + // whatever the session's search_path happens to be. + expectedRangeType.MultiRangeName = "testschema.timemultirange" + } structmatcher.ExpectStructsToMatchExcluding(&expectedRangeType, &results[0], "Oid") }) }) diff --git a/testutils/functions.go b/testutils/functions.go index 653e589d..9eeb0719 100644 --- a/testutils/functions.go +++ b/testutils/functions.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/DATA-DOG/go-sqlmock" + "github.com/blang/semver" "github.com/greenplum-db/gpbackup/backup" "github.com/greenplum-db/gpbackup/filepath" "github.com/greenplum-db/gpbackup/restore" @@ -43,9 +44,41 @@ func SetupTestEnvironment() (*dbconn.DBConn, sqlmock.Sqlmock, *Buffer, *Buffer, SetupTestCluster() backup.SetVersion("0.1.0") + SetTestVersion(connectionPool) return connectionPool, mock, testStdout, testStderr, testLogfile } +// testConnection is whichever connection the running suite describes. +var testConnection *dbconn.DBConn + +// SetTestVersion tells the fixtures below which server they are describing. +// SetupTestEnvironment calls it for the unit suites; the integration suite +// builds its own connection and calls it from BeforeSuite. +func SetTestVersion(connectionPool *dbconn.DBConn) { + testConnection = connectionPool +} + +// relationPrivilegesIncludeMaintain reports whether a relation's full privilege +// set includes MAINTAIN, which PG17 (WHPG19) added to ACL_ALL_RIGHTS_RELATION. +// It decides what DefaultACLForType means by "all privileges on a table". +// +// The live connection is consulted first, because the integration suite +// compares these fixtures against the ACL a real server hands back. The unit +// suites fall through to the environment variable they are driven by: their +// fixtures are built while Ginkgo is still constructing the spec tree, before +// any BeforeEach has run and so before a connection exists. +func relationPrivilegesIncludeMaintain() bool { + if testConnection != nil && testConnection.Version.VersionString != "" { + return testConnection.Version.AtLeast("19") + } + envVersion := os.Getenv("TEST_GPDB_VERSION") + if envVersion == "" { + return false + } + parsed, err := semver.Parse(envVersion) + return err == nil && parsed.Major >= 19 +} + func SetupTestCluster() *cluster.Cluster { testCluster := SetDefaultSegmentConfiguration() backup.SetCluster(testCluster) @@ -285,6 +318,8 @@ func DefaultACLForType(grantee string, objType string) backup.ACL { Create: objType == toc.OBJ_DATABASE || objType == toc.OBJ_SCHEMA || objType == toc.OBJ_TABLESPACE, Temporary: objType == toc.OBJ_DATABASE, Connect: objType == toc.OBJ_DATABASE, + Maintain: relationPrivilegesIncludeMaintain() && + (objType == toc.OBJ_TABLE || objType == toc.OBJ_VIEW || objType == toc.OBJ_MATERIALIZED_VIEW), } } @@ -303,6 +338,8 @@ func DefaultACLForTypeWithGrant(grantee string, objType string) backup.ACL { CreateWithGrant: objType == toc.OBJ_DATABASE || objType == toc.OBJ_SCHEMA || objType == toc.OBJ_TABLESPACE, TemporaryWithGrant: objType == toc.OBJ_DATABASE, ConnectWithGrant: objType == toc.OBJ_DATABASE, + MaintainWithGrant: relationPrivilegesIncludeMaintain() && + (objType == toc.OBJ_TABLE || objType == toc.OBJ_VIEW || objType == toc.OBJ_MATERIALIZED_VIEW), } } @@ -561,6 +598,37 @@ func SkipIfBefore7(connectionPool *dbconn.DBConn) { } } +// IsGPDB7OrLater reports whether the server is GPDB7 or any later major. +// Written out rather than spelled Version.AtLeast("7") because WHPG19 reports +// major 19, so "7 or later" has to name both the 7 line and everything above +// it -- and a bare AtLeast("7") would also be true for a hypothetical 8. +// Having it in one place keeps the specs from drifting apart when the next +// major lands. +func IsGPDB7OrLater(connectionPool *dbconn.DBConn) bool { + return connectionPool.Version.Is("7") || connectionPool.Version.AtLeast("19") +} + +// SkipIfNoS3Protocol skips a spec that needs the s3 external protocol library. +// Six specs declare functions as '$libdir/gps3ext.so', and CREATE FUNCTION ... +// LANGUAGE C validates that the library exists and exports the symbol, so a +// stub will not do. WHPG19 does not ship it: warehouse-pg-next has no +// gpcontrib/gpcloud. Drop this helper once it ships there again. +func SkipIfNoS3Protocol(connectionPool *dbconn.DBConn) { + if connectionPool.Version.AtLeast("19") { + Skip("s3 protocol fixture library gps3ext.so is not available on WHPG19") + } +} + +// SkipIfLanguagesAreExtensionManaged skips a spec that creates or drops a +// procedural language standalone. WHPG19 ships plpython3u as a real extension, +// so CREATE LANGUAGE auto-converts to CREATE EXTENSION and a plain DROP +// LANGUAGE then fails; GPDB6/7 do not package it that way. +func SkipIfLanguagesAreExtensionManaged(connectionPool *dbconn.DBConn) { + if connectionPool.Version.AtLeast("19") { + Skip("procedural languages are extension-managed on WHPG19") + } +} + func InitializeTestTOC(buffer io.Writer, which string) (*toc.TOC, *utils.FileWithByteCount) { tocfile := &toc.TOC{} tocfile.InitializeMetadataEntryMap()