Build in some knowledge about foreign-key relationships in the catalogs.

This follows in the spirit of commit dfb75e4, which created primary
key and uniqueness constraints to improve the visibility of constraints
imposed on the system catalogs.  While our catalogs contain many
foreign-key-like relationships, they don't quite follow SQL semantics,
in that the convention for an omitted reference is to write zero not
NULL.  Plus, we have some cases in which there are arrays each of whose
elements is supposed to be an FK reference; SQL has no way to model that.
So we can't create actual foreign key constraints to describe the
situation.  Nonetheless, we can collect and use knowledge about these
relationships.

This patch therefore adds annotations to the catalog header files to
declare foreign-key relationships.  (The BKI_LOOKUP annotations cover
simple cases, but we weren't previously distinguishing which such
columns are allowed to contain zeroes; we also need new markings for
multi-column FK references.)  Then, Catalog.pm and genbki.pl are
taught to collect this information into a table in a new generated
header "system_fk_info.h".  The only user of that at the moment is
a new SQL function pg_get_catalog_foreign_keys(), which exposes the
table to SQL.  The oidjoins regression test is rewritten to use
pg_get_catalog_foreign_keys() to find out which columns to check.
Aside from removing the need for manual maintenance of that test
script, this allows it to cover numerous relationships that were not
checked by the old implementation based on findoidjoins.  (As of this
commit, 217 relationships are checked by the test, versus 181 before.)

Discussion: https://postgr.es/m/3240355.1612129197@sss.pgh.pa.us

Author: tglsfdc

doc/src/sgml/bki.sgml

@@ -474,10 +474,15 @@
 
     <listitem>
      <para>
-      In such a column, all entries must use the symbolic format except
-      when writing <literal>0</literal> for InvalidOid.  (If the column is
+      In some catalog columns, it's allowed for entries to be zero instead
+      of a valid reference.  If this is allowed, write
+      <literal>BKI_LOOKUP_OPT</literal> instead
+      of <literal>BKI_LOOKUP</literal>.  Then you can
+      write <literal>0</literal> for an entry.  (If the column is
       declared <type>regproc</type>, you can optionally
       write <literal>-</literal> instead of <literal>0</literal>.)
+      Except for this special case, all entries in
+      a <literal>BKI_LOOKUP</literal> column must be symbolic references.
       <filename>genbki.pl</filename> will warn about unrecognized names.
      </para>
     </listitem>
@@ -554,6 +559,22 @@
     therefore no need for the bootstrap backend to deal with symbolic
     references.
    </para>
+
+   <para>
+    It's desirable to mark OID reference columns
+    with <literal>BKI_LOOKUP</literal> or <literal>BKI_LOOKUP_OPT</literal>
+    even if the catalog has no initial data that requires lookup.  This
+    allows <filename>genbki.pl</filename> to record the foreign key
+    relationships that exist in the system catalogs.  That information is
+    used in the regression tests to check for incorrect entries.  See also
+    the macros <literal>DECLARE_FOREIGN_KEY</literal>,
+    <literal>DECLARE_FOREIGN_KEY_OPT</literal>,
+    <literal>DECLARE_ARRAY_FOREIGN_KEY</literal>,
+    and <literal>DECLARE_ARRAY_FOREIGN_KEY_OPT</literal>, which are
+    used to declare foreign key relationships that are too complex
+    for <literal>BKI_LOOKUP</literal> (typically, multi-column foreign
+    keys).
+   </para>
   </sect2>
 
   <sect2 id="system-catalog-auto-array-types">

doc/src/sgml/func.sgml

@@ -22789,6 +22789,38 @@ SELECT pg_type_is_visible('myschema.widget'::regtype);
        </para></entry>
       </row>
 
+      <row>
+       <entry role="func_table_entry"><para role="func_signature">
+        <indexterm>
+         <primary>pg_get_catalog_foreign_keys</primary>
+        </indexterm>
+        <function>pg_get_catalog_foreign_keys</function> ()
+        <returnvalue>setof record</returnvalue>
+        ( <parameter>fktable</parameter> <type>regclass</type>,
+          <parameter>fkcols</parameter> <type>text[]</type>,
+          <parameter>pktable</parameter> <type>regclass</type>,
+          <parameter>pkcols</parameter> <type>text[]</type>,
+          <parameter>is_array</parameter> <type>boolean</type>,
+          <parameter>is_opt</parameter> <type>boolean</type> )
+       </para>
+       <para>
+        Returns a set of records describing the foreign key relationships
+        that exist within the <productname>PostgreSQL</productname> system
+        catalogs.
+        The <parameter>fktable</parameter> column contains the name of the
+        referencing catalog, and the <parameter>fkcols</parameter> column
+        contains the name(s) of the referencing column(s).  Similarly,
+        the <parameter>pktable</parameter> column contains the name of the
+        referenced catalog, and the <parameter>pkcols</parameter> column
+        contains the name(s) of the referenced column(s).
+        If <parameter>is_array</parameter> is true, the last referencing
+        column is an array, each of whose elements should match some entry
+        in the referenced catalog.
+        If <parameter>is_opt</parameter> is true, the referencing column(s)
+        are allowed to contain zeroes instead of a valid reference.
+       </para></entry>
+      </row>
+
       <row>
        <entry role="func_table_entry"><para role="func_signature">
         <indexterm>

src/backend/catalog/.gitignore

@@ -1,5 +1,6 @@
 /postgres.bki
 /schemapg.h
+/system_fk_info.h
 /system_constraints.sql
 /pg_*_d.h
 /bki-stamp

src/backend/catalog/Catalog.pm

@@ -105,6 +105,17 @@ sub ParseHeader
 				index_decl => $5
 			  };
 		}
+		elsif (/^DECLARE_(ARRAY_)?FOREIGN_KEY(_OPT)?\(\s*\(([^)]+)\),\s*(\w+),\s*\(([^)]+)\)\)/)
+		{
+			push @{ $catalog{foreign_keys} },
+			  {
+				is_array => $1 ? 1 : 0,
+				is_opt   => $2 ? 1 : 0,
+				fk_cols  => $3,
+				pk_table => $4,
+				pk_cols  => $5
+			  };
+		}
 		elsif (/^CATALOG\((\w+),(\d+),(\w+)\)/)
 		{
 			$catalog{catname}            = $1;
@@ -197,9 +208,22 @@ sub ParseHeader
 					{
 						$column{array_default} = $1;
 					}
-					elsif ($attopt =~ /BKI_LOOKUP\((\w+)\)/)
+					elsif ($attopt =~ /BKI_LOOKUP(_OPT)?\((\w+)\)/)
 					{
-						$column{lookup} = $1;
+						$column{lookup} = $2;
+						$column{lookup_opt} = $1 ? 1 : 0;
+						# BKI_LOOKUP implicitly makes an FK reference
+						push @{ $catalog{foreign_keys} },
+						  {
+							is_array =>
+							  ($atttype eq 'oidvector' || $atttype eq '_oid')
+							? 1
+							: 0,
+							is_opt   => $column{lookup_opt},
+							fk_cols  => $attname,
+							pk_table => $column{lookup},
+							pk_cols  => 'oid'
+						  };
 					}
 					else
 					{

src/backend/catalog/Makefile

@@ -70,7 +70,7 @@ CATALOG_HEADERS := \
 	pg_sequence.h pg_publication.h pg_publication_rel.h pg_subscription.h \
 	pg_subscription_rel.h
 
-GENERATED_HEADERS := $(CATALOG_HEADERS:%.h=%_d.h) schemapg.h
+GENERATED_HEADERS := $(CATALOG_HEADERS:%.h=%_d.h) schemapg.h system_fk_info.h
 
 POSTGRES_BKI_SRCS := $(addprefix $(top_srcdir)/src/include/catalog/, $(CATALOG_HEADERS))
 

src/backend/catalog/genbki.pl

@@ -213,6 +213,12 @@
 	$amoids{ $row->{amname} } = $row->{oid};
 }
 
+# There is only one authid at bootstrap time, and we handle it specially:
+# the usually-defaulted symbol PGUID becomes the bootstrap superuser's OID.
+# (We could drop this in favor of writing out BKI_DEFAULT(POSTGRES) ...)
+my %authidoids;
+$authidoids{'PGUID'} = $BOOTSTRAP_SUPERUSERID;
+
 # class (relation) OID lookup (note this only covers bootstrap catalogs!)
 my %classoids;
 foreach my $row (@{ $catalog_data{pg_class} })
@@ -234,6 +240,12 @@
 	$langoids{ $row->{lanname} } = $row->{oid};
 }
 
+# There is only one namespace at bootstrap time, and we handle it specially:
+# the usually-defaulted symbol PGNSP becomes the pg_catalog namespace's OID.
+# (We could drop this in favor of writing out BKI_DEFAULT(pg_catalog) ...)
+my %namespaceoids;
+$namespaceoids{'PGNSP'} = $PG_CATALOG_NAMESPACE;
+
 # opclass OID lookup
 my %opcoids;
 foreach my $row (@{ $catalog_data{pg_opclass} })
@@ -376,9 +388,11 @@
 # Map lookup name to the corresponding hash table.
 my %lookup_kind = (
 	pg_am          => \%amoids,
+	pg_authid      => \%authidoids,
 	pg_class       => \%classoids,
 	pg_collation   => \%collationoids,
 	pg_language    => \%langoids,
+	pg_namespace   => \%namespaceoids,
 	pg_opclass     => \%opcoids,
 	pg_operator    => \%operoids,
 	pg_opfamily    => \%opfoids,
@@ -400,6 +414,9 @@
 my $schemafile = $output_path . 'schemapg.h';
 open my $schemapg, '>', $schemafile . $tmpext
   or die "can't open $schemafile$tmpext: $!";
+my $fk_info_file = $output_path . 'system_fk_info.h';
+open my $fk_info, '>', $fk_info_file . $tmpext
+  or die "can't open $fk_info_file$tmpext: $!";
 my $constraints_file = $output_path . 'system_constraints.sql';
 open my $constraints, '>', $constraints_file . $tmpext
   or die "can't open $constraints_file$tmpext: $!";
@@ -554,18 +571,14 @@
 				$GenbkiNextOid++;
 			}
 
-			# Substitute constant values we acquired above.
-			# (It's intentional that this can apply to parts of a field).
-			$bki_values{$attname} =~ s/\bPGUID\b/$BOOTSTRAP_SUPERUSERID/g;
-			$bki_values{$attname} =~ s/\bPGNSP\b/$PG_CATALOG_NAMESPACE/g;
-
 			# Replace OID synonyms with OIDs per the appropriate lookup rule.
 			#
 			# If the column type is oidvector or _oid, we have to replace
 			# each element of the array as per the lookup rule.
 			if ($column->{lookup})
 			{
-				my $lookup = $lookup_kind{ $column->{lookup} };
+				my $lookup     = $lookup_kind{ $column->{lookup} };
+				my $lookup_opt = $column->{lookup_opt};
 				my @lookupnames;
 				my @lookupoids;
 
@@ -575,8 +588,9 @@
 				if ($atttype eq 'oidvector')
 				{
 					@lookupnames = split /\s+/, $bki_values{$attname};
-					@lookupoids = lookup_oids($lookup, $catname, \%bki_values,
-						@lookupnames);
+					@lookupoids =
+					  lookup_oids($lookup, $catname, $attname, $lookup_opt,
+						\%bki_values, @lookupnames);
 					$bki_values{$attname} = join(' ', @lookupoids);
 				}
 				elsif ($atttype eq '_oid')
@@ -586,17 +600,18 @@
 						$bki_values{$attname} =~ s/[{}]//g;
 						@lookupnames = split /,/, $bki_values{$attname};
 						@lookupoids =
-						  lookup_oids($lookup, $catname, \%bki_values,
-							@lookupnames);
+						  lookup_oids($lookup, $catname, $attname,
+							$lookup_opt, \%bki_values, @lookupnames);
 						$bki_values{$attname} = sprintf "{%s}",
 						  join(',', @lookupoids);
 					}
 				}
 				else
 				{
 					$lookupnames[0] = $bki_values{$attname};
-					@lookupoids = lookup_oids($lookup, $catname, \%bki_values,
-						@lookupnames);
+					@lookupoids =
+					  lookup_oids($lookup, $catname, $attname, $lookup_opt,
+						\%bki_values, @lookupnames);
 					$bki_values{$attname} = $lookupoids[0];
 				}
 			}
@@ -706,14 +721,78 @@
 # Closing boilerplate for schemapg.h
 print $schemapg "\n#endif\t\t\t\t\t\t\t/* SCHEMAPG_H */\n";
 
+# Now generate system_fk_info.h
+
+# Opening boilerplate for system_fk_info.h
+print $fk_info <<EOM;
+/*-------------------------------------------------------------------------
+ *
+ * system_fk_info.h
+ *    Data about the foreign-key relationships in the system catalogs
+ *
+ * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * NOTES
+ *  ******************************
+ *  *** DO NOT EDIT THIS FILE! ***
+ *  ******************************
+ *
+ *  It has been GENERATED by src/backend/catalog/genbki.pl
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef SYSTEM_FK_INFO_H
+#define SYSTEM_FK_INFO_H
+
+typedef struct SysFKRelationship
+{
+	Oid			fk_table;		/* referencing catalog */
+	Oid			pk_table;		/* referenced catalog */
+	const char *fk_columns;		/* referencing column name(s) */
+	const char *pk_columns;		/* referenced column name(s) */
+	bool		is_array;		/* if true, last fk_column is an array */
+	bool		is_opt;			/* if true, fk_column can be zero */
+} SysFKRelationship;
+
+static const SysFKRelationship sys_fk_relationships[] = {
+EOM
+
+# Emit system_fk_info data
+foreach my $catname (@catnames)
+{
+	my $catalog = $catalogs{$catname};
+	foreach my $fkinfo (@{ $catalog->{foreign_keys} })
+	{
+		my $pktabname = $fkinfo->{pk_table};
+
+		# We use BKI_LOOKUP for encodings, but there's no real catalog there
+		next if $pktabname eq 'encoding';
+
+		printf $fk_info
+		  "\t{ /* %s */ %s, /* %s */ %s, \"{%s}\", \"{%s}\", %s, %s},\n",
+		  $catname,   $catalog->{relation_oid},
+		  $pktabname, $catalogs{$pktabname}->{relation_oid},
+		  $fkinfo->{fk_cols},
+		  $fkinfo->{pk_cols},
+		  ($fkinfo->{is_array} ? "true" : "false"),
+		  ($fkinfo->{is_opt}   ? "true" : "false");
+	}
+}
+
+# Closing boilerplate for system_fk_info.h
+print $fk_info "};\n\n#endif\t\t\t\t\t\t\t/* SYSTEM_FK_INFO_H */\n";
+
 # We're done emitting data
 close $bki;
 close $schemapg;
+close $fk_info;
 close $constraints;
 
 # Finally, rename the completed files into place.
 Catalog::RenameTempFile($bkifile,    $tmpext);
 Catalog::RenameTempFile($schemafile, $tmpext);
+Catalog::RenameTempFile($fk_info_file, $tmpext);
 Catalog::RenameTempFile($constraints_file, $tmpext);
 
 exit 0;
@@ -948,7 +1027,8 @@ sub morph_row_for_schemapg
 # within this genbki.pl run.)
 sub lookup_oids
 {
-	my ($lookup, $catname, $bki_values, @lookupnames) = @_;
+	my ($lookup, $catname, $attname, $lookup_opt, $bki_values, @lookupnames)
+	  = @_;
 
 	my @lookupoids;
 	foreach my $lookupname (@lookupnames)
@@ -961,10 +1041,19 @@ sub lookup_oids
 		else
 		{
 			push @lookupoids, $lookupname;
-			warn sprintf
-			  "unresolved OID reference \"%s\" in %s.dat line %s\n",
-			  $lookupname, $catname, $bki_values->{line_number}
-			  if $lookupname ne '-' and $lookupname ne '0';
+			if ($lookupname eq '-' or $lookupname eq '0')
+			{
+				warn sprintf
+				  "invalid zero OID reference in %s.dat field %s line %s\n",
+				  $catname, $attname, $bki_values->{line_number}
+				  if !$lookup_opt;
+			}
+			else
+			{
+				warn sprintf
+				  "unresolved OID reference \"%s\" in %s.dat field %s line %s\n",
+				  $lookupname, $catname, $attname, $bki_values->{line_number};
+			}
 		}
 	}
 	return @lookupoids;