[CF 6099] v31 - Support EXCEPT tables and EXCEPT (column-list) in publications

This branch was automatically generated by a robot using patches from an
email thread registered at:

https://commitfest.postgresql.org/patch/6099

The branch will be overwritten each time a new patch version is posted to
the thread, and also periodically to check for bitrot caused by changes
on the master branch.

Patch(es): https://www.postgresql.org/message-id/CANhcyEV_EVi5cgJ6WPvmeVAqjCS7Of+VAWuRHZtsVf8PQb_z7g@mail.gmail.com
Author(s): Shlok Kyal

Author: Commitfest Bot

doc/src/sgml/catalogs.sgml

@@ -6581,6 +6581,15 @@ SCRAM-SHA-256$<replaceable>&lt;iteration count&gt;</replaceable>:<replaceable>&l
       if there is no publication qualifying condition.</para></entry>
      </row>
 
+     <row>
+      <entry role="catalog_table_entry"><para role="column_definition">
+      <structfield>prexcept</structfield> <type>bool</type>
+      </para>
+      <para>
+       True if the relation must be excluded
+      </para></entry>
+     </row>
+
      <row>
       <entry role="catalog_table_entry"><para role="column_definition">
        <structfield>prattrs</structfield> <type>int2vector</type>

doc/src/sgml/logical-replication.sgml

@@ -2550,10 +2550,12 @@ CONTEXT:  processing remote data for replication origin "pg_16395" during "INSER
   </para>
 
   <para>
-   To add tables to a publication, the user must have ownership rights on the
-   table. To add all tables in schema to a publication, the user must be a
-   superuser. To create a publication that publishes all tables, all tables in
-   schema, or all sequences automatically, the user must be a superuser.
+   To create a publication using <literal>FOR ALL TABLES</literal>,
+   <literal>FOR ALL SEQUENCES</literal> or
+   <literal>FOR TABLES IN SCHEMA</literal>, the user must be a superuser. To add
+   <literal>ALL TABLES</literal> or <literal>TABLES IN SCHEMA</literal> to a
+   publication, the user must be a superuser. To add tables to a publication,
+   the user must have ownership rights on the table.
   </para>
 
   <para>

doc/src/sgml/ref/create_publication.sgml

@@ -32,12 +32,16 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable>
 
 <phrase>and <replaceable class="parameter">publication_all_object</replaceable> is one of:</phrase>
 
-    ALL TABLES
+    ALL TABLES [ EXCEPT [ TABLE ] ( <replaceable class="parameter">table_exception_object</replaceable> [, ... ] ) ]
     ALL SEQUENCES
 
 <phrase>and <replaceable class="parameter">table_and_columns</replaceable> is:</phrase>
 
     [ ONLY ] <replaceable class="parameter">table_name</replaceable> [ * ] [ ( <replaceable class="parameter">column_name</replaceable> [, ... ] ) ] [ WHERE ( <replaceable class="parameter">expression</replaceable> ) ]
+
+<phrase>where <replaceable class="parameter">table_exception_object</replaceable> is:</phrase>
+
+    [ ONLY ] <replaceable class="parameter">table_name</replaceable> [ * ]
 </synopsis>
  </refsynopsisdiv>
 
@@ -164,7 +168,9 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable>
     <listitem>
      <para>
       Marks the publication as one that replicates changes for all tables in
-      the database, including tables created in the future.
+      the database, including tables created in the future. If
+      <literal>EXCEPT TABLE</literal> is specified, then exclude replicating
+      the changes for the specified tables.
      </para>
     </listitem>
    </varlistentry>
@@ -184,6 +190,35 @@ CREATE PUBLICATION <replaceable class="parameter">name</replaceable>
     </listitem>
    </varlistentry>
 
+   <varlistentry id="sql-createpublication-params-for-except-table">
+    <term><literal>EXCEPT TABLE</literal></term>
+    <listitem>
+     <para>
+      This clause specifies a list of tables to be excluded from the
+      publication. It can only be used with <literal>FOR ALL TABLES</literal>.
+      If <literal>ONLY</literal> is specified before the table name, only
+      that table is excluded from the publication. If <literal>ONLY</literal> is
+      not specified, the table and all its descendant tables (if any) are
+      excluded. Optionally, <literal>*</literal> can be specified after the
+      table name to explicitly indicate that descendant tables are excluded.
+      This does not apply to a partitioned table, however.
+     </para>
+     <para>
+      The partitioned table or its partitions are excluded from the publication
+      based on the parameter <literal>publish_via_partition_root</literal>.
+      When <literal>publish_via_partition_root</literal> is set to
+      <literal>true</literal>, specifying a root partitioned table in
+      <literal>EXCEPT TABLE</literal> excludes it and all its partitions from
+      replication. Specifying a leaf partition has no effect, as its changes are
+      still replicated via the root partitioned table. When
+      <literal>publish_via_partition_root</literal> is set to
+      <literal>false</literal>, specifying a root partitioned table has no
+      effect, as changes are replicated via the leaf partitions. Specifying a
+      leaf partition excludes only that partition from replication.
+     </para>
+    </listitem>
+   </varlistentry>
+
    <varlistentry id="sql-createpublication-params-with">
     <term><literal>WITH ( <replaceable class="parameter">publication_parameter</replaceable> [= <replaceable class="parameter">value</replaceable>] [, ... ] )</literal></term>
     <listitem>
@@ -487,6 +522,23 @@ CREATE PUBLICATION all_sequences FOR ALL SEQUENCES;
    all sequences for synchronization:
 <programlisting>
 CREATE PUBLICATION all_tables_sequences FOR ALL TABLES, ALL SEQUENCES;
+</programlisting>
+  </para>
+
+  <para>
+   Create a publication that publishes all changes in all the tables except
+   <structname>users</structname> and <structname>departments</structname>:
+<programlisting>
+CREATE PUBLICATION all_tables_except FOR ALL TABLES EXCEPT (users, departments);
+</programlisting>
+  </para>
+
+  <para>
+   Create a publication that publishes all changes in all sequences and all the
+   tables except tables <structname>users</structname> and
+   <structname>departments</structname>:
+<programlisting>
+CREATE PUBLICATION all_sequences_tables_except FOR ALL SEQUENCES, ALL TABLES EXCEPT (users, departments);
 </programlisting>
   </para>
  </refsect1>

doc/src/sgml/ref/psql-ref.sgml

@@ -2103,8 +2103,9 @@ SELECT $1 \parse stmt1
         listed.
         If <literal>x</literal> is appended to the command name, the results
         are displayed in expanded mode.
-        If <literal>+</literal> is appended to the command name, the tables and
-        schemas associated with each publication are shown as well.
+        If <literal>+</literal> is appended to the command name, the tables,
+        excluded tables, and schemas associated with each publication are shown
+        as well.
         </para>
         </listitem>
       </varlistentry>

src/backend/catalog/pg_publication.c

@@ -354,7 +354,8 @@ GetPubPartitionOptionRelations(List *result, PublicationPartOpt pub_partopt,
  * ancestor is at the end of the list.
  */
 Oid
-GetTopMostAncestorInPublication(Oid puboid, List *ancestors, int *ancestor_level)
+GetTopMostAncestorInPublication(Oid puboid, List *ancestors,
+								int *ancestor_level, bool puballtables)
 {
 	ListCell   *lc;
 	Oid			topmost_relid = InvalidOid;
@@ -366,32 +367,43 @@ GetTopMostAncestorInPublication(Oid puboid, List *ancestors, int *ancestor_level
 	foreach(lc, ancestors)
 	{
 		Oid			ancestor = lfirst_oid(lc);
-		List	   *apubids = GetRelationPublications(ancestor);
-		List	   *aschemaPubids = NIL;
+		List	   *apubids = NIL;
+		List	   *aexceptpubids = NIL;
+		List	   *aschemapubids = NIL;
+		bool		set_top = false;
+
+		GetRelationPublications(ancestor, &apubids, &aexceptpubids);
 
 		level++;
 
-		if (list_member_oid(apubids, puboid))
+		/* check if member of table publications */
+		set_top = list_member_oid(apubids, puboid);
+		if (!set_top)
 		{
-			topmost_relid = ancestor;
+			aschemapubids = GetSchemaPublications(get_rel_namespace(ancestor));
 
-			if (ancestor_level)
-				*ancestor_level = level;
+			/* check if member of schema publications */
+			set_top = list_member_oid(aschemapubids, puboid);
+
+			/*
+			 * If the publication is all tables publication and the table is
+			 * not part of exception tables.
+			 */
+			if (!set_top && puballtables)
+				set_top = !list_member_oid(aexceptpubids, puboid);
 		}
-		else
+
+		if (set_top)
 		{
-			aschemaPubids = GetSchemaPublications(get_rel_namespace(ancestor));
-			if (list_member_oid(aschemaPubids, puboid))
-			{
-				topmost_relid = ancestor;
+			topmost_relid = ancestor;
 
-				if (ancestor_level)
-					*ancestor_level = level;
-			}
+			if (ancestor_level)
+				*ancestor_level = level;
 		}
 
 		list_free(apubids);
-		list_free(aschemaPubids);
+		list_free(aschemapubids);
+		list_free(aexceptpubids);
 	}
 
 	return topmost_relid;
@@ -466,6 +478,26 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
 						RelationGetRelationName(targetrel), pub->name)));
 	}
 
+	/*
+	 * Check when a partition is excluded via EXCEPT TABLE while the
+	 * publication has publish_via_partition_root = true.
+	 */
+	if (pub->alltables && pub->pubviaroot && pri->except &&
+		targetrel->rd_rel->relispartition)
+		ereport(WARNING,
+				(errmsg("partition \"%s\" might be replicated as publish_via_partition_root is \"%s\"",
+						RelationGetRelationName(targetrel), "true")));
+
+	/*
+	 * Check when a partitioned table is excluded via EXCEPT TABLE while the
+	 * publication has publish_via_partition_root = false.
+	 */
+	if (pub->alltables && !pub->pubviaroot && pri->except &&
+		targetrel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE)
+		ereport(WARNING,
+				(errmsg("partitioned table \"%s\" might be replicated as publish_via_partition_root is \"%s\"",
+						RelationGetRelationName(targetrel), "false")));
+
 	check_publication_add_relation(targetrel);
 
 	/* Validate and translate column names into a Bitmapset of attnums. */
@@ -482,6 +514,8 @@ publication_add_relation(Oid pubid, PublicationRelInfo *pri,
 		ObjectIdGetDatum(pubid);
 	values[Anum_pg_publication_rel_prrelid - 1] =
 		ObjectIdGetDatum(relid);
+	values[Anum_pg_publication_rel_prexcept - 1] =
+		BoolGetDatum(pri->except);
 
 	/* Add qualifications, if available */
 	if (pri->whereClause != NULL)
@@ -749,35 +783,59 @@ publication_add_schema(Oid pubid, Oid schemaid, bool if_not_exists)
 	return myself;
 }
 
-/* Gets list of publication oids for a relation */
-List *
-GetRelationPublications(Oid relid)
+/*
+ * Get the list of publication oids associated with a specified relation.
+ * pubids is filled with the list of publication oids the relation is part of.
+ * except_pubids is filled with the list of publication oids the relation is
+ * excluded from.
+ *
+ * This function returns true if the relation is part of any publication.
+ */
+bool
+GetRelationPublications(Oid relid, List **pubids, List **except_pubids)
 {
-	List	   *result = NIL;
 	CatCList   *pubrellist;
-	int			i;
+	bool		found = false;
 
 	/* Find all publications associated with the relation. */
 	pubrellist = SearchSysCacheList1(PUBLICATIONRELMAP,
 									 ObjectIdGetDatum(relid));
-	for (i = 0; i < pubrellist->n_members; i++)
+	for (int i = 0; i < pubrellist->n_members; i++)
 	{
 		HeapTuple	tup = &pubrellist->members[i]->tuple;
-		Oid			pubid = ((Form_pg_publication_rel) GETSTRUCT(tup))->prpubid;
+		Form_pg_publication_rel pubrel = (Form_pg_publication_rel) GETSTRUCT(tup);
+		Oid			pubid = pubrel->prpubid;
 
-		result = lappend_oid(result, pubid);
+		if (pubrel->prexcept)
+		{
+			if (except_pubids)
+				*except_pubids = lappend_oid(*except_pubids, pubid);
+		}
+		else
+		{
+			if (pubids)
+				*pubids = lappend_oid(*pubids, pubid);
+			found = true;
+		}
 	}
 
 	ReleaseSysCacheList(pubrellist);
 
-	return result;
+	return found;
 }
 
 /*
- * Gets list of relation oids for a publication.
+ * Return the list of relation OIDs for a publication.
+ *
+ * For a FOR ALL TABLES publication, this returns the list of tables that were
+ * explicitly excluded via an EXCEPT TABLE clause.
+ *
+ * For a FOR TABLE publication, this returns the list of tables explicitly
+ * included in the publication.
  *
- * This should only be used FOR TABLE publications, the FOR ALL TABLES/SEQUENCES
- * should use GetAllPublicationRelations().
+ * Publications declared with FOR ALL TABLES or FOR ALL SEQUENCES should use
+ * GetAllPublicationRelations() to obtain the complete set of tables covered by
+ * the publication.
  */
 List *
 GetPublicationRelations(Oid pubid, PublicationPartOpt pub_partopt)
@@ -864,15 +922,23 @@ GetAllTablesPublications(void)
  * partitioned tables, we must exclude partitions in favor of including the
  * root partitioned tables. This is not applicable to FOR ALL SEQUENCES
  * publication.
+ *
+ * The list does not include relations that are explicitly excluded via the
+ * EXCEPT TABLE clause of the publication specified by pubid.
  */
 List *
-GetAllPublicationRelations(char relkind, bool pubviaroot)
+GetAllPublicationRelations(Oid pubid, char relkind, bool pubviaroot)
 {
 	Relation	classRel;
 	ScanKeyData key[1];
 	TableScanDesc scan;
 	HeapTuple	tuple;
 	List	   *result = NIL;
+	List	   *exceptlist;
+
+	exceptlist = GetPublicationRelations(pubid, pubviaroot ?
+										 PUBLICATION_PART_ALL :
+										 PUBLICATION_PART_ROOT);
 
 	Assert(!(relkind == RELKIND_SEQUENCE && pubviaroot));
 
@@ -891,7 +957,8 @@ GetAllPublicationRelations(char relkind, bool pubviaroot)
 		Oid			relid = relForm->oid;
 
 		if (is_publishable_class(relid, relForm) &&
-			!(relForm->relispartition && pubviaroot))
+			!(relForm->relispartition && pubviaroot) &&
+			!list_member_oid(exceptlist, relid))
 			result = lappend_oid(result, relid);
 	}
 
@@ -912,7 +979,8 @@ GetAllPublicationRelations(char relkind, bool pubviaroot)
 			Oid			relid = relForm->oid;
 
 			if (is_publishable_class(relid, relForm) &&
-				!relForm->relispartition)
+				!relForm->relispartition &&
+				!list_member_oid(exceptlist, relid))
 				result = lappend_oid(result, relid);
 		}
 
@@ -1168,7 +1236,8 @@ pg_get_publication_tables(PG_FUNCTION_ARGS)
 			 * those. Otherwise, get the partitioned table itself.
 			 */
 			if (pub_elem->alltables)
-				pub_elem_tables = GetAllPublicationRelations(RELKIND_RELATION,
+				pub_elem_tables = GetAllPublicationRelations(pub_elem->oid,
+															 RELKIND_RELATION,
 															 pub_elem->pubviaroot);
 			else
 			{
@@ -1367,7 +1436,7 @@ pg_get_publication_sequences(PG_FUNCTION_ARGS)
 		publication = GetPublicationByName(pubname, false);
 
 		if (publication->allsequences)
-			sequences = GetAllPublicationRelations(RELKIND_SEQUENCE, false);
+			sequences = GetAllPublicationRelations(publication->oid, RELKIND_SEQUENCE, false);
 
 		funcctx->user_fctx = sequences;