Skip to content

Commit b82ea98

Browse files
Dilip KumarCommitfest Bot
authored andcommitted
Add configurable conflict log table for Logical Replication
This patch adds a feature to provide a structured, queryable record of all logical replication conflicts. The current approach of logging conflicts as plain text in the server logs makes it difficult to query, analyze, and use for external monitoring and automation. This patch addresses these limitations by introducing a configurable conflict_log_table option in the CREATE SUBSCRIPTION command. Key design decisions include: User-Managed Table: The conflict log is stored in a user-managed table rather than a system catalog. Structured Data: Conflict details, including the original and remote tuples, are stored in JSON columns, providing a flexible format to accommodate different table schemas. Comprehensive Information: The log table captures essential attributes such as local and remote transaction IDs, LSNs, commit timestamps, and conflict type, providing a complete record for post-mortem analysis. This feature will make logical replication conflicts easier to monitor and manage, significantly improving the overall resilience and operability of replication setups. The conflict log tables will not be included in a publication, even if the publication is configured to include ALL TABLES or ALL TABLES IN SCHEMA. Note: A single remote tuple may conflict with multiple local tuples when conflict type is CT_MULTIPLE_UNIQUE_CONFLICTS, so for handling this case we create a single row in conflict log table with respect to each remote conflict tuple even if it conflicts with multiple local tuples and we store the multiple conflict tuples as a single JSON array element in format as [ { "xid": "1001", "commit_ts": "...", "origin": "...", "tuple": {...} }, ... ] We can extract the elements of the local tuples from the conflict log table row as given in below example. SELECT remote_xid, relname, remote_origin, local_conflicts[1] ->> 'xid' AS local_xid, local_conflicts[1] ->> 'tuple' AS local_tuple FROM myschema.conflict_log_history2; remote_xid | relname | remote_origin | local_xid | local_tuple ------------+----------+---------------+-----------+--------------------- 760 | test | pg_16406 | 771 | {"a":1,"b":10} 765 | conf_tab | pg_16406 | 775 | {"a":2,"b":2,"c":2}
1 parent 795e94c commit b82ea98

File tree

15 files changed

+1381
-128
lines changed

15 files changed

+1381
-128
lines changed

src/backend/catalog/pg_publication.c

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
#include "catalog/pg_publication_rel.h"
3232
#include "catalog/pg_type.h"
3333
#include "commands/publicationcmds.h"
34+
#include "commands/subscriptioncmds.h"
3435
#include "funcapi.h"
3536
#include "utils/array.h"
3637
#include "utils/builtins.h"
@@ -85,6 +86,15 @@ check_publication_add_relation(Relation targetrel)
8586
errmsg("cannot add relation \"%s\" to publication",
8687
RelationGetRelationName(targetrel)),
8788
errdetail("This operation is not supported for unlogged tables.")));
89+
90+
/* Can't be conflict log table */
91+
if (IsConflictLogTable(RelationGetRelid(targetrel)))
92+
ereport(ERROR,
93+
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
94+
errmsg("cannot add relation \"%s.%s\" to publication",
95+
get_namespace_name(RelationGetNamespace(targetrel)),
96+
RelationGetRelationName(targetrel)),
97+
errdetail("This operation is not supported for conflict log tables.")));
8898
}
8999

90100
/*
@@ -145,6 +155,13 @@ is_publishable_class(Oid relid, Form_pg_class reltuple)
145155

146156
/*
147157
* Another variant of is_publishable_class(), taking a Relation.
158+
*
159+
* Note: Conflict log tables are not publishable. However, we intentionally
160+
* skip this check here because this function is called for every change and
161+
* performing this check during every change publication is costly. To ensure
162+
* unpublishable entries are ignored without incurring performance overhead,
163+
* tuples inserted into the conflict log table uses the HEAP_INSERT_NO_LOGICAL
164+
* flag. This allows the decoding layer to bypass these entries automatically.
148165
*/
149166
bool
150167
is_publishable_relation(Relation rel)
@@ -169,7 +186,10 @@ pg_relation_is_publishable(PG_FUNCTION_ARGS)
169186
tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid));
170187
if (!HeapTupleIsValid(tuple))
171188
PG_RETURN_NULL();
172-
result = is_publishable_class(relid, (Form_pg_class) GETSTRUCT(tuple));
189+
190+
/* Subscription conflict log tables are not published */
191+
result = is_publishable_class(relid, (Form_pg_class) GETSTRUCT(tuple)) &&
192+
!IsConflictLogTable(relid);
173193
ReleaseSysCache(tuple);
174194
PG_RETURN_BOOL(result);
175195
}
@@ -890,7 +910,9 @@ GetAllPublicationRelations(char relkind, bool pubviaroot)
890910
Form_pg_class relForm = (Form_pg_class) GETSTRUCT(tuple);
891911
Oid relid = relForm->oid;
892912

913+
/* Subscription conflict log tables are not published */
893914
if (is_publishable_class(relid, relForm) &&
915+
!IsConflictLogTable(relid) &&
894916
!(relForm->relispartition && pubviaroot))
895917
result = lappend_oid(result, relid);
896918
}
@@ -1018,7 +1040,7 @@ GetSchemaPublicationRelations(Oid schemaid, PublicationPartOpt pub_partopt)
10181040
Oid relid = relForm->oid;
10191041
char relkind;
10201042

1021-
if (!is_publishable_class(relid, relForm))
1043+
if (!is_publishable_class(relid, relForm) || IsConflictLogTable(relid))
10221044
continue;
10231045

10241046
relkind = get_rel_relkind(relid);

0 commit comments

Comments
 (0)