Fix issues with wide tuples being updated and REPLICA IDENTITY FULL.

When replica identity full is being used with a wide tuple (above 2^16
bytes after compression) it lead to errors and/or crashes during
decoding because the length field used to store such tuples doesn't
fit into the variable used to store the width in the WAL record.

To fix, discontinue use of xl_heap_header_len.t_len when decoding the
old tuple version, instead compute length of the old tuple by
subtracting the new tuple's length from the record length.

In newer version of postgres this issue is moot because the length is
stored by the new WAL machinery, instead of a xl_heap_header_len
struct.  A separate commit will forward-patch the regression test.

Reported-By: "anderson"
Discussion: http://postgr.es/m/20170105144819.f6i5o64vfvy4bn5i@alap3.anarazel.de

diff --git a/contrib/test_decoding/expected/toast.out b/contrib/test_decoding/expected/toast.out
index b7bae65ee82d7bdd8a6cd58cb3d03a077c0e9958..c0a19878091ab15f19871223a881db64e131a7e8 100644 (file)
--- a/contrib/test_decoding/expected/toast.out
+++ b/contrib/test_decoding/expected/toast.out
@@ -298,7 +298,13 @@ ALTER TABLE toasted_several REPLICA IDENTITY FULL;
 ALTER TABLE toasted_several ALTER COLUMN toasted_key SET STORAGE EXTERNAL;
 ALTER TABLE toasted_several ALTER COLUMN toasted_col1 SET STORAGE EXTERNAL;
 ALTER TABLE toasted_several ALTER COLUMN toasted_col2 SET STORAGE EXTERNAL;
-INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 2000));
+INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 10000));
+SELECT pg_column_size(toasted_key) > 2^16 FROM toasted_several;
+ ?column? 
+----------
+ t
+(1 row)
+
 SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
                                                                                                regexp_replace                                                                                               
 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

diff --git a/contrib/test_decoding/sql/toast.sql b/contrib/test_decoding/sql/toast.sql
index a333d99abcefd61dd9d2b6156b1b42cf57f7edfa..428816ff4980d59ccc977f7b7fedcbbde58f22d8 100644 (file)
--- a/contrib/test_decoding/sql/toast.sql
+++ b/contrib/test_decoding/sql/toast.sql
@@ -274,7 +274,8 @@ ALTER TABLE toasted_several ALTER COLUMN toasted_key SET STORAGE EXTERNAL;
 ALTER TABLE toasted_several ALTER COLUMN toasted_col1 SET STORAGE EXTERNAL;
 ALTER TABLE toasted_several ALTER COLUMN toasted_col2 SET STORAGE EXTERNAL;
 
-INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 2000));
+INSERT INTO toasted_several(toasted_key) VALUES(repeat('9876543210', 10000));
+SELECT pg_column_size(toasted_key) > 2^16 FROM toasted_several;
 
 SELECT regexp_replace(data, '^(.{100}).*(.{100})$', '\1..\2') FROM pg_logical_slot_peek_changes('regression_slot', NULL, NULL, 'include-xids', '0', 'skip-empty-xacts', '1');
 

diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c
index 1848bc5f2c01f627ad41eb02b66c1261af573289..26d96fd7fc0ce0b786d9e0367aa03b563306d2f6 100644 (file)
--- a/src/backend/access/heap/heapam.c
+++ b/src/backend/access/heap/heapam.c
@@ -7086,7 +7086,13 @@ log_heap_update(Relation reln, Buffer oldbuf,
        /* We need to log a tuple identity */
        if (old_key_tuple)
        {
-           /* don't really need this, but its more comfy to decode */
+           /*
+            * This isn't needed, and can't actually capture the contents of
+            * the tuple accurately (because t_len isn't guaranteed to be big
+            * enough to contain old tuples which can be up to 1 GB long). But
+            * previous versions of 9.4 used this, so we can't change the WAL
+            * format.
+            */
            xlhdr_idx.header.t_infomask2 = old_key_tuple->t_data->t_infomask2;
            xlhdr_idx.header.t_infomask = old_key_tuple->t_data->t_infomask;
            xlhdr_idx.header.t_hoff = old_key_tuple->t_data->t_hoff;

diff --git a/src/backend/replication/logical/decode.c b/src/backend/replication/logical/decode.c
index 3a6d6ffab1a09f505da5d94e14649a70b9d783ad..09a334232e16e5bcd3367337731b102389375019 100644 (file)
--- a/src/backend/replication/logical/decode.c
+++ b/src/backend/replication/logical/decode.c
@@ -653,6 +653,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
    xl_heap_update *xlrec;
    ReorderBufferChange *change;
    char       *data;
+   size_t      remlen = r->xl_len;
 
    xlrec = (xl_heap_update *) buf->record_data;
 
@@ -666,6 +667,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
 
    /* caution, remaining data in record is not aligned */
    data = buf->record_data + SizeOfHeapUpdate;
+   remlen -= SizeOfHeapUpdate;
 
    if (xlrec->flags & XLOG_HEAP_CONTAINS_NEW_TUPLE)
    {
@@ -677,6 +679,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
 
        memcpy(&xlhdr, data, sizeof(xlhdr));
        data += offsetof(xl_heap_header_len, header);
+       remlen -= offsetof(xl_heap_header_len, header);
 
        datalen = xlhdr.t_len + SizeOfHeapHeader;
        tuplelen = xlhdr.t_len;
@@ -687,8 +690,10 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
        DecodeXLogTuple(data, datalen, change->data.tp.newtuple);
        /* skip over the rest of the tuple header */
        data += SizeOfHeapHeader;
+       remlen -= SizeOfHeapHeader;
        /* skip over the tuple data */
        data += xlhdr.t_len;
+       remlen -= xlhdr.t_len;
    }
 
    if (xlrec->flags & XLOG_HEAP_CONTAINS_OLD)
@@ -699,10 +704,17 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
 
        memcpy(&xlhdr, data, sizeof(xlhdr));
        data += offsetof(xl_heap_header_len, header);
+       remlen -= offsetof(xl_heap_header_len, header);
 
-       /* t_len is inconsistent with other cases, see log_heap_update */
-       tuplelen = xlhdr.t_len - offsetof(HeapTupleHeaderData, t_bits);
-       datalen = tuplelen + SizeOfHeapHeader;
+       /*
+        * NB: Even though xl_heap_header_len contains the tuple's length,
+        * it's length field is not wide enough. Use the whole record length
+        * minus the new tuple's length instead. We can't remove the record
+        * length from the WAL record format in 9.4 due to compatibility
+        * concerns - later versions don't have it anyway.
+        */
+       datalen = remlen;
+       tuplelen = datalen - SizeOfHeapHeader;
 
        change->data.tp.oldtuple =
            ReorderBufferGetTupleBuf(ctx->reorder, tuplelen);
@@ -710,6 +722,7 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
        DecodeXLogTuple(data, datalen, change->data.tp.oldtuple);
 #ifdef NOT_USED
        data += datalen;
+       remlen -= datalen;
 #endif
    }