Skip to content

Commit 1b86c81

Browse files
anarazelanarazel
committed
Fix decoding of MULTI_INSERTs when rows other than the last are toasted.
When decoding the results of a HEAP2_MULTI_INSERT (currently only generated by COPY FROM) toast columns for all but the last tuple weren't replaced by their actual contents before being handed to the output plugin. The reassembled toast datums where disregarded after every REORDER_BUFFER_CHANGE_(INSERT|UPDATE|DELETE) which is correct for plain inserts, updates, deletes, but not multi inserts - there we generate several REORDER_BUFFER_CHANGE_INSERTs for a single xl_heap_multi_insert record. To solve the problem add a clear_toast_afterwards boolean to ReorderBufferChange's union member that's used by modifications. All row changes but multi_inserts always set that to true, but multi_insert sets it only for the last change generated. Add a regression test covering decoding of multi_inserts - there was none at all before. Backpatch to 9.4 where logical decoding was introduced. Bug found by Petr Jelinek.
1 parent 333b7db commit 1b86c81

File tree

5 files changed

+54
-2
lines changed

5 files changed

+54
-2
lines changed

‎contrib/test_decoding/expected/toast.out

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,14 @@ UPDATE toasted_key SET toasted_col2 = toasted_col1;
4040
-- test update of a toasted key, changing it
4141
UPDATE toasted_key SET toasted_key = toasted_key || '1';
4242
DELETE FROM toasted_key;
43+
-- Test that HEAP2_MULTI_INSERT insertions with and without toasted
44+
-- columns are handled correctly
45+
CREATE TABLE toasted_copy (
46+
id int primary key, -- no default, copy didn't use to handle that with multi inserts
47+
data text
48+
);
49+
ALTER TABLE toasted_copy ALTER COLUMN data SET STORAGE EXTERNAL;
50+
\copy toasted_copy FROM STDIN
4351
SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0');
4452
substr
4553
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
@@ -80,7 +88,17 @@ SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot',
8088
BEGIN
8189
table public.toasted_key: DELETE: toasted_key[text]:'123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
8290
COMMIT
83-
(37 rows)
91+
BEGIN
92+
COMMIT
93+
BEGIN
94+
COMMIT
95+
BEGIN
96+
table public.toasted_copy: INSERT: id[integer]:1 data[text]:'untoasted1'
97+
table public.toasted_copy: INSERT: id[integer]:2 data[text]:'toasted1-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
98+
table public.toasted_copy: INSERT: id[integer]:3 data[text]:'untoasted2'
99+
table public.toasted_copy: INSERT: id[integer]:4 data[text]:'toasted2-1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
100+
COMMIT
101+
(47 rows)
84102

85103
SELECT pg_drop_replication_slot('regression_slot');
86104
pg_drop_replication_slot

‎contrib/test_decoding/sql/toast.sql

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,5 +47,18 @@ UPDATE toasted_key SET toasted_key = toasted_key || '1';
4747

4848
DELETE FROM toasted_key;
4949

50+
-- Test that HEAP2_MULTI_INSERT insertions with and without toasted
51+
-- columns are handled correctly
52+
CREATE TABLE toasted_copy (
53+
id int primary key, -- no default, copy didn't use to handle that with multi inserts
54+
data text
55+
);
56+
ALTER TABLE toasted_copy ALTER COLUMN data SET STORAGE EXTERNAL;
57+
\copy toasted_copy FROM STDIN
58+
1 untoasted1
59+
2 toasted1-12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
60+
3 untoasted2
61+
4 toasted2-12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
62+
\.
5063
SELECT substr(data, 1, 200) FROM pg_logical_slot_get_changes('regression_slot', NULL, NULL, 'include-xids', '0');
5164
SELECT pg_drop_replication_slot('regression_slot');

‎src/backend/replication/logical/decode.c

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -608,6 +608,8 @@ DecodeInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
608608
change->data.tp.newtuple);
609609
}
610610

611+
change->data.tp.clear_toast_afterwards = true;
612+
611613
ReorderBufferQueueChange(ctx->reorder, r->xl_xid, buf->origptr, change);
612614
}
613615

@@ -673,6 +675,8 @@ DecodeUpdate(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
673675
#endif
674676
}
675677

678+
change->data.tp.clear_toast_afterwards = true;
679+
676680
ReorderBufferQueueChange(ctx->reorder, r->xl_xid, buf->origptr, change);
677681
}
678682

@@ -710,6 +714,9 @@ DecodeDelete(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
710714
r->xl_len - SizeOfHeapDelete,
711715
change->data.tp.oldtuple);
712716
}
717+
718+
change->data.tp.clear_toast_afterwards = true;
719+
713720
ReorderBufferQueueChange(ctx->reorder, r->xl_xid, buf->origptr, change);
714721
}
715722

@@ -795,6 +802,9 @@ DecodeMultiInsert(LogicalDecodingContext *ctx, XLogRecordBuffer *buf)
795802
tuple->header.t_hoff = xlhdr->t_hoff;
796803
}
797804

805+
/* reset toast reassembly only after the last chunk */
806+
change->data.tp.clear_toast_afterwards = (i + 1) == xlrec->ntuples;
807+
798808
ReorderBufferQueueChange(ctx->reorder, r->xl_xid,
799809
buf->origptr, change);
800810
}

‎src/backend/replication/logical/reorderbuffer.c

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1383,7 +1383,14 @@ ReorderBufferCommit(ReorderBuffer *rb, TransactionId xid,
13831383
{
13841384
ReorderBufferToastReplace(rb, txn, relation, change);
13851385
rb->apply_change(rb, txn, relation, change);
1386-
ReorderBufferToastReset(rb, txn);
1386+
1387+
/*
1388+
* Only clear reassembled toast chunks if we're
1389+
* sure they're not required anymore. The creator
1390+
* of the tuple tells us.
1391+
*/
1392+
if (change->data.tp.clear_toast_afterwards)
1393+
ReorderBufferToastReset(rb, txn);
13871394
}
13881395
/* we're not interested in toast deletions */
13891396
else if (change->action == REORDER_BUFFER_CHANGE_INSERT)

‎src/include/replication/reorderbuffer.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,10 @@ typedef struct ReorderBufferChange
7575
{
7676
/* relation that has been changed */
7777
RelFileNode relnode;
78+
79+
/* no previously reassembled toast chunks are necessary anymore */
80+
bool clear_toast_afterwards;
81+
7882
/* valid for DELETE || UPDATE */
7983
ReorderBufferTupleBuf *oldtuple;
8084
/* valid for INSERT || UPDATE */

0 commit comments

Comments
 (0)