Skip to content

Commit 9ecb134

Browse files
michaelpqmichaelpq
committed
pg_walinspect: pg_get_wal_fpi_info() -> pg_get_wal_block_info()
This commit reworks pg_get_wal_fpi_info() to become aware of all the block information that can be attached to a record rather than just its full-page writes: - Addition of the block id as assigned by XLogRegisterBuffer(), XLogRegisterBlock() or XLogRegisterBufData(). - Addition of the block data, as bytea, or NULL if none. The length of the block data can be guessed with length(), so there is no need to store its length in a separate field. - Addition of the full-page image length, as counted without a hole or even compressed. - Modification of the handling of the full-page image data. This is still a bytea, but it could become NULL if none is assigned to a block. - Addition of the full-page image flags, tracking if a page is stored with a hole, if it needs to be applied and the type of compression applied to it, as of all the BKPIMAGE_* values in xlogrecord.h. The information of each block is returned as one single record, with the record's ReadRecPtr included to be able to join the block information with the existing pg_get_wal_records_info(). Note that it is perfectly possible for a block to hold both data and full-page image. Thanks also to Kyotaro Horiguchi and Matthias van de Meent for the discussion. This commit uses some of the work proposed by Melanie, though it has been largely redesigned and rewritten by me. Bharath has helped in refining a bit the whole. Reported-by: Melanie Plageman Author: Michael Paquier, Melanie Plageman, Bharath Rupireddy Discussion: https://postgr.es/m/CAAKRu_bORebdZmcV8V4cZBzU8M_C6tDDdbiPhCZ6i-iuSXW9TA@mail.gmail.com
1 parent 8da2ec3 commit 9ecb134

File tree

5 files changed

+174
-86
lines changed

5 files changed

+174
-86
lines changed

‎contrib/pg_walinspect/expected/pg_walinspect.out

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -74,17 +74,28 @@ SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_records_info(:'wal_lsn1', :'wal_lsn2'
7474
(1 row)
7575

7676
-- ===================================================================
77-
-- Tests to get full page image (FPI) from WAL record
77+
-- Tests to get block information from WAL record
7878
-- ===================================================================
79+
-- Update table to generate some block data
7980
SELECT pg_current_wal_lsn() AS wal_lsn3 \gset
80-
-- Force FPI on the next update.
81-
CHECKPOINT;
82-
-- Update table to generate an FPI.
83-
UPDATE sample_tbl SET col1 = col1 * 100 WHERE col1 = 1;
81+
UPDATE sample_tbl SET col1 = col1 + 1 WHERE col1 = 1;
8482
SELECT pg_current_wal_lsn() AS wal_lsn4 \gset
83+
-- Check if we get block data from WAL record.
84+
SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_block_info(:'wal_lsn3', :'wal_lsn4')
85+
WHERE relfilenode = :'sample_tbl_oid' AND blockdata IS NOT NULL;
86+
ok
87+
----
88+
t
89+
(1 row)
90+
91+
-- Force full-page image on the next update.
92+
SELECT pg_current_wal_lsn() AS wal_lsn5 \gset
93+
CHECKPOINT;
94+
UPDATE sample_tbl SET col1 = col1 + 1 WHERE col1 = 2;
95+
SELECT pg_current_wal_lsn() AS wal_lsn6 \gset
8596
-- Check if we get FPI from WAL record.
86-
SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_fpi_info(:'wal_lsn3', :'wal_lsn4')
87-
WHERE relfilenode = :'sample_tbl_oid';
97+
SELECT COUNT(*) >= 1 AS ok FROM pg_get_wal_block_info(:'wal_lsn5', :'wal_lsn6')
98+
WHERE relfilenode = :'sample_tbl_oid' AND fpi IS NOT NULL;
8899
ok
89100
----
90101
t
@@ -116,7 +127,7 @@ SELECT has_function_privilege('regress_pg_walinspect',
116127
(1 row)
117128

118129
SELECT has_function_privilege('regress_pg_walinspect',
119-
'pg_get_wal_fpi_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- no
130+
'pg_get_wal_block_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- no
120131
has_function_privilege
121132
------------------------
122133
f
@@ -146,7 +157,7 @@ SELECT has_function_privilege('regress_pg_walinspect',
146157
(1 row)
147158

148159
SELECT has_function_privilege('regress_pg_walinspect',
149-
'pg_get_wal_fpi_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes
160+
'pg_get_wal_block_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes
150161
has_function_privilege
151162
------------------------
152163
t
@@ -160,7 +171,7 @@ GRANT EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn)
160171
TO regress_pg_walinspect;
161172
GRANT EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean)
162173
TO regress_pg_walinspect;
163-
GRANT EXECUTE ON FUNCTION pg_get_wal_fpi_info(pg_lsn, pg_lsn)
174+
GRANT EXECUTE ON FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn)
164175
TO regress_pg_walinspect;
165176
SELECT has_function_privilege('regress_pg_walinspect',
166177
'pg_get_wal_record_info(pg_lsn)', 'EXECUTE'); -- yes
@@ -184,7 +195,7 @@ SELECT has_function_privilege('regress_pg_walinspect',
184195
(1 row)
185196

186197
SELECT has_function_privilege('regress_pg_walinspect',
187-
'pg_get_wal_fpi_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes
198+
'pg_get_wal_block_info(pg_lsn, pg_lsn) ', 'EXECUTE'); -- yes
188199
has_function_privilege
189200
------------------------
190201
t
@@ -196,7 +207,7 @@ REVOKE EXECUTE ON FUNCTION pg_get_wal_records_info(pg_lsn, pg_lsn)
196207
FROM regress_pg_walinspect;
197208
REVOKE EXECUTE ON FUNCTION pg_get_wal_stats(pg_lsn, pg_lsn, boolean)
198209
FROM regress_pg_walinspect;
199-
REVOKE EXECUTE ON FUNCTION pg_get_wal_fpi_info(pg_lsn, pg_lsn)
210+
REVOKE EXECUTE ON FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn)
200211
FROM regress_pg_walinspect;
201212
-- ===================================================================
202213
-- Clean up

‎contrib/pg_walinspect/pg_walinspect--1.0--1.1.sql

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,25 @@
44
\echo Use "ALTER EXTENSION pg_walinspect UPDATE TO '1.1'" to load this file. \quit
55

66
--
7-
-- pg_get_wal_fpi_info()
7+
-- pg_get_wal_block_info()
88
--
9-
CREATE FUNCTION pg_get_wal_fpi_info(IN start_lsn pg_lsn,
9+
CREATE FUNCTION pg_get_wal_block_info(IN start_lsn pg_lsn,
1010
IN end_lsn pg_lsn,
1111
OUT lsn pg_lsn,
12+
OUT blockid int2,
1213
OUT reltablespace oid,
1314
OUT reldatabase oid,
1415
OUT relfilenode oid,
1516
OUT relblocknumber int8,
1617
OUT forkname text,
17-
OUT fpi bytea
18+
OUT blockdata bytea,
19+
OUT fpi bytea,
20+
OUT fpilen int4,
21+
OUT fpiinfo text[]
1822
)
1923
RETURNS SETOF record
20-
AS 'MODULE_PATHNAME', 'pg_get_wal_fpi_info'
24+
AS 'MODULE_PATHNAME', 'pg_get_wal_block_info'
2125
LANGUAGE C STRICT PARALLEL SAFE;
2226

23-
REVOKE EXECUTE ON FUNCTION pg_get_wal_fpi_info(pg_lsn, pg_lsn) FROM PUBLIC;
24-
GRANT EXECUTE ON FUNCTION pg_get_wal_fpi_info(pg_lsn, pg_lsn) TO pg_read_server_files;
27+
REVOKE EXECUTE ON FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn) FROM PUBLIC;
28+
GRANT EXECUTE ON FUNCTION pg_get_wal_block_info(pg_lsn, pg_lsn) TO pg_read_server_files;

‎contrib/pg_walinspect/pg_walinspect.c

Lines changed: 100 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "access/xlogutils.h"
2121
#include "funcapi.h"
2222
#include "miscadmin.h"
23+
#include "utils/array.h"
2324
#include "utils/builtins.h"
2425
#include "utils/pg_lsn.h"
2526

@@ -30,7 +31,7 @@
3031

3132
PG_MODULE_MAGIC;
3233

33-
PG_FUNCTION_INFO_V1(pg_get_wal_fpi_info);
34+
PG_FUNCTION_INFO_V1(pg_get_wal_block_info);
3435
PG_FUNCTION_INFO_V1(pg_get_wal_record_info);
3536
PG_FUNCTION_INFO_V1(pg_get_wal_records_info);
3637
PG_FUNCTION_INFO_V1(pg_get_wal_records_info_till_end_of_wal);
@@ -56,7 +57,7 @@ static void FillXLogStatsRow(const char *name, uint64 n, uint64 total_count,
5657
Datum *values, bool *nulls, uint32 ncols);
5758
static void GetWalStats(FunctionCallInfo fcinfo, XLogRecPtr start_lsn,
5859
XLogRecPtr end_lsn, bool stats_per_record);
59-
static void GetWALFPIInfo(FunctionCallInfo fcinfo, XLogReaderState *record);
60+
static void GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record);
6061

6162
/*
6263
* Check if the given LSN is in future. Also, return the LSN up to which the
@@ -221,49 +222,40 @@ GetWALRecordInfo(XLogReaderState *record, Datum *values,
221222

222223

223224
/*
224-
* Store a set of full page images from a single record.
225+
* Store a set of block information from a single record (FPI and block
226+
* information).
225227
*/
226228
static void
227-
GetWALFPIInfo(FunctionCallInfo fcinfo, XLogReaderState *record)
229+
GetWALBlockInfo(FunctionCallInfo fcinfo, XLogReaderState *record)
228230
{
229-
#define PG_GET_WAL_FPI_INFO_COLS 7
231+
#define PG_GET_WAL_BLOCK_INFO_COLS 11
230232
int block_id;
231233
ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
232234

233235
for (block_id = 0; block_id <= XLogRecMaxBlockId(record); block_id++)
234236
{
235-
PGAlignedBlock buf;
236-
Page page;
237-
bytea *raw_page;
238-
BlockNumber blk;
237+
DecodedBkpBlock *blk;
238+
BlockNumber blkno;
239239
RelFileLocator rnode;
240240
ForkNumber fork;
241-
Datum values[PG_GET_WAL_FPI_INFO_COLS] = {0};
242-
bool nulls[PG_GET_WAL_FPI_INFO_COLS] = {0};
241+
Datum values[PG_GET_WAL_BLOCK_INFO_COLS] = {0};
242+
bool nulls[PG_GET_WAL_BLOCK_INFO_COLS] = {0};
243243
int i = 0;
244244

245245
if (!XLogRecHasBlockRef(record, block_id))
246246
continue;
247247

248-
if (!XLogRecHasBlockImage(record, block_id))
249-
continue;
250-
251-
page = (Page) buf.data;
252-
253-
if (!RestoreBlockImage(record, block_id, page))
254-
ereport(ERROR,
255-
(errcode(ERRCODE_INTERNAL_ERROR),
256-
errmsg_internal("%s", record->errormsg_buf)));
248+
blk = XLogRecGetBlock(record, block_id);
257249

258-
/* Full page exists, so let's save it. */
259250
(void) XLogRecGetBlockTagExtended(record, block_id,
260-
&rnode, &fork, &blk, NULL);
251+
&rnode, &fork, &blkno, NULL);
261252

262253
values[i++] = LSNGetDatum(record->ReadRecPtr);
263-
values[i++] = ObjectIdGetDatum(rnode.spcOid);
264-
values[i++] = ObjectIdGetDatum(rnode.dbOid);
265-
values[i++] = ObjectIdGetDatum(rnode.relNumber);
266-
values[i++] = Int64GetDatum((int64) blk);
254+
values[i++] = Int16GetDatum(block_id);
255+
values[i++] = ObjectIdGetDatum(blk->rlocator.spcOid);
256+
values[i++] = ObjectIdGetDatum(blk->rlocator.dbOid);
257+
values[i++] = ObjectIdGetDatum(blk->rlocator.relNumber);
258+
values[i++] = Int64GetDatum((int64) blkno);
267259

268260
if (fork >= 0 && fork <= MAX_FORKNUM)
269261
values[i++] = CStringGetTextDatum(forkNames[fork]);
@@ -272,34 +264,102 @@ GetWALFPIInfo(FunctionCallInfo fcinfo, XLogReaderState *record)
272264
(errcode(ERRCODE_INTERNAL_ERROR),
273265
errmsg_internal("invalid fork number: %u", fork)));
274266

275-
/* Initialize bytea buffer to copy the FPI to. */
276-
raw_page = (bytea *) palloc(BLCKSZ + VARHDRSZ);
277-
SET_VARSIZE(raw_page, BLCKSZ + VARHDRSZ);
267+
/* Block data */
268+
if (blk->has_data)
269+
{
270+
bytea *raw_data;
271+
272+
/* Initialize bytea buffer to copy the data to */
273+
raw_data = (bytea *) palloc(blk->data_len + VARHDRSZ);
274+
SET_VARSIZE(raw_data, blk->data_len + VARHDRSZ);
278275

279-
/* Take a verbatim copy of the FPI. */
280-
memcpy(VARDATA(raw_page), page, BLCKSZ);
276+
/* Copy the data */
277+
memcpy(VARDATA(raw_data), blk->data, blk->data_len);
278+
values[i++] = PointerGetDatum(raw_data);
279+
}
280+
else
281+
{
282+
/* No data, so set this field to NULL */
283+
nulls[i++] = true;
284+
}
281285

282-
values[i++] = PointerGetDatum(raw_page);
286+
if (blk->has_image)
287+
{
288+
PGAlignedBlock buf;
289+
Page page;
290+
bytea *raw_page;
291+
int bitcnt;
292+
int cnt = 0;
293+
Datum *flags;
294+
ArrayType *a;
295+
296+
page = (Page) buf.data;
297+
298+
/* Full page image exists, so let's save it */
299+
if (!RestoreBlockImage(record, block_id, page))
300+
ereport(ERROR,
301+
(errcode(ERRCODE_INTERNAL_ERROR),
302+
errmsg_internal("%s", record->errormsg_buf)));
303+
304+
/* Initialize bytea buffer to copy the FPI to */
305+
raw_page = (bytea *) palloc(BLCKSZ + VARHDRSZ);
306+
SET_VARSIZE(raw_page, BLCKSZ + VARHDRSZ);
307+
308+
/* Take a verbatim copy of the FPI */
309+
memcpy(VARDATA(raw_page), page, BLCKSZ);
310+
311+
values[i++] = PointerGetDatum(raw_page);
312+
values[i++] = UInt32GetDatum(blk->bimg_len);
313+
314+
/* FPI flags */
315+
bitcnt = pg_popcount((const char *) &blk->bimg_info,
316+
sizeof(uint8));
317+
/* Build set of raw flags */
318+
flags = (Datum *) palloc0(sizeof(Datum) * bitcnt);
319+
320+
if ((blk->bimg_info & BKPIMAGE_HAS_HOLE) != 0)
321+
flags[cnt++] = CStringGetTextDatum("HAS_HOLE");
322+
if (blk->apply_image)
323+
flags[cnt++] = CStringGetTextDatum("APPLY");
324+
if ((blk->bimg_info & BKPIMAGE_COMPRESS_PGLZ) != 0)
325+
flags[cnt++] = CStringGetTextDatum("COMPRESS_PGLZ");
326+
if ((blk->bimg_info & BKPIMAGE_COMPRESS_LZ4) != 0)
327+
flags[cnt++] = CStringGetTextDatum("COMPRESS_LZ4");
328+
if ((blk->bimg_info & BKPIMAGE_COMPRESS_ZSTD) != 0)
329+
flags[cnt++] = CStringGetTextDatum("COMPRESS_ZSTD");
330+
331+
Assert(cnt <= bitcnt);
332+
a = construct_array_builtin(flags, cnt, TEXTOID);
333+
values[i++] = PointerGetDatum(a);
334+
}
335+
else
336+
{
337+
/* No full page image, so store NULLs for all its fields */
338+
memset(&nulls[i], true, 3 * sizeof(bool));
339+
i += 3;
340+
}
283341

284-
Assert(i == PG_GET_WAL_FPI_INFO_COLS);
342+
Assert(i == PG_GET_WAL_BLOCK_INFO_COLS);
285343

286344
tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
287345
values, nulls);
288346
}
289347

290-
#undef PG_GET_WAL_FPI_INFO_COLS
348+
#undef PG_GET_WAL_FPI_BLOCK_COLS
291349
}
292350

293351
/*
294-
* Get full page images with their relation information for all the WAL
295-
* records between start and end LSNs. Decompression is applied to the
296-
* blocks, if necessary.
352+
* Get information about all the blocks saved in WAL records between start
353+
* and end LSNs. This produces information about the full page images with
354+
* their relation information, and the data saved in each block associated
355+
* to a record. Decompression is applied to the full page images, if
356+
* necessary.
297357
*
298358
* This function emits an error if a future start or end WAL LSN i.e. WAL LSN
299359
* the database system doesn't know about is specified.
300360
*/
301361
Datum
302-
pg_get_wal_fpi_info(PG_FUNCTION_ARGS)
362+
pg_get_wal_block_info(PG_FUNCTION_ARGS)
303363
{
304364
XLogRecPtr start_lsn;
305365
XLogRecPtr end_lsn;
@@ -317,7 +377,7 @@ pg_get_wal_fpi_info(PG_FUNCTION_ARGS)
317377
xlogreader = InitXLogReaderState(start_lsn);
318378

319379
tmp_cxt = AllocSetContextCreate(CurrentMemoryContext,
320-
"pg_get_wal_fpi_info temporary cxt",
380+
"pg_get_block_fpi_info temporary cxt",
321381
ALLOCSET_DEFAULT_SIZES);
322382

323383
while (ReadNextXLogRecord(xlogreader) &&
@@ -326,7 +386,7 @@ pg_get_wal_fpi_info(PG_FUNCTION_ARGS)
326386
/* Use the tmp context so we can clean up after each tuple is done */
327387
old_cxt = MemoryContextSwitchTo(tmp_cxt);
328388

329-
GetWALFPIInfo(fcinfo, xlogreader);
389+
GetWALBlockInfo(fcinfo, xlogreader);
330390

331391
/* clean up and switch back */
332392
MemoryContextSwitchTo(old_cxt);

0 commit comments

Comments
 (0)