Revamp the WAL record format.

Each WAL record now carries information about the modified relation and
block(s) in a standardized format. That makes it easier to write tools that
need that information, like pg_rewind, prefetching the blocks to speed up
recovery, etc.

There's a whole new API for building WAL records, replacing the XLogRecData
chains used previously. The new API consists of XLogRegister* functions,
which are called for each buffer and chunk of data that is added to the
record. The new API also gives more control over when a full-page image is
written, by passing flags to the XLogRegisterBuffer function.

This also simplifies the XLogReadBufferForRedo() calls. The function can dig
the relation and block number from the WAL record, so they no longer need to
be passed as arguments.

For the convenience of redo routines, XLogReader now disects each WAL record
after reading it, copying the main data part and the per-block data into
MAXALIGNed buffers. The data chunks are not aligned within the WAL record,
but the redo routines can assume that the pointers returned by XLogRecGet*
functions are. Redo routines are now passed the XLogReaderState, which
contains the record in the already-disected format, instead of the plain
XLogRecord.

The new record format also makes the fixed size XLogRecord header smaller,
by removing the xl_len field. The length of the "main data" portion is now
stored at the end of the WAL record, and there's a separate header after
XLogRecord for it. The alignment padding at the end of XLogRecord is also
removed. This compansates for the fact that the new format would otherwise
be more bulky than the old format.

Reviewed by Andres Freund, Amit Kapila, Michael Paquier, Alvaro Herrera,
Fujii Masao.

Author: hlinnaka

contrib/pg_xlogdump/pg_xlogdump.c

@@ -17,6 +17,7 @@
 
 #include "access/xlogreader.h"
 #include "access/xlogrecord.h"
+#include "access/xlog_internal.h"
 #include "access/transam.h"
 #include "common/fe_memutils.h"
 #include "getopt_long.h"
@@ -343,90 +344,117 @@ XLogDumpReadPage(XLogReaderState *state, XLogRecPtr targetPagePtr, int reqLen,
  * Store per-rmgr and per-record statistics for a given record.
  */
 static void
-XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats, XLogRecPtr ReadRecPtr, XLogRecord *record)
+XLogDumpCountRecord(XLogDumpConfig *config, XLogDumpStats *stats,
+					XLogReaderState *record)
 {
 	RmgrId		rmid;
 	uint8		recid;
+	uint32		rec_len;
+	uint32		fpi_len;
 
 	stats->count++;
 
 	/* Update per-rmgr statistics */
 
-	rmid = record->xl_rmid;
+	rmid = XLogRecGetRmid(record);
+	rec_len = XLogRecGetDataLen(record) + SizeOfXLogRecord;
+	fpi_len = record->decoded_record->xl_tot_len - rec_len;
 
 	stats->rmgr_stats[rmid].count++;
-	stats->rmgr_stats[rmid].rec_len +=
-		record->xl_len + SizeOfXLogRecord;
-	stats->rmgr_stats[rmid].fpi_len +=
-		record->xl_tot_len - (record->xl_len + SizeOfXLogRecord);
+	stats->rmgr_stats[rmid].rec_len += rec_len;
+	stats->rmgr_stats[rmid].fpi_len += fpi_len;
 
 	/*
 	 * Update per-record statistics, where the record is identified by a
-	 * combination of the RmgrId and the four bits of the xl_info field
-	 * that are the rmgr's domain (resulting in sixteen possible entries
-	 * per RmgrId).
+	 * combination of the RmgrId and the four bits of the xl_info field that
+	 * are the rmgr's domain (resulting in sixteen possible entries per
+	 * RmgrId).
 	 */
 
-	recid = record->xl_info >> 4;
+	recid = XLogRecGetInfo(record) >> 4;
 
 	stats->record_stats[rmid][recid].count++;
-	stats->record_stats[rmid][recid].rec_len +=
-		record->xl_len + SizeOfXLogRecord;
-	stats->record_stats[rmid][recid].fpi_len +=
-		record->xl_tot_len - (record->xl_len + SizeOfXLogRecord);
+	stats->record_stats[rmid][recid].rec_len += rec_len;
+	stats->record_stats[rmid][recid].fpi_len += fpi_len;
 }
 
 /*
  * Print a record to stdout
  */
 static void
-XLogDumpDisplayRecord(XLogDumpConfig *config, XLogRecPtr ReadRecPtr, XLogRecord *record)
+XLogDumpDisplayRecord(XLogDumpConfig *config, XLogReaderState *record)
 {
-	const char	   *id;
-	const RmgrDescData *desc = &RmgrDescTable[record->xl_rmid];
-
-	id = desc->rm_identify(record->xl_info);
+	const char *id;
+	const RmgrDescData *desc = &RmgrDescTable[XLogRecGetRmid(record)];
+	RelFileNode rnode;
+	ForkNumber	forknum;
+	BlockNumber blk;
+	int			block_id;
+	uint8		info = XLogRecGetInfo(record);
+	XLogRecPtr	xl_prev = XLogRecGetPrev(record);
+
+	id = desc->rm_identify(info);
 	if (id == NULL)
-		id = psprintf("UNKNOWN (%x)", record->xl_info & ~XLR_INFO_MASK);
+		id = psprintf("UNKNOWN (%x)", info & ~XLR_INFO_MASK);
 
-	printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, bkp: %u%u%u%u, desc: %s ",
+	printf("rmgr: %-11s len (rec/tot): %6u/%6u, tx: %10u, lsn: %X/%08X, prev %X/%08X, ",
 		   desc->rm_name,
-		   record->xl_len, record->xl_tot_len,
-		   record->xl_xid,
-		   (uint32) (ReadRecPtr >> 32), (uint32) ReadRecPtr,
-		   (uint32) (record->xl_prev >> 32), (uint32) record->xl_prev,
-		   !!(XLR_BKP_BLOCK(0) & record->xl_info),
-		   !!(XLR_BKP_BLOCK(1) & record->xl_info),
-		   !!(XLR_BKP_BLOCK(2) & record->xl_info),
-		   !!(XLR_BKP_BLOCK(3) & record->xl_info),
-		   id);
+		   XLogRecGetDataLen(record), XLogRecGetTotalLen(record),
+		   XLogRecGetXid(record),
+		   (uint32) (record->ReadRecPtr >> 32), (uint32) record->ReadRecPtr,
+		   (uint32) (xl_prev >> 32), (uint32) xl_prev);
+	printf("desc: %s ", id);
 
 	/* the desc routine will printf the description directly to stdout */
 	desc->rm_desc(NULL, record);
 
-	putchar('\n');
-
-	if (config->bkp_details)
+	if (!config->bkp_details)
 	{
-		int			bkpnum;
-		char	   *blk = (char *) XLogRecGetData(record) + record->xl_len;
-
-		for (bkpnum = 0; bkpnum < XLR_MAX_BKP_BLOCKS; bkpnum++)
+		/* print block references (short format) */
+		for (block_id = 0; block_id <= record->max_block_id; block_id++)
 		{
-			BkpBlock	bkpb;
-
-			if (!(XLR_BKP_BLOCK(bkpnum) & record->xl_info))
+			if (!XLogRecHasBlockRef(record, block_id))
 				continue;
 
-			memcpy(&bkpb, blk, sizeof(BkpBlock));
-			blk += sizeof(BkpBlock);
-			blk += BLCKSZ - bkpb.hole_length;
+			XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk);
+			if (forknum != MAIN_FORKNUM)
+				printf(", blkref #%u: rel %u/%u/%u fork %s blk %u",
+					   block_id,
+					   rnode.spcNode, rnode.dbNode, rnode.relNode,
+					   forkNames[forknum],
+					   blk);
+			else
+				printf(", blkref #%u: rel %u/%u/%u blk %u",
+					   block_id,
+					   rnode.spcNode, rnode.dbNode, rnode.relNode,
+					   blk);
+			if (XLogRecHasBlockImage(record, block_id))
+				printf(" FPW");
+		}
+		putchar('\n');
+	}
+	else
+	{
+		/* print block references (detailed format) */
+		putchar('\n');
+		for (block_id = 0; block_id <= record->max_block_id; block_id++)
+		{
+			if (!XLogRecHasBlockRef(record, block_id))
+				continue;
 
-			printf("\tbackup bkp #%u; rel %u/%u/%u; fork: %s; block: %u; hole: offset: %u, length: %u\n",
-				   bkpnum,
-				   bkpb.node.spcNode, bkpb.node.dbNode, bkpb.node.relNode,
-				   forkNames[bkpb.fork],
-				   bkpb.block, bkpb.hole_offset, bkpb.hole_length);
+			XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blk);
+			printf("\tblkref #%u: rel %u/%u/%u fork %s blk %u",
+				   block_id,
+				   rnode.spcNode, rnode.dbNode, rnode.relNode,
+				   forkNames[forknum],
+				   blk);
+			if (XLogRecHasBlockImage(record, block_id))
+			{
+				printf(" (FPW); hole: offset: %u, length: %u\n",
+					   record->blocks[block_id].hole_offset,
+					   record->blocks[block_id].hole_length);
+			}
+			putchar('\n');
 		}
 	}
 }
@@ -924,9 +952,9 @@ main(int argc, char **argv)
 
 		/* process the record */
 		if (config.stats == true)
-			XLogDumpCountRecord(&config, &stats, xlogreader_state->ReadRecPtr, record);
+			XLogDumpCountRecord(&config, &stats, xlogreader_state);
 		else
-			XLogDumpDisplayRecord(&config, xlogreader_state->ReadRecPtr, record);
+			XLogDumpDisplayRecord(&config, xlogreader_state);
 
 		/* check whether we printed enough */
 		config.already_displayed_records++;

contrib/pg_xlogdump/rmgrdesc.h

@@ -13,7 +13,7 @@
 typedef struct RmgrDescData
 {
 	const char *rm_name;
-	void		(*rm_desc) (StringInfo buf, XLogRecord *record);
+	void		(*rm_desc) (StringInfo buf, XLogReaderState *record);
 	const char *(*rm_identify) (uint8 info);
 } RmgrDescData;
 

src/backend/access/brin/brin.c

@@ -666,19 +666,16 @@ brinbuild(PG_FUNCTION_ARGS)
 	{
 		xl_brin_createidx xlrec;
 		XLogRecPtr	recptr;
-		XLogRecData rdata;
 		Page		page;
 
-		xlrec.node = index->rd_node;
 		xlrec.version = BRIN_CURRENT_VERSION;
 		xlrec.pagesPerRange = BrinGetPagesPerRange(index);
 
-		rdata.buffer = InvalidBuffer;
-		rdata.data = (char *) &xlrec;
-		rdata.len = SizeOfBrinCreateIdx;
-		rdata.next = NULL;
+		XLogBeginInsert();
+		XLogRegisterData((char *) &xlrec, SizeOfBrinCreateIdx);
+		XLogRegisterBuffer(0, meta, REGBUF_WILL_INIT);
 
-		recptr = XLogInsert(RM_BRIN_ID, XLOG_BRIN_CREATE_INDEX, &rdata);
+		recptr = XLogInsert(RM_BRIN_ID, XLOG_BRIN_CREATE_INDEX);
 
 		page = BufferGetPage(meta);
 		PageSetLSN(page, recptr);

src/backend/access/brin/brin_pageops.c

@@ -140,27 +140,19 @@ brin_doupdate(Relation idxrel, BlockNumber pagesPerRange,
 		/* XLOG stuff */
 		if (RelationNeedsWAL(idxrel))
 		{
-			BlockNumber blk = BufferGetBlockNumber(oldbuf);
 			xl_brin_samepage_update xlrec;
 			XLogRecPtr	recptr;
-			XLogRecData rdata[2];
 			uint8		info = XLOG_BRIN_SAMEPAGE_UPDATE;
 
-			xlrec.node = idxrel->rd_node;
-			ItemPointerSetBlockNumber(&xlrec.tid, blk);
-			ItemPointerSetOffsetNumber(&xlrec.tid, oldoff);
-			rdata[0].data = (char *) &xlrec;
-			rdata[0].len = SizeOfBrinSamepageUpdate;
-			rdata[0].buffer = InvalidBuffer;
-			rdata[0].next = &(rdata[1]);
+			xlrec.offnum = oldoff;
 
-			rdata[1].data = (char *) newtup;
-			rdata[1].len = newsz;
-			rdata[1].buffer = oldbuf;
-			rdata[1].buffer_std = true;
-			rdata[1].next = NULL;
+			XLogBeginInsert();
+			XLogRegisterData((char *) &xlrec, SizeOfBrinSamepageUpdate);
 
-			recptr = XLogInsert(RM_BRIN_ID, info, rdata);
+			XLogRegisterBuffer(0, oldbuf, REGBUF_STANDARD);
+			XLogRegisterBufData(0, (char *) newtup, newsz);
+
+			recptr = XLogInsert(RM_BRIN_ID, info);
 
 			PageSetLSN(oldpage, recptr);
 		}
@@ -211,43 +203,30 @@ brin_doupdate(Relation idxrel, BlockNumber pagesPerRange,
 		{
 			xl_brin_update xlrec;
 			XLogRecPtr	recptr;
-			XLogRecData rdata[4];
 			uint8		info;
 
 			info = XLOG_BRIN_UPDATE | (extended ? XLOG_BRIN_INIT_PAGE : 0);
 
-			xlrec.insert.node = idxrel->rd_node;
-			ItemPointerSet(&xlrec.insert.tid, BufferGetBlockNumber(newbuf), newoff);
+			xlrec.insert.offnum = newoff;
 			xlrec.insert.heapBlk = heapBlk;
-			xlrec.insert.tuplen = newsz;
-			xlrec.insert.revmapBlk = BufferGetBlockNumber(revmapbuf);
 			xlrec.insert.pagesPerRange = pagesPerRange;
-			ItemPointerSet(&xlrec.oldtid, BufferGetBlockNumber(oldbuf), oldoff);
+			xlrec.oldOffnum = oldoff;
+
+			XLogBeginInsert();
 
-			rdata[0].data = (char *) &xlrec;
-			rdata[0].len = SizeOfBrinUpdate;
-			rdata[0].buffer = InvalidBuffer;
-			rdata[0].next = &(rdata[1]);
+			/* new page */
+			XLogRegisterData((char *) &xlrec, SizeOfBrinUpdate);
 
-			rdata[1].data = (char *) newtup;
-			rdata[1].len = newsz;
-			rdata[1].buffer = extended ? InvalidBuffer : newbuf;
-			rdata[1].buffer_std = true;
-			rdata[1].next = &(rdata[2]);
+			XLogRegisterBuffer(0, newbuf, REGBUF_STANDARD | (extended ? REGBUF_WILL_INIT : 0));
+			XLogRegisterBufData(0, (char *) newtup, newsz);
 
-			rdata[2].data = (char *) NULL;
-			rdata[2].len = 0;
-			rdata[2].buffer = revmapbuf;
-			rdata[2].buffer_std = true;
-			rdata[2].next = &(rdata[3]);
+			/* revmap page */
+			XLogRegisterBuffer(1, revmapbuf, REGBUF_STANDARD);
 
-			rdata[3].data = (char *) NULL;
-			rdata[3].len = 0;
-			rdata[3].buffer = oldbuf;
-			rdata[3].buffer_std = true;
-			rdata[3].next = NULL;
+			/* old page */
+			XLogRegisterBuffer(2, oldbuf, REGBUF_STANDARD);
 
-			recptr = XLogInsert(RM_BRIN_ID, info, rdata);
+			recptr = XLogInsert(RM_BRIN_ID, info);
 
 			PageSetLSN(oldpage, recptr);
 			PageSetLSN(newpage, recptr);
@@ -354,36 +333,22 @@ brin_doinsert(Relation idxrel, BlockNumber pagesPerRange,
 	{
 		xl_brin_insert xlrec;
 		XLogRecPtr	recptr;
-		XLogRecData rdata[3];
 		uint8		info;
 
 		info = XLOG_BRIN_INSERT | (extended ? XLOG_BRIN_INIT_PAGE : 0);
-		xlrec.node = idxrel->rd_node;
 		xlrec.heapBlk = heapBlk;
 		xlrec.pagesPerRange = pagesPerRange;
-		xlrec.revmapBlk = BufferGetBlockNumber(revmapbuf);
-		xlrec.tuplen = itemsz;
-		ItemPointerSet(&xlrec.tid, blk, off);
-
-		rdata[0].data = (char *) &xlrec;
-		rdata[0].len = SizeOfBrinInsert;
-		rdata[0].buffer = InvalidBuffer;
-		rdata[0].buffer_std = false;
-		rdata[0].next = &(rdata[1]);
-
-		rdata[1].data = (char *) tup;
-		rdata[1].len = itemsz;
-		rdata[1].buffer = extended ? InvalidBuffer : *buffer;
-		rdata[1].buffer_std = true;
-		rdata[1].next = &(rdata[2]);
-
-		rdata[2].data = (char *) NULL;
-		rdata[2].len = 0;
-		rdata[2].buffer = revmapbuf;
-		rdata[2].buffer_std = false;
-		rdata[2].next = NULL;
-
-		recptr = XLogInsert(RM_BRIN_ID, info, rdata);
+		xlrec.offnum = off;
+
+		XLogBeginInsert();
+		XLogRegisterData((char *) &xlrec, SizeOfBrinInsert);
+
+		XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD | (extended ? REGBUF_WILL_INIT : 0));
+		XLogRegisterBufData(0, (char *) tup, itemsz);
+
+		XLogRegisterBuffer(1, revmapbuf, 0);
+
+		recptr = XLogInsert(RM_BRIN_ID, info);
 
 		PageSetLSN(page, recptr);
 		PageSetLSN(BufferGetPage(revmapbuf), recptr);

src/backend/access/brin/brin_revmap.c

@@ -477,23 +477,16 @@ revmap_physical_extend(BrinRevmap *revmap)
 	{
 		xl_brin_revmap_extend xlrec;
 		XLogRecPtr	recptr;
-		XLogRecData rdata[2];
 
-		xlrec.node = revmap->rm_irel->rd_node;
 		xlrec.targetBlk = mapBlk;
-		rdata[0].data = (char *) &xlrec;
-		rdata[0].len = SizeOfBrinRevmapExtend;
-		rdata[0].buffer = InvalidBuffer;
-		rdata[0].buffer_std = false;
-		rdata[0].next = &(rdata[1]);
-
-		rdata[1].data = (char *) NULL;
-		rdata[1].len = 0;
-		rdata[1].buffer = revmap->rm_metaBuf;
-		rdata[1].buffer_std = false;
-		rdata[1].next = NULL;
-
-		recptr = XLogInsert(RM_BRIN_ID, XLOG_BRIN_REVMAP_EXTEND, rdata);
+
+		XLogBeginInsert();
+		XLogRegisterData((char *) &xlrec, SizeOfBrinRevmapExtend);
+		XLogRegisterBuffer(0, revmap->rm_metaBuf, 0);
+
+		XLogRegisterBuffer(1, buf, REGBUF_WILL_INIT);
+
+		recptr = XLogInsert(RM_BRIN_ID, XLOG_BRIN_REVMAP_EXTEND);
 		PageSetLSN(metapage, recptr);
 		PageSetLSN(page, recptr);
 	}