Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 56c7d8d

Browse files
committed
Allow pg_basebackup to stream transaction log in tar mode
This will write the received transaction log into a file called pg_wal.tar(.gz) next to the other tarfiles instead of writing it to base.tar. When using fetch mode, the transaction log is still written to base.tar like before, and when used against a pre-10 server, the file is named pg_xlog.tar. To do this, implement a new concept of a "walmethod", which is responsible for writing the WAL. Two implementations exist, one that writes to a plain directory (which is also used by pg_receivexlog) and one that writes to a tar file with optional compression. Reviewed by Michael Paquier
1 parent 1885c88 commit 56c7d8d

File tree

11 files changed

+1107
-244
lines changed

11 files changed

+1107
-244
lines changed

doc/src/sgml/ref/pg_basebackup.sgml

+15-3
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,8 @@ PostgreSQL documentation
180180
target directory, the tar contents will be written to
181181
standard output, suitable for piping to for example
182182
<productname>gzip</productname>. This is only possible if
183-
the cluster has no additional tablespaces.
183+
the cluster has no additional tablespaces and transaction
184+
log streaming is not used.
184185
</para>
185186
</listitem>
186187
</varlistentry>
@@ -323,6 +324,10 @@ PostgreSQL documentation
323324
If the log has been rotated when it's time to transfer it, the
324325
backup will fail and be unusable.
325326
</para>
327+
<para>
328+
The transaction log files will be written to
329+
the <filename>base.tar</filename> file.
330+
</para>
326331
</listitem>
327332
</varlistentry>
328333

@@ -339,6 +344,11 @@ PostgreSQL documentation
339344
client can keep up with transaction log received, using this mode
340345
requires no extra transaction logs to be saved on the master.
341346
</para>
347+
<para>
348+
The transaction log files are written to a separate file
349+
named <filename>pg_wal.tar</filename> (if the server is a version
350+
earlier than 10, the file will be named <filename>pg_xlog.tar</filename>).
351+
</para>
342352
</listitem>
343353
</varlistentry>
344354
</variablelist>
@@ -353,7 +363,8 @@ PostgreSQL documentation
353363
<para>
354364
Enables gzip compression of tar file output, with the default
355365
compression level. Compression is only available when using
356-
the tar format.
366+
the tar format, and the suffix <filename>.gz</filename> will
367+
automatically be added to all tar filenames.
357368
</para>
358369
</listitem>
359370
</varlistentry>
@@ -366,7 +377,8 @@ PostgreSQL documentation
366377
Enables gzip compression of tar file output, and specifies the
367378
compression level (0 through 9, 0 being no compression and 9 being best
368379
compression). Compression is only available when using the tar
369-
format.
380+
format, and the suffix <filename>.gz</filename> will
381+
automatically be added to all tar filenames.
370382
</para>
371383
</listitem>
372384
</varlistentry>

src/bin/pg_basebackup/Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ include $(top_builddir)/src/Makefile.global
1919
override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
2020
LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils -lpq
2121

22-
OBJS=receivelog.o streamutil.o $(WIN32RES)
22+
OBJS=receivelog.o streamutil.o walmethods.o $(WIN32RES)
2323

2424
all: pg_basebackup pg_receivexlog pg_recvlogical
2525

src/bin/pg_basebackup/pg_basebackup.c

+34-28
Original file line numberDiff line numberDiff line change
@@ -449,7 +449,7 @@ typedef struct
449449
{
450450
PGconn *bgconn;
451451
XLogRecPtr startptr;
452-
char xlogdir[MAXPGPATH];
452+
char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
453453
char *sysidentifier;
454454
int timeline;
455455
} logstreamer_param;
@@ -470,9 +470,13 @@ LogStreamerMain(logstreamer_param *param)
470470
stream.synchronous = false;
471471
stream.do_sync = do_sync;
472472
stream.mark_done = true;
473-
stream.basedir = param->xlogdir;
474473
stream.partial_suffix = NULL;
475474

475+
if (format == 'p')
476+
stream.walmethod = CreateWalDirectoryMethod(param->xlog, do_sync);
477+
else
478+
stream.walmethod = CreateWalTarMethod(param->xlog, compresslevel, do_sync);
479+
476480
if (!ReceiveXlogStream(param->bgconn, &stream))
477481

478482
/*
@@ -482,6 +486,14 @@ LogStreamerMain(logstreamer_param *param)
482486
*/
483487
return 1;
484488

489+
if (!stream.walmethod->finish())
490+
{
491+
fprintf(stderr,
492+
_("%s: could not finish writing WAL files: %s\n"),
493+
progname, strerror(errno));
494+
return 1;
495+
}
496+
485497
PQfinish(param->bgconn);
486498
return 0;
487499
}
@@ -533,28 +545,32 @@ StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
533545
exit(1);
534546

535547
/* In post-10 cluster, pg_xlog has been renamed to pg_wal */
536-
snprintf(param->xlogdir, sizeof(param->xlogdir), "%s/%s",
548+
snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
537549
basedir,
538550
PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
539551
"pg_xlog" : "pg_wal");
540552

541-
/*
542-
* Create pg_wal/archive_status or pg_xlog/archive_status (and thus
543-
* pg_wal or pg_xlog) depending on the target server so we can write to
544-
* basedir/pg_wal or basedir/pg_xlog as the directory entry in the tar
545-
* file may arrive later.
546-
*/
547-
snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
548-
basedir,
549-
PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
550-
"pg_xlog" : "pg_wal");
551553

552-
if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST)
554+
if (format == 'p')
553555
{
554-
fprintf(stderr,
555-
_("%s: could not create directory \"%s\": %s\n"),
556-
progname, statusdir, strerror(errno));
557-
disconnect_and_exit(1);
556+
/*
557+
* Create pg_wal/archive_status or pg_xlog/archive_status (and thus
558+
* pg_wal or pg_xlog) depending on the target server so we can write to
559+
* basedir/pg_wal or basedir/pg_xlog as the directory entry in the tar
560+
* file may arrive later.
561+
*/
562+
snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
563+
basedir,
564+
PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
565+
"pg_xlog" : "pg_wal");
566+
567+
if (pg_mkdir_p(statusdir, S_IRWXU) != 0 && errno != EEXIST)
568+
{
569+
fprintf(stderr,
570+
_("%s: could not create directory \"%s\": %s\n"),
571+
progname, statusdir, strerror(errno));
572+
disconnect_and_exit(1);
573+
}
558574
}
559575

560576
/*
@@ -2245,16 +2261,6 @@ main(int argc, char **argv)
22452261
exit(1);
22462262
}
22472263

2248-
if (format != 'p' && streamwal)
2249-
{
2250-
fprintf(stderr,
2251-
_("%s: WAL streaming can only be used in plain mode\n"),
2252-
progname);
2253-
fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2254-
progname);
2255-
exit(1);
2256-
}
2257-
22582264
if (replication_slot && !streamwal)
22592265
{
22602266
fprintf(stderr,

src/bin/pg_basebackup/pg_receivexlog.c

+9-1
Original file line numberDiff line numberDiff line change
@@ -338,11 +338,19 @@ StreamLog(void)
338338
stream.synchronous = synchronous;
339339
stream.do_sync = true;
340340
stream.mark_done = false;
341-
stream.basedir = basedir;
341+
stream.walmethod = CreateWalDirectoryMethod(basedir, stream.do_sync);
342342
stream.partial_suffix = ".partial";
343343

344344
ReceiveXlogStream(conn, &stream);
345345

346+
if (!stream.walmethod->finish())
347+
{
348+
fprintf(stderr,
349+
_("%s: could not finish writing WAL files: %s\n"),
350+
progname, strerror(errno));
351+
return;
352+
}
353+
346354
PQfinish(conn);
347355
conn = NULL;
348356
}

0 commit comments

Comments
 (0)