Server-side gzip compression.

pg_basebackup's --compression option now lets you write either
"client-gzip" or "server-gzip" instead of just "gzip" to specify
where the compression should be performed. If you write simply
"gzip" it's taken to mean "client-gzip" unless you also use
--target, in which case it is interpreted to mean "server-gzip",
because that's the only thing that makes any sense in that case.

To make this work, the BASE_BACKUP command now takes new
COMPRESSION and COMPRESSION_LEVEL options.

At present, pg_basebackup cannot decompress .gz files, so
server-side compression will cause a failure if (1) -Ft is not
used or (2) -R is used or (3) -D- is used without --no-manifest.

Along the way, I removed the information message added by commit
5c649fe which occurred if you
specified no compression level and told you that the default level
had been used instead. That seemed like more output than most
people would want.

Also along the way, this adds a check to the server for
unrecognized base backup options. This repairs a bug introduced
by commit 0ba281c.

This commit also adds some new test cases for pg_verifybackup.
They take a server-side backup with and without compression, and
then extract the backup if we have the OS facilities available
to do so, and then run pg_verifybackup on the extracted
directory. That is a good test of the functionality added by
this commit and also improves test coverage for the backup target
patch (commit 3500ccc) and for
pg_verifybackup itself.

Patch by me, with a bug fix by Jeevan Ladhe.  The patch set of which
this is a part has also had review and/or testing from Tushar Ahuja,
Suraj Kharage, Dipesh Pandit, and Mark Dilger.

Discussion: http://postgr.es/m/CA+Tgmoa-ST7fMLsVJduOB7Eub=2WjfpHS+QxHVEpUoinf4bOSg@mail.gmail.com

Author: robertmhaas

doc/src/sgml/protocol.sgml

@@ -2719,6 +2719,28 @@ The commands accepted in replication mode are:
         </listitem>
        </varlistentry>
 
+       <varlistentry>
+        <term><literal>COMPRESSION</literal> <replaceable>'method'</replaceable></term>
+        <listitem>
+         <para>
+          Instructs the server to compress the backup using the specified
+          method. Currently, the only supported method is
+          <literal>gzip</literal>.
+         </para>
+        </listitem>
+       </varlistentry>
+
+       <varlistentry>
+        <term><literal>COMPRESSION_LEVEL</literal> <replaceable>level</replaceable></term>
+        <listitem>
+         <para>
+          Specifies the compression level to be used. This should only be
+          used in conjunction with the <literal>COMPRESSION</literal> option.
+          The value should be an integer between 1 and 9.
+         </para>
+        </listitem>
+       </varlistentry>
+
        <varlistentry>
         <term><literal>MAX_RATE</literal> <replaceable>rate</replaceable></term>
         <listitem>

doc/src/sgml/ref/pg_basebackup.sgml

@@ -400,21 +400,36 @@ PostgreSQL documentation
       <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
       <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
       <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
-      <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
+      <term><option>--compress=[[{<replaceable class="parameter">client|server</replaceable>-}]<replaceable class="parameter">method</replaceable></option>[:<replaceable>level</replaceable>]</term>
       <listitem>
        <para>
-        Enables compression of tar file output, and specifies the
-        compression level (0 through 9, 0 being no compression and 9 being best
-        compression). Compression is only available when using the tar
-        format, and the suffix <filename>.gz</filename> will
-        automatically be added to all tar filenames.
+        Requests compression of the backup. If <literal>client</literal> or
+        <literal>server</literal> is included, it specifies where the
+        compression is to be performed. Compressing on the server will reduce
+        transfer bandwidth but will increase server CPU consumption.  The
+        default is <literal>client</literal> except when
+        <literal>--target</literal> is used. In that case, the backup is not
+        being sent to the client, so only server compression is sensible.
+        When <literal>-Xstream</literal>, which is the default, is used,
+        server-side compression will not be applied to the WAL. To compress
+        the WAL, use client-side compression, or
+        specify <literal>-Xfetch</literal>.
        </para>
        <para>
         The compression method can be set to either <literal>gzip</literal>
         for compression with <application>gzip</application>, or
         <literal>none</literal> for no compression. A compression level
         can be optionally specified, by appending the level number after a
-        colon (<literal>:</literal>).
+        colon (<literal>:</literal>). If no level is specified, the default
+        compression level will be used. If only a level is specified without
+        mentioning an algorithm, <literal>gzip</literal> compression will
+        be used if the level is greater than 0, and no compression will be
+        used if the level is 0.
+       </para>
+       <para>
+        When the tar format is used, the suffix <filename>.gz</filename> will
+        automatically be added to all tar filenames. Compression is not
+        available in plain format.
        </para>
       </listitem>
      </varlistentry>

src/backend/Makefile

@@ -48,7 +48,7 @@ OBJS = \
 LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS)
 
 # The backend doesn't need everything that's in LIBS, however
-LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
+LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS))
 
 ifeq ($(with_systemd),yes)
 LIBS += -lsystemd

src/backend/replication/Makefile

@@ -18,6 +18,7 @@ OBJS = \
 	backup_manifest.o \
 	basebackup.o \
 	basebackup_copy.o \
+	basebackup_gzip.o \
 	basebackup_progress.o \
 	basebackup_server.o \
 	basebackup_sink.o \

src/backend/replication/basebackup.c

@@ -61,6 +61,12 @@ typedef enum
 	BACKUP_TARGET_SERVER
 } backup_target_type;
 
+typedef enum
+{
+	BACKUP_COMPRESSION_NONE,
+	BACKUP_COMPRESSION_GZIP
+} basebackup_compression_type;
+
 typedef struct
 {
 	const char *label;
@@ -73,6 +79,8 @@ typedef struct
 	backup_target_type target;
 	char	   *target_detail;
 	backup_manifest_option manifest;
+	basebackup_compression_type	compression;
+	int			compression_level;
 	pg_checksum_type manifest_checksum_type;
 } basebackup_options;
 
@@ -707,11 +715,14 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 	bool		o_target = false;
 	bool		o_target_detail = false;
 	char	   *target_str = "compat";	/* placate compiler */
+	bool		o_compression = false;
+	bool		o_compression_level = false;
 
 	MemSet(opt, 0, sizeof(*opt));
 	opt->target = BACKUP_TARGET_COMPAT;
 	opt->manifest = MANIFEST_OPTION_NO;
 	opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
+	opt->compression = BACKUP_COMPRESSION_NONE;
 
 	foreach(lopt, options)
 	{
@@ -881,7 +892,41 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 			opt->target_detail = optval;
 			o_target_detail = true;
 		}
+		else if (strcmp(defel->defname, "compression") == 0)
+		{
+			char	   *optval = defGetString(defel);
+
+			if (o_compression)
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("duplicate option \"%s\"", defel->defname)));
+			if (strcmp(optval, "none") == 0)
+				opt->compression = BACKUP_COMPRESSION_NONE;
+			else if (strcmp(optval, "gzip") == 0)
+				opt->compression = BACKUP_COMPRESSION_GZIP;
+			else
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("unrecognized compression algorithm: \"%s\"",
+								optval)));
+			o_compression = true;
+		}
+		else if (strcmp(defel->defname, "compression_level") == 0)
+		{
+			if (o_compression_level)
+				ereport(ERROR,
+						(errcode(ERRCODE_SYNTAX_ERROR),
+						 errmsg("duplicate option \"%s\"", defel->defname)));
+			opt->compression_level = defGetInt32(defel);
+			o_compression_level = true;
+		}
+		else
+			ereport(ERROR,
+					(errcode(ERRCODE_SYNTAX_ERROR),
+					 errmsg("unrecognized base backup option: \"%s\"",
+							defel->defname)));
 	}
+
 	if (opt->label == NULL)
 		opt->label = "base backup";
 	if (opt->manifest == MANIFEST_OPTION_NO)
@@ -908,6 +953,11 @@ parse_basebackup_options(List *options, basebackup_options *opt)
 					 errmsg("target '%s' does not accept a target detail",
 							target_str)));
 	}
+
+	if (o_compression_level && !o_compression)
+		ereport(ERROR,
+				(errcode(ERRCODE_SYNTAX_ERROR),
+				 errmsg("compression level requires compression")));
 }
 
 
@@ -975,6 +1025,10 @@ SendBaseBackup(BaseBackupCmd *cmd)
 	if (opt.maxrate > 0)
 		sink = bbsink_throttle_new(sink, opt.maxrate);
 
+	/* Set up server-side compression, if client requested it */
+	if (opt.compression == BACKUP_COMPRESSION_GZIP)
+		sink = bbsink_gzip_new(sink, opt.compression_level);
+
 	/* Set up progress reporting. */
 	sink = bbsink_progress_new(sink, opt.progress);