Switch pg_dump to use compression specifications

Compression specifications are currently used by pg_basebackup and
pg_receivewal, and are able to let the user control in an extended way
the method and level of compression used.  As an effect of this commit,
pg_dump's -Z/--compress is now able to use more than just an integer, as
of the grammar "method[:detail]".

The method can be either "none" or "gzip", and can optionally take a
detail string.  If the detail string is only an integer, it defines the
compression level.  A comma-separated list of keywords can also be used
method allows for more options, the only keyword supported now is
"level".

The change is backward-compatible, hence specifying only an integer
leads to no compression for a level of 0 and gzip compression when the
level is greater than 0.

Most of the code changes are straight-forward, as pg_dump was relying on
an integer tracking the compression level to check for gzip or no
compression.  These are changed to use a compression specification and
the algorithm stored in it.

As of this change, note that the dump format is not bumped because there
is no need yet to track the compression algorithm in the TOC entries.
Hence, we still rely on the compression level to make the difference
when reading them.  This will be mandatory once a new compression method
is added, though.

In order to keep the code simpler when parsing the compression
specification, the code is changed so as pg_dump now fails hard when
using gzip on -Z/--compress without its support compiled, rather than
enforcing no compression without the user knowing about it except
through a warning.  Like before this commit, archive and custom formats
are compressed by default when the code is compiled with gzip, and left
uncompressed without gzip.

Author: Georgios Kokolatos
Reviewed-by: Michael Paquier
Discussion: https://postgr.es/m/O4mutIrCES8ZhlXJiMvzsivT7ztAMja2lkdL1LJx6O5f22I2W8PBIeLKz7mDLwxHoibcnRAYJXm1pH4tyUNC4a8eDzLn22a6Pb1S74Niexg=@pm.me

Author: michaelpq

doc/src/sgml/ref/pg_dump.sgml

@@ -644,17 +644,39 @@ PostgreSQL documentation
      </varlistentry>
 
      <varlistentry>
-      <term><option>-Z <replaceable class="parameter">0..9</replaceable></option></term>
-      <term><option>--compress=<replaceable class="parameter">0..9</replaceable></option></term>
+      <term><option>-Z <replaceable class="parameter">level</replaceable></option></term>
+      <term><option>-Z <replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
+      <term><option>--compress=<replaceable class="parameter">level</replaceable></option></term>
+      <term><option>--compress=<replaceable class="parameter">method</replaceable></option>[:<replaceable>detail</replaceable>]</term>
       <listitem>
        <para>
-        Specify the compression level to use.  Zero means no compression.
+        Specify the compression method and/or the compression level to use.
+        The compression method can be set to <literal>gzip</literal> or
+        <literal>none</literal> for no compression.
+        A compression detail string can optionally be specified.  If the
+        detail string is an integer, it specifies the compression level.
+        Otherwise, it should be a comma-separated list of items, each of the
+        form <literal>keyword</literal> or <literal>keyword=value</literal>.
+        Currently, the only supported keyword is <literal>level</literal>.
+       </para>
+
+       <para>
+        If no compression level is specified, the default compression
+        level will be used. If only a level is specified without mentioning
+        an algorithm, <literal>gzip</literal> compression will be used if
+        the level is greater than <literal>0</literal>, and no compression
+        will be used if the level is <literal>0</literal>.
+       </para>
+
+       <para>
         For the custom and directory archive formats, this specifies compression of
-        individual table-data segments, and the default is to compress
-        at a moderate level.
-        For plain text output, setting a nonzero compression level causes
-        the entire output file to be compressed, as though it had been
-        fed through <application>gzip</application>; but the default is not to compress.
+        individual table-data segments, and the default is to compress using
+        <literal>gzip</literal> at a moderate level. For plain text output,
+        setting a nonzero compression level causes the entire output file to be compressed,
+        as though it had been fed through <application>gzip</application>; but the default
+        is not to compress.
+       </para>
+       <para>
         The tar archive format currently does not support compression at all.
        </para>
       </listitem>

src/bin/pg_dump/compress_io.c

@@ -64,7 +64,7 @@
 /* typedef appears in compress_io.h */
 struct CompressorState
 {
-	CompressionAlgorithm comprAlg;
+	pg_compress_specification compression_spec;
 	WriteFunc	writeF;
 
 #ifdef HAVE_LIBZ
@@ -74,9 +74,6 @@ struct CompressorState
 #endif
 };
 
-static void ParseCompressionOption(int compression, CompressionAlgorithm *alg,
-								   int *level);
-
 /* Routines that support zlib compressed data I/O */
 #ifdef HAVE_LIBZ
 static void InitCompressorZlib(CompressorState *cs, int level);
@@ -93,57 +90,30 @@ static void ReadDataFromArchiveNone(ArchiveHandle *AH, ReadFunc readF);
 static void WriteDataToArchiveNone(ArchiveHandle *AH, CompressorState *cs,
 								   const char *data, size_t dLen);
 
-/*
- * Interprets a numeric 'compression' value. The algorithm implied by the
- * value (zlib or none at the moment), is returned in *alg, and the
- * zlib compression level in *level.
- */
-static void
-ParseCompressionOption(int compression, CompressionAlgorithm *alg, int *level)
-{
-	if (compression == Z_DEFAULT_COMPRESSION ||
-		(compression > 0 && compression <= 9))
-		*alg = COMPR_ALG_LIBZ;
-	else if (compression == 0)
-		*alg = COMPR_ALG_NONE;
-	else
-	{
-		pg_fatal("invalid compression code: %d", compression);
-		*alg = COMPR_ALG_NONE;	/* keep compiler quiet */
-	}
-
-	/* The level is just the passed-in value. */
-	if (level)
-		*level = compression;
-}
-
 /* Public interface routines */
 
 /* Allocate a new compressor */
 CompressorState *
-AllocateCompressor(int compression, WriteFunc writeF)
+AllocateCompressor(const pg_compress_specification compression_spec,
+				   WriteFunc writeF)
 {
 	CompressorState *cs;
-	CompressionAlgorithm alg;
-	int			level;
-
-	ParseCompressionOption(compression, &alg, &level);
 
 #ifndef HAVE_LIBZ
-	if (alg == COMPR_ALG_LIBZ)
+	if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
 		pg_fatal("not built with zlib support");
 #endif
 
 	cs = (CompressorState *) pg_malloc0(sizeof(CompressorState));
 	cs->writeF = writeF;
-	cs->comprAlg = alg;
+	cs->compression_spec = compression_spec;
 
 	/*
 	 * Perform compression algorithm specific initialization.
 	 */
 #ifdef HAVE_LIBZ
-	if (alg == COMPR_ALG_LIBZ)
-		InitCompressorZlib(cs, level);
+	if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
+		InitCompressorZlib(cs, cs->compression_spec.level);
 #endif
 
 	return cs;
@@ -154,15 +124,12 @@ AllocateCompressor(int compression, WriteFunc writeF)
  * out with ahwrite().
  */
 void
-ReadDataFromArchive(ArchiveHandle *AH, int compression, ReadFunc readF)
+ReadDataFromArchive(ArchiveHandle *AH, pg_compress_specification compression_spec,
+					ReadFunc readF)
 {
-	CompressionAlgorithm alg;
-
-	ParseCompressionOption(compression, &alg, NULL);
-
-	if (alg == COMPR_ALG_NONE)
+	if (compression_spec.algorithm == PG_COMPRESSION_NONE)
 		ReadDataFromArchiveNone(AH, readF);
-	if (alg == COMPR_ALG_LIBZ)
+	if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
 	{
 #ifdef HAVE_LIBZ
 		ReadDataFromArchiveZlib(AH, readF);
@@ -179,18 +146,23 @@ void
 WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
 				   const void *data, size_t dLen)
 {
-	switch (cs->comprAlg)
+	switch (cs->compression_spec.algorithm)
 	{
-		case COMPR_ALG_LIBZ:
+		case PG_COMPRESSION_GZIP:
 #ifdef HAVE_LIBZ
 			WriteDataToArchiveZlib(AH, cs, data, dLen);
 #else
 			pg_fatal("not built with zlib support");
 #endif
 			break;
-		case COMPR_ALG_NONE:
+		case PG_COMPRESSION_NONE:
 			WriteDataToArchiveNone(AH, cs, data, dLen);
 			break;
+		case PG_COMPRESSION_LZ4:
+			/* fallthrough */
+		case PG_COMPRESSION_ZSTD:
+			pg_fatal("invalid compression method");
+			break;
 	}
 }
 
@@ -201,7 +173,7 @@ void
 EndCompressor(ArchiveHandle *AH, CompressorState *cs)
 {
 #ifdef HAVE_LIBZ
-	if (cs->comprAlg == COMPR_ALG_LIBZ)
+	if (cs->compression_spec.algorithm == PG_COMPRESSION_GZIP)
 		EndCompressorZlib(AH, cs);
 #endif
 	free(cs);
@@ -453,20 +425,27 @@ cfopen_read(const char *path, const char *mode)
 {
 	cfp		   *fp;
 
+	pg_compress_specification compression_spec = {0};
+
 #ifdef HAVE_LIBZ
 	if (hasSuffix(path, ".gz"))
-		fp = cfopen(path, mode, 1);
+	{
+		compression_spec.algorithm = PG_COMPRESSION_GZIP;
+		fp = cfopen(path, mode, compression_spec);
+	}
 	else
 #endif
 	{
-		fp = cfopen(path, mode, 0);
+		compression_spec.algorithm = PG_COMPRESSION_NONE;
+		fp = cfopen(path, mode, compression_spec);
 #ifdef HAVE_LIBZ
 		if (fp == NULL)
 		{
 			char	   *fname;
 
 			fname = psprintf("%s.gz", path);
-			fp = cfopen(fname, mode, 1);
+			compression_spec.algorithm = PG_COMPRESSION_GZIP;
+			fp = cfopen(fname, mode, compression_spec);
 			free_keep_errno(fname);
 		}
 #endif
@@ -479,26 +458,27 @@ cfopen_read(const char *path, const char *mode)
  * be a filemode as accepted by fopen() and gzopen() that indicates writing
  * ("w", "wb", "a", or "ab").
  *
- * If 'compression' is non-zero, a gzip compressed stream is opened, and
- * 'compression' indicates the compression level used. The ".gz" suffix
- * is automatically added to 'path' in that case.
+ * If 'compression_spec.algorithm' is GZIP, a gzip compressed stream is opened,
+ * and 'compression_spec.level' used. The ".gz" suffix is automatically added to
+ * 'path' in that case.
  *
  * On failure, return NULL with an error code in errno.
  */
 cfp *
-cfopen_write(const char *path, const char *mode, int compression)
+cfopen_write(const char *path, const char *mode,
+			 const pg_compress_specification compression_spec)
 {
 	cfp		   *fp;
 
-	if (compression == 0)
-		fp = cfopen(path, mode, 0);
+	if (compression_spec.algorithm == PG_COMPRESSION_NONE)
+		fp = cfopen(path, mode, compression_spec);
 	else
 	{
 #ifdef HAVE_LIBZ
 		char	   *fname;
 
 		fname = psprintf("%s.gz", path);
-		fp = cfopen(fname, mode, compression);
+		fp = cfopen(fname, mode, compression_spec);
 		free_keep_errno(fname);
 #else
 		pg_fatal("not built with zlib support");
@@ -509,26 +489,27 @@ cfopen_write(const char *path, const char *mode, int compression)
 }
 
 /*
- * Opens file 'path' in 'mode'. If 'compression' is non-zero, the file
+ * Opens file 'path' in 'mode'. If compression is GZIP, the file
  * is opened with libz gzopen(), otherwise with plain fopen().
  *
  * On failure, return NULL with an error code in errno.
  */
 cfp *
-cfopen(const char *path, const char *mode, int compression)
+cfopen(const char *path, const char *mode,
+	   const pg_compress_specification compression_spec)
 {
 	cfp		   *fp = pg_malloc(sizeof(cfp));
 
-	if (compression != 0)
+	if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
 	{
 #ifdef HAVE_LIBZ
-		if (compression != Z_DEFAULT_COMPRESSION)
+		if (compression_spec.level != Z_DEFAULT_COMPRESSION)
 		{
 			/* user has specified a compression level, so tell zlib to use it */
 			char		mode_compression[32];
 
 			snprintf(mode_compression, sizeof(mode_compression), "%s%d",
-					 mode, compression);
+					 mode, compression_spec.level);
 			fp->compressedfp = gzopen(path, mode_compression);
 		}
 		else

src/bin/pg_dump/compress_io.h

@@ -21,12 +21,6 @@
 #define ZLIB_OUT_SIZE	4096
 #define ZLIB_IN_SIZE	4096
 
-typedef enum
-{
-	COMPR_ALG_NONE,
-	COMPR_ALG_LIBZ
-} CompressionAlgorithm;
-
 /* Prototype for callback function to WriteDataToArchive() */
 typedef void (*WriteFunc) (ArchiveHandle *AH, const char *buf, size_t len);
 
@@ -46,8 +40,10 @@ typedef size_t (*ReadFunc) (ArchiveHandle *AH, char **buf, size_t *buflen);
 /* struct definition appears in compress_io.c */
 typedef struct CompressorState CompressorState;
 
-extern CompressorState *AllocateCompressor(int compression, WriteFunc writeF);
-extern void ReadDataFromArchive(ArchiveHandle *AH, int compression,
+extern CompressorState *AllocateCompressor(const pg_compress_specification compression_spec,
+										   WriteFunc writeF);
+extern void ReadDataFromArchive(ArchiveHandle *AH,
+								const pg_compress_specification compression_spec,
 								ReadFunc readF);
 extern void WriteDataToArchive(ArchiveHandle *AH, CompressorState *cs,
 							   const void *data, size_t dLen);
@@ -56,9 +52,13 @@ extern void EndCompressor(ArchiveHandle *AH, CompressorState *cs);
 
 typedef struct cfp cfp;
 
-extern cfp *cfopen(const char *path, const char *mode, int compression);
+extern cfp *cfopen(const char *path, const char *mode,
+				   const pg_compress_specification compression_spec);
+extern cfp *cfdopen(int fd, const char *mode,
+					pg_compress_specification compression_spec);
 extern cfp *cfopen_read(const char *path, const char *mode);
-extern cfp *cfopen_write(const char *path, const char *mode, int compression);
+extern cfp *cfopen_write(const char *path, const char *mode,
+						 const pg_compress_specification compression_spec);
 extern int	cfread(void *ptr, int size, cfp *fp);
 extern int	cfwrite(const void *ptr, int size, cfp *fp);
 extern int	cfgetc(cfp *fp);

src/bin/pg_dump/pg_backup.h

@@ -23,6 +23,7 @@
 #ifndef PG_BACKUP_H
 #define PG_BACKUP_H
 
+#include "common/compression.h"
 #include "fe_utils/simple_list.h"
 #include "libpq-fe.h"
 
@@ -143,7 +144,8 @@ typedef struct _restoreOptions
 
 	int			noDataForFailedTables;
 	int			exit_on_error;
-	int			compression;
+	pg_compress_specification compression_spec; /* Specification for
+												 * compression */
 	int			suppressDumpWarnings;	/* Suppress output of WARNING entries
 										 * to stderr */
 	bool		single_txn;
@@ -303,7 +305,8 @@ extern Archive *OpenArchive(const char *FileSpec, const ArchiveFormat fmt);
 
 /* Create a new archive */
 extern Archive *CreateArchive(const char *FileSpec, const ArchiveFormat fmt,
-							  const int compression, bool dosync, ArchiveMode mode,
+							  const pg_compress_specification compression_spec,
+							  bool dosync, ArchiveMode mode,
 							  SetupWorkerPtrType setupDumpWorker);
 
 /* The --list option */