Merge with pgpro-727

Author: za-arthur

.gitignore

@@ -27,6 +27,7 @@
 /results
 /env
 /tests/__pycache__/
+/tests/helpers/__pycache__/
 /tests/tmp_dirs/
 /tests/*pyc
 /tests/helpers/*pyc
@@ -40,3 +41,5 @@
 /src/streamutil.c
 /src/streamutil.h
 /src/xlogreader.c
+/src/walmethods.c
+/src/walmethods.h

Makefile

@@ -9,46 +9,41 @@ OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \
 EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \
 	src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h
 
-all: checksrcdir src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h pg_probackup
+INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
-ifndef top_srcdir
-	@echo "You must have PostgreSQL source tree available to compile."
-	@echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir"
-	@echo "variable: \"make top_srcdir=<path to PostgreSQL source tree>\""
-	@exit 1
-endif
-# Those files are symlinked from the PostgreSQL sources.
-src/xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/xlogreader.c
-	rm -f $@ && $(LN_S) $< ./src/xlogreader.c
-src/datapagemap.c: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
-	rm -f $@ && $(LN_S) $< ./src/datapagemap.c
-src/datapagemap.h: % : $(top_srcdir)/src/bin/pg_rewind/datapagemap.h
-	rm -f $@ && $(LN_S) $< src/datapagemap.h
-src/logging.h: % : $(top_srcdir)/src/bin/pg_rewind/logging.h
-	rm -f $@ && $(LN_S) $< ./src
-src/receivelog.c: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.c
-	rm -f $@ && $(LN_S) $< ./src
-src/receivelog.h: % : $(top_srcdir)/src/bin/pg_basebackup/receivelog.h
-	rm -f $@ && $(LN_S) $< ./src
-src/streamutil.c: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.c
-	rm -f $@ && $(LN_S) $< ./src
-src/streamutil.h: % : $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
-	rm -f $@ && $(LN_S) $< ./src
+# !USE_PGXS
 else
 subdir=contrib/pg_probackup
 top_builddir=../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
-ifeq ("$(top_srcdir)","../..")
-srchome="$(top_srcdir)/.."
+endif # USE_PGXS
+
+ifeq ($(top_srcdir),../..)
+srchome=$(top_srcdir)/..
 else
-srchome="$(top_srcdir)"
+srchome=$(top_srcdir)
+endif
+
+ifeq ($(MAJORVERSION),10)
+OBJS += src/walmethods.o
+EXTRA_CLEAN += src/walmethods.c src/walmethods.h
+INCLUDES += src/walmethods.h
 endif
-# Those files are symlinked from the PostgreSQL sources.
+
+PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc
+override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS)
+PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS}
+
+all: checksrcdir $(INCLUDES) $(PROGRAM);
+
+$(PROGRAM): $(OBJS)
+	$(CC) $(CFLAGS) $(OBJS) $(PG_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
 src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
 	rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
 src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
@@ -65,20 +60,18 @@ src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c
 	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@
 src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
 	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
-endif
 
-PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc
-override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS)
-PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS}
+ifeq ($(MAJORVERSION),10)
+src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c
+	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@
+src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h
+	rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@
+endif
 
 ifeq ($(PORTNAME), aix)
 	CC=xlc_r
 endif
 
-envtest:
-	: top_srcdir=$( )
-	: libpq_srcdir = $(libpq_srcdir)
-
 # This rule's only purpose is to give the user instructions on how to pass
 # the path to PostgreSQL source tree to the makefile.
 .PHONY: checksrcdir

src/backup.c

@@ -23,8 +23,8 @@
 #include "libpq/pqsignal.h"
 #include "storage/bufpage.h"
 #include "datapagemap.h"
-#include "streamutil.h"
 #include "receivelog.h"
+#include "streamutil.h"
 
 static int	standby_message_timeout = 10 * 1000;	/* 10 sec = default */
 static XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr;
@@ -848,7 +848,7 @@ pg_ptrack_get_and_clear(Oid tablespace_oid, Oid db_oid, Oid rel_oid,
  * If current backup started in archive mode wait for 'lsn' to be archived in
  * archive 'wal' directory with WAL segment file.
  * If current backup started in stream mode wait for 'lsn' to be streamed in
- * 'pg_xlog' directory.
+ * 'pg_wal' directory.
  *
  * If 'wait_prev_segment' wait for previous segment.
  */
@@ -1938,7 +1938,7 @@ StreamLog(void *arg)
 {
 	XLogRecPtr	startpos;
 	TimeLineID	starttli;
-	char *basedir = (char *)arg;
+	char	   *basedir = (char *)arg;
 
 	/*
 	 * Connect in replication mode to the server
@@ -1995,18 +1995,33 @@ StreamLog(void *arg)
 
 #if PG_VERSION_NUM >= 90600
 	{
-		StreamCtl ctl;
+		StreamCtl	ctl;
+
 		ctl.startpos = startpos;
 		ctl.timeline = starttli;
 		ctl.sysidentifier = NULL;
+
+#if PG_VERSION_NUM >= 100000
+		ctl.walmethod = CreateWalDirectoryMethod(basedir, 0, true);
+		ctl.replication_slot = replication_slot;
+#else
 		ctl.basedir = basedir;
+#endif
+
 		ctl.stream_stop = stop_streaming;
 		ctl.standby_message_timeout = standby_message_timeout;
 		ctl.partial_suffix = NULL;
 		ctl.synchronous = false;
 		ctl.mark_done = false;
+
 		if(ReceiveXlogStream(conn, &ctl) == false)
 			elog(ERROR, "Problem in receivexlog");
+
+#if PG_VERSION_NUM >= 100000
+		if (!ctl.walmethod->finish())
+			elog(ERROR, "Could not finish writing WAL files: %s",
+				 strerror(errno));
+#endif
 	}
 #else
 	if(ReceiveXlogStream(conn, startpos, starttli, NULL, basedir,

src/parsexlog.c

@@ -24,8 +24,13 @@
  * RmgrNames is an array of resource manager names, to make error messages
  * a bit nicer.
  */
+#if PG_VERSION_NUM >= 100000
+#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
+  name,
+#else
 #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \
   name,
+#endif
 
 static const char *RmgrNames[RM_MAX_ID + 1] = {
 #include "access/rmgrlist.h"

src/pg_probackup.c

@@ -40,6 +40,9 @@ char	   *backup_id_string_param = NULL;
 int			num_threads = 1;
 bool		stream_wal = false;
 bool		progress = false;
+#if PG_VERSION_NUM >= 100000
+char	   *replication_slot = NULL;
+#endif
 
 /* backup options */
 bool		backup_logs = false;

src/pg_probackup.h

@@ -44,7 +44,11 @@
 /* Directory/File names */
 #define DATABASE_DIR			"database"
 #define BACKUPS_DIR				"backups"
+#if PG_VERSION_NUM >= 100000
+#define PG_XLOG_DIR				"pg_wal"
+#else
 #define PG_XLOG_DIR				"pg_xlog"
+#endif
 #define PG_TBLSPC_DIR			"pg_tblspc"
 #define BACKUP_CONTROL_FILE		"backup.control"
 #define BACKUP_CATALOG_CONF_FILE	"pg_probackup.conf"
@@ -271,6 +275,10 @@ extern char arclog_path[MAXPGPATH];
 extern int num_threads;
 extern bool stream_wal;
 extern bool progress;
+#if PG_VERSION_NUM >= 100000
+/* In pre-10 'replication_slot' is defined in receivelog.h */
+extern char *replication_slot;
+#endif
 
 /* backup options */
 extern bool	smooth_checkpoint;

src/util.c

@@ -65,9 +65,15 @@ checkControlFile(ControlFileData *ControlFile)
 static void
 digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
 {
-	if (size != PG_CONTROL_SIZE)
+#if PG_VERSION_NUM >= 100000
+	int			ControlFileSize = PG_CONTROL_FILE_SIZE;
+#else
+	int			ControlFileSize = PG_CONTROL_SIZE;
+#endif
+
+	if (size != ControlFileSize)
 		elog(ERROR, "unexpected control file size %d, expected %d",
-			 (int) size, PG_CONTROL_SIZE);
+			 (int) size, ControlFileSize);
 
 	memcpy(ControlFile, src, sizeof(ControlFileData));