Merge branch 'PGPROEE9_6' into PGPROEE9_6_partition

Author: vbwagner

contrib/pg_probackup/README.md

@@ -101,14 +101,16 @@ some overhead to PostgreSQL performance.  On our experiments it appears to be
 less than 3%.
 
 These two approaches were implemented in this fork of pg_probackup.  The second
-approach requires [patch for PostgreSQL 9.5](https://gist.github.com/stalkerg/44703dbcbac1da08f448b7e6966646c0) or
-[patch for PostgreSQL 10](https://gist.github.com/stalkerg/ab833d94e2f64df241f1835651e06e4b).
+approach requires [patch for PostgreSQL 9.6.2](https://gist.github.com/alubennikova/9daacf35790eca1a09b63a1bca86d836) or
+[patch for PostgreSQL 10 (master)](https://gist.github.com/alubennikova/d24f61804525f0248fa71a1075158c21).
 
 Testing block level incremental backup
 --------------------------------------
 
-You need build and install [PGPRO9_5 or PGPRO9_6 branch of PostgreSQL](https://github.com/postgrespro/postgrespro) or apply this patch to
-[PostgreSQL 9.5](https://gist.github.com/stalkerg/44703dbcbac1da08f448b7e6966646c0) or [PostgreSQL 10](https://gist.github.com/stalkerg/ab833d94e2f64df241f1835651e06e4b).
+You need to apply ptrack patch to [PostgreSQL 9.6.2](https://gist.github.com/alubennikova/9daacf35790eca1a09b63a1bca86d836)
+or [PostgreSQL 10 (master)](https://gist.github.com/alubennikova/d24f61804525f0248fa71a1075158c21).
+Or you can build and install [PGPRO9_5 or PGPRO9_6 branch of PostgreSQL](https://github.com/postgrespro/postgrespro).
+Note that PGPRO branches currently contain old version of ptrack.
 
 ### Retrieving changed blocks from WAL archive
 

contrib/pg_probackup/backup.c

@@ -111,11 +111,6 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
 	/* repack the options */
 	pgBackup   *prev_backup = NULL;
 
-	/* Block backup operations on a standby */
-	from_replica = pg_is_in_recovery();
-	if (pg_is_standby() && !from_replica)
-		elog(ERROR, "Backup cannot run on a standby.");
-
 	elog(LOG, "database backup start");
 
 	/* Initialize size summary */
@@ -423,6 +418,15 @@ do_backup(bool smooth_checkpoint)
 	/* setup cleanup callback function */
 	in_backup = true;
 
+	/* Block backup operations on a standby */
+	from_replica = pg_is_in_recovery();
+	if (pg_is_standby() && !from_replica)
+		elog(ERROR, "backup is not allowed for standby");
+
+	/* Page backup is not allowed for replica instance */
+	if (from_replica && current.backup_mode == BACKUP_MODE_DIFF_PAGE)
+		elog(ERROR, "page backup is not allowed for standby");
+
 	/* show configuration actually used */
 	elog(LOG, "========================================");
 	elog(LOG, "backup start");

contrib/pg_probackup/catalog.c

@@ -42,7 +42,7 @@ unlink_lock_atexit(void)
 /*
  * Create a lockfile.
  */
-int
+void
 catalog_lock(bool check_catalog)
 {
 	int			fd;
@@ -220,8 +220,6 @@ catalog_lock(bool check_catalog)
 			elog(ERROR, "Backup directory was initialized for system id = %ld, but target system id = %ld",
 				 system_identifier, id);
 	}
-
-	return 0;
 }
 
 /*

contrib/pg_probackup/dir.c

@@ -20,18 +20,59 @@
 #include "pgut/pgut-port.h"
 #include "datapagemap.h"
 
-/* directory exclusion list for backup mode listing */
+/*
+ * The contents of these directories are removed or recreated during server
+ * start so they are not included in backups.  The directories themselves are
+ * kept and included as empty to preserve access permissions.
+ */
 const char *pgdata_exclude_dir[] =
 {
-	"pg_xlog",
-	"pg_stat_tmp",
-	"pgsql_tmp",
-	NULL,			/* pg_log will be set later */
-	NULL
+		"pg_xlog",
+       /*
+        * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
+        * when stats_temp_directory is set because PGSS_TEXT_FILE is always created
+        * there.
+        */
+        "pg_stat_tmp",
+        "pgsql_tmp",
+
+       /*
+        * It is generally not useful to backup the contents of this directory even
+        * if the intention is to restore to another master. See backup.sgml for a
+        * more detailed description.
+        */
+       "pg_replslot",
+
+       /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
+       "pg_dynshmem",
+
+       /* Contents removed on startup, see AsyncShmemInit(). */
+       "pg_notify",
+
+       /*
+        * Old contents are loaded for possible debugging but are not required for
+        * normal operation, see OldSerXidInit().
+        */
+       "pg_serial",
+
+       /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
+       "pg_snapshots",
+
+       /* Contents zeroed on startup, see StartupSUBTRANS(). */
+       "pg_subtrans",
+
+       /* end of list */
+        NULL,                   /* pg_log will be set later */
+        NULL
 };
 
 static char *pgdata_exclude_files[] =
 {
+	/* Skip auto conf temporary file. */
+	"postgresql.auto.conf.tmp",
+
+	/* Skip current log file temporary file */
+	"current_logfiles.tmp",
 	"recovery.conf",
 	"postmaster.pid",
 	"postmaster.opts",
@@ -515,13 +556,7 @@ list_data_directories(parray *files, const char *path, bool is_root,
 			elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno));
 
 		if (!S_ISDIR(st.st_mode))
-		{
-			/* Stop reading the directory if we met file */
-			if (!is_root)
-				break;
-			else
-				continue;
-		}
+			continue;
 
 		/* Check for exclude for the first level of listing */
 		if (is_root && exclude)

contrib/pg_probackup/doc/pg_probackup.md

@@ -291,13 +291,13 @@ This mode should be used with caution as it allows to delete WAL files required
 
 ### Backup from Standby
 
-If replication is in use, starting with PostgreSQL 9.6 a backup can be taken not only from primary server, but also from standby. Backup taken from standby is absolutely interchangeable with backup taken from primary (bearing in mind possible replication delay).
+If replication is in use, starting with PostgreSQL 9.6 a backup can be taken not only from primary server, but also from standby. Backup taken from standby is absolutely interchangeable with backup taken from primary (bearing in mind possible replication delay). Page-level incremental backups are not allowed to perform from standby.
 
-Currently it is required for primary database server to have full\_page\_writes turned on (in future this requirement may be relaxed in the case checksums are enabled on data pages).
+Currently it is required for primary database server to have `full_page_writes` turned `on` (in future this requirement may be relaxed in the case checksums are enabled on data pages).
 
-The same backup directory can be used for pg\_probackup on both servers, primary and standby, as long as it is accessible in both server's file systems. This way all backups, taken from either primary or standby, are shown together and could be managed from one server or from the other.
+The same backup directory can be used for `pg_probackup` on both servers, primary and standby, as long as it is accessible in both server's file systems. This way all backups, taken from either primary or standby, are shown together and could be managed from one server or from the other.
 
-A backup can be used to restore primary database server as well as standby. It depends on the server on which pg\_probackup is executed with restore command. Note that recovered PostgreSQL will always run as primary server if started right after the pg\_probackup. To run it as standby, edit recovery.conf file created by pg\_probackup: at least delete every parameter that specify recovery target (recovery\_target, recovery\_target\_time, and recovery\_target\_xid), change target timeline to 'latest', and add standby\_mode = 'on'. Probably primary\_conninfo should be added too for streaming replication, and hot\_standby = 'on' in database configuration parameters for hot standby mode.
+A backup can be used to restore primary database server as well as standby. It depends on the server on which `pg_probackup` is executed with restore command. Note that recovered PostgreSQL will always run as primary server if started right after the `pg_probackup`. To run it as standby, edit `recovery.conf` file created by `pg_probackup`: at least delete every parameter that specify recovery target (`recovery_target`, `recovery_target_time`, and `recovery_target_xid`), change target timeline to `latest`, and add `standby_mode = on`. Probably `primary_conninfo` should be added too for streaming replication, and `hot_standby = on` in database configuration parameters for hot standby mode.
 
 ### Backup Retention Policy
 
@@ -480,11 +480,11 @@ Specifies whether to stop just after the specified recovery target (true), or ju
 
 Specifies recovering into a particular timeline.
 
--T OLDDIR=NEWDIR  
---tablespace-mapping=OLDDIR=NEWDIR
+-T olddir=newdir
+--tablespace-mapping=olddir=newdir
 
-Relocate the tablespace in directory `OLDDIR` to `NEWDIR` during restore. Both
-`OLDDIR` and `NEWDIR` must be absolute paths.
+Relocate the tablespace in directory `olddir` to `newdir` during restore. Both
+`olddir` and `newdir` must be absolute paths.
 
 ### Delete options:
 

contrib/pg_probackup/pg_probackup.c

@@ -16,7 +16,7 @@
 #include <time.h>
 #include <sys/stat.h>
 
-const char *PROGRAM_VERSION	= "1.0.1";
+const char *PROGRAM_VERSION	= "1.1.0";
 const char *PROGRAM_URL		= "https://github.com/postgrespro/pg_probackup";
 const char *PROGRAM_EMAIL	= "https://github.com/postgrespro/pg_probackup/issues";
 

contrib/pg_probackup/pg_probackup.h

@@ -300,7 +300,7 @@ extern parray *catalog_get_backup_list(time_t backup_id);
 extern pgBackup *catalog_get_last_data_backup(parray *backup_list,
 											  TimeLineID tli);
 
-extern int catalog_lock(bool check_catalog);
+extern void catalog_lock(bool check_catalog);
 
 extern void pgBackupWriteConfigSection(FILE *out, pgBackup *backup);
 extern void pgBackupWriteResultSection(FILE *out, pgBackup *backup);

contrib/pg_probackup/pgut/pgut.c

@@ -614,12 +614,12 @@ pgut_getopt(int argc, char **argv, pgut_option options[])
 		if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
 		{
 			help(true);
-			exit_or_abort(1);
+			exit_or_abort(0);
 		}
 		if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
 		{
 			fprintf(stderr, "%s %s\n", PROGRAM_NAME, PROGRAM_VERSION);
-			exit_or_abort(1);
+			exit_or_abort(0);
 		}
 	}
 

contrib/pg_probackup/restore.c

@@ -91,7 +91,7 @@ do_restore(time_t backup_id,
 	pgBackup   *base_backup = NULL;
 	pgBackup   *dest_backup = NULL;
 	pgRecoveryTarget *rt = NULL;
-	bool		need_recovery_conf = false;
+	bool		need_recovery_conf = true;
 
 	/* PGDATA and ARCLOG_PATH are always required */
 	if (pgdata == NULL)
@@ -200,16 +200,16 @@ do_restore(time_t backup_id,
 	/* Tablespace directories checking */
 	check_tablespace_mapping((pgBackup *) parray_get(backups, last_diff_index));
 
+	if (dest_backup && dest_backup->stream)
+		need_recovery_conf = target_time != NULL || target_xid != NULL;
+
 	/* Restore backups from base_index to last_diff_index */
-	need_recovery_conf = target_time != NULL || target_xid != NULL;
 	for (i = base_index; i >= last_diff_index; i--)
 	{
 		pgBackup   *backup = (pgBackup *) parray_get(backups, i);
 
 		if (backup->status == BACKUP_STATUS_OK)
 		{
-			need_recovery_conf = need_recovery_conf || !backup->stream;
-
 			print_backup_lsn(backup);
 			restore_database(backup);
 		}
@@ -416,7 +416,7 @@ restore_directories(const char *pg_data_dir, const char *backup_dir)
 				link_name[len] = '\0';
 			}
 			else
-				strcpy(link_name, link_ptr);
+				goto create_directory;
 
 			tmp_ptr = dir->path;
 			dir->path = link_name;
@@ -442,7 +442,13 @@ restore_directories(const char *pg_data_dir, const char *backup_dir)
 					 * create it second time.
 					 */
 					if (strcmp(dir_created, linked_path) == 0)
-						continue;
+					{
+						/* Create rest of directories */
+						if (link_sep && (link_sep + 1))
+							goto create_directory;
+						else
+							continue;
+					}
 					else
 						elog(ERROR, "tablespace directory \"%s\" of page backup does not "
 							 "match with previous created tablespace directory \"%s\" of symlink \"%s\"",
@@ -467,12 +473,6 @@ restore_directories(const char *pg_data_dir, const char *backup_dir)
 
 				/* Firstly, create linked directory */
 				dir_create_dir(linked_path, DIR_PERMISSION);
-				/* Create rest of directories */
-				if (link_sep && (link_sep + 1))
-				{
-					join_path_components(to_path, linked_path, link_sep + 1);
-					dir_create_dir(to_path, DIR_PERMISSION);
-				}
 
 				join_path_components(to_path, pg_data_dir, PG_TBLSPC_DIR);
 				/* Create pg_tblspc directory just in case */
@@ -487,10 +487,15 @@ restore_directories(const char *pg_data_dir, const char *backup_dir)
 				/* Save linked directory */
 				set_tablespace_created(link_name, linked_path);
 
+				/* Create rest of directories */
+				if (link_sep && (link_sep + 1))
+					goto create_directory;
+
 				continue;
 			}
 		}
 
+create_directory:
 		elog(LOG, "create directory \"%s\"", relative_ptr);
 
 		/* This is not symlink, create directory */

contrib/pg_probackup/show.c

@@ -28,7 +28,7 @@ do_show(time_t backup_id)
 	 */
 	if (backup_id != 0)
 	{
-		pgBackup *backup;
+		pgBackup   *backup;
 
 		backup = read_backup(backup_id);
 		if (backup == NULL)
@@ -40,6 +40,7 @@ do_show(time_t backup_id)
 			/* This is not error case */
 			return 0;
 		}
+
 		show_backup_detail(stdout, backup);
 
 		/* cleanup */
@@ -184,7 +185,7 @@ get_parent_tli(TimeLineID child_tli)
 static void
 show_backup_list(FILE *out, parray *backup_list)
 {
-	int i;
+	int			i;
 
 	/* show header */
 	fputs("=========================================================================================\n", out);
@@ -193,14 +194,12 @@ show_backup_list(FILE *out, parray *backup_list)
 
 	for (i = 0; i < parray_num(backup_list); i++)
 	{
-		pgBackup *backup;
-		const char *modes[] = { "", "PAGE", "PTRACK", "FULL", "", "PAGE+STREAM", "PTRACK+STREAM", "FULL+STREAM"};
-		TimeLineID  parent_tli;
-		char timestamp[20] = "----";
-		char duration[20] = "----";
-		char data_bytes_str[10] = "----";
-
-		backup = parray_get(backup_list, i);
+		pgBackup   *backup = parray_get(backup_list, i);
+		const char *modes[] = {"", "PAGE", "PTRACK", "FULL", "", "PAGE+STREAM", "PTRACK+STREAM", "FULL+STREAM"};
+		TimeLineID	parent_tli;
+		char		timestamp[20] = "----";
+		char		duration[20] = "----";
+		char		data_bytes_str[10] = "----";
 
 		if (backup->recovery_time != (time_t) 0)
 			time2iso(timestamp, lengthof(timestamp), backup->recovery_time);

contrib/pg_probackup/tests/expected/option_version.out

@@ -1 +1 @@
-pg_probackup 1.0
+pg_probackup 1.1.0