Merge commit '94540661566677a456e927050664b7e03db601d0' into PGPRO9_6

Conflicts:
	contrib/pg_probackup/pgut/getopt_long.c

Author: dmpgpro

contrib/pg_probackup/backup.c

@@ -242,6 +242,10 @@ pgBackupCreateDir(pgBackup *backup)
 	char   *subdirs[] = { DATABASE_DIR, NULL };
 
 	pgBackupGetPath(backup, path, lengthof(path), NULL);
+
+	if (!dir_is_empty(path))
+		elog(ERROR, "backup destination is not empty \"%s\"", path);
+
 	dir_create_dir(path, DIR_PERMISSION);
 
 	/* create directories for actual backup files */
@@ -525,6 +529,8 @@ pgBackupGetPath(const pgBackup *backup, char *path, size_t len, const char *subd
 	else
 		snprintf(path, len, "%s/%s/%s", backup_path, BACKUPS_DIR, datetime);
 	free(datetime);
+
+	make_native_path(path);
 }
 
 void

contrib/pg_probackup/catalog.c

@@ -25,7 +25,6 @@ const char *pgdata_exclude_dir[] =
 	"pg_xlog",
 	"pg_stat_tmp",
 	"pgsql_tmp",
-	NULL,			/* arclog_path will be set later */
 	NULL,			/* pg_log will be set later */
 	NULL
 };
@@ -41,12 +40,18 @@ static char *pgdata_exclude_files[] =
 pgFile *pgFileNew(const char *path, bool omit_symlink);
 static int BlackListCompare(const void *str1, const void *str2);
 
-/* create directory, also create parent directories if necessary */
+static void dir_list_file_internal(parray *files, const char *root,
+								   bool exclude, bool omit_symlink,
+								   bool add_root, parray *black_list);
+
+/*
+ * Create directory, also create parent directories if necessary.
+ */
 int
 dir_create_dir(const char *dir, mode_t mode)
 {
-	char copy[MAXPGPATH];
-	char parent[MAXPGPATH];
+	char		copy[MAXPGPATH];
+	char		parent[MAXPGPATH];
 
 	strncpy(copy, dir, MAXPGPATH);
 	strncpy(parent, dirname(copy), MAXPGPATH);
@@ -60,8 +65,7 @@ dir_create_dir(const char *dir, mode_t mode)
 	{
 		if (errno == EEXIST)	/* already exist */
 			return 0;
-		elog(ERROR, "cannot create directory \"%s\": %s", dir,
-			strerror(errno));
+		elog(ERROR, "cannot create directory \"%s\": %s", dir, strerror(errno));
 	}
 
 	return 0;
@@ -295,7 +299,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
 	parray_qsort(files, pgFileComparePath);
 }
 
-void
+static void
 dir_list_file_internal(parray *files, const char *root, bool exclude,
 					   bool omit_symlink, bool add_root, parray *black_list)
 {
@@ -327,11 +331,7 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 			else
 				file_name++;
 
-			/*
-			 * If the item in the exclude list starts with '/', compare to the
-			 * absolute path of the directory. Otherwise compare to the directory
-			 * name portion.
-			 */
+			/* Check if we need to exclude file by name */
 			for (i = 0; pgdata_exclude_files[i]; i++)
 				if (strcmp(file_name, pgdata_exclude_files[i]) == 0)
 					/* Skip */
@@ -416,13 +416,10 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 						break;
 					}
 				}
-				else
+				else if (strcmp(dirname, pgdata_exclude_dir[i]) == 0)
 				{
-					if (strcmp(dirname, pgdata_exclude_dir[i]) == 0)
-					{
-						skip = true;
-						break;
-					}
+					skip = true;
+					break;
 				}
 			}
 			if (skip)
@@ -468,74 +465,159 @@ dir_list_file_internal(parray *files, const char *root, bool exclude,
 	}
 }
 
-/* print mkdirs.sh */
+/*
+ * List data directories excluding directories from
+ * pgdata_exclude_dir array.
+ *
+ * **is_root** is a little bit hack. We exclude only first level of directories
+ * and on the first level we check all files and directories.
+ */
 void
-dir_print_mkdirs_sh(FILE *out, const parray *files, const char *root)
+list_data_directories(parray *files, const char *path, bool is_root,
+					  bool exclude)
 {
-	int i;
-
-	for (i = 0; i < parray_num(files); i++)
+	DIR		   *dir;
+	struct dirent *dent;
+	int			prev_errno;
+	bool		has_child_dirs = false;
+
+	/* open directory and list contents */
+	dir = opendir(path);
+	if (dir == NULL)
+		elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno));
+
+	errno = 0;
+	while ((dent = readdir(dir)))
 	{
-		pgFile *file = (pgFile *) parray_get(files, i);
-		if (S_ISDIR(file->mode))
+		char		child[MAXPGPATH];
+		bool		skip = false;
+		struct stat	st;
+
+		/* skip entries point current dir or parent dir */
+		if (strcmp(dent->d_name, ".") == 0 ||
+			strcmp(dent->d_name, "..") == 0)
+			continue;
+
+		join_path_components(child, path, dent->d_name);
+
+		if (lstat(child, &st) == -1)
+			elog(ERROR, "cannot stat file \"%s\": %s", child, strerror(errno));
+
+		if (!S_ISDIR(st.st_mode))
 		{
-			if (strstr(file->path, root) == file->path &&
-				*(file->path + strlen(root)) == '/')
-			{
-				fprintf(out, "mkdir -m 700 -p %s\n", file->path + strlen(root) + 1);
-			}
+			/* Stop reading the directory if we met file */
+			if (!is_root)
+				break;
 			else
+				continue;
+		}
+
+		/* Check for exclude for the first level of listing */
+		if (is_root && exclude)
+		{
+			int			i;
+
+			for (i = 0; pgdata_exclude_dir[i]; i++)
 			{
-				fprintf(out, "mkdir -m 700 -p %s\n", file->path);
+				if (strcmp(dent->d_name, pgdata_exclude_dir[i]) == 0)
+				{
+					skip = true;
+					break;
+				}
 			}
 		}
-	}
+		if (skip)
+			continue;
 
-	fprintf(out, "\n");
+		has_child_dirs = true;
+		list_data_directories(files, child, false, exclude);
+	}
 
-	for (i = 0; i < parray_num(files); i++)
+	/* List only full and last directories */
+	if (!is_root && !has_child_dirs)
 	{
-		pgFile *file = (pgFile *) parray_get(files, i);
-		if (S_ISLNK(file->mode))
-		{
-			fprintf(out, "rm -f %s\n", file->path + strlen(root) + 1);
-			fprintf(out, "ln -s %s %s\n", file->linked, file->path + strlen(root) + 1);
-		}
+		pgFile	   *dir;
+
+		dir = pgFileNew(path, false);
+		parray_append(files, dir);
 	}
+
+	prev_errno = errno;
+	closedir(dir);
+
+	if (prev_errno && prev_errno != ENOENT)
+		elog(ERROR, "cannot read directory \"%s\": %s",
+			 path, strerror(prev_errno));
 }
 
-/* print file list */
+/*
+ * Read names of symbolik names of tablespaces with links to directories from
+ * tablespace_map or tablespace_map.txt.
+ */
 void
-dir_print_file_list(FILE *out, const parray *files, const char *root, const char *prefix)
+read_tablespace_map(parray *files, const char *backup_dir)
 {
-	int i;
-	int root_len = 0;
+	FILE	   *fp;
+	char		db_path[MAXPGPATH],
+				map_path[MAXPGPATH];
+	char		buf[MAXPGPATH * 2];
 
-	/* calculate length of root directory portion */
-	if (root)
+	join_path_components(db_path, backup_dir, DATABASE_DIR);
+	join_path_components(map_path, db_path, "tablespace_map");
+
+	/* Exit if database/tablespace_map don't exists */
+	if (!fileExists(map_path))
 	{
-		root_len = strlen(root);
-		if (root[root_len - 1] != '/')
-			root_len++;
+		elog(LOG, "there is no file tablespace_map");
+		return;
 	}
 
+	fp = fopen(map_path, "rt");
+	if (fp == NULL)
+		elog(ERROR, "cannot open \"%s\": %s", map_path, strerror(errno));
+
+	while (fgets(buf, lengthof(buf), fp))
+	{
+		char		link_name[MAXPGPATH],
+					path[MAXPGPATH];
+		pgFile	   *file;
+
+		if (sscanf(buf, "%s %s", link_name, path) != 2)
+			elog(ERROR, "invalid format found in \"%s\"", map_path);
+
+		file = pgut_new(pgFile);
+		memset(file, 0, sizeof(pgFile));
+
+		file->path = pgut_malloc(strlen(link_name) + 1);
+		strcpy(file->path, link_name);
+
+		file->linked = pgut_malloc(strlen(path) + 1);
+		strcpy(file->linked, path);
+
+		parray_append(files, file);
+	}
+
+	fclose(fp);
+}
+
+/*
+ * Print file list.
+ */
+void
+print_file_list(FILE *out, const parray *files, const char *root)
+{
+	size_t		i;
+
 	/* print each file in the list */
 	for (i = 0; i < parray_num(files); i++)
 	{
-		pgFile *file = (pgFile *)parray_get(files, i);
-		char path[MAXPGPATH];
-		char *ptr = file->path;
-		char type;
+		pgFile	   *file = (pgFile *) parray_get(files, i);
+		char	   *path = file->path;
+		char		type;
 
 		/* omit root directory portion */
-		if (root && strstr(ptr, root) == ptr)
-			ptr = JoinPathEnd(ptr, root);
-
-		/* append prefix if not NULL */
-		if (prefix)
-			join_path_components(path, prefix, ptr);
-		else
-			strcpy(path, ptr);
+		if (root && strstr(path, root) == path)
+			path = JoinPathEnd(path, root);
 
 		if (S_ISREG(file->mode) && file->is_datafile)
 			type = 'F';
@@ -556,7 +638,8 @@ dir_print_file_list(FILE *out, const parray *files, const char *root, const char
 			fprintf(out, " %s", file->linked);
 		else
 		{
-			char timestamp[20];
+			char		timestamp[20];
+
 			time2iso(timestamp, 20, file->mtime);
 			fprintf(out, " %s", timestamp);
 		}
@@ -683,45 +766,39 @@ dir_read_file_list(const char *root, const char *file_txt)
 }
 
 /*
- * Copy contents of directory from_root into to_root.
+ * Check if directory empty.
  */
-void
-dir_copy_files(const char *from_root, const char *to_root)
+bool
+dir_is_empty(const char *path)
 {
-	size_t		i;
-	parray	   *files = parray_new();
+	DIR		   *dir;
+	struct dirent *dir_ent;
 
-	/* don't copy root directory */
-	dir_list_file(files, from_root, false, true, false);
-
-	for (i = 0; i < parray_num(files); i++)
+	dir = opendir(path);
+	if (dir == NULL)
 	{
-		pgFile	   *file = (pgFile *) parray_get(files, i);
-
-		if (S_ISDIR(file->mode))
-		{
-			char		to_path[MAXPGPATH];
+		/* Directory in path doesn't exist */
+		if (errno == ENOENT)
+			return true;
+		elog(ERROR, "cannot open directory \"%s\": %s", path, strerror(errno));
+	}
 
-			join_path_components(to_path, to_root,
-								 file->path + strlen(from_root) + 1);
+	errno = 0;
+	while ((dir_ent = readdir(dir)))
+	{
+		/* Skip entries point current dir or parent dir */
+		if (strcmp(dir_ent->d_name, ".") == 0 ||
+			strcmp(dir_ent->d_name, "..") == 0)
+			continue;
 
-			if (verbose && !check)
-				elog(LOG, "creating directory \"%s\"",
-					 file->path + strlen(from_root) + 1);
-			if (!check)
-				dir_create_dir(to_path, DIR_PERMISSION);
-		}
-		else if (S_ISREG(file->mode))
-		{
-			if (verbose && !check)
-				elog(LOG, "copying \"%s\"",
-					file->path + strlen(from_root) + 1);
-			if (!check)
-				copy_file(from_root, to_root, file);
-		}
+		/* Directory is not empty */
+		closedir(dir);
+		return false;
 	}
+	if (errno)
+		elog(ERROR, "cannot read directory \"%s\": %s", path, strerror(errno));
+
+	closedir(dir);
 
-	/* cleanup */
-	parray_walk(files, pgFileFree);
-	parray_free(files);
+	return true;
 }

contrib/pg_probackup/dir.c

@@ -477,6 +477,12 @@ Specifies whether to stop just after the specified recovery target (true), or ju
 
 Specifies recovering into a particular timeline.
 
+-T  
+--tablespace-mapping=OLDDIR=NEWDIR
+
+Relocate the tablespace in directory `OLDDIR` to `NEWDIR` during restore. Both
+`OLDDIR` and `NEWDIR` must be absolute paths.
+
 ### Delete options:
 
 --wal