Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 3b40215

Browse files
dmpgprodmpgpro
committed
Add 'contrib/pg_probackup/' from commit 'ed8f9bebb0597839535f0e69ac634e296665e795'
git-subtree-dir: contrib/pg_probackup git-subtree-mainline: a595350 git-subtree-split: ed8f9be
2 parents a595350 + ed8f9be commit 3b40215

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

62 files changed

+18444
-0
lines changed

‎contrib/pg_probackup/.gitignore

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# Object files
2+
*.o
3+
4+
# Libraries
5+
*.lib
6+
*.a
7+
8+
# Shared objects (inc. Windows DLLs)
9+
*.dll
10+
*.so
11+
*.so.*
12+
*.dylib
13+
14+
# Executables
15+
*.exe
16+
*.app
17+
18+
# Dependencies
19+
.deps
20+
21+
# Binaries
22+
/pg_probackup
23+
24+
# Generated by test suite
25+
/regression.diffs
26+
/regression.out
27+
/results
28+
/env
29+
/tests/__pycache__/
30+
/tests/helpers/__pycache__/
31+
/tests/tmp_dirs/
32+
/tests/*pyc
33+
/tests/helpers/*pyc
34+
35+
# Extra files
36+
/src/datapagemap.c
37+
/src/datapagemap.h
38+
/src/logging.h
39+
/src/receivelog.c
40+
/src/receivelog.h
41+
/src/streamutil.c
42+
/src/streamutil.h
43+
/src/xlogreader.c
44+
/src/walmethods.c
45+
/src/walmethods.h

‎contrib/pg_probackup/.travis.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
sudo: required
2+
3+
services:
4+
- docker
5+
6+
script:
7+
- docker run -v $(pwd):/tests --rm centos:7 /tests/travis/backup_restore.sh

‎contrib/pg_probackup/COPYRIGHT

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
Copyright (c) 2015-2017, Postgres Professional
2+
Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION
3+
4+
Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
5+
Portions Copyright (c) 1994, The Regents of the University of California
6+
7+
Redistribution and use in source and binary forms, with or without
8+
modification, are permitted provided that the following conditions are met:
9+
10+
* Redistributions of source code must retain the above copyright notice,
11+
this list of conditions and the following disclaimer.
12+
* Redistributions in binary form must reproduce the above copyright
13+
notice, this list of conditions and the following disclaimer in the
14+
documentation and/or other materials provided with the distribution.
15+
* Neither the name of the NIPPON TELEGRAPH AND TELEPHONE CORPORATION
16+
(NTT) nor the names of its contributors may be used to endorse or
17+
promote products derived from this software without specific prior
18+
written permission.
19+
20+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

‎contrib/pg_probackup/Makefile

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
PROGRAM = pg_probackup
2+
OBJS = src/backup.o src/catalog.o src/configure.o src/data.o \
3+
src/delete.o src/dir.o src/fetch.o src/help.o src/init.o \
4+
src/pg_probackup.o src/restore.o src/show.o src/status.o \
5+
src/util.o src/validate.o src/datapagemap.o src/parsexlog.o \
6+
src/xlogreader.o src/streamutil.o src/receivelog.o \
7+
src/archive.o src/utils/parray.o src/utils/pgut.o src/utils/logger.o
8+
9+
EXTRA_CLEAN = src/datapagemap.c src/datapagemap.h src/xlogreader.c \
10+
src/receivelog.c src/receivelog.h src/streamutil.c src/streamutil.h src/logging.h
11+
12+
INCLUDES = src/datapagemap.h src/logging.h src/receivelog.h src/streamutil.h
13+
14+
ifdef USE_PGXS
15+
PG_CONFIG = pg_config
16+
PGXS := $(shell $(PG_CONFIG) --pgxs)
17+
include $(PGXS)
18+
# !USE_PGXS
19+
else
20+
subdir=contrib/pg_probackup
21+
top_builddir=../..
22+
include $(top_builddir)/src/Makefile.global
23+
include $(top_srcdir)/contrib/contrib-global.mk
24+
endif # USE_PGXS
25+
26+
ifeq ($(top_srcdir),../..)
27+
srchome=$(top_srcdir)/..
28+
else
29+
srchome=$(top_srcdir)
30+
endif
31+
32+
ifeq ($(MAJORVERSION),10)
33+
OBJS += src/walmethods.o
34+
EXTRA_CLEAN += src/walmethods.c src/walmethods.h
35+
INCLUDES += src/walmethods.h
36+
endif
37+
38+
PG_CPPFLAGS = -I$(libpq_srcdir) ${PTHREAD_CFLAGS} -Isrc
39+
override CPPFLAGS := -DFRONTEND $(CPPFLAGS) $(PG_CPPFLAGS)
40+
PG_LIBS = $(libpq_pgport) ${PTHREAD_CFLAGS}
41+
42+
all: checksrcdir $(INCLUDES);
43+
44+
$(PROGRAM): $(OBJS)
45+
46+
src/xlogreader.c: $(top_srcdir)/src/backend/access/transam/xlogreader.c
47+
rm -f $@ && $(LN_S) $(srchome)/src/backend/access/transam/xlogreader.c $@
48+
src/datapagemap.c: $(top_srcdir)/src/bin/pg_rewind/datapagemap.c
49+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.c $@
50+
src/datapagemap.h: $(top_srcdir)/src/bin/pg_rewind/datapagemap.h
51+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/datapagemap.h $@
52+
src/logging.h: $(top_srcdir)/src/bin/pg_rewind/logging.h
53+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_rewind/logging.h $@
54+
src/receivelog.c: $(top_srcdir)/src/bin/pg_basebackup/receivelog.c
55+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.c $@
56+
src/receivelog.h: $(top_srcdir)/src/bin/pg_basebackup/receivelog.h
57+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/receivelog.h $@
58+
src/streamutil.c: $(top_srcdir)/src/bin/pg_basebackup/streamutil.c
59+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.c $@
60+
src/streamutil.h: $(top_srcdir)/src/bin/pg_basebackup/streamutil.h
61+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/streamutil.h $@
62+
63+
ifeq ($(MAJORVERSION),10)
64+
src/walmethods.c: $(top_srcdir)/src/bin/pg_basebackup/walmethods.c
65+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.c $@
66+
src/walmethods.h: $(top_srcdir)/src/bin/pg_basebackup/walmethods.h
67+
rm -f $@ && $(LN_S) $(srchome)/src/bin/pg_basebackup/walmethods.h $@
68+
endif
69+
70+
ifeq ($(PORTNAME), aix)
71+
CC=xlc_r
72+
endif
73+
74+
# This rule's only purpose is to give the user instructions on how to pass
75+
# the path to PostgreSQL source tree to the makefile.
76+
.PHONY: checksrcdir
77+
checksrcdir:
78+
ifndef top_srcdir
79+
@echo "You must have PostgreSQL source tree available to compile."
80+
@echo "Pass the path to the PostgreSQL source tree to make, in the top_srcdir"
81+
@echo "variable: \"make top_srcdir=<path to PostgreSQL source tree>\""
82+
@exit 1
83+
endif

‎contrib/pg_probackup/README.md

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
# pg_probackup
2+
3+
`pg_probackup` is a utility to manage backup and recovery of PostgreSQL database clusters. It is designed to perform periodic backups of the PostgreSQL instance that enable you to restore the server in case of a failure.
4+
5+
The extension is compatible with:
6+
* PostgreSQL 9.5, 9.6, 10;
7+
8+
`PTRACK` backup requires:
9+
* Postgres Pro Standard 9.5, 9.6;
10+
or
11+
* Postgres Pro Enterprise;
12+
13+
As compared to other backup solutions, `pg_probackup` offers the following benefits that can help you implement different backup strategies and deal with large amounts of data:
14+
* Choosing between full and page-level incremental backups to speed up backup and recovery
15+
* Implementing a single backup strategy for multi-server PostgreSQL clusters
16+
* Automatic data consistency checks and on-demand backup validation without actual data recovery
17+
* Managing backups in accordance with retention policy
18+
* Running backup, restore, and validation processes on multiple parallel threads
19+
* Storing backup data in a compressed state to save disk space
20+
* Taking backups from a standby server to avoid extra load on the master server
21+
* Extended logging settings
22+
* Custom commands to simplify WAL log archiving
23+
24+
To manage backup data, `pg_probackup` creates a backup catalog. This directory stores all backup files with additional meta information, as well as WAL archives required for [point-in-time recovery](https://postgrespro.com/docs/postgresql/current/continuous-archiving.html). You can store backups for different instances in separate subdirectories of a single backup catalog.
25+
26+
Using `pg_probackup`, you can take full or incremental backups:
27+
* `Full` backups contain all the data files required to restore the database cluster from scratch.
28+
* `Incremental` backups only store the data that has changed since the previous backup. It allows to decrease the backup size and speed up backup operations. `pg_probackup` supports the following modes of incremental backups:
29+
* `PAGE` backup. In this mode, `pg_probackup` scans all WAL files in the archive from the moment the previous full or incremental backup was taken. Newly created backups contain only the pages that were mentioned in WAL records. This requires all the WAL files since the previous backup to be present in the WAL archive. If the size of these files is comparable to the total size of the database cluster files, speedup is smaller, but the backup still takes less space.
30+
* `PTRACK` backup. In this mode, PostgreSQL tracks page changes on the fly. Continuous archiving is not necessary for it to operate. Each time a relation page is updated, this page is marked in a special `PTRACK` bitmap for this relation. As one page requires just one bit in the `PTRACK` fork, such bitmaps are quite small. Tracking implies some minor overhead on the database server operation, but speeds up incremental backups significantly.
31+
32+
Regardless of the chosen backup type, all backups taken with `pg_probackup` support the following archiving strategies:
33+
* `Autonomous backups` include all the files required to restore the cluster to a consistent state at the time the backup was taken. Even if continuous archiving is not set up, the required WAL segments are included into the backup.
34+
* `Archive backups` rely on continuous archiving. Such backups enable cluster recovery to an arbitrary point after the backup was taken (point-in-time recovery).
35+
36+
## Limitations
37+
38+
`pg_probackup` currently has the following limitations:
39+
* Creating backups from a remote server is currently not supported.
40+
* The server from which the backup was taken and the restored server must be compatible by the [block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-block-size) and [wal_block_size](https://postgrespro.com/docs/postgresql/current/runtime-config-preset#guc-wal-block-size) parameters and have the same major release number.
41+
* Microsoft Windows operating system is not supported.
42+
* Configuration files outside of PostgreSQL data directory are not included into the backup and should be backed up separately.
43+
44+
## Installation and Setup
45+
46+
To compile `pg_probackup`, you must have a PostgreSQL installation and raw source tree. To install `pg_probackup`, execute this in the module's directory:
47+
48+
```shell
49+
make USE_PGXS=1 PG_CONFIG=<path_to_pg_config> top_srcdir=<path_to_PostgreSQL_source_tree>
50+
```
51+
52+
Once you have `pg_probackup` installed, complete [the setup](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup.html#pg-probackup-install-and-setup).
53+
54+
## Documentation
55+
56+
Currently the latest documentation can be found at [Postgres Pro Enterprise documentation](https://postgrespro.com/docs/postgrespro/current/app-pgprobackup).
57+
58+
## Licence
59+
60+
This module available under the same license as [PostgreSQL](https://www.postgresql.org/about/licence/).
61+
62+
## Feedback
63+
64+
Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_probackup/issues) page.
65+
66+
## Authors
67+
68+
Postgres Professional, Moscow, Russia.
69+
70+
## Credits
71+
72+
`pg_probackup` utility is based on `pg_arman`, that was originally written by NTT and then developed and maintained by Michael Paquier.

‎contrib/pg_probackup/src/archive.c

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
/*-------------------------------------------------------------------------
2+
*
3+
* archive.c: - pg_probackup specific archive commands for archive backups.
4+
*
5+
*
6+
* Portions Copyright (c) 2017, Postgres Professional
7+
*
8+
*-------------------------------------------------------------------------
9+
*/
10+
#include "pg_probackup.h"
11+
12+
#include <unistd.h>
13+
#include <sys/stat.h>
14+
15+
/*
16+
* pg_probackup specific archive command for archive backups
17+
* set archive_command = 'pg_probackup archive-push -B /home/anastasia/backup
18+
* --wal-file-path %p --wal-file-name %f', to move backups into arclog_path.
19+
* Where archlog_path is $BACKUP_PATH/wal/system_id.
20+
* Currently it just copies wal files to the new location.
21+
* TODO: Planned options: compress, list the arclog content,
22+
* compute and validate checksums.
23+
*/
24+
int
25+
do_archive_push(char *wal_file_path, char *wal_file_name)
26+
{
27+
char backup_wal_file_path[MAXPGPATH];
28+
char absolute_wal_file_path[MAXPGPATH];
29+
char current_dir[MAXPGPATH];
30+
int64 system_id;
31+
pgBackupConfig *config;
32+
33+
if (wal_file_name == NULL && wal_file_path == NULL)
34+
elog(ERROR, "required parameters are not specified: --wal_file_name %%f --wal_file_path %%p");
35+
36+
if (wal_file_name == NULL)
37+
elog(ERROR, "required parameter not specified: --wal_file_name %%f");
38+
39+
if (wal_file_path == NULL)
40+
elog(ERROR, "required parameter not specified: --wal_file_path %%p");
41+
42+
if (!getcwd(current_dir, sizeof(current_dir)))
43+
elog(ERROR, "getcwd() error");
44+
45+
/* verify that archive-push --instance parameter is valid */
46+
config = readBackupCatalogConfigFile();
47+
system_id = get_system_identifier(current_dir);
48+
49+
if (config->pgdata == NULL)
50+
elog(ERROR, "cannot read pg_probackup.conf for this instance");
51+
52+
if(system_id != config->system_identifier)
53+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
54+
"Instance '%s' should have SYSTEM_ID = %ld instead of %ld",
55+
wal_file_name, instance_name, config->system_identifier, system_id);
56+
57+
if (strcmp(current_dir, config->pgdata) != 0)
58+
elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch."
59+
"Instance '%s' should have PGDATA = %s instead of %s",
60+
wal_file_name, instance_name, config->pgdata, current_dir);
61+
62+
/* Create 'archlog_path' directory. Do nothing if it already exists. */
63+
dir_create_dir(arclog_path, DIR_PERMISSION);
64+
65+
join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
66+
join_path_components(backup_wal_file_path, arclog_path, wal_file_name);
67+
68+
elog(INFO, "pg_probackup archive-push from %s to %s", absolute_wal_file_path, backup_wal_file_path);
69+
if (access(backup_wal_file_path, F_OK) != -1)
70+
elog(ERROR, "file '%s', already exists.", backup_wal_file_path);
71+
72+
copy_wal_file(absolute_wal_file_path, backup_wal_file_path);
73+
elog(INFO, "pg_probackup archive-push completed successfully");
74+
75+
return 0;
76+
}
77+
78+
/*
79+
* pg_probackup specific restore command.
80+
* Move files from arclog_path to pgdata/wal_file_path.
81+
*/
82+
int
83+
do_archive_get(char *wal_file_path, char *wal_file_name)
84+
{
85+
char backup_wal_file_path[MAXPGPATH];
86+
char absolute_wal_file_path[MAXPGPATH];
87+
char current_dir[MAXPGPATH];
88+
89+
if (wal_file_name == NULL && wal_file_path == NULL)
90+
elog(ERROR, "required parameters are not specified: --wal_file_name %%f --wal_file_path %%p");
91+
92+
if (wal_file_name == NULL)
93+
elog(ERROR, "required parameter not specified: --wal_file_name %%f");
94+
95+
if (wal_file_path == NULL)
96+
elog(ERROR, "required parameter not specified: --wal_file_path %%p");
97+
98+
if (!getcwd(current_dir, sizeof(current_dir)))
99+
elog(ERROR, "getcwd() error");
100+
101+
join_path_components(absolute_wal_file_path, current_dir, wal_file_path);
102+
join_path_components(backup_wal_file_path, arclog_path, wal_file_name);
103+
104+
elog(INFO, "pg_probackup archive-get from %s to %s", backup_wal_file_path, absolute_wal_file_path);
105+
copy_wal_file(backup_wal_file_path, absolute_wal_file_path);
106+
elog(INFO, "pg_probackup archive-get completed successfully");
107+
108+
return 0;
109+
}

0 commit comments

Comments
 (0)