Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 8694cc9

Browse files
committed
Exclude unlogged tables from base backups
Exclude unlogged tables from base backup entirely except init fork which marks created unlogged table. The next question is do not backup temp table but it's a story for separate patch. Author: David Steele Review by: Adam Brightwell, Masahiko Sawada Discussion: https://www.postgresql.org/message-id/flat/04791bab-cb04-ba43-e9c0-664a4c1ffb2c@pgmasters.net
1 parent 7ba7986 commit 8694cc9

File tree

5 files changed

+113
-5
lines changed

5 files changed

+113
-5
lines changed

doc/src/sgml/protocol.sgml

+6
Original file line numberDiff line numberDiff line change
@@ -2568,6 +2568,12 @@ The commands accepted in replication mode are:
25682568
with <filename>pgsql_tmp</filename>.
25692569
</para>
25702570
</listitem>
2571+
<listitem>
2572+
<para>
2573+
Unlogged relations, except for the init fork which is required to
2574+
recreate the (empty) unlogged relation on recovery.
2575+
</para>
2576+
</listitem>
25712577
<listitem>
25722578
<para>
25732579
<filename>pg_wal</filename>, including subdirectories. If the backup is run

src/backend/replication/basebackup.c

+64
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,15 @@
2626
#include "nodes/pg_list.h"
2727
#include "pgtar.h"
2828
#include "pgstat.h"
29+
#include "port.h"
2930
#include "postmaster/syslogger.h"
3031
#include "replication/basebackup.h"
3132
#include "replication/walsender.h"
3233
#include "replication/walsender_private.h"
3334
#include "storage/dsm_impl.h"
3435
#include "storage/fd.h"
3536
#include "storage/ipc.h"
37+
#include "storage/reinit.h"
3638
#include "utils/builtins.h"
3739
#include "utils/ps_status.h"
3840
#include "utils/relcache.h"
@@ -958,12 +960,44 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
958960
char pathbuf[MAXPGPATH * 2];
959961
struct stat statbuf;
960962
int64 size = 0;
963+
const char *lastDir; /* Split last dir from parent path. */
964+
bool isDbDir = false; /* Does this directory contain relations? */
965+
966+
/*
967+
* Determine if the current path is a database directory that can
968+
* contain relations.
969+
*
970+
* Start by finding the location of the delimiter between the parent
971+
* path and the current path.
972+
*/
973+
lastDir = last_dir_separator(path);
974+
975+
/* Does this path look like a database path (i.e. all digits)? */
976+
if (lastDir != NULL &&
977+
strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
978+
{
979+
/* Part of path that contains the parent directory. */
980+
int parentPathLen = lastDir - path;
981+
982+
/*
983+
* Mark path as a database directory if the parent path is either
984+
* $PGDATA/base or a tablespace version path.
985+
*/
986+
if (strncmp(path, "./base", parentPathLen) == 0 ||
987+
(parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
988+
strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
989+
TABLESPACE_VERSION_DIRECTORY,
990+
sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
991+
isDbDir = true;
992+
}
961993

962994
dir = AllocateDir(path);
963995
while ((de = ReadDir(dir, path)) != NULL)
964996
{
965997
int excludeIdx;
966998
bool excludeFound;
999+
ForkNumber relForkNum; /* Type of fork if file is a relation */
1000+
int relOidChars; /* Chars in filename that are the rel oid */
9671001

9681002
/* Skip special stuff */
9691003
if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
@@ -1007,6 +1041,36 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
10071041
if (excludeFound)
10081042
continue;
10091043

1044+
/* Exclude all forks for unlogged tables except the init fork */
1045+
if (isDbDir &&
1046+
parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
1047+
&relForkNum))
1048+
{
1049+
/* Never exclude init forks */
1050+
if (relForkNum != INIT_FORKNUM)
1051+
{
1052+
char initForkFile[MAXPGPATH];
1053+
char relOid[OIDCHARS + 1];
1054+
1055+
/*
1056+
* If any other type of fork, check if there is an init fork
1057+
* with the same OID. If so, the file can be excluded.
1058+
*/
1059+
strncpy(relOid, de->d_name, relOidChars);
1060+
snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
1061+
path, relOid);
1062+
1063+
if (lstat(initForkFile, &statbuf) == 0)
1064+
{
1065+
elog(DEBUG2,
1066+
"unlogged relation file \"%s\" excluded from backup",
1067+
de->d_name);
1068+
1069+
continue;
1070+
}
1071+
}
1072+
}
1073+
10101074
snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
10111075

10121076
/* Skip pg_control here to back up it last */

src/backend/storage/file/reinit.c

+1-3
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@ static void ResetUnloggedRelationsInTablespaceDir(const char *tsdirname,
2828
int op);
2929
static void ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname,
3030
int op);
31-
static bool parse_filename_for_nontemp_relation(const char *name,
32-
int *oidchars, ForkNumber *fork);
3331

3432
typedef struct
3533
{
@@ -373,7 +371,7 @@ ResetUnloggedRelationsInDbspaceDir(const char *dbspacedirname, int op)
373371
* portion of the filename. This is critical to protect against a possible
374372
* buffer overrun.
375373
*/
376-
static bool
374+
bool
377375
parse_filename_for_nontemp_relation(const char *name, int *oidchars,
378376
ForkNumber *fork)
379377
{

src/bin/pg_basebackup/t/010_pg_basebackup.pl

+40-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
use Config;
55
use PostgresNode;
66
use TestLib;
7-
use Test::More tests => 79;
7+
use Test::More tests => 87;
88

99
program_help_ok('pg_basebackup');
1010
program_version_ok('pg_basebackup');
@@ -66,6 +66,16 @@
6666
# positive.
6767
$node->safe_psql('postgres', 'SELECT 1;');
6868

69+
# Create an unlogged table to test that forks other than init are not copied.
70+
$node->safe_psql('postgres', 'CREATE UNLOGGED TABLE base_unlogged (id int)');
71+
72+
my $baseUnloggedPath = $node->safe_psql('postgres',
73+
q{select pg_relation_filepath('base_unlogged')});
74+
75+
# Make sure main and init forks exist
76+
ok(-f "$pgdata/${baseUnloggedPath}_init", 'unlogged init fork in base');
77+
ok(-f "$pgdata/$baseUnloggedPath", 'unlogged main fork in base');
78+
6979
$node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup", '-X', 'none' ],
7080
'pg_basebackup runs');
7181
ok(-f "$tempdir/backup/PG_VERSION", 'backup was created');
@@ -96,6 +106,12 @@
96106
ok(!-f "$tempdir/backup/$filename", "$filename not copied");
97107
}
98108

109+
# Unlogged relation forks other than init should not be copied
110+
ok(-f "$tempdir/backup/${baseUnloggedPath}_init",
111+
'unlogged init fork in backup');
112+
ok(!-f "$tempdir/backup/$baseUnloggedPath",
113+
'unlogged main fork not in backup');
114+
99115
# Make sure existing backup_label was ignored.
100116
isnt(slurp_file("$tempdir/backup/backup_label"),
101117
'DONOTCOPY', 'existing backup_label not copied');
@@ -147,7 +163,7 @@
147163
# skip on Windows.
148164
SKIP:
149165
{
150-
skip "symlinks not supported on Windows", 11 if ($windows_os);
166+
skip "symlinks not supported on Windows", 15 if ($windows_os);
151167

152168
# Move pg_replslot out of $pgdata and create a symlink to it.
153169
$node->stop;
@@ -177,6 +193,19 @@
177193
my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar";
178194
is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');
179195

196+
# Create an unlogged table to test that forks other than init are not copied.
197+
$node->safe_psql('postgres',
198+
'CREATE UNLOGGED TABLE tblspc1_unlogged (id int) TABLESPACE tblspc1;');
199+
200+
my $tblspc1UnloggedPath = $node->safe_psql(
201+
'postgres', q{select pg_relation_filepath('tblspc1_unlogged')});
202+
203+
# Make sure main and init forks exist
204+
ok(-f "$pgdata/${tblspc1UnloggedPath}_init",
205+
'unlogged init fork in tablespace');
206+
ok(-f "$pgdata/$tblspc1UnloggedPath",
207+
'unlogged main fork in tablespace');
208+
180209
$node->command_fails(
181210
[ 'pg_basebackup', '-D', "$tempdir/backup1", '-Fp' ],
182211
'plain format with tablespaces fails without tablespace mapping');
@@ -195,11 +224,20 @@
195224
"tablespace symlink was updated");
196225
closedir $dh;
197226

227+
# Unlogged relation forks other than init should not be copied
228+
my ($tblspc1UnloggedBackupPath) = $tblspc1UnloggedPath =~ /[^\/]*\/[^\/]*\/[^\/]*$/g;
229+
230+
ok(-f "$tempdir/tbackup/tblspc1/${tblspc1UnloggedBackupPath}_init",
231+
'unlogged init fork in tablespace backup');
232+
ok(!-f "$tempdir/tbackup/tblspc1/$tblspc1UnloggedBackupPath",
233+
'unlogged main fork not in tablespace backup');
234+
198235
ok( -d "$tempdir/backup1/pg_replslot",
199236
'pg_replslot symlink copied as directory');
200237

201238
mkdir "$tempdir/tbl=spc2";
202239
$node->safe_psql('postgres', "DROP TABLE test1;");
240+
$node->safe_psql('postgres', "DROP TABLE tblspc1_unlogged;");
203241
$node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");
204242
$node->safe_psql('postgres',
205243
"CREATE TABLESPACE tblspc2 LOCATION '$shorter_tempdir/tbl=spc2';");

src/include/storage/reinit.h

+2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
#define REINIT_H
1717

1818
extern void ResetUnloggedRelations(int op);
19+
extern bool parse_filename_for_nontemp_relation(
20+
const char *name, int *oidchars, ForkNumber *fork);
1921

2022
#define UNLOGGED_RELATION_CLEANUP 0x0001
2123
#define UNLOGGED_RELATION_INIT 0x0002

0 commit comments

Comments
 (0)