Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit d93627b

Browse files
committed
Add --copy-file-range option to pg_upgrade.
The copy_file_range() system call is available on at least Linux and FreeBSD, and asks the kernel to use efficient ways to copy ranges of a file. Options available to the kernel include sharing block ranges (similar to --clone mode), and pushing down block copies to the storage layer. For automated testing, see PG_TEST_PG_UPGRADE_MODE. (Perhaps in a later commit we could consider setting this mode for one of the CI targets.) Reviewed-by: Peter Eisentraut <peter@eisentraut.org> Discussion: https://postgr.es/m/CA%2BhUKGKe7Hb0-UNih8VD5UNZy5-ojxFb3Pr3xSBBL8qj2M2%3DdQ%40mail.gmail.com
1 parent 2bce0ad commit d93627b

File tree

11 files changed

+120
-4
lines changed

11 files changed

+120
-4
lines changed

configure

+1-1
Original file line numberDiff line numberDiff line change
@@ -15259,7 +15259,7 @@ fi
1525915259
LIBS_including_readline="$LIBS"
1526015260
LIBS=`echo "$LIBS" | sed -e 's/-ledit//g' -e 's/-lreadline//g'`
1526115261

15262-
for ac_func in backtrace_symbols copyfile getifaddrs getpeerucred inet_pton kqueue mbstowcs_l memset_s posix_fallocate ppoll pthread_is_threaded_np setproctitle setproctitle_fast strchrnul strsignal syncfs sync_file_range uselocale wcstombs_l
15262+
for ac_func in backtrace_symbols copyfile copy_file_range getifaddrs getpeerucred inet_pton kqueue mbstowcs_l memset_s posix_fallocate ppoll pthread_is_threaded_np setproctitle setproctitle_fast strchrnul strsignal syncfs sync_file_range uselocale wcstombs_l
1526315263
do :
1526415264
as_ac_var=`$as_echo "ac_cv_func_$ac_func" | $as_tr_sh`
1526515265
ac_fn_c_check_func "$LINENO" "$ac_func" "$as_ac_var"

configure.ac

+1
Original file line numberDiff line numberDiff line change
@@ -1749,6 +1749,7 @@ LIBS=`echo "$LIBS" | sed -e 's/-ledit//g' -e 's/-lreadline//g'`
17491749
AC_CHECK_FUNCS(m4_normalize([
17501750
backtrace_symbols
17511751
copyfile
1752+
copy_file_range
17521753
getifaddrs
17531754
getpeerucred
17541755
inet_pton

doc/src/sgml/ref/pgupgrade.sgml

+13
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,19 @@ PostgreSQL documentation
263263
</listitem>
264264
</varlistentry>
265265

266+
<varlistentry>
267+
<term><option>--copy-file-range</option></term>
268+
<listitem>
269+
<para>
270+
Use the <function>copy_file_range</function> system call for efficient
271+
copying. On some file systems this gives results similar to
272+
<option>--clone</option>, sharing physical disk blocks, while on others
273+
it may still copy blocks, but do so via an optimized path. At present,
274+
it is supported on Linux and FreeBSD.
275+
</para>
276+
</listitem>
277+
</varlistentry>
278+
266279
<varlistentry>
267280
<term><option>-?</option></term>
268281
<term><option>--help</option></term>

meson.build

+1
Original file line numberDiff line numberDiff line change
@@ -2420,6 +2420,7 @@ func_checks = [
24202420
['backtrace_symbols', {'dependencies': [execinfo_dep]}],
24212421
['clock_gettime', {'dependencies': [rt_dep], 'define': false}],
24222422
['copyfile'],
2423+
['copy_file_range'],
24232424
# gcc/clang's sanitizer helper library provides dlopen but not dlsym, thus
24242425
# when enabling asan the dlopen check doesn't notice that -ldl is actually
24252426
# required. Just checking for dlsym() ought to suffice.

src/bin/pg_upgrade/TESTING

+2-2
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,8 @@ export oldinstall=...otherversion/ (old version's install base path)
2020
See DETAILS below for more information about creation of the dump.
2121

2222
You can also test the different transfer modes (--copy, --link,
23-
--clone) by setting the environment variable PG_TEST_PG_UPGRADE_MODE
24-
to the respective command-line option, like
23+
--clone, --copy-file-range) by setting the environment variable
24+
PG_TEST_PG_UPGRADE_MODE to the respective command-line option, like
2525

2626
make check PG_TEST_PG_UPGRADE_MODE=--link
2727

src/bin/pg_upgrade/check.c

+3
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,9 @@ check_new_cluster(void)
235235
break;
236236
case TRANSFER_MODE_COPY:
237237
break;
238+
case TRANSFER_MODE_COPY_FILE_RANGE:
239+
check_copy_file_range();
240+
break;
238241
case TRANSFER_MODE_LINK:
239242
check_hard_link();
240243
break;

src/bin/pg_upgrade/file.c

+78
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include "postgres_fe.h"
1111

1212
#include <sys/stat.h>
13+
#include <limits.h>
1314
#include <fcntl.h>
1415
#ifdef HAVE_COPYFILE_H
1516
#include <copyfile.h>
@@ -140,6 +141,45 @@ copyFile(const char *src, const char *dst,
140141
}
141142

142143

144+
/*
145+
* copyFileByRange()
146+
*
147+
* Copies a relation file from src to dst.
148+
* schemaName/relName are relation's SQL name (used for error messages only).
149+
*/
150+
void
151+
copyFileByRange(const char *src, const char *dst,
152+
const char *schemaName, const char *relName)
153+
{
154+
#ifdef HAVE_COPY_FILE_RANGE
155+
int src_fd;
156+
int dest_fd;
157+
ssize_t nbytes;
158+
159+
if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
160+
pg_fatal("error while copying relation \"%s.%s\": could not open file \"%s\": %s",
161+
schemaName, relName, src, strerror(errno));
162+
163+
if ((dest_fd = open(dst, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
164+
pg_file_create_mode)) < 0)
165+
pg_fatal("error while copying relation \"%s.%s\": could not create file \"%s\": %s",
166+
schemaName, relName, dst, strerror(errno));
167+
168+
do
169+
{
170+
nbytes = copy_file_range(src_fd, NULL, dest_fd, NULL, SSIZE_MAX, 0);
171+
if (nbytes < 0)
172+
pg_fatal("error while copying relation \"%s.%s\": could not copy file range from \"%s\" to \"%s\": %s",
173+
schemaName, relName, src, dst, strerror(errno));
174+
}
175+
while (nbytes > 0);
176+
177+
close(src_fd);
178+
close(dest_fd);
179+
#endif
180+
}
181+
182+
143183
/*
144184
* linkFile()
145185
*
@@ -358,6 +398,44 @@ check_file_clone(void)
358398
unlink(new_link_file);
359399
}
360400

401+
void
402+
check_copy_file_range(void)
403+
{
404+
char existing_file[MAXPGPATH];
405+
char new_link_file[MAXPGPATH];
406+
407+
snprintf(existing_file, sizeof(existing_file), "%s/PG_VERSION", old_cluster.pgdata);
408+
snprintf(new_link_file, sizeof(new_link_file), "%s/PG_VERSION.copy_file_range_test", new_cluster.pgdata);
409+
unlink(new_link_file); /* might fail */
410+
411+
#if defined(HAVE_COPY_FILE_RANGE)
412+
{
413+
int src_fd;
414+
int dest_fd;
415+
416+
if ((src_fd = open(existing_file, O_RDONLY | PG_BINARY, 0)) < 0)
417+
pg_fatal("could not open file \"%s\": %s",
418+
existing_file, strerror(errno));
419+
420+
if ((dest_fd = open(new_link_file, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
421+
pg_file_create_mode)) < 0)
422+
pg_fatal("could not create file \"%s\": %s",
423+
new_link_file, strerror(errno));
424+
425+
if (copy_file_range(src_fd, NULL, dest_fd, NULL, SSIZE_MAX, 0) < 0)
426+
pg_fatal("could not copy file range between old and new data directories: %s",
427+
strerror(errno));
428+
429+
close(src_fd);
430+
close(dest_fd);
431+
}
432+
#else
433+
pg_fatal("copy_file_range not supported on this platform");
434+
#endif
435+
436+
unlink(new_link_file);
437+
}
438+
361439
void
362440
check_hard_link(void)
363441
{

src/bin/pg_upgrade/option.c

+6-1
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ parseCommandLine(int argc, char *argv[])
5858
{"verbose", no_argument, NULL, 'v'},
5959
{"clone", no_argument, NULL, 1},
6060
{"copy", no_argument, NULL, 2},
61-
{"sync-method", required_argument, NULL, 3},
61+
{"copy-file-range", no_argument, NULL, 3},
62+
{"sync-method", required_argument, NULL, 4},
6263

6364
{NULL, 0, NULL, 0}
6465
};
@@ -203,6 +204,9 @@ parseCommandLine(int argc, char *argv[])
203204
break;
204205

205206
case 3:
207+
user_opts.transfer_mode = TRANSFER_MODE_COPY_FILE_RANGE;
208+
break;
209+
case 4:
206210
if (!parse_sync_method(optarg, &unused))
207211
exit(1);
208212
user_opts.sync_method = pg_strdup(optarg);
@@ -301,6 +305,7 @@ usage(void)
301305
printf(_(" -V, --version display version information, then exit\n"));
302306
printf(_(" --clone clone instead of copying files to new cluster\n"));
303307
printf(_(" --copy copy files to new cluster (default)\n"));
308+
printf(_(" --copy-file-range copy files to new cluster with copy_file_range\n"));
304309
printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
305310
printf(_(" -?, --help show this help, then exit\n"));
306311
printf(_("\n"

src/bin/pg_upgrade/pg_upgrade.h

+4
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@ typedef enum
256256
{
257257
TRANSFER_MODE_CLONE,
258258
TRANSFER_MODE_COPY,
259+
TRANSFER_MODE_COPY_FILE_RANGE,
259260
TRANSFER_MODE_LINK,
260261
} transferMode;
261262

@@ -402,11 +403,14 @@ void cloneFile(const char *src, const char *dst,
402403
const char *schemaName, const char *relName);
403404
void copyFile(const char *src, const char *dst,
404405
const char *schemaName, const char *relName);
406+
void copyFileByRange(const char *src, const char *dst,
407+
const char *schemaName, const char *relName);
405408
void linkFile(const char *src, const char *dst,
406409
const char *schemaName, const char *relName);
407410
void rewriteVisibilityMap(const char *fromfile, const char *tofile,
408411
const char *schemaName, const char *relName);
409412
void check_file_clone(void);
413+
void check_copy_file_range(void);
410414
void check_hard_link(void);
411415

412416
/* fopen_priv() is no longer different from fopen() */

src/bin/pg_upgrade/relfilenumber.c

+8
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@ transfer_all_new_tablespaces(DbInfoArr *old_db_arr, DbInfoArr *new_db_arr,
3737
case TRANSFER_MODE_COPY:
3838
prep_status_progress("Copying user relation files");
3939
break;
40+
case TRANSFER_MODE_COPY_FILE_RANGE:
41+
prep_status_progress("Copying user relation files with copy_file_range");
42+
break;
4043
case TRANSFER_MODE_LINK:
4144
prep_status_progress("Linking user relation files");
4245
break;
@@ -250,6 +253,11 @@ transfer_relfile(FileNameMap *map, const char *type_suffix, bool vm_must_add_fro
250253
old_file, new_file);
251254
copyFile(old_file, new_file, map->nspname, map->relname);
252255
break;
256+
case TRANSFER_MODE_COPY_FILE_RANGE:
257+
pg_log(PG_VERBOSE, "copying \"%s\" to \"%s\" with copy_file_range",
258+
old_file, new_file);
259+
copyFileByRange(old_file, new_file, map->nspname, map->relname);
260+
break;
253261
case TRANSFER_MODE_LINK:
254262
pg_log(PG_VERBOSE, "linking \"%s\" to \"%s\"",
255263
old_file, new_file);

src/include/pg_config.h.in

+3
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@
7878
/* Define to 1 if you have the <copyfile.h> header file. */
7979
#undef HAVE_COPYFILE_H
8080

81+
/* Define to 1 if you have the `copy_file_range' function. */
82+
#undef HAVE_COPY_FILE_RANGE
83+
8184
/* Define to 1 if you have the <crtdefs.h> header file. */
8285
#undef HAVE_CRTDEFS_H
8386

0 commit comments

Comments
 (0)