Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 73fbf3d

Browse files
committed
pg_rewind: Fix some problems when copying files >2GB.
When incrementally updating a file larger than 2GB, the old code could either fail outright (if the client asked the server for bytes beyond the 2GB boundary) or fail to copy all the blocks that had actually been modified (if the server reported a file size to the client in excess of 2GB), resulting in data corruption. Generally, such files won't occur anyway, but they might if using a non-default segment size or if there the directory contains stray files unrelated to PostgreSQL. Fix by a more prudent choice of data types. Even with these improvements, this code still uses a mix of different types (off_t, size_t, uint64, int64) to represent file sizes and offsets, not all of which necessarily have the same width or signedness, so further cleanup might be in order here. However, at least now they all have the potential to be 64 bits wide on 64-bit platforms. Kuntal Ghosh and Michael Paquier, with a tweak by me. Discussion: http://postgr.es/m/CAGz5QC+8gbkz=Brp0TgoKNqHWTzonbPtPex80U0O6Uh_bevbaA@mail.gmail.com
1 parent afd56b8 commit 73fbf3d

File tree

1 file changed

+35
-12
lines changed

1 file changed

+35
-12
lines changed

src/bin/pg_rewind/libpq_fetch.c

+35-12
Original file line numberDiff line numberDiff line change
@@ -195,7 +195,7 @@ libpqProcessFileList(void)
195195
for (i = 0; i < PQntuples(res); i++)
196196
{
197197
char *path = PQgetvalue(res, i, 0);
198-
int filesize = atoi(PQgetvalue(res, i, 1));
198+
int64 filesize = atol(PQgetvalue(res, i, 1));
199199
bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0);
200200
char *link_target = PQgetvalue(res, i, 3);
201201
file_type_t type;
@@ -221,13 +221,35 @@ libpqProcessFileList(void)
221221
PQclear(res);
222222
}
223223

224+
/*
225+
* Converts an int64 from network byte order to native format.
226+
*/
227+
static int64
228+
pg_recvint64(int64 value)
229+
{
230+
union
231+
{
232+
int64 i64;
233+
uint32 i32[2];
234+
} swap;
235+
int64 result;
236+
237+
swap.i64 = value;
238+
239+
result = (uint32) ntohl(swap.i32[0]);
240+
result <<= 32;
241+
result |= (uint32) ntohl(swap.i32[1]);
242+
243+
return result;
244+
}
245+
224246
/*----
225247
* Runs a query, which returns pieces of files from the remote source data
226248
* directory, and overwrites the corresponding parts of target files with
227249
* the received parts. The result set is expected to be of format:
228250
*
229251
* path text -- path in the data directory, e.g "base/1/123"
230-
* begin int4 -- offset within the file
252+
* begin int8 -- offset within the file
231253
* chunk bytea -- file content
232254
*----
233255
*/
@@ -248,7 +270,7 @@ receiveFileChunks(const char *sql)
248270
{
249271
char *filename;
250272
int filenamelen;
251-
int chunkoff;
273+
int64 chunkoff;
252274
int chunksize;
253275
char *chunk;
254276

@@ -271,7 +293,7 @@ receiveFileChunks(const char *sql)
271293
pg_fatal("unexpected result set size while fetching remote files\n");
272294

273295
if (PQftype(res, 0) != TEXTOID ||
274-
PQftype(res, 1) != INT4OID ||
296+
PQftype(res, 1) != INT8OID ||
275297
PQftype(res, 2) != BYTEAOID)
276298
{
277299
pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n",
@@ -291,12 +313,12 @@ receiveFileChunks(const char *sql)
291313
pg_fatal("unexpected null values in result while fetching remote files\n");
292314
}
293315

294-
if (PQgetlength(res, 0, 1) != sizeof(int32))
316+
if (PQgetlength(res, 0, 1) != sizeof(int64))
295317
pg_fatal("unexpected result length while fetching remote files\n");
296318

297319
/* Read result set to local variables */
298-
memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int32));
299-
chunkoff = ntohl(chunkoff);
320+
memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int64));
321+
chunkoff = pg_recvint64(chunkoff);
300322
chunksize = PQgetlength(res, 0, 2);
301323

302324
filenamelen = PQgetlength(res, 0, 0);
@@ -321,7 +343,7 @@ receiveFileChunks(const char *sql)
321343
continue;
322344
}
323345

324-
pg_log(PG_DEBUG, "received chunk for file \"%s\", offset %d, size %d\n",
346+
pg_log(PG_DEBUG, "received chunk for file \"%s\", offset " INT64_FORMAT ", size %d\n",
325347
filename, chunkoff, chunksize);
326348

327349
open_target_file(filename, false);
@@ -381,7 +403,7 @@ libpqGetFile(const char *filename, size_t *filesize)
381403
* function to actually fetch the data.
382404
*/
383405
static void
384-
fetch_file_range(const char *path, unsigned int begin, unsigned int end)
406+
fetch_file_range(const char *path, uint64 begin, uint64 end)
385407
{
386408
char linebuf[MAXPGPATH + 23];
387409

@@ -390,12 +412,13 @@ fetch_file_range(const char *path, unsigned int begin, unsigned int end)
390412
{
391413
unsigned int len;
392414

415+
/* Fine as long as CHUNKSIZE is not bigger than UINT32_MAX */
393416
if (end - begin > CHUNKSIZE)
394417
len = CHUNKSIZE;
395418
else
396-
len = end - begin;
419+
len = (unsigned int) (end - begin);
397420

398-
snprintf(linebuf, sizeof(linebuf), "%s\t%u\t%u\n", path, begin, len);
421+
snprintf(linebuf, sizeof(linebuf), "%s\t" UINT64_FORMAT "\t%u\n", path, begin, len);
399422

400423
if (PQputCopyData(conn, linebuf, strlen(linebuf)) != 1)
401424
pg_fatal("could not send COPY data: %s",
@@ -420,7 +443,7 @@ libpq_executeFileMap(filemap_t *map)
420443
* First create a temporary table, and load it with the blocks that we
421444
* need to fetch.
422445
*/
423-
sql = "CREATE TEMPORARY TABLE fetchchunks(path text, begin int4, len int4);";
446+
sql = "CREATE TEMPORARY TABLE fetchchunks(path text, begin int8, len int4);";
424447
res = PQexec(conn, sql);
425448

426449
if (PQresultStatus(res) != PGRES_COMMAND_OK)

0 commit comments

Comments
 (0)