Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 96d1f42

Browse files
committed
Read until EOF vice stat-reported size in read_binary_file
read_binary_file(), used by SQL functions pg_read_file() and friends, uses stat to determine file length to read, when not passed an explicit length as an argument. This is problematic, for example, if the file being read is a virtual file with a stat-reported length of zero. Arrange to read until EOF, or StringInfo data string lenth limit, is reached instead. Original complaint and patch by me, with significant review, corrections, advice, and code optimizations by Tom Lane. Backpatched to v11. Prior to that only paths relative to the data and log dirs were allowed for files, so no "zero length" files were reachable anyway. Reviewed-By: Tom Lane Discussion: https://postgr.es/m/flat/969b8d82-5bb2-5fa8-4eb1-f0e685c5d736%40joeconway.com Backpatch-through: 11
1 parent ca5e93f commit 96d1f42

File tree

2 files changed

+66
-29
lines changed

2 files changed

+66
-29
lines changed

contrib/adminpack/expected/adminpack.out

+2-2
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ SELECT pg_file_rename('test_file1', 'test_file2');
7979
(1 row)
8080

8181
SELECT pg_read_file('test_file1'); -- not there
82-
ERROR: could not stat file "test_file1": No such file or directory
82+
ERROR: could not open file "test_file1" for reading: No such file or directory
8383
SELECT pg_read_file('test_file2');
8484
pg_read_file
8585
--------------
@@ -108,7 +108,7 @@ SELECT pg_file_rename('test_file2', 'test_file3', 'test_file3_archive');
108108
(1 row)
109109

110110
SELECT pg_read_file('test_file2'); -- not there
111-
ERROR: could not stat file "test_file2": No such file or directory
111+
ERROR: could not open file "test_file2" for reading: No such file or directory
112112
SELECT pg_read_file('test_file3');
113113
pg_read_file
114114
--------------

src/backend/utils/adt/genfile.c

+64-27
Original file line numberDiff line numberDiff line change
@@ -106,33 +106,11 @@ read_binary_file(const char *filename, int64 seek_offset, int64 bytes_to_read,
106106
bool missing_ok)
107107
{
108108
bytea *buf;
109-
size_t nbytes;
109+
size_t nbytes = 0;
110110
FILE *file;
111111

112-
if (bytes_to_read < 0)
113-
{
114-
if (seek_offset < 0)
115-
bytes_to_read = -seek_offset;
116-
else
117-
{
118-
struct stat fst;
119-
120-
if (stat(filename, &fst) < 0)
121-
{
122-
if (missing_ok && errno == ENOENT)
123-
return NULL;
124-
else
125-
ereport(ERROR,
126-
(errcode_for_file_access(),
127-
errmsg("could not stat file \"%s\": %m", filename)));
128-
}
129-
130-
bytes_to_read = fst.st_size - seek_offset;
131-
}
132-
}
133-
134-
/* not sure why anyone thought that int64 length was a good idea */
135-
if (bytes_to_read > (MaxAllocSize - VARHDRSZ))
112+
/* clamp request size to what we can actually deliver */
113+
if (bytes_to_read > (int64) (MaxAllocSize - VARHDRSZ))
136114
ereport(ERROR,
137115
(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
138116
errmsg("requested length too large")));
@@ -154,9 +132,68 @@ read_binary_file(const char *filename, int64 seek_offset, int64 bytes_to_read,
154132
(errcode_for_file_access(),
155133
errmsg("could not seek in file \"%s\": %m", filename)));
156134

157-
buf = (bytea *) palloc((Size) bytes_to_read + VARHDRSZ);
135+
if (bytes_to_read >= 0)
136+
{
137+
/* If passed explicit read size just do it */
138+
buf = (bytea *) palloc((Size) bytes_to_read + VARHDRSZ);
139+
140+
nbytes = fread(VARDATA(buf), 1, (size_t) bytes_to_read, file);
141+
}
142+
else
143+
{
144+
/* Negative read size, read rest of file */
145+
StringInfoData sbuf;
146+
147+
initStringInfo(&sbuf);
148+
/* Leave room in the buffer for the varlena length word */
149+
sbuf.len += VARHDRSZ;
150+
Assert(sbuf.len < sbuf.maxlen);
151+
152+
while (!(feof(file) || ferror(file)))
153+
{
154+
size_t rbytes;
155+
156+
/* Minimum amount to read at a time */
157+
#define MIN_READ_SIZE 4096
158+
159+
/*
160+
* If not at end of file, and sbuf.len is equal to
161+
* MaxAllocSize - 1, then either the file is too large, or
162+
* there is nothing left to read. Attempt to read one more
163+
* byte to see if the end of file has been reached. If not,
164+
* the file is too large; we'd rather give the error message
165+
* for that ourselves.
166+
*/
167+
if (sbuf.len == MaxAllocSize - 1)
168+
{
169+
char rbuf[1];
158170

159-
nbytes = fread(VARDATA(buf), 1, (size_t) bytes_to_read, file);
171+
fread(rbuf, 1, 1, file);
172+
if (!feof(file))
173+
ereport(ERROR,
174+
(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
175+
errmsg("file length too large")));
176+
else
177+
break;
178+
}
179+
180+
/* OK, ensure that we can read at least MIN_READ_SIZE */
181+
enlargeStringInfo(&sbuf, MIN_READ_SIZE);
182+
183+
/*
184+
* stringinfo.c likes to allocate in powers of 2, so it's likely
185+
* that much more space is available than we asked for. Use all
186+
* of it, rather than making more fread calls than necessary.
187+
*/
188+
rbytes = fread(sbuf.data + sbuf.len, 1,
189+
(size_t) (sbuf.maxlen - sbuf.len - 1), file);
190+
sbuf.len += rbytes;
191+
nbytes += rbytes;
192+
}
193+
194+
/* Now we can commandeer the stringinfo's buffer as the result */
195+
buf = (bytea *) sbuf.data;
196+
}
160197

161198
if (ferror(file))
162199
ereport(ERROR,

0 commit comments

Comments
 (0)