Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 8442317

Browse files
committed
Make the overflow guards in ExecChooseHashTableSize be more protective.
The original coding ensured nbuckets and nbatch didn't exceed INT_MAX, which while not insane on its own terms did nothing to protect subsequent code like "palloc(nbatch * sizeof(BufFile *))". Since enormous join size estimates might well be planner error rather than reality, it seems best to constrain the initial sizes to be not more than work_mem/sizeof(pointer), thus ensuring the allocated arrays don't exceed work_mem. We will allow nbatch to get bigger than that during subsequent ExecHashIncreaseNumBatches calls, but we should still guard against integer overflow in those palloc requests. Per bug #5145 from Bernt Marius Johnsen. Although the given test case only seems to fail back to 8.2, previous releases have variants of this issue, so patch all supported branches.
1 parent 9041222 commit 8442317

File tree

1 file changed

+25
-18
lines changed

1 file changed

+25
-18
lines changed

src/backend/executor/nodeHash.c

+25-18
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*
1010
* IDENTIFICATION
11-
* $PostgreSQL: pgsql/src/backend/executor/nodeHash.c,v 1.122 2009/09/27 21:10:53 tgl Exp $
11+
* $PostgreSQL: pgsql/src/backend/executor/nodeHash.c,v 1.123 2009/10/30 20:58:45 tgl Exp $
1212
*
1313
*-------------------------------------------------------------------------
1414
*/
@@ -394,6 +394,7 @@ ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew,
394394
double inner_rel_bytes;
395395
long hash_table_bytes;
396396
long skew_table_bytes;
397+
long max_pointers;
397398
int nbatch;
398399
int nbuckets;
399400
int i;
@@ -435,17 +436,18 @@ ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew,
435436
{
436437
skew_table_bytes = hash_table_bytes * SKEW_WORK_MEM_PERCENT / 100;
437438

438-
*num_skew_mcvs = skew_table_bytes / (
439-
/* size of a hash tuple */
440-
tupsize +
441-
/* worst-case size of skewBucket[] per MCV */
439+
/*----------
440+
* Divisor is:
441+
* size of a hash tuple +
442+
* worst-case size of skewBucket[] per MCV +
443+
* size of skewBucketNums[] entry +
444+
* size of skew bucket struct itself
445+
*----------
446+
*/
447+
*num_skew_mcvs = skew_table_bytes / (tupsize +
442448
(8 * sizeof(HashSkewBucket *)) +
443-
/* size of skewBucketNums[] entry */
444449
sizeof(int) +
445-
/* size of skew bucket struct itself */
446-
SKEW_BUCKET_OVERHEAD
447-
);
448-
450+
SKEW_BUCKET_OVERHEAD);
449451
if (*num_skew_mcvs > 0)
450452
hash_table_bytes -= skew_table_bytes;
451453
}
@@ -455,8 +457,13 @@ ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew,
455457
/*
456458
* Set nbuckets to achieve an average bucket load of NTUP_PER_BUCKET when
457459
* memory is filled. Set nbatch to the smallest power of 2 that appears
458-
* sufficient.
460+
* sufficient. The Min() steps limit the results so that the pointer
461+
* arrays we'll try to allocate do not exceed work_mem.
459462
*/
463+
max_pointers = (work_mem * 1024L) / sizeof(void *);
464+
/* also ensure we avoid integer overflow in nbatch and nbuckets */
465+
max_pointers = Min(max_pointers, INT_MAX / 2);
466+
460467
if (inner_rel_bytes > hash_table_bytes)
461468
{
462469
/* We'll need multiple batches */
@@ -465,11 +472,11 @@ ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew,
465472
int minbatch;
466473

467474
lbuckets = (hash_table_bytes / tupsize) / NTUP_PER_BUCKET;
468-
lbuckets = Min(lbuckets, INT_MAX / 2);
475+
lbuckets = Min(lbuckets, max_pointers);
469476
nbuckets = (int) lbuckets;
470477

471478
dbatch = ceil(inner_rel_bytes / hash_table_bytes);
472-
dbatch = Min(dbatch, INT_MAX / 2);
479+
dbatch = Min(dbatch, max_pointers);
473480
minbatch = (int) dbatch;
474481
nbatch = 2;
475482
while (nbatch < minbatch)
@@ -481,7 +488,7 @@ ExecChooseHashTableSize(double ntuples, int tupwidth, bool useskew,
481488
double dbuckets;
482489

483490
dbuckets = ceil(ntuples / NTUP_PER_BUCKET);
484-
dbuckets = Min(dbuckets, INT_MAX / 2);
491+
dbuckets = Min(dbuckets, max_pointers);
485492
nbuckets = (int) dbuckets;
486493

487494
nbatch = 1;
@@ -555,7 +562,7 @@ ExecHashIncreaseNumBatches(HashJoinTable hashtable)
555562
return;
556563

557564
/* safety check to avoid overflow */
558-
if (oldnbatch > INT_MAX / 2)
565+
if (oldnbatch > Min(INT_MAX / 2, MaxAllocSize / (sizeof(void *) * 2)))
559566
return;
560567

561568
nbatch = oldnbatch * 2;
@@ -1033,9 +1040,9 @@ ExecHashBuildSkewHash(HashJoinTable hashtable, Hash *node, int mcvsToUse)
10331040
* will be at least one null entry, so searches will always
10341041
* terminate.)
10351042
*
1036-
* Note: this code could fail if mcvsToUse exceeds INT_MAX/8, but that
1037-
* is not currently possible since we limit pg_statistic entries to
1038-
* much less than that.
1043+
* Note: this code could fail if mcvsToUse exceeds INT_MAX/8 or
1044+
* MaxAllocSize/sizeof(void *)/8, but that is not currently possible
1045+
* since we limit pg_statistic entries to much less than that.
10391046
*/
10401047
nbuckets = 2;
10411048
while (nbuckets <= mcvsToUse)

0 commit comments

Comments
 (0)