Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit d8a7ce2

Browse files
committed
HashAgg: before spilling tuples, set unneeded columns to NULL.
This is a replacement for 4cad253. Instead of projecting all tuples going into a HashAgg, only remove unnecessary attributes when actually spilling. This avoids the regression for the in-memory case. Discussion: https://postgr.es/m/a2fb7dfeb4f50aa0a123e42151ee3013933cb802.camel%40j-davis.com Backpatch-through: 13
1 parent 926ecf8 commit d8a7ce2

File tree

2 files changed

+96
-35
lines changed

2 files changed

+96
-35
lines changed

src/backend/executor/nodeAgg.c

+90-33
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,14 @@ typedef struct HashAggBatch
359359
int64 input_tuples; /* number of tuples in this batch */
360360
} HashAggBatch;
361361

362+
/* used to find referenced colnos */
363+
typedef struct FindColsContext
364+
{
365+
bool is_aggref; /* is under an aggref */
366+
Bitmapset *aggregated; /* column references under an aggref */
367+
Bitmapset *unaggregated; /* other column references */
368+
} FindColsContext;
369+
362370
static void select_current_set(AggState *aggstate, int setno, bool is_hash);
363371
static void initialize_phase(AggState *aggstate, int newphase);
364372
static TupleTableSlot *fetch_input_tuple(AggState *aggstate);
@@ -391,8 +399,9 @@ static void finalize_aggregates(AggState *aggstate,
391399
AggStatePerAgg peragg,
392400
AggStatePerGroup pergroup);
393401
static TupleTableSlot *project_aggregates(AggState *aggstate);
394-
static Bitmapset *find_unaggregated_cols(AggState *aggstate);
395-
static bool find_unaggregated_cols_walker(Node *node, Bitmapset **colnos);
402+
static void find_cols(AggState *aggstate, Bitmapset **aggregated,
403+
Bitmapset **unaggregated);
404+
static bool find_cols_walker(Node *node, FindColsContext *context);
396405
static void build_hash_tables(AggState *aggstate);
397406
static void build_hash_table(AggState *aggstate, int setno, long nbuckets);
398407
static void hashagg_recompile_expressions(AggState *aggstate, bool minslot,
@@ -425,8 +434,8 @@ static MinimalTuple hashagg_batch_read(HashAggBatch *batch, uint32 *hashp);
425434
static void hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo,
426435
int used_bits, uint64 input_tuples,
427436
double hashentrysize);
428-
static Size hashagg_spill_tuple(HashAggSpill *spill, TupleTableSlot *slot,
429-
uint32 hash);
437+
static Size hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
438+
TupleTableSlot *slot, uint32 hash);
430439
static void hashagg_spill_finish(AggState *aggstate, HashAggSpill *spill,
431440
int setno);
432441
static void hashagg_tapeinfo_init(AggState *aggstate);
@@ -1375,26 +1384,28 @@ project_aggregates(AggState *aggstate)
13751384
}
13761385

13771386
/*
1378-
* find_unaggregated_cols
1379-
* Construct a bitmapset of the column numbers of un-aggregated Vars
1380-
* appearing in our targetlist and qual (HAVING clause)
1387+
* Walk tlist and qual to find referenced colnos, dividing them into
1388+
* aggregated and unaggregated sets.
13811389
*/
1382-
static Bitmapset *
1383-
find_unaggregated_cols(AggState *aggstate)
1390+
static void
1391+
find_cols(AggState *aggstate, Bitmapset **aggregated, Bitmapset **unaggregated)
13841392
{
1385-
Agg *node = (Agg *) aggstate->ss.ps.plan;
1386-
Bitmapset *colnos;
1387-
1388-
colnos = NULL;
1389-
(void) find_unaggregated_cols_walker((Node *) node->plan.targetlist,
1390-
&colnos);
1391-
(void) find_unaggregated_cols_walker((Node *) node->plan.qual,
1392-
&colnos);
1393-
return colnos;
1393+
Agg *agg = (Agg *) aggstate->ss.ps.plan;
1394+
FindColsContext context;
1395+
1396+
context.is_aggref = false;
1397+
context.aggregated = NULL;
1398+
context.unaggregated = NULL;
1399+
1400+
(void) find_cols_walker((Node *) agg->plan.targetlist, &context);
1401+
(void) find_cols_walker((Node *) agg->plan.qual, &context);
1402+
1403+
*aggregated = context.aggregated;
1404+
*unaggregated = context.unaggregated;
13941405
}
13951406

13961407
static bool
1397-
find_unaggregated_cols_walker(Node *node, Bitmapset **colnos)
1408+
find_cols_walker(Node *node, FindColsContext *context)
13981409
{
13991410
if (node == NULL)
14001411
return false;
@@ -1405,16 +1416,24 @@ find_unaggregated_cols_walker(Node *node, Bitmapset **colnos)
14051416
/* setrefs.c should have set the varno to OUTER_VAR */
14061417
Assert(var->varno == OUTER_VAR);
14071418
Assert(var->varlevelsup == 0);
1408-
*colnos = bms_add_member(*colnos, var->varattno);
1419+
if (context->is_aggref)
1420+
context->aggregated = bms_add_member(context->aggregated,
1421+
var->varattno);
1422+
else
1423+
context->unaggregated = bms_add_member(context->unaggregated,
1424+
var->varattno);
14091425
return false;
14101426
}
1411-
if (IsA(node, Aggref) || IsA(node, GroupingFunc))
1427+
if (IsA(node, Aggref))
14121428
{
1413-
/* do not descend into aggregate exprs */
1429+
Assert(!context->is_aggref);
1430+
context->is_aggref = true;
1431+
expression_tree_walker(node, find_cols_walker, (void *) context);
1432+
context->is_aggref = false;
14141433
return false;
14151434
}
1416-
return expression_tree_walker(node, find_unaggregated_cols_walker,
1417-
(void *) colnos);
1435+
return expression_tree_walker(node, find_cols_walker,
1436+
(void *) context);
14181437
}
14191438

14201439
/*
@@ -1532,13 +1551,27 @@ static void
15321551
find_hash_columns(AggState *aggstate)
15331552
{
15341553
Bitmapset *base_colnos;
1554+
Bitmapset *aggregated_colnos;
1555+
TupleDesc scanDesc = aggstate->ss.ss_ScanTupleSlot->tts_tupleDescriptor;
15351556
List *outerTlist = outerPlanState(aggstate)->plan->targetlist;
15361557
int numHashes = aggstate->num_hashes;
15371558
EState *estate = aggstate->ss.ps.state;
15381559
int j;
15391560

15401561
/* Find Vars that will be needed in tlist and qual */
1541-
base_colnos = find_unaggregated_cols(aggstate);
1562+
find_cols(aggstate, &aggregated_colnos, &base_colnos);
1563+
aggstate->colnos_needed = bms_union(base_colnos, aggregated_colnos);
1564+
aggstate->max_colno_needed = 0;
1565+
aggstate->all_cols_needed = true;
1566+
1567+
for (int i = 0; i < scanDesc->natts; i++)
1568+
{
1569+
int colno = i + 1;
1570+
if (bms_is_member(colno, aggstate->colnos_needed))
1571+
aggstate->max_colno_needed = colno;
1572+
else
1573+
aggstate->all_cols_needed = false;
1574+
}
15421575

15431576
for (j = 0; j < numHashes; ++j)
15441577
{
@@ -2097,7 +2130,7 @@ lookup_hash_entries(AggState *aggstate)
20972130
perhash->aggnode->numGroups,
20982131
aggstate->hashentrysize);
20992132

2100-
hashagg_spill_tuple(spill, slot, hash);
2133+
hashagg_spill_tuple(aggstate, spill, slot, hash);
21012134
}
21022135
}
21032136
}
@@ -2619,7 +2652,7 @@ agg_refill_hash_table(AggState *aggstate)
26192652
HASHAGG_READ_BUFFER_SIZE);
26202653
for (;;)
26212654
{
2622-
TupleTableSlot *slot = aggstate->hash_spill_slot;
2655+
TupleTableSlot *slot = aggstate->hash_spill_rslot;
26232656
MinimalTuple tuple;
26242657
uint32 hash;
26252658
bool in_hash_table;
@@ -2655,7 +2688,7 @@ agg_refill_hash_table(AggState *aggstate)
26552688
ngroups_estimate, aggstate->hashentrysize);
26562689
}
26572690
/* no memory for a new group, spill */
2658-
hashagg_spill_tuple(&spill, slot, hash);
2691+
hashagg_spill_tuple(aggstate, &spill, slot, hash);
26592692
}
26602693

26612694
/*
@@ -2934,9 +2967,11 @@ hashagg_spill_init(HashAggSpill *spill, HashTapeInfo *tapeinfo, int used_bits,
29342967
* partition.
29352968
*/
29362969
static Size
2937-
hashagg_spill_tuple(HashAggSpill *spill, TupleTableSlot *slot, uint32 hash)
2970+
hashagg_spill_tuple(AggState *aggstate, HashAggSpill *spill,
2971+
TupleTableSlot *inputslot, uint32 hash)
29382972
{
29392973
LogicalTapeSet *tapeset = spill->tapeset;
2974+
TupleTableSlot *spillslot;
29402975
int partition;
29412976
MinimalTuple tuple;
29422977
int tapenum;
@@ -2945,8 +2980,28 @@ hashagg_spill_tuple(HashAggSpill *spill, TupleTableSlot *slot, uint32 hash)
29452980

29462981
Assert(spill->partitions != NULL);
29472982

2948-
/* XXX: may contain unnecessary attributes, should project */
2949-
tuple = ExecFetchSlotMinimalTuple(slot, &shouldFree);
2983+
/* spill only attributes that we actually need */
2984+
if (!aggstate->all_cols_needed)
2985+
{
2986+
spillslot = aggstate->hash_spill_wslot;
2987+
slot_getsomeattrs(inputslot, aggstate->max_colno_needed);
2988+
ExecClearTuple(spillslot);
2989+
for (int i = 0; i < spillslot->tts_tupleDescriptor->natts; i++)
2990+
{
2991+
if (bms_is_member(i + 1, aggstate->colnos_needed))
2992+
{
2993+
spillslot->tts_values[i] = inputslot->tts_values[i];
2994+
spillslot->tts_isnull[i] = inputslot->tts_isnull[i];
2995+
}
2996+
else
2997+
spillslot->tts_isnull[i] = true;
2998+
}
2999+
ExecStoreVirtualTuple(spillslot);
3000+
}
3001+
else
3002+
spillslot = inputslot;
3003+
3004+
tuple = ExecFetchSlotMinimalTuple(spillslot, &shouldFree);
29503005

29513006
partition = (hash & spill->mask) >> spill->shift;
29523007
spill->ntuples[partition]++;
@@ -3563,8 +3618,10 @@ ExecInitAgg(Agg *node, EState *estate, int eflags)
35633618
aggstate->hash_metacxt = AllocSetContextCreate(aggstate->ss.ps.state->es_query_cxt,
35643619
"HashAgg meta context",
35653620
ALLOCSET_DEFAULT_SIZES);
3566-
aggstate->hash_spill_slot = ExecInitExtraTupleSlot(estate, scanDesc,
3567-
&TTSOpsMinimalTuple);
3621+
aggstate->hash_spill_rslot = ExecInitExtraTupleSlot(estate, scanDesc,
3622+
&TTSOpsMinimalTuple);
3623+
aggstate->hash_spill_wslot = ExecInitExtraTupleSlot(estate, scanDesc,
3624+
&TTSOpsVirtual);
35683625

35693626
/* this is an array of pointers, not structures */
35703627
aggstate->hash_pergroup = pergroups;

src/include/nodes/execnodes.h

+6-2
Original file line numberDiff line numberDiff line change
@@ -2169,6 +2169,9 @@ typedef struct AggState
21692169
int current_set; /* The current grouping set being evaluated */
21702170
Bitmapset *grouped_cols; /* grouped cols in current projection */
21712171
List *all_grouped_cols; /* list of all grouped cols in DESC order */
2172+
Bitmapset *colnos_needed; /* all columns needed from the outer plan */
2173+
int max_colno_needed; /* highest colno needed from outer plan */
2174+
bool all_cols_needed; /* are all cols from outer plan needed? */
21722175
/* These fields are for grouping set phase data */
21732176
int maxsets; /* The max number of sets in any phase */
21742177
AggStatePerPhase phases; /* array of all phases */
@@ -2186,7 +2189,8 @@ typedef struct AggState
21862189
struct HashTapeInfo *hash_tapeinfo; /* metadata for spill tapes */
21872190
struct HashAggSpill *hash_spills; /* HashAggSpill for each grouping set,
21882191
* exists only during first pass */
2189-
TupleTableSlot *hash_spill_slot; /* slot for reading from spill files */
2192+
TupleTableSlot *hash_spill_rslot; /* for reading spill files */
2193+
TupleTableSlot *hash_spill_wslot; /* for writing spill files */
21902194
List *hash_batches; /* hash batches remaining to be processed */
21912195
bool hash_ever_spilled; /* ever spilled during this execution? */
21922196
bool hash_spill_mode; /* we hit a limit during the current batch
@@ -2207,7 +2211,7 @@ typedef struct AggState
22072211
* per-group pointers */
22082212

22092213
/* support for evaluation of agg input expressions: */
2210-
#define FIELDNO_AGGSTATE_ALL_PERGROUPS 49
2214+
#define FIELDNO_AGGSTATE_ALL_PERGROUPS 53
22112215
AggStatePerGroup *all_pergroups; /* array of first ->pergroups, than
22122216
* ->hash_pergroup */
22132217
ProjectionInfo *combinedproj; /* projection machinery */

0 commit comments

Comments
 (0)