Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 1f39bce

Browse files
committed
Disk-based Hash Aggregation.
While performing hash aggregation, track memory usage when adding new groups to a hash table. If the memory usage exceeds work_mem, enter "spill mode". In spill mode, new groups are not created in the hash table(s), but existing groups continue to be advanced if input tuples match. Tuples that would cause a new group to be created are instead spilled to a logical tape to be processed later. The tuples are spilled in a partitioned fashion. When all tuples from the outer plan are processed (either by advancing the group or spilling the tuple), finalize and emit the groups from the hash table. Then, create new batches of work from the spilled partitions, and select one of the saved batches and process it (possibly spilling recursively). Author: Jeff Davis Reviewed-by: Tomas Vondra, Adam Lee, Justin Pryzby, Taylor Vesely, Melanie Plageman Discussion: https://postgr.es/m/507ac540ec7c20136364b5272acbcd4574aa76ef.camel@j-davis.com
1 parent e00912e commit 1f39bce

File tree

18 files changed

+1950
-38
lines changed

18 files changed

+1950
-38
lines changed

doc/src/sgml/config.sgml

+32
Original file line numberDiff line numberDiff line change
@@ -4482,6 +4482,23 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
44824482
</listitem>
44834483
</varlistentry>
44844484

4485+
<varlistentry id="guc-enable-groupingsets-hash-disk" xreflabel="enable_groupingsets_hash_disk">
4486+
<term><varname>enable_groupingsets_hash_disk</varname> (<type>boolean</type>)
4487+
<indexterm>
4488+
<primary><varname>enable_groupingsets_hash_disk</varname> configuration parameter</primary>
4489+
</indexterm>
4490+
</term>
4491+
<listitem>
4492+
<para>
4493+
Enables or disables the query planner's use of hashed aggregation plan
4494+
types for grouping sets when the total size of the hash tables is
4495+
expected to exceed <varname>work_mem</varname>. See <xref
4496+
linkend="queries-grouping-sets"/>. The default is
4497+
<literal>off</literal>.
4498+
</para>
4499+
</listitem>
4500+
</varlistentry>
4501+
44854502
<varlistentry id="guc-enable-hashagg" xreflabel="enable_hashagg">
44864503
<term><varname>enable_hashagg</varname> (<type>boolean</type>)
44874504
<indexterm>
@@ -4496,6 +4513,21 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
44964513
</listitem>
44974514
</varlistentry>
44984515

4516+
<varlistentry id="guc-enable-hashagg-disk" xreflabel="enable_hashagg_disk">
4517+
<term><varname>enable_hashagg_disk</varname> (<type>boolean</type>)
4518+
<indexterm>
4519+
<primary><varname>enable_hashagg_disk</varname> configuration parameter</primary>
4520+
</indexterm>
4521+
</term>
4522+
<listitem>
4523+
<para>
4524+
Enables or disables the query planner's use of hashed aggregation plan
4525+
types when the memory usage is expected to exceed
4526+
<varname>work_mem</varname>. The default is <literal>on</literal>.
4527+
</para>
4528+
</listitem>
4529+
</varlistentry>
4530+
44994531
<varlistentry id="guc-enable-hashjoin" xreflabel="enable_hashjoin">
45004532
<term><varname>enable_hashjoin</varname> (<type>boolean</type>)
45014533
<indexterm>

src/backend/commands/explain.c

+37
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ static void show_tablesample(TableSampleClause *tsc, PlanState *planstate,
104104
List *ancestors, ExplainState *es);
105105
static void show_sort_info(SortState *sortstate, ExplainState *es);
106106
static void show_hash_info(HashState *hashstate, ExplainState *es);
107+
static void show_hashagg_info(AggState *hashstate, ExplainState *es);
107108
static void show_tidbitmap_info(BitmapHeapScanState *planstate,
108109
ExplainState *es);
109110
static void show_instrumentation_count(const char *qlabel, int which,
@@ -1882,6 +1883,7 @@ ExplainNode(PlanState *planstate, List *ancestors,
18821883
case T_Agg:
18831884
show_agg_keys(castNode(AggState, planstate), ancestors, es);
18841885
show_upper_qual(plan->qual, "Filter", planstate, ancestors, es);
1886+
show_hashagg_info((AggState *) planstate, es);
18851887
if (plan->qual)
18861888
show_instrumentation_count("Rows Removed by Filter", 1,
18871889
planstate, es);
@@ -2769,6 +2771,41 @@ show_hash_info(HashState *hashstate, ExplainState *es)
27692771
}
27702772
}
27712773

2774+
/*
2775+
* Show information on hash aggregate memory usage and batches.
2776+
*/
2777+
static void
2778+
show_hashagg_info(AggState *aggstate, ExplainState *es)
2779+
{
2780+
Agg *agg = (Agg *)aggstate->ss.ps.plan;
2781+
long memPeakKb = (aggstate->hash_mem_peak + 1023) / 1024;
2782+
2783+
Assert(IsA(aggstate, AggState));
2784+
2785+
if (agg->aggstrategy != AGG_HASHED &&
2786+
agg->aggstrategy != AGG_MIXED)
2787+
return;
2788+
2789+
if (es->costs && aggstate->hash_planned_partitions > 0)
2790+
{
2791+
ExplainPropertyInteger("Planned Partitions", NULL,
2792+
aggstate->hash_planned_partitions, es);
2793+
}
2794+
2795+
if (!es->analyze)
2796+
return;
2797+
2798+
/* EXPLAIN ANALYZE */
2799+
ExplainPropertyInteger("Peak Memory Usage", "kB", memPeakKb, es);
2800+
if (aggstate->hash_batches_used > 0)
2801+
{
2802+
ExplainPropertyInteger("Disk Usage", "kB",
2803+
aggstate->hash_disk_used, es);
2804+
ExplainPropertyInteger("HashAgg Batches", NULL,
2805+
aggstate->hash_batches_used, es);
2806+
}
2807+
}
2808+
27722809
/*
27732810
* If it's EXPLAIN ANALYZE, show exact/lossy pages for a BitmapHeapScan node
27742811
*/

0 commit comments

Comments
 (0)