Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit1f39bce

Browse files
committed
Disk-based Hash Aggregation.
While performing hash aggregation, track memory usage when adding newgroups to a hash table. If the memory usage exceeds work_mem, enter"spill mode".In spill mode, new groups are not created in the hash table(s), butexisting groups continue to be advanced if input tuples match. Tuplesthat would cause a new group to be created are instead spilled to alogical tape to be processed later.The tuples are spilled in a partitioned fashion. When all tuples fromthe outer plan are processed (either by advancing the group orspilling the tuple), finalize and emit the groups from the hashtable. Then, create new batches of work from the spilled partitions,and select one of the saved batches and process it (possibly spillingrecursively).Author: Jeff DavisReviewed-by: Tomas Vondra, Adam Lee, Justin Pryzby, Taylor Vesely, Melanie PlagemanDiscussion:https://postgr.es/m/507ac540ec7c20136364b5272acbcd4574aa76ef.camel@j-davis.com
1 parente00912e commit1f39bce

File tree

18 files changed

+1950
-38
lines changed

18 files changed

+1950
-38
lines changed

‎doc/src/sgml/config.sgml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4482,6 +4482,23 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
44824482
</listitem>
44834483
</varlistentry>
44844484

4485+
<varlistentry id="guc-enable-groupingsets-hash-disk" xreflabel="enable_groupingsets_hash_disk">
4486+
<term><varname>enable_groupingsets_hash_disk</varname> (<type>boolean</type>)
4487+
<indexterm>
4488+
<primary><varname>enable_groupingsets_hash_disk</varname> configuration parameter</primary>
4489+
</indexterm>
4490+
</term>
4491+
<listitem>
4492+
<para>
4493+
Enables or disables the query planner's use of hashed aggregation plan
4494+
types for grouping sets when the total size of the hash tables is
4495+
expected to exceed <varname>work_mem</varname>. See <xref
4496+
linkend="queries-grouping-sets"/>. The default is
4497+
<literal>off</literal>.
4498+
</para>
4499+
</listitem>
4500+
</varlistentry>
4501+
44854502
<varlistentry id="guc-enable-hashagg" xreflabel="enable_hashagg">
44864503
<term><varname>enable_hashagg</varname> (<type>boolean</type>)
44874504
<indexterm>
@@ -4496,6 +4513,21 @@ ANY <replaceable class="parameter">num_sync</replaceable> ( <replaceable class="
44964513
</listitem>
44974514
</varlistentry>
44984515

4516+
<varlistentry id="guc-enable-hashagg-disk" xreflabel="enable_hashagg_disk">
4517+
<term><varname>enable_hashagg_disk</varname> (<type>boolean</type>)
4518+
<indexterm>
4519+
<primary><varname>enable_hashagg_disk</varname> configuration parameter</primary>
4520+
</indexterm>
4521+
</term>
4522+
<listitem>
4523+
<para>
4524+
Enables or disables the query planner's use of hashed aggregation plan
4525+
types when the memory usage is expected to exceed
4526+
<varname>work_mem</varname>. The default is <literal>on</literal>.
4527+
</para>
4528+
</listitem>
4529+
</varlistentry>
4530+
44994531
<varlistentry id="guc-enable-hashjoin" xreflabel="enable_hashjoin">
45004532
<term><varname>enable_hashjoin</varname> (<type>boolean</type>)
45014533
<indexterm>

‎src/backend/commands/explain.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ static void show_tablesample(TableSampleClause *tsc, PlanState *planstate,
104104
List*ancestors,ExplainState*es);
105105
staticvoidshow_sort_info(SortState*sortstate,ExplainState*es);
106106
staticvoidshow_hash_info(HashState*hashstate,ExplainState*es);
107+
staticvoidshow_hashagg_info(AggState*hashstate,ExplainState*es);
107108
staticvoidshow_tidbitmap_info(BitmapHeapScanState*planstate,
108109
ExplainState*es);
109110
staticvoidshow_instrumentation_count(constchar*qlabel,intwhich,
@@ -1882,6 +1883,7 @@ ExplainNode(PlanState *planstate, List *ancestors,
18821883
caseT_Agg:
18831884
show_agg_keys(castNode(AggState,planstate),ancestors,es);
18841885
show_upper_qual(plan->qual,"Filter",planstate,ancestors,es);
1886+
show_hashagg_info((AggState*)planstate,es);
18851887
if (plan->qual)
18861888
show_instrumentation_count("Rows Removed by Filter",1,
18871889
planstate,es);
@@ -2769,6 +2771,41 @@ show_hash_info(HashState *hashstate, ExplainState *es)
27692771
}
27702772
}
27712773

2774+
/*
2775+
* Show information on hash aggregate memory usage and batches.
2776+
*/
2777+
staticvoid
2778+
show_hashagg_info(AggState*aggstate,ExplainState*es)
2779+
{
2780+
Agg*agg= (Agg*)aggstate->ss.ps.plan;
2781+
longmemPeakKb= (aggstate->hash_mem_peak+1023) /1024;
2782+
2783+
Assert(IsA(aggstate,AggState));
2784+
2785+
if (agg->aggstrategy!=AGG_HASHED&&
2786+
agg->aggstrategy!=AGG_MIXED)
2787+
return;
2788+
2789+
if (es->costs&&aggstate->hash_planned_partitions>0)
2790+
{
2791+
ExplainPropertyInteger("Planned Partitions",NULL,
2792+
aggstate->hash_planned_partitions,es);
2793+
}
2794+
2795+
if (!es->analyze)
2796+
return;
2797+
2798+
/* EXPLAIN ANALYZE */
2799+
ExplainPropertyInteger("Peak Memory Usage","kB",memPeakKb,es);
2800+
if (aggstate->hash_batches_used>0)
2801+
{
2802+
ExplainPropertyInteger("Disk Usage","kB",
2803+
aggstate->hash_disk_used,es);
2804+
ExplainPropertyInteger("HashAgg Batches",NULL,
2805+
aggstate->hash_batches_used,es);
2806+
}
2807+
}
2808+
27722809
/*
27732810
* If it's EXPLAIN ANALYZE, show exact/lossy pages for a BitmapHeapScan node
27742811
*/

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp