Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 4ae351a

Browse files
Daniil AnisimovAnisimov-ds
Daniil Anisimov
authored and
Anisimov-ds
committed
Try reducing the memory overhead.
Free some allocated memory right after use. Reset AQOPredictMemCtx as soon as posible. Remove learning attempts on SubPlan nodes. Bugfix. Free allocated memory on save/load data. Add memory context for storage. Change copyright to 2016-2023.
1 parent c809878 commit 4ae351a

13 files changed

+183
-49
lines changed

‎aqo.c

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
* aqo.c
33
* Adaptive query optimization extension
44
*
5-
* Copyright (c) 2016-2022, Postgres Professional
5+
* Copyright (c) 2016-2023, Postgres Professional
66
*
77
* IDENTIFICATION
88
* aqo/aqo.c
@@ -92,6 +92,9 @@ MemoryContext AQOPredictMemCtx = NULL;
9292
/* Is released at the end of learning */
9393
MemoryContext AQOLearnMemCtx = NULL;
9494

95+
/* Is released at the end of load/store routines */
96+
MemoryContext AQOStorageMemCtx = NULL;
97+
9598
/* Additional plan info */
9699
int njoins;
97100

@@ -343,6 +346,12 @@ _PG_init(void)
343346
AQOLearnMemCtx = AllocSetContextCreate(AQOTopMemCtx,
344347
"AQOLearnMemoryContext",
345348
ALLOCSET_DEFAULT_SIZES);
349+
/*
350+
* AQOStorageMemoryContext containe data for load/store routines.
351+
*/
352+
AQOStorageMemCtx = AllocSetContextCreate(AQOTopMemCtx,
353+
"AQOStorageMemoryContext",
354+
ALLOCSET_DEFAULT_SIZES);
346355
RegisterResourceReleaseCallback(aqo_free_callback, NULL);
347356
RegisterAQOPlanNodeMethods();
348357

‎aqo.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@
105105
* Module storage.c is responsible for storage query settings and models
106106
* (i. e. all information which is used in extension).
107107
*
108-
* Copyright (c) 2016-2022, Postgres Professional
108+
* Copyright (c) 2016-2023, Postgres Professional
109109
*
110110
* IDENTIFICATION
111111
* aqo/aqo.h
@@ -232,6 +232,7 @@ extern MemoryContext AQOTopMemCtx;
232232
extern MemoryContext AQOCacheMemCtx;
233233
extern MemoryContext AQOPredictMemCtx;
234234
extern MemoryContext AQOLearnMemCtx;
235+
extern MemoryContext AQOStorageMemCtx;
235236

236237
extern int aqo_statement_timeout;
237238

‎auto_tuning.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*******************************************************************************
1010
*
11-
* Copyright (c) 2016-2022, Postgres Professional
11+
* Copyright (c) 2016-2023, Postgres Professional
1212
*
1313
* IDENTIFICATION
1414
* aqo/auto_tuning.c

‎cardinality_estimation.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
*
99
*******************************************************************************
1010
*
11-
* Copyright (c) 2016-2022, Postgres Professional
11+
* Copyright (c) 2016-2023, Postgres Professional
1212
*
1313
* IDENTIFICATION
1414
* aqo/cardinality_estimation.c

‎cardinality_hooks.c

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
*
1919
*******************************************************************************
2020
*
21-
* Copyright (c) 2016-2022, Postgres Professional
21+
* Copyright (c) 2016-2023, Postgres Professional
2222
*
2323
* IDENTIFICATION
2424
* aqo/cardinality_hooks.c
@@ -81,6 +81,7 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
8181
if (!query_context.use_aqo)
8282
{
8383
MemoryContextSwitchTo(old_ctx_m);
84+
MemoryContextReset(AQOPredictMemCtx);
8485
goto default_estimator;
8586
}
8687

@@ -99,6 +100,7 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
99100

100101
/* Return to the caller's memory context. */
101102
MemoryContextSwitchTo(old_ctx_m);
103+
MemoryContextReset(AQOPredictMemCtx);
102104

103105
if (predicted < 0)
104106
goto default_estimator;
@@ -190,12 +192,15 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
190192
cache_selectivity(current_hash, rel->relid, rte->relid,
191193
*((double *) lfirst(l2)));
192194
}
195+
196+
pfree(args_hash);
197+
pfree(eclass_hash);
193198
}
194199

195200
if (!query_context.use_aqo)
196201
{
197202
MemoryContextSwitchTo(oldctx);
198-
203+
MemoryContextReset(AQOPredictMemCtx);
199204
goto default_estimator;
200205
}
201206

@@ -210,6 +215,7 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
210215

211216
/* Return to the caller's memory context */
212217
MemoryContextSwitchTo(oldctx);
218+
MemoryContextReset(AQOPredictMemCtx);
213219

214220
predicted_ppi_rows = predicted;
215221
fss_ppi_hash = fss;
@@ -264,6 +270,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
264270
if (!query_context.use_aqo)
265271
{
266272
MemoryContextSwitchTo(old_ctx_m);
273+
MemoryContextReset(AQOPredictMemCtx);
267274
goto default_estimator;
268275
}
269276

@@ -283,6 +290,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel,
283290

284291
/* Return to the caller's memory context */
285292
MemoryContextSwitchTo(old_ctx_m);
293+
MemoryContextReset(AQOPredictMemCtx);
286294

287295
rel->fss_hash = fss;
288296

@@ -342,6 +350,7 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
342350
if (!query_context.use_aqo)
343351
{
344352
MemoryContextSwitchTo(old_ctx_m);
353+
MemoryContextReset(AQOPredictMemCtx);
345354
goto default_estimator;
346355
}
347356

@@ -358,6 +367,7 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root,
358367
&fss);
359368
/* Return to the caller's memory context */
360369
MemoryContextSwitchTo(old_ctx_m);
370+
MemoryContextReset(AQOPredictMemCtx);
361371

362372
predicted_ppi_rows = predicted;
363373
fss_ppi_hash = fss;
@@ -445,6 +455,7 @@ aqo_estimate_num_groups(PlannerInfo *root, List *groupExprs,
445455
grouped_rel->rows = predicted;
446456
grouped_rel->fss_hash = fss;
447457
MemoryContextSwitchTo(old_ctx_m);
458+
MemoryContextReset(AQOPredictMemCtx);
448459
return predicted;
449460
}
450461
else
@@ -455,6 +466,7 @@ aqo_estimate_num_groups(PlannerInfo *root, List *groupExprs,
455466
grouped_rel->predicted_cardinality = -1;
456467

457468
MemoryContextSwitchTo(old_ctx_m);
469+
MemoryContextReset(AQOPredictMemCtx);
458470

459471
default_estimator:
460472
if (aqo_estimate_num_groups_next)

‎expected/unsupported.out

Lines changed: 59 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,59 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
311311
JOINS: 0
312312
(23 rows)
313313

314+
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
315+
SELECT * FROM t WHERE
316+
x = (SELECT x FROM t t0 WHERE t0.x = t.x LIMIT 1) AND
317+
x IN (SELECT x FROM t t0 WHERE t0.x = t.x);
318+
QUERY PLAN
319+
-----------------------------------------------------------
320+
Seq Scan on t (actual rows=1000 loops=1)
321+
AQO not used
322+
Filter: ((x = (SubPlan 1)) AND (SubPlan 2))
323+
SubPlan 1
324+
-> Limit (actual rows=1 loops=1000)
325+
AQO not used
326+
-> Seq Scan on t t0 (actual rows=1 loops=1000)
327+
AQO not used
328+
Filter: (x = t.x)
329+
Rows Removed by Filter: 475
330+
SubPlan 2
331+
-> Seq Scan on t t0_1 (actual rows=1 loops=1000)
332+
AQO not used
333+
Filter: (x = t.x)
334+
Rows Removed by Filter: 475
335+
Using aqo: true
336+
AQO mode: LEARN
337+
JOINS: 0
338+
(18 rows)
339+
340+
-- No prediction for top SeqScan, because it fss is changed
341+
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
342+
SELECT * FROM t WHERE
343+
x = (SELECT x FROM t t0 WHERE t0.x = t.x LIMIT 1) AND
344+
x IN (SELECT x FROM t t0 WHERE t0.x = t.x);
345+
QUERY PLAN
346+
-----------------------------------------------------------
347+
Seq Scan on t (actual rows=1000 loops=1)
348+
AQO not used
349+
Filter: ((SubPlan 2) AND (x = (SubPlan 1)))
350+
SubPlan 2
351+
-> Seq Scan on t t0_1 (actual rows=1 loops=1000)
352+
AQO: rows=1, error=0%
353+
Filter: (x = t.x)
354+
Rows Removed by Filter: 475
355+
SubPlan 1
356+
-> Limit (actual rows=1 loops=1000)
357+
AQO not used
358+
-> Seq Scan on t t0 (actual rows=1 loops=1000)
359+
AQO: rows=1, error=0%
360+
Filter: (x = t.x)
361+
Rows Removed by Filter: 475
362+
Using aqo: true
363+
AQO mode: LEARN
364+
JOINS: 0
365+
(18 rows)
366+
314367
-- It's OK to use the knowledge for a query with different constants.
315368
EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF)
316369
SELECT count(*) FROM t WHERE
@@ -580,6 +633,10 @@ ORDER BY (md5(query_text),error) DESC;
580633
-------+------------------------------------------------------------------------------------------------
581634
0.768 | SELECT count(*) FROM (SELECT count(*) FROM t1 GROUP BY (x,y)) AS q1;
582635
0.070 | SELECT count(*) FROM (SELECT * FROM t GROUP BY (x) HAVING x > 3) AS q1;
636+
1.554 | EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +
637+
| SELECT * FROM t WHERE +
638+
| x = (SELECT x FROM t t0 WHERE t0.x = t.x LIMIT 1) AND +
639+
| x IN (SELECT x FROM t t0 WHERE t0.x = t.x);
583640
0.000 | SELECT count(*) FROM t WHERE x < 3 AND mod(x,3) = 1;
584641
0.000 | SELECT * FROM +
585642
| (SELECT * FROM t WHERE x < 0) AS t0 +
@@ -612,13 +669,13 @@ ORDER BY (md5(query_text),error) DESC;
612669
| JOIN +
613670
| (SELECT * FROM t WHERE x % 3 < (SELECT avg(x) FROM t t0 WHERE t0.x <> t.x)) AS q2 +
614671
| ON q1.x = q2.x+1;
615-
(13 rows)
672+
(14 rows)
616673

617674
DROP TABLE t,t1 CASCADE; -- delete all tables used in the test
618675
SELECT count(*) FROM aqo_data; -- Just to detect some changes in the logic. May some false positives really bother us here?
619676
count
620677
-------
621-
44
678+
48
622679
(1 row)
623680

624681
SELECT true AS success FROM aqo_cleanup();

‎hash.c

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*
1313
*******************************************************************************
1414
*
15-
* Copyright (c) 2016-2022, Postgres Professional
15+
* Copyright (c) 2016-2023, Postgres Professional
1616
*
1717
* IDENTIFICATION
1818
* aqo/hash.c
@@ -175,6 +175,8 @@ get_grouped_exprs_hash(int child_fss, List *group_exprs)
175175
final_hashes[0] = child_fss;
176176
final_hashes[1] = get_int_array_hash(hashes, i);
177177

178+
pfree(hashes);
179+
178180
return get_int_array_hash(final_hashes, 2);
179181
}
180182

@@ -242,6 +244,7 @@ get_fss_for_object(List *relsigns, List *clauselist,
242244
clause_has_consts[i] = (args != NULL && has_consts(*args));
243245
i++;
244246
}
247+
pfree(args_hash);
245248

246249
idx = argsort(clause_hashes, n, sizeof(*clause_hashes), int_cmp);
247250
inverse_idx = inverse_permutation(idx, n);
@@ -252,6 +255,7 @@ get_fss_for_object(List *relsigns, List *clauselist,
252255
sorted_clauses[inverse_idx[i]] = clause_hashes[i];
253256
i++;
254257
}
258+
pfree(clause_hashes);
255259

256260
i = 0;
257261
foreach(lc, selectivities)
@@ -267,6 +271,7 @@ get_fss_for_object(List *relsigns, List *clauselist,
267271
}
268272
i++;
269273
}
274+
pfree(inverse_idx);
270275

271276
for (i = 0; i < n;)
272277
{
@@ -290,6 +295,8 @@ get_fss_for_object(List *relsigns, List *clauselist,
290295
sizeof(**features), double_cmp);
291296
i = j;
292297
}
298+
pfree(idx);
299+
pfree(clause_has_consts);
293300

294301
/*
295302
* Generate feature subspace hash.
@@ -299,6 +306,8 @@ get_fss_for_object(List *relsigns, List *clauselist,
299306
eclasses_hash = get_int_array_hash(eclass_hash, nargs);
300307
relations_hash = get_relations_hash(relsigns);
301308
fss_hash = get_fss_hash(clauses_hash, eclasses_hash, relations_hash);
309+
pfree(sorted_clauses);
310+
pfree(eclass_hash);
302311

303312
if (nfeatures != NULL)
304313
{
@@ -358,11 +367,17 @@ static int
358367
get_node_hash(Node *node)
359368
{
360369
char *str;
370+
char *no_consts;
371+
char *no_locations;
361372
int hash;
362373

363-
str = remove_locations(remove_consts(nodeToString(node)));
364-
hash = get_str_hash(str);
374+
str = nodeToString(node);
375+
no_consts = remove_consts(str);
365376
pfree(str);
377+
no_locations = remove_locations(no_consts);
378+
pfree(no_consts);
379+
hash = get_str_hash(no_locations);
380+
pfree(no_locations);
366381
return hash;
367382
}
368383

@@ -485,6 +500,7 @@ get_relations_hash(List *relsigns)
485500

486501
result = DatumGetInt32(hash_any((const unsigned char *) hashes,
487502
nhashes * sizeof(uint32)));
503+
pfree(hashes);
488504

489505
return result;
490506
}
@@ -497,9 +513,11 @@ static char *
497513
remove_consts(const char *str)
498514
{
499515
char *res;
516+
char *tmp;
500517

501-
res = replace_patterns(str, "{CONST", is_brace);
502-
res = replace_patterns(res, ":stmt_len", is_brace);
518+
tmp = replace_patterns(str, "{CONST", is_brace);
519+
res = replace_patterns(tmp, ":stmt_len", is_brace);
520+
pfree(tmp);
503521
return res;
504522
}
505523

@@ -701,6 +719,8 @@ get_eclasses(List *clauselist, int *nargs, int **args_hash, int **eclass_hash)
701719

702720
for (i = 0; i < *nargs; ++i)
703721
(*eclass_hash)[i] = e_hashes[disjoint_set_get_parent(p, i)];
722+
723+
pfree(e_hashes);
704724
}
705725

706726
/*

‎machine_learning.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
*
1313
*******************************************************************************
1414
*
15-
* Copyright (c) 2016-2022, Postgres Professional
15+
* Copyright (c) 2016-2023, Postgres Professional
1616
*
1717
* IDENTIFICATION
1818
* aqo/machine_learning.c

0 commit comments

Comments
 (0)