Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitc05385e

Browse files
danolivoAndrey Lepikhov
authored and
Andrey Lepikhov
committed
One big commit for the next changes, induced by 1C-benchmarking activity:
1. Rewrite access to ML-storage to fix visibility and deadlocks problems.2. Add log_ignorance table for as a storage for cases that couldn't processedby AQO and needed to study & fix further.3. Add pgbench TAP test4. Add GUC's aqo.show_details and aqo.show_hash
1 parent6cd32af commitc05385e

File tree

15 files changed

+1113
-572
lines changed

15 files changed

+1113
-572
lines changed

‎Makefile‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@
22

33
EXTENSION = aqo
44
EXTVERSION = 1.2
5-
PGFILEDESC = "AQO -adaptive query optimization"
6-
MODULES = aqo
5+
PGFILEDESC = "AQO -Adaptive Query Optimization"
6+
MODULE_big = aqo
77
OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o\
88
hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o\
9-
selectivity_cache.o storage.o utils.o$(WIN32RES)
9+
selectivity_cache.o storage.o utils.o ignorance.o$(WIN32RES)
10+
11+
TAP_TESTS = 1
1012

1113
REGRESS =aqo_disabled\
1214
aqo_controlled\
@@ -15,7 +17,8 @@ REGRESS =aqo_disabled \
1517
aqo_learn\
1618
schema\
1719
aqo_fdw\
18-
aqo_CVE-2020-14350
20+
aqo_CVE-2020-14350\
21+
gucs
1922

2023
fdw_srcdir =$(top_srcdir)/contrib/postgres_fdw
2124
PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir)
@@ -24,7 +27,6 @@ EXTRA_INSTALL = contrib/postgres_fdw
2427

2528
DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql
2629

27-
MODULE_big = aqo
2830
ifdefUSE_PGXS
2931
PG_CONFIG ?= pg_config
3032
PGXS :=$(shell$(PG_CONFIG) --pgxs)

‎aqo.c‎

Lines changed: 111 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,35 @@
99
*/
1010

1111
#include"aqo.h"
12+
#include"ignorance.h"
13+
14+
#include"access/table.h"
15+
#include"catalog/pg_extension.h"
16+
#include"commands/extension.h"
1217

13-
PG_MODULE_MAGIC;
1418

19+
PG_MODULE_MAGIC;
1520
void_PG_init(void);
1621

1722

23+
#defineAQO_MODULE_MAGIC(1234)
24+
1825
/* Strategy of determining feature space for new queries. */
1926
intaqo_mode;
2027
boolforce_collect_stat;
21-
boolaqo_show_hash;
22-
boolaqo_details;
28+
29+
/*
30+
* Show special info in EXPLAIN mode.
31+
*
32+
* aqo_show_hash - show query class (hash) and a feature space value (hash)
33+
* of each plan node. This is instance-dependent value and can't be used
34+
* in regression and TAP tests.
35+
*
36+
* aqo_show_details - show AQO settings for this class and prediction
37+
* for each plan node.
38+
*/
39+
boolaqo_show_hash;
40+
boolaqo_show_details;
2341

2442
/* GUC variables */
2543
staticconststructconfig_enum_entryformat_options[]= {
@@ -138,7 +156,7 @@ _PG_init(void)
138156
"aqo.show_details",
139157
"Show AQO state on a query.",
140158
NULL,
141-
&aqo_details,
159+
&aqo_show_details,
142160
false,
143161
PGC_USERSET,
144162
0,
@@ -147,6 +165,32 @@ _PG_init(void)
147165
NULL
148166
);
149167

168+
DefineCustomBoolVariable(
169+
"aqo.log_ignorance",
170+
"Log in a special table all feature spaces for which the AQO prediction was not successful.",
171+
NULL,
172+
&aqo_log_ignorance,
173+
false,
174+
PGC_SUSET,
175+
0,
176+
NULL,
177+
set_ignorance,
178+
NULL
179+
);
180+
181+
DefineCustomIntVariable("aqo.query_text_limit",
182+
"Sets the maximum size of logged query text.",
183+
"Zero logs full query text.",
184+
&aqo_query_text_limit,
185+
1024,
186+
0,INT_MAX,
187+
PGC_SUSET,
188+
0,
189+
NULL,
190+
NULL,
191+
NULL
192+
);
193+
150194
prev_planner_hook=planner_hook;
151195
planner_hook=aqo_planner;
152196
prev_post_parse_analyze_hook=post_parse_analyze_hook;
@@ -191,3 +235,66 @@ invalidate_deactivated_queries_cache(PG_FUNCTION_ARGS)
191235
init_deactivated_queries_storage();
192236
PG_RETURN_POINTER(NULL);
193237
}
238+
239+
/*
240+
* Return AQO schema's Oid or InvalidOid if that's not possible.
241+
*/
242+
Oid
243+
get_aqo_schema(void)
244+
{
245+
Oidresult;
246+
Relationrel;
247+
SysScanDescscandesc;
248+
HeapTupletuple;
249+
ScanKeyDataentry[1];
250+
Oidext_oid;
251+
252+
/* It's impossible to fetch pg_aqo's schema now */
253+
if (!IsTransactionState())
254+
returnInvalidOid;
255+
256+
ext_oid=get_extension_oid("aqo", true);
257+
if (ext_oid==InvalidOid)
258+
returnInvalidOid;/* exit if pg_aqo does not exist */
259+
260+
ScanKeyInit(&entry[0],
261+
#ifPG_VERSION_NUM >=120000
262+
Anum_pg_extension_oid,
263+
#else
264+
ObjectIdAttributeNumber,
265+
#endif
266+
BTEqualStrategyNumber,F_OIDEQ,
267+
ObjectIdGetDatum(ext_oid));
268+
269+
rel=heap_open(ExtensionRelationId,AccessShareLock);
270+
scandesc=systable_beginscan(rel,ExtensionOidIndexId, true,
271+
NULL,1,entry);
272+
273+
tuple=systable_getnext(scandesc);
274+
275+
/* We assume that there can be at most one matching tuple */
276+
if (HeapTupleIsValid(tuple))
277+
result= ((Form_pg_extension)GETSTRUCT(tuple))->extnamespace;
278+
else
279+
result=InvalidOid;
280+
281+
systable_endscan(scandesc);
282+
283+
heap_close(rel,AccessShareLock);
284+
285+
returnresult;
286+
}
287+
288+
/*
289+
* Init userlock
290+
*/
291+
void
292+
init_lock_tag(LOCKTAG*tag,uint32key1,uint32key2)
293+
{
294+
tag->locktag_field1=AQO_MODULE_MAGIC;
295+
tag->locktag_field2=key1;
296+
tag->locktag_field3=key2;
297+
tag->locktag_field4=0;
298+
tag->locktag_type=LOCKTAG_USERLOCK;
299+
tag->locktag_lockmethodid=USER_LOCKMETHOD;
300+
}

‎aqo.h‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -175,7 +175,8 @@ typedef enum
175175
externintaqo_mode;
176176
externboolforce_collect_stat;
177177
externboolaqo_show_hash;
178-
externboolaqo_details;
178+
externboolaqo_show_details;
179+
externintaqo_query_text_limit;
179180

180181
/*
181182
* It is mostly needed for auto tuning of query. with auto tuning mode aqo
@@ -267,7 +268,7 @@ externget_parameterized_joinrel_size_hook_type
267268
prev_get_parameterized_joinrel_size_hook;
268269
externcopy_generic_path_info_hook_type
269270
prev_copy_generic_path_info_hook;
270-
externExplainOnePlan_hook_typeprev_ExplainOnePlan_hook;
271+
externExplainOnePlan_hook_typeprev_ExplainOnePlan_hook;
271272

272273
externvoidppi_hook(ParamPathInfo*ppi);
273274

@@ -282,20 +283,19 @@ intget_clause_hash(Expr *clause, int nargs,
282283

283284

284285
/* Storage interaction */
285-
boolfind_query(intquery_hash,
286-
Datum*search_values,
287-
bool*search_nulls);
288-
booladd_query(intquery_hash,boollearn_aqo,booluse_aqo,
289-
intfspace_hash,boolauto_tuning);
290-
boolupdate_query(intquery_hash,boollearn_aqo,booluse_aqo,
291-
intfspace_hash,boolauto_tuning);
292-
booladd_query_text(intquery_hash,constchar*query_text);
293-
boolload_fss(intfss_hash,intncols,
294-
double**matrix,double*targets,int*rows);
295-
externboolupdate_fss(intfss_hash,intnrows,intncols,
286+
externboolfind_query(intqhash,Datum*search_values,bool*search_nulls);
287+
externboolupdate_query(intqhash,intfhash,
288+
boollearn_aqo,booluse_aqo,boolauto_tuning);
289+
externbooladd_query_text(intquery_hash,char*query_text);
290+
externboolload_fss(intfhash,intfss_hash,
291+
intncols,double**matrix,double*targets,int*rows);
292+
externboolupdate_fss(intfhash,intfss_hash,intnrows,intncols,
296293
double**matrix,double*targets);
297294
QueryStat*get_aqo_stat(intquery_hash);
298295
voidupdate_aqo_stat(intquery_hash,QueryStat*stat);
296+
externboolmy_index_insert(RelationindexRelation,Datum*values,bool*isnull,
297+
ItemPointerheap_t_ctid,RelationheapRelation,
298+
IndexUniqueCheckcheckUnique);
299299
voidinit_deactivated_queries_storage(void);
300300
voidfini_deactivated_queries_storage(void);
301301
boolquery_is_deactivated(intquery_hash);
@@ -382,4 +382,7 @@ void cache_selectivity(int clause_hash,
382382
double*selectivity_cache_find_global_relid(intclause_hash,intglobal_relid);
383383
voidselectivity_cache_clear(void);
384384

385+
externOidget_aqo_schema(void);
386+
externvoidinit_lock_tag(LOCKTAG*tag,uint32key1,uint32key2);
387+
385388
#endif

‎auto_tuning.c‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -187,8 +187,11 @@ automatical_query_tuning(int query_hash, QueryStat * stat)
187187
}
188188

189189
if (num_iterations <=auto_tuning_max_iterations||p_use>0.5)
190-
update_query(query_hash,query_context.learn_aqo,query_context.use_aqo,
191-
query_context.fspace_hash, true);
190+
update_query(query_hash,
191+
query_context.fspace_hash,
192+
query_context.learn_aqo,
193+
query_context.use_aqo,
194+
true);
192195
else
193-
update_query(query_hash,false, false,query_context.fspace_hash, false);
196+
update_query(query_hash,query_context.fspace_hash, false,false, false);
194197
}

‎cardinality_estimation.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ predict_for_relation(List *restrict_clauses, List *selectivities,
4040
for (i=0;i<aqo_K;++i)
4141
matrix[i]=palloc0(sizeof(**matrix)*nfeatures);
4242

43-
if (load_fss(*fss_hash,nfeatures,matrix,targets,&rows))
43+
if (load_fss(query_context.fspace_hash,*fss_hash,nfeatures,matrix,
44+
targets,&rows))
4445
result=OkNNr_predict(rows,nfeatures,matrix,targets,features);
4546
else
4647
{

‎cardinality_hooks.c‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
156156
relids=list_make1_int(relid);
157157

158158
restrict_clauses=list_copy(rel->baserestrictinfo);
159-
predicted=predict_for_relation(restrict_clauses,selectivities,relids,&fss);
159+
predicted=predict_for_relation(restrict_clauses,selectivities,
160+
relids,&fss);
160161
rel->fss_hash=fss;
161162

162163
if (predicted >=0)
@@ -208,12 +209,16 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
208209

209210
if (query_context.use_aqo||query_context.learn_aqo)
210211
{
212+
MemoryContextmcxt;
213+
211214
allclauses=list_concat(list_copy(param_clauses),
212215
list_copy(rel->baserestrictinfo));
213216
selectivities=get_selectivities(root,allclauses,rel->relid,
214217
JOIN_INNER,NULL);
215218
relid=planner_rt_fetch(rel->relid,root)->relid;
216219
get_eclasses(allclauses,&nargs,&args_hash,&eclass_hash);
220+
221+
mcxt=MemoryContextSwitchTo(CacheMemoryContext);
217222
forboth(l,allclauses,l2,selectivities)
218223
{
219224
current_hash=get_clause_hash(
@@ -222,6 +227,8 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
222227
cache_selectivity(current_hash,rel->relid,relid,
223228
*((double*)lfirst(l2)));
224229
}
230+
MemoryContextSwitchTo(mcxt);
231+
225232
pfree(args_hash);
226233
pfree(eclass_hash);
227234
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp