One big commit for the next changes, induced by 1C-benchmarking activity:

1. Rewrite access to ML-storage to fix visibility and deadlocks problems.
2. Add log_ignorance table for as a storage for cases that couldn't processed
by AQO and needed to study & fix further.
3. Add pgbench TAP test
4. Add GUC's aqo.show_details and aqo.show_hash

Author: danolivo

Makefile

@@ -2,11 +2,13 @@
 
 EXTENSION = aqo
 EXTVERSION = 1.2
-PGFILEDESC = "AQO - adaptive query optimization"
-MODULES = aqo
+PGFILEDESC = "AQO - Adaptive Query Optimization"
+MODULE_big = aqo
 OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
 hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
-selectivity_cache.o storage.o utils.o $(WIN32RES)
+selectivity_cache.o storage.o utils.o ignorance.o $(WIN32RES)
+
+TAP_TESTS = 1
 
 REGRESS =	aqo_disabled \
 			aqo_controlled \
@@ -15,7 +17,8 @@ REGRESS =	aqo_disabled \
 			aqo_learn \
 			schema \
 			aqo_fdw \
-			aqo_CVE-2020-14350
+			aqo_CVE-2020-14350 \
+			gucs
 
 fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw
 PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir)
@@ -24,7 +27,6 @@ EXTRA_INSTALL = contrib/postgres_fdw
 
 DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql
 
-MODULE_big = aqo
 ifdef USE_PGXS
 PG_CONFIG ?= pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)

aqo.c

@@ -9,17 +9,35 @@
  */
 
 #include "aqo.h"
+#include "ignorance.h"
+
+#include "access/table.h"
+#include "catalog/pg_extension.h"
+#include "commands/extension.h"
 
-PG_MODULE_MAGIC;
 
+PG_MODULE_MAGIC;
 void _PG_init(void);
 
 
+#define AQO_MODULE_MAGIC	(1234)
+
 /* Strategy of determining feature space for new queries. */
 int		aqo_mode;
 bool	force_collect_stat;
-bool	aqo_show_hash;
-bool	aqo_details;
+
+/*
+ * Show special info in EXPLAIN mode.
+ *
+ * aqo_show_hash - show query class (hash) and a feature space value (hash)
+ * of each plan node. This is instance-dependent value and can't be used
+ * in regression and TAP tests.
+ *
+ * aqo_show_details - show AQO settings for this class and prediction
+ * for each plan node.
+ */
+bool   aqo_show_hash;
+bool   aqo_show_details;
 
 /* GUC variables */
 static const struct config_enum_entry format_options[] = {
@@ -138,7 +156,7 @@ _PG_init(void)
 							 "aqo.show_details",
 							 "Show AQO state on a query.",
 							 NULL,
-							 &aqo_details,
+							 &aqo_show_details,
 							 false,
 							 PGC_USERSET,
 							 0,
@@ -147,6 +165,32 @@ _PG_init(void)
 							 NULL
 	);
 
+	DefineCustomBoolVariable(
+							 "aqo.log_ignorance",
+							 "Log in a special table all feature spaces for which the AQO prediction was not successful.",
+							 NULL,
+							 &aqo_log_ignorance,
+							 false,
+							 PGC_SUSET,
+							 0,
+							 NULL,
+							 set_ignorance,
+							 NULL
+	);
+
+	DefineCustomIntVariable("aqo.query_text_limit",
+							"Sets the maximum size of logged query text.",
+							"Zero logs full query text.",
+							&aqo_query_text_limit,
+							1024,
+							0, INT_MAX,
+							PGC_SUSET,
+							0,
+							NULL,
+							NULL,
+							NULL
+	);
+
 	prev_planner_hook							= planner_hook;
 	planner_hook								= aqo_planner;
 	prev_post_parse_analyze_hook				= post_parse_analyze_hook;
@@ -191,3 +235,66 @@ invalidate_deactivated_queries_cache(PG_FUNCTION_ARGS)
 	init_deactivated_queries_storage();
 	PG_RETURN_POINTER(NULL);
 }
+
+/*
+ * Return AQO schema's Oid or InvalidOid if that's not possible.
+ */
+Oid
+get_aqo_schema(void)
+{
+	Oid				result;
+	Relation		rel;
+	SysScanDesc		scandesc;
+	HeapTuple		tuple;
+	ScanKeyData		entry[1];
+	Oid				ext_oid;
+
+	/* It's impossible to fetch pg_aqo's schema now */
+	if (!IsTransactionState())
+		return InvalidOid;
+
+	ext_oid = get_extension_oid("aqo", true);
+	if (ext_oid == InvalidOid)
+		return InvalidOid; /* exit if pg_aqo does not exist */
+
+	ScanKeyInit(&entry[0],
+#if PG_VERSION_NUM >= 120000
+				Anum_pg_extension_oid,
+#else
+				ObjectIdAttributeNumber,
+#endif
+				BTEqualStrategyNumber, F_OIDEQ,
+				ObjectIdGetDatum(ext_oid));
+
+	rel = heap_open(ExtensionRelationId, AccessShareLock);
+	scandesc = systable_beginscan(rel, ExtensionOidIndexId, true,
+								  NULL, 1, entry);
+
+	tuple = systable_getnext(scandesc);
+
+	/* We assume that there can be at most one matching tuple */
+	if (HeapTupleIsValid(tuple))
+		result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace;
+	else
+		result = InvalidOid;
+
+	systable_endscan(scandesc);
+
+	heap_close(rel, AccessShareLock);
+
+	return result;
+}
+
+/*
+ * Init userlock
+ */
+void
+init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2)
+{
+	tag->locktag_field1 = AQO_MODULE_MAGIC;
+	tag->locktag_field2 = key1;
+	tag->locktag_field3 = key2;
+	tag->locktag_field4 = 0;
+	tag->locktag_type = LOCKTAG_USERLOCK;
+	tag->locktag_lockmethodid = USER_LOCKMETHOD;
+}

aqo.h

@@ -175,7 +175,8 @@ typedef enum
 extern int	aqo_mode;
 extern bool	force_collect_stat;
 extern bool aqo_show_hash;
-extern bool aqo_details;
+extern bool aqo_show_details;
+extern int aqo_query_text_limit;
 
 /*
  * It is mostly needed for auto tuning of query. with auto tuning mode aqo
@@ -267,7 +268,7 @@ extern		get_parameterized_joinrel_size_hook_type
 			prev_get_parameterized_joinrel_size_hook;
 extern		copy_generic_path_info_hook_type
 			prev_copy_generic_path_info_hook;
-extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
+extern		ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
 
 extern void ppi_hook(ParamPathInfo *ppi);
 
@@ -282,20 +283,19 @@ int			get_clause_hash(Expr *clause, int nargs,
 
 
 /* Storage interaction */
-bool find_query(int query_hash,
-		   Datum *search_values,
-		   bool *search_nulls);
-bool add_query(int query_hash, bool learn_aqo, bool use_aqo,
-		  int fspace_hash, bool auto_tuning);
-bool update_query(int query_hash, bool learn_aqo, bool use_aqo,
-			 int fspace_hash, bool auto_tuning);
-bool		add_query_text(int query_hash, const char *query_text);
-bool load_fss(int fss_hash, int ncols,
-		 double **matrix, double *targets, int *rows);
-extern bool update_fss(int fss_hash, int nrows, int ncols,
+extern bool find_query(int qhash, Datum *search_values, bool *search_nulls);
+extern bool update_query(int qhash, int fhash,
+						 bool learn_aqo, bool use_aqo, bool auto_tuning);
+extern bool add_query_text(int query_hash, char *query_text);
+extern bool load_fss(int fhash, int fss_hash,
+					 int ncols, double **matrix, double *targets, int *rows);
+extern bool update_fss(int fhash, int fss_hash, int nrows, int ncols,
 					   double **matrix, double *targets);
 QueryStat  *get_aqo_stat(int query_hash);
 void		update_aqo_stat(int query_hash, QueryStat * stat);
+extern bool my_index_insert(Relation indexRelation,	Datum *values, bool *isnull,
+							ItemPointer heap_t_ctid, Relation heapRelation,
+							IndexUniqueCheck checkUnique);
 void		init_deactivated_queries_storage(void);
 void		fini_deactivated_queries_storage(void);
 bool		query_is_deactivated(int query_hash);
@@ -382,4 +382,7 @@ void cache_selectivity(int clause_hash,
 double	   *selectivity_cache_find_global_relid(int clause_hash, int global_relid);
 void		selectivity_cache_clear(void);
 
+extern Oid get_aqo_schema(void);
+extern void init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2);
+
 #endif

auto_tuning.c

@@ -187,8 +187,11 @@ automatical_query_tuning(int query_hash, QueryStat * stat)
 	}
 
 	if (num_iterations <= auto_tuning_max_iterations || p_use > 0.5)
-		update_query(query_hash, query_context.learn_aqo, query_context.use_aqo,
-											query_context.fspace_hash, true);
+		update_query(query_hash,
+					 query_context.fspace_hash,
+					 query_context.learn_aqo,
+					 query_context.use_aqo,
+					 true);
 	else
-		update_query(query_hash, false, false, query_context.fspace_hash, false);
+		update_query(query_hash, query_context.fspace_hash, false, false, false);
 }

cardinality_estimation.c

@@ -40,7 +40,8 @@ predict_for_relation(List *restrict_clauses, List *selectivities,
 		for (i = 0; i < aqo_K; ++i)
 			matrix[i] = palloc0(sizeof(**matrix) * nfeatures);
 
-	if (load_fss(*fss_hash, nfeatures, matrix, targets, &rows))
+	if (load_fss(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
+				 targets, &rows))
 		result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
 	else
 	{

cardinality_hooks.c

@@ -156,7 +156,8 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel)
 	relids = list_make1_int(relid);
 
 	restrict_clauses = list_copy(rel->baserestrictinfo);
-	predicted = predict_for_relation(restrict_clauses, selectivities, relids, &fss);
+	predicted = predict_for_relation(restrict_clauses,selectivities,
+									 relids, &fss);
 	rel->fss_hash = fss;
 
 	if (predicted >= 0)
@@ -208,12 +209,16 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
 
 	if (query_context.use_aqo || query_context.learn_aqo)
 	{
+		MemoryContext mcxt;
+
 		allclauses = list_concat(list_copy(param_clauses),
 								 list_copy(rel->baserestrictinfo));
 		selectivities = get_selectivities(root, allclauses, rel->relid,
 										  JOIN_INNER, NULL);
 		relid = planner_rt_fetch(rel->relid, root)->relid;
 		get_eclasses(allclauses, &nargs, &args_hash, &eclass_hash);
+
+		mcxt = MemoryContextSwitchTo(CacheMemoryContext);
 		forboth(l, allclauses, l2, selectivities)
 		{
 			current_hash = get_clause_hash(
@@ -222,6 +227,8 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root,
 			cache_selectivity(current_hash, rel->relid, relid,
 							  *((double *) lfirst(l2)));
 		}
+		MemoryContextSwitchTo(mcxt);
+
 		pfree(args_hash);
 		pfree(eclass_hash);
 	}