Add basic regression test on FDW support.

Switch from AQO_EXPLAIN define to the two user GUCS: aqo.show_details
and aqo.show_hash.

New ExplainOneNode hook allows to add info into explain of each node.
The changes concern the core and the extension.

Author: danolivo

Makefile

@@ -14,11 +14,13 @@ REGRESS =	aqo_disabled \
 			aqo_forced \
 			aqo_learn \
 			schema \
+			aqo_fdw \
 			aqo_CVE-2020-14350
 
 fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw
 PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir)
 EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add
+EXTRA_INSTALL = contrib/postgres_fdw
 
 DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql
 

aqo.c

@@ -18,6 +18,8 @@ void _PG_init(void);
 /* Strategy of determining feature space for new queries. */
 int		aqo_mode;
 bool	force_collect_stat;
+bool	aqo_show_hash;
+bool	aqo_details;
 
 /* GUC variables */
 static const struct config_enum_entry format_options[] = {
@@ -83,6 +85,7 @@ set_joinrel_size_estimates_hook_type		prev_set_joinrel_size_estimates_hook;
 get_parameterized_joinrel_size_hook_type	prev_get_parameterized_joinrel_size_hook;
 copy_generic_path_info_hook_type			prev_copy_generic_path_info_hook;
 ExplainOnePlan_hook_type					prev_ExplainOnePlan_hook;
+ExplainOneNode_hook_type					prev_ExplainOneNode_hook;
 
 /*****************************************************************************
  *
@@ -116,7 +119,33 @@ _PG_init(void)
 							 NULL,
 							 NULL,
 							 NULL
-		);
+	);
+
+	DefineCustomBoolVariable(
+							 "aqo.show_hash",
+							 "Show query and node hash on explain.",
+							 "Hash value depend on each instance and is not good to enable it in regression or TAP tests.",
+							 &aqo_show_hash,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL
+	);
+
+	DefineCustomBoolVariable(
+							 "aqo.show_details",
+							 "Show AQO state on a query.",
+							 NULL,
+							 &aqo_details,
+							 false,
+							 PGC_USERSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL
+	);
 
 	prev_planner_hook							= planner_hook;
 	planner_hook								= aqo_planner;
@@ -139,6 +168,8 @@ _PG_init(void)
 	copy_generic_path_info_hook					= aqo_copy_generic_path_info;
 	prev_ExplainOnePlan_hook					= ExplainOnePlan_hook;
 	ExplainOnePlan_hook							= print_into_explain;
+	prev_ExplainOneNode_hook					= ExplainOneNode_hook;
+	ExplainOneNode_hook							= print_node_explain;
 	parampathinfo_postinit_hook					= ppi_hook;
 
 	init_deactivated_queries_storage();

aqo.h

@@ -174,6 +174,8 @@ typedef enum
 
 extern int	aqo_mode;
 extern bool	force_collect_stat;
+extern bool aqo_show_hash;
+extern bool aqo_details;
 
 /*
  * It is mostly needed for auto tuning of query. with auto tuning mode aqo
@@ -311,6 +313,10 @@ void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into,
 			   ExplainState *es, const char *queryString,
 			   ParamListInfo params, const instr_time *planduration,
 			   QueryEnvironment *queryEnv);
+extern void print_node_explain(ExplainState *es,
+							   PlanState *ps,
+							   Plan *plan,
+							   double rows);
 void		disable_aqo_for_query(void);
 
 /* Cardinality estimation hooks */

aqo_pg12.patch

@@ -11,7 +11,7 @@ index 92184ed487..ac8abbfd03 100644
  		auto_explain	\
  		bloom		\
 diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c
-index a40bc14ec5..28ec281a70 100644
+index a40bc14ec5..d157c3043e 100644
 --- a/src/backend/commands/explain.c
 +++ b/src/backend/commands/explain.c
 @@ -24,6 +24,7 @@
@@ -22,17 +22,20 @@ index a40bc14ec5..28ec281a70 100644
  #include "parser/parsetree.h"
  #include "rewrite/rewriteHandler.h"
  #include "storage/bufmgr.h"
-@@ -46,6 +47,9 @@ ExplainOneQuery_hook_type ExplainOneQuery_hook = NULL;
+@@ -46,6 +47,12 @@ ExplainOneQuery_hook_type ExplainOneQuery_hook = NULL;
  /* Hook for plugins to get control in explain_get_index_name() */
  explain_get_index_name_hook_type explain_get_index_name_hook = NULL;
 
 +/* Hook for plugins to get control in ExplainOnePlan() */
 +ExplainOnePlan_hook_type ExplainOnePlan_hook = NULL;
++
++/* Hook for plugins to get control in ExplainOnePlan() */
++ExplainOneNode_hook_type ExplainOneNode_hook = NULL;
 +
 
  /* OR-able flags for ExplainXMLTag() */
  #define X_OPENING 0
-@@ -597,6 +601,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
+@@ -597,6 +604,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es,
  		ExplainPropertyFloat("Execution Time", "ms", 1000.0 * totaltime, 3,
  							 es);
 
@@ -43,42 +46,13 @@ index a40bc14ec5..28ec281a70 100644
  	ExplainCloseGroup("Query", NULL, true, es);
  }
 
-@@ -1521,6 +1529,38 @@ ExplainNode(PlanState *planstate, List *ancestors,
+@@ -1521,6 +1532,9 @@ ExplainNode(PlanState *planstate, List *ancestors,
  				appendStringInfo(es->str,
  								 " (actual rows=%.0f loops=%.0f)",
  								 rows, nloops);
 +
-+#ifdef AQO_EXPLAIN
-+			if (es->verbose && plan && planstate->instrument)
-+			{
-+				int wrkrs = 1;
-+				double error = -1.;
-+
-+				if (planstate->worker_instrument && IsParallelTuplesProcessing(plan))
-+				{
-+					int i;
-+					for (i = 0; i < planstate->worker_instrument->num_workers; i++)
-+					{
-+						Instrumentation *instrument = &planstate->worker_instrument->instrument[i];
-+						if (instrument->nloops <= 0)
-+							continue;
-+						wrkrs++;
-+					}
-+				}
-+
-+				if (plan->predicted_cardinality > 0.)
-+				{
-+					error = 100. * (plan->predicted_cardinality - (rows*wrkrs))
-+												/ plan->predicted_cardinality;
-+					appendStringInfo(es->str,
-+					" (AQO: cardinality=%.0lf, error=%.0lf%%, fsspace_hash=%d)",
-+							plan->predicted_cardinality, error, plan->fss_hash);
-+				}
-+				else
-+					appendStringInfo(es->str, " (AQO not used, fsspace_hash=%d)",
-+																plan->fss_hash);
-+			}
-+#endif
++			if (ExplainOneNode_hook)
++				ExplainOneNode_hook(es, planstate, plan, rows);
  		}
  		else
  		{
@@ -909,10 +883,10 @@ index 7758a49fa4..4f221a000b 100644
 
  	return ppi;
 diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h
-index f8b79ec120..b5eda01907 100644
+index f8b79ec120..f970971a4a 100644
 --- a/src/include/commands/explain.h
 +++ b/src/include/commands/explain.h
-@@ -62,6 +62,12 @@ extern PGDLLIMPORT ExplainOneQuery_hook_type ExplainOneQuery_hook;
+@@ -62,6 +62,19 @@ extern PGDLLIMPORT ExplainOneQuery_hook_type ExplainOneQuery_hook;
  typedef const char *(*explain_get_index_name_hook_type) (Oid indexId);
  extern PGDLLIMPORT explain_get_index_name_hook_type explain_get_index_name_hook;
 
@@ -922,6 +896,13 @@ index f8b79ec120..b5eda01907 100644
 +			   ParamListInfo params, const instr_time *planduration,
 +			   QueryEnvironment *queryEnv);
 +extern PGDLLIMPORT ExplainOnePlan_hook_type ExplainOnePlan_hook;
++
++/* Explain a plan node info */
++typedef void (*ExplainOneNode_hook_type) (ExplainState *es,
++										 PlanState *ps,
++										 Plan *plan,
++										 double rows);
++extern PGDLLIMPORT ExplainOneNode_hook_type ExplainOneNode_hook;
 
  extern void ExplainQuery(ParseState *pstate, ExplainStmt *stmt, const char *queryString,
  						 ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest);

conf.add

@@ -1 +1,3 @@
-shared_preload_libraries = 'aqo'
+autovacuum = off
+shared_preload_libraries = 'postgres_fdw, aqo'
+max_parallel_workers = 0 # switch off parallel workers because of unsteadiness

expected/aqo_fdw.out

@@ -0,0 +1,74 @@
+-- Tests on cardinality estimation of FDW-queries:
+-- simple ForeignScan.
+-- JOIN push-down (check push of baserestrictinfo and joininfo)
+-- Aggregate push-down
+-- Push-down of groupings with HAVING clause.
+CREATE EXTENSION aqo;
+CREATE EXTENSION postgres_fdw;
+SET aqo.mode = 'learn';
+SET aqo.show_details = 'true'; -- show AQO info for each node and entire query.
+SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it.
+DO $d$
+    BEGIN
+        EXECUTE $$CREATE SERVER loopback FOREIGN DATA WRAPPER postgres_fdw
+            OPTIONS (dbname '$$||current_database()||$$',
+                     port '$$||current_setting('port')||$$'
+            )$$;
+    END;
+$d$;
+CREATE USER MAPPING FOR PUBLIC SERVER loopback;
+CREATE TABLE local (x int);
+CREATE FOREIGN TABLE frgn(x int) SERVER loopback OPTIONS (table_name 'local');
+INSERT INTO frgn (x) VALUES (1);
+ANALYZE local;
+EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn;
+                         QUERY PLAN                          
+-------------------------------------------------------------
+ Foreign Scan on frgn (actual rows=1 loops=1) (AQO not used)
+ Using aqo: true
+ AQO mode: LEARN
+ JOINS: 0
+(4 rows)
+
+EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn;
+                                 QUERY PLAN                                  
+-----------------------------------------------------------------------------
+ Foreign Scan on frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%)
+ Using aqo: true
+ AQO mode: LEARN
+ JOINS: 0
+(4 rows)
+
+-- Push down base filters.
+EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10;
+                             QUERY PLAN                             
+--------------------------------------------------------------------
+ Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO not used)
+   Output: x
+   Remote SQL: SELECT x FROM public.local WHERE ((x < 10))
+ Using aqo: true
+ AQO mode: LEARN
+ JOINS: 0
+(6 rows)
+
+EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10;
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%)
+   Output: x
+   Remote SQL: SELECT x FROM public.local WHERE ((x < 10))
+ Using aqo: true
+ AQO mode: LEARN
+ JOINS: 0
+(6 rows)
+
+EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants
+                                  QUERY PLAN                                   
+-------------------------------------------------------------------------------
+ Foreign Scan on frgn (actual rows=0 loops=1) (AQO: cardinality=1, error=100%)
+ Using aqo: true
+ AQO mode: LEARN
+ JOINS: 0
+(4 rows)
+
+DROP EXTENSION aqo;