Merge branch 'PGPROEE10_DEV' into PGPROEE10-pgpro-1078

Author: vbwagner

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for PostgreSQL 10.0.
+# Generated by GNU Autoconf 2.69 for PostgreSQL 10.1.
 #
 # Report bugs to <bugs@postgrespro.ru>.
 #
@@ -583,8 +583,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='PostgreSQL'
 PACKAGE_TARNAME='postgrespro'
-PACKAGE_VERSION='10.0'
-PACKAGE_STRING='PostgreSQL 10.0'
+PACKAGE_VERSION='10.1'
+PACKAGE_STRING='PostgreSQL 10.1'
 PACKAGE_BUGREPORT='bugs@postgrespro.ru'
 PACKAGE_URL=''
 
@@ -1428,7 +1428,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures PostgreSQL 10.0 to adapt to many kinds of systems.
+\`configure' configures PostgreSQL 10.1 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1494,7 +1494,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of PostgreSQL 10.0:";;
+     short | recursive ) echo "Configuration of PostgreSQL 10.1:";;
    esac
   cat <<\_ACEOF
 
@@ -1655,7 +1655,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-PostgreSQL configure 10.0
+PostgreSQL configure 10.1
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2367,7 +2367,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by PostgreSQL $as_me 10.0, which was
+It was created by PostgreSQL $as_me 10.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -16989,7 +16989,7 @@ _ACEOF
 # awk -F is a regex on some platforms, and not on others, so make "." a tab
 PG_VERSION_NUM="`echo "$PACKAGE_VERSION" | sed 's/[A-Za-z].*$//' |
 tr '.' '	' |
-$AWK '{printf "%d%02d%02d", $1, $2, (NF >= 3) ? $3 : 0}'`"
+$AWK '{printf "%d%04d", $1, $2}'`"
 
 cat >>confdefs.h <<_ACEOF
 #define PG_VERSION_NUM $PG_VERSION_NUM
@@ -17557,7 +17557,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by PostgreSQL $as_me 10.0, which was
+This file was extended by PostgreSQL $as_me 10.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -17627,7 +17627,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-PostgreSQL config.status 10.0
+PostgreSQL config.status 10.1
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.in

@@ -17,7 +17,7 @@ dnl Read the Autoconf manual for details.
 dnl
 m4_pattern_forbid(^PGAC_)dnl to catch undefined macros
 
-AC_INIT([PostgreSQL], [10.0], [bugs@postgrespro.ru], [postgrespro])
+AC_INIT([PostgreSQL], [10.1], [bugs@postgrespro.ru], [postgrespro])
 PACKAGE_TARNAME=postgrespro
 
 m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.69], [], [m4_fatal([Autoconf version 2.69 is required.
@@ -2232,7 +2232,7 @@ AC_DEFINE_UNQUOTED(PGPRO_VERSION_STR,
 # awk -F is a regex on some platforms, and not on others, so make "." a tab
 [PG_VERSION_NUM="`echo "$PACKAGE_VERSION" | sed 's/[A-Za-z].*$//' |
 tr '.' '	' |
-$AWK '{printf "%d%02d%02d", $1, $2, (NF >= 3) ? $3 : 0}'`"]
+$AWK '{printf "%d%04d", $1, $2}'`"]
 AC_DEFINE_UNQUOTED(PG_VERSION_NUM, $PG_VERSION_NUM, [PostgreSQL version as a number])
 AC_SUBST(PG_VERSION_NUM)
 

contrib/Makefile

@@ -7,6 +7,7 @@ include $(top_builddir)/src/Makefile.global
 SUBDIRS = \
 		adminpack	\
 		amcheck		\
+		aqo			\
 		auth_delay	\
 		auto_explain	\
 		bloom		\

contrib/aqo/Makefile

@@ -0,0 +1,22 @@
+# contrib/aqo/Makefile
+
+EXTENSION = aqo
+PGFILEDESC = "AQO - adaptive query optimization"
+MODULES = aqo
+DATA = aqo--1.0.sql
+OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
+hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
+selectivity_cache.o storage.o utils.o $(WIN32RES)
+REGRESS = aqo_disabled aqo_controlled aqo_intelligent aqo_forced
+
+MODULE_big = aqo
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/aqo
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif

contrib/aqo/aqo--1.0.sql

@@ -0,0 +1,67 @@
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION aqo" to load this file. \quit
+
+CREATE TABLE aqo_queries (
+	query_hash		int PRIMARY KEY,
+	learn_aqo		boolean NOT NULL,
+	use_aqo			boolean NOT NULL,
+	fspace_hash		int NOT NULL,
+	auto_tuning		boolean NOT NULL
+);
+
+CREATE TABLE aqo_query_texts (
+	query_hash		int PRIMARY KEY REFERENCES aqo_queries ON DELETE CASCADE,
+	query_text		varchar NOT NULL
+);
+
+CREATE TABLE aqo_query_stat (
+	query_hash		int PRIMARY KEY REFERENCES aqo_queries ON DELETE CASCADE,
+	execution_time_with_aqo					double precision[],
+	execution_time_without_aqo				double precision[],
+	planning_time_with_aqo					double precision[],
+	planning_time_without_aqo				double precision[],
+	cardinality_error_with_aqo				double precision[],
+	cardinality_error_without_aqo			double precision[],
+	executions_with_aqo						bigint,
+	executions_without_aqo					bigint
+);
+
+CREATE TABLE aqo_data (
+	fspace_hash		int NOT NULL REFERENCES aqo_queries ON DELETE CASCADE,
+	fsspace_hash	int NOT NULL,
+	nfeatures		int NOT NULL,
+	features		double precision[][],
+	targets			double precision[],
+	UNIQUE (fspace_hash, fsspace_hash)
+);
+
+CREATE INDEX aqo_queries_query_hash_idx ON aqo_queries (query_hash);
+CREATE INDEX aqo_query_texts_query_hash_idx ON aqo_query_texts (query_hash);
+CREATE INDEX aqo_query_stat_idx ON aqo_query_stat (query_hash);
+CREATE INDEX aqo_fss_access_idx ON aqo_data (fspace_hash, fsspace_hash);
+
+ALTER TABLE aqo_data ALTER COLUMN features SET STORAGE MAIN;
+ALTER TABLE aqo_data ALTER COLUMN targets SET STORAGE MAIN;
+ALTER TABLE aqo_query_stat
+ALTER COLUMN execution_time_with_aqo SET STORAGE MAIN;
+ALTER TABLE aqo_query_stat
+ALTER COLUMN execution_time_without_aqo SET STORAGE MAIN;
+ALTER TABLE aqo_query_stat
+ALTER COLUMN planning_time_with_aqo SET STORAGE MAIN;
+ALTER TABLE aqo_query_stat
+ALTER COLUMN planning_time_without_aqo SET STORAGE MAIN;
+ALTER TABLE aqo_query_stat
+ALTER COLUMN cardinality_error_without_aqo SET STORAGE MAIN;
+ALTER TABLE aqo_query_stat
+ALTER COLUMN cardinality_error_with_aqo SET STORAGE MAIN;
+
+INSERT INTO aqo_queries VALUES (0, false, false, 0, false);
+INSERT INTO aqo_query_texts VALUES (0, 'COMMON feature space (do not delete!)');
+-- a virtual query for COMMON feature space
+
+CREATE FUNCTION invalidate_deactivated_queries_cache() RETURNS trigger
+	AS 'MODULE_PATHNAME' LANGUAGE C;
+
+CREATE TRIGGER aqo_queries_invalidate AFTER UPDATE OR DELETE OR TRUNCATE
+	ON aqo_queries FOR EACH STATEMENT
+	EXECUTE PROCEDURE invalidate_deactivated_queries_cache();

contrib/aqo/aqo.c

@@ -0,0 +1,142 @@
+#include "aqo.h"
+
+PG_MODULE_MAGIC;
+
+void		_PG_init(void);
+void		_PG_fini(void);
+
+/* Strategy of determining feature space for new queries. */
+int			aqo_mode;
+
+/* GUC variables */
+static const struct config_enum_entry format_options[] = {
+	{"intelligent", AQO_MODE_INTELLIGENT, false},
+	{"forced", AQO_MODE_FORCED, false},
+	{"controlled", AQO_MODE_CONTROLLED, false},
+	{"disabled", AQO_MODE_DISABLED, false},
+	{NULL, 0, false}
+};
+
+/* Parameters of autotuning */
+int			aqo_stat_size = 20;
+int			auto_tuning_window_size = 5;
+double		auto_tuning_exploration = 0.1;
+int			auto_tuning_max_iterations = 50;
+int			auto_tuning_infinite_loop = 8;
+
+/* stat_size > infinite_loop + window_size + 3 is required for auto_tuning*/
+
+/* Machine learning parameters */
+double		object_selection_prediction_threshold = 0.3;
+double		object_selection_object_threshold = 0.1;
+double		learning_rate = 1e-1;
+int			aqo_k = 3;
+int			aqo_K = 30;
+double		log_selectivity_lower_bound = -30;
+
+/* Parameters for current query */
+int			query_hash;
+bool		learn_aqo;
+bool		use_aqo;
+int			fspace_hash;
+bool		auto_tuning;
+bool		collect_stat;
+bool		adding_query;
+bool		explain_only;
+bool		explain_aqo;
+
+/* Query execution time */
+instr_time	query_starttime;
+double		query_planning_time;
+
+/* Saved hook values in case of unload */
+post_parse_analyze_hook_type prev_post_parse_analyze_hook;
+planner_hook_type prev_planner_hook;
+ExecutorStart_hook_type prev_ExecutorStart_hook;
+ExecutorEnd_hook_type prev_ExecutorEnd_hook;
+set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook;
+get_parameterized_baserel_size_hook_type prev_get_parameterized_baserel_size_hook;
+set_joinrel_size_estimates_hook_type prev_set_joinrel_size_estimates_hook;
+get_parameterized_joinrel_size_hook_type prev_get_parameterized_joinrel_size_hook;
+copy_generic_path_info_hook_type prev_copy_generic_path_info_hook;
+ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
+
+/*****************************************************************************
+ *
+ *	CREATE/DROP EXTENSION FUNCTIONS
+ *
+ *****************************************************************************/
+
+void
+_PG_init(void)
+{
+	DefineCustomEnumVariable("aqo.mode",
+							 "Mode of aqo usage.",
+							 NULL,
+							 &aqo_mode,
+							 AQO_MODE_CONTROLLED,
+							 format_options,
+							 PGC_SUSET,
+							 0,
+							 NULL,
+							 NULL,
+							 NULL);
+
+	prev_planner_hook = planner_hook;
+	planner_hook = &aqo_planner;
+	prev_post_parse_analyze_hook = post_parse_analyze_hook;
+	post_parse_analyze_hook = &get_query_text;
+	prev_ExecutorStart_hook = ExecutorStart_hook;
+	ExecutorStart_hook = &aqo_ExecutorStart;
+	prev_ExecutorEnd_hook = ExecutorEnd_hook;
+	ExecutorEnd_hook = &learn_query_stat;
+	prev_set_baserel_rows_estimate_hook = set_baserel_rows_estimate_hook;
+	set_baserel_rows_estimate_hook = &aqo_set_baserel_rows_estimate;
+	prev_get_parameterized_baserel_size_hook =
+		get_parameterized_baserel_size_hook;
+	get_parameterized_baserel_size_hook =
+		&aqo_get_parameterized_baserel_size;
+	prev_set_joinrel_size_estimates_hook = set_joinrel_size_estimates_hook;
+	set_joinrel_size_estimates_hook = &aqo_set_joinrel_size_estimates;
+	prev_get_parameterized_joinrel_size_hook =
+		get_parameterized_joinrel_size_hook;
+	get_parameterized_joinrel_size_hook =
+		&aqo_get_parameterized_joinrel_size;
+	prev_copy_generic_path_info_hook = copy_generic_path_info_hook;
+	copy_generic_path_info_hook = &aqo_copy_generic_path_info;
+	prev_ExplainOnePlan_hook = ExplainOnePlan_hook;
+	ExplainOnePlan_hook = print_into_explain;
+	init_deactivated_queries_storage();
+}
+
+void
+_PG_fini(void)
+{
+	planner_hook = prev_planner_hook;
+	post_parse_analyze_hook = prev_post_parse_analyze_hook;
+	ExecutorStart_hook = prev_ExecutorStart_hook;
+	ExecutorEnd_hook = prev_ExecutorEnd_hook;
+	set_baserel_rows_estimate_hook = prev_set_baserel_rows_estimate_hook;
+	get_parameterized_baserel_size_hook =
+		prev_get_parameterized_baserel_size_hook;
+	set_joinrel_size_estimates_hook = prev_set_joinrel_size_estimates_hook;
+	get_parameterized_joinrel_size_hook =
+		prev_get_parameterized_joinrel_size_hook;
+	copy_generic_path_info_hook = prev_copy_generic_path_info_hook;
+	ExplainOnePlan_hook = prev_ExplainOnePlan_hook;
+	fini_deactivated_queries_storage();
+}
+
+PG_FUNCTION_INFO_V1(invalidate_deactivated_queries_cache);
+
+/*
+ * Clears the cache of deactivated queries if the user changed aqo_queries
+ * manually.
+ */
+Datum
+invalidate_deactivated_queries_cache(PG_FUNCTION_ARGS)
+{
+	fini_deactivated_queries_storage();
+	init_deactivated_queries_storage();
+	PG_RETURN_POINTER(NULL);
+}

contrib/aqo/aqo.control

@@ -0,0 +1,5 @@
+# AQO extension
+comment = 'machine learning for cardinality estimation in optimizer'
+default_version = '1.0'
+module_pathname = '$libdir/aqo'
+relocatable = false