Merge branch 'master' into rel_future_beta

Author: funbringer

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,28 @@
+<!--
+### IMPORTANT: make sure that you:
+
+- are familiar with [known limitations](https://github.com/postgrespro/pg_pathman/wiki/Known-limitations).
+- checked the [faq](https://github.com/postgrespro/pg_pathman/wiki/What-the-FAQ) for common problems.
+-->
+
+
+### Problem description
+
+Explain your problem here (it's always better to provide reproduction steps) ...
+
+
+
+### Environment
+
+<!-- Put the result of (SELECT * FROM pg_extension) below -->
+
+<!-- Put the result of (SELECT version()) below -->
+
+<!-- For Postgres Pro: put the result of (SELECT pgpro_version()) below -->
+
+<!-- For Postgres Pro: put the result of (SELECT pgpro_edition()) below -->
+
+<!-- For pg_pathman 1.4: put the result of (SELECT get_pathman_lib_version()) below -->
+
+<!-- For pg_pathman 1.5: put the result of (SELECT pathman_version()) below -->
+

expected/pathman_utility_stmt.out

@@ -24,70 +24,6 @@ VACUUM FULL copy_stmt_hooking.test_1;
 VACUUM FULL copy_stmt_hooking.test_2;
 VACUUM FULL copy_stmt_hooking.test_3;
 VACUUM FULL copy_stmt_hooking.test_4;
-/* COPY TO */
-COPY copy_stmt_hooking.test TO stdout;
-1	comment	\N	\N
-2	comment	\N	\N
-3	comment	\N	\N
-4	comment	\N	\N
-5	comment	\N	\N
-6	comment	\N	\N
-7	comment	\N	\N
-8	comment	\N	\N
-9	comment	\N	\N
-10	comment	\N	\N
-11	comment	\N	\N
-12	comment	\N	\N
-13	comment	\N	\N
-14	comment	\N	\N
-15	comment	\N	\N
-16	comment	\N	\N
-17	comment	\N	\N
-18	comment	\N	\N
-19	comment	\N	\N
-20	comment	\N	\N
-\copy copy_stmt_hooking.test to stdout (format csv)
-1,comment,,
-2,comment,,
-3,comment,,
-4,comment,,
-5,comment,,
-6,comment,,
-7,comment,,
-8,comment,,
-9,comment,,
-10,comment,,
-11,comment,,
-12,comment,,
-13,comment,,
-14,comment,,
-15,comment,,
-16,comment,,
-17,comment,,
-18,comment,,
-19,comment,,
-20,comment,,
-\copy copy_stmt_hooking.test(comment) to stdout
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
-comment
 /* DELETE ROWS, COPY FROM */
 DELETE FROM copy_stmt_hooking.test;
 COPY copy_stmt_hooking.test FROM stdin;
@@ -113,32 +49,30 @@ VACUUM FULL copy_stmt_hooking.test_1;
 VACUUM FULL copy_stmt_hooking.test_2;
 VACUUM FULL copy_stmt_hooking.test_3;
 VACUUM FULL copy_stmt_hooking.test_4;
-/* COPY FROM (specified columns) */
-COPY copy_stmt_hooking.test (val) TO stdout;
-1
-6
-7
-11
-16
-COPY copy_stmt_hooking.test (val, comment) TO stdout;
-1	test_1
-6	test_2
-7	test_2
-11	test_3
-16	test_4
-COPY copy_stmt_hooking.test (c3, val, comment) TO stdout;
-0	1	test_1
-0	6	test_2
-0	7	test_2
-0	11	test_3
-0	16	test_4
-COPY copy_stmt_hooking.test (val, comment, c3, c4) TO stdout;
+/* COPY TO */
+COPY copy_stmt_hooking.test TO stdout;			/* not ok */
+WARNING:  COPY TO will only select rows from parent table "test"
+COPY copy_stmt_hooking.test (val) TO stdout;	/* not ok */
+WARNING:  COPY TO will only select rows from parent table "test"
+COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
+1	test_1	0	0
+6	test_2	0	0
+7	test_2	0	0
+11	test_3	0	0
+16	test_4	0	0
+COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout (FORMAT CSV);
+1,test_1,0,0
+6,test_2,0,0
+7,test_2,0,0
+11,test_3,0,0
+16,test_4,0,0
+\copy (SELECT * FROM copy_stmt_hooking.test) TO stdout
 1	test_1	0	0
 6	test_2	0	0
 7	test_2	0	0
 11	test_3	0	0
 16	test_4	0	0
-/* COPY TO (partition does not exist, NOT allowed to create partitions) */
+/* COPY FROM (partition does not exist, NOT allowed to create partitions) */
 SET pg_pathman.enable_auto_partition = OFF;
 COPY copy_stmt_hooking.test FROM stdin;
 ERROR:  no suitable partition for key '21'
@@ -147,7 +81,7 @@ SELECT * FROM copy_stmt_hooking.test WHERE val > 20;
 -----+---------+----+----
 (0 rows)
 
-/* COPY TO (partition does not exist, allowed to create partitions) */
+/* COPY FROM (partition does not exist, allowed to create partitions) */
 SET pg_pathman.enable_auto_partition = ON;
 COPY copy_stmt_hooking.test FROM stdin;
 SELECT * FROM copy_stmt_hooking.test WHERE val > 20;
@@ -194,18 +128,18 @@ WHERE attnum > 0 AND attrelid = 'copy_stmt_hooking.test_6'::REGCLASS;
      3
 (1 row)
 
-/* COPY FROM (test transformed tuples) */
-COPY copy_stmt_hooking.test (val, c3, c4) TO stdout;
+/* test transformed tuples */
+COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
 1	0	0
 6	0	0
 7	0	0
 11	0	0
 16	0	0
 21	0	0
 26	1	2
-/* COPY TO (insert into table with dropped column) */
+/* COPY FROM (insert into table with dropped column) */
 COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
-/* COPY TO (insert into table without dropped column) */
+/* COPY FROM (insert into table without dropped column) */
 COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
 /* check tuples from last partition (without dropped column) */
 SELECT *, tableoid::REGCLASS FROM copy_stmt_hooking.test ORDER BY val;

sql/pathman_utility_stmt.sql

@@ -28,11 +28,6 @@ VACUUM FULL copy_stmt_hooking.test_2;
 VACUUM FULL copy_stmt_hooking.test_3;
 VACUUM FULL copy_stmt_hooking.test_4;
 
-/* COPY TO */
-COPY copy_stmt_hooking.test TO stdout;
-\copy copy_stmt_hooking.test to stdout (format csv)
-\copy copy_stmt_hooking.test(comment) to stdout
-
 /* DELETE ROWS, COPY FROM */
 DELETE FROM copy_stmt_hooking.test;
 COPY copy_stmt_hooking.test FROM stdin;
@@ -52,20 +47,21 @@ VACUUM FULL copy_stmt_hooking.test_2;
 VACUUM FULL copy_stmt_hooking.test_3;
 VACUUM FULL copy_stmt_hooking.test_4;
 
-/* COPY FROM (specified columns) */
-COPY copy_stmt_hooking.test (val) TO stdout;
-COPY copy_stmt_hooking.test (val, comment) TO stdout;
-COPY copy_stmt_hooking.test (c3, val, comment) TO stdout;
-COPY copy_stmt_hooking.test (val, comment, c3, c4) TO stdout;
+/* COPY TO */
+COPY copy_stmt_hooking.test TO stdout;			/* not ok */
+COPY copy_stmt_hooking.test (val) TO stdout;	/* not ok */
+COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
+COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout (FORMAT CSV);
+\copy (SELECT * FROM copy_stmt_hooking.test) TO stdout
 
-/* COPY TO (partition does not exist, NOT allowed to create partitions) */
+/* COPY FROM (partition does not exist, NOT allowed to create partitions) */
 SET pg_pathman.enable_auto_partition = OFF;
 COPY copy_stmt_hooking.test FROM stdin;
 21	test_no_part	0	0
 \.
 SELECT * FROM copy_stmt_hooking.test WHERE val > 20;
 
-/* COPY TO (partition does not exist, allowed to create partitions) */
+/* COPY FROM (partition does not exist, allowed to create partitions) */
 SET pg_pathman.enable_auto_partition = ON;
 COPY copy_stmt_hooking.test FROM stdin;
 21	test_no_part	0	0
@@ -98,16 +94,16 @@ WHERE attnum > 0 AND attrelid = 'copy_stmt_hooking.test'::REGCLASS;
 SELECT count(*) FROM pg_attribute
 WHERE attnum > 0 AND attrelid = 'copy_stmt_hooking.test_6'::REGCLASS;
 
+/* test transformed tuples */
+COPY (SELECT * FROM copy_stmt_hooking.test) TO stdout;
 
-/* COPY FROM (test transformed tuples) */
-COPY copy_stmt_hooking.test (val, c3, c4) TO stdout;
 
-/* COPY TO (insert into table with dropped column) */
+/* COPY FROM (insert into table with dropped column) */
 COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
 2	1	2
 \.
 
-/* COPY TO (insert into table without dropped column) */
+/* COPY FROM (insert into table without dropped column) */
 COPY copy_stmt_hooking.test(val, c3, c4) FROM stdin;
 27	1	2
 \.

src/hooks.c

@@ -37,6 +37,11 @@
 #include "utils/lsyscache.h"
 
 
+#ifdef USE_ASSERT_CHECKING
+#define USE_RELCACHE_LOGGING
+#endif
+
+
 /* Borrowed from joinpath.c */
 #define PATH_PARAM_BY_REL(path, rel)  \
 	((path)->param_info && bms_overlap(PATH_REQ_OUTER(path), (rel)->relids))
@@ -816,6 +821,18 @@ pathman_relcache_hook(Datum arg, Oid relid)
 	if (!IsPathmanReady())
 		return;
 
+	/* Special case: flush whole relcache */
+	if (relid == InvalidOid)
+	{
+		delay_invalidation_whole_cache();
+
+#ifdef USE_RELCACHE_LOGGING
+		elog(DEBUG2, "Invalidation message for all relations [%u]", MyProcPid);
+#endif
+
+		return;
+	}
+
 	/* We shouldn't even consider special OIDs */
 	if (relid < FirstNormalObjectId)
 		return;
@@ -835,16 +852,20 @@ pathman_relcache_hook(Datum arg, Oid relid)
 	{
 		delay_invalidation_parent_rel(parent_relid);
 
+#ifdef USE_RELCACHE_LOGGING
 		elog(DEBUG2, "Invalidation message for partition %u [%u]",
 			 relid, MyProcPid);
+#endif
 	}
 	/* We can't say, perform full invalidation procedure */
 	else
 	{
 		delay_invalidation_vague_rel(relid);
 
-		elog(DEBUG2, "Invalidation message for vague relation %u [%u]",
+#ifdef USE_RELCACHE_LOGGING
+		elog(DEBUG2, "Invalidation message for vague rel %u [%u]",
 			 relid, MyProcPid);
+#endif
 	}
 }
 
@@ -894,7 +915,7 @@ pathman_process_utility_hook(Node *first_arg,
 						  stmt_location, stmt_len, &processed);
 			if (completionTag)
 				snprintf(completionTag, COMPLETION_TAG_BUFSIZE,
-						 "PATHMAN COPY " UINT64_FORMAT, processed);
+						 "COPY " UINT64_FORMAT, processed);
 
 			return; /* don't call standard_ProcessUtility() or hooks */
 		}

src/include/init.h

@@ -54,6 +54,7 @@ typedef struct
 
 #define PATHMAN_MCXT_COUNT	4
 extern MemoryContext		TopPathmanContext;
+extern MemoryContext		PathmanInvalJobsContext;
 extern MemoryContext		PathmanRelationCacheContext;
 extern MemoryContext		PathmanParentCacheContext;
 extern MemoryContext		PathmanBoundCacheContext;

src/include/relation_info.h

@@ -278,6 +278,7 @@ const PartRelationInfo *refresh_pathman_relation_info(Oid relid,
 													  Datum *values,
 													  bool allow_incomplete);
 PartRelationInfo *invalidate_pathman_relation_info(Oid relid, bool *found);
+void invalidate_pathman_relation_info_cache(const Oid *parents, int parents_count);
 void remove_pathman_relation_info(Oid relid);
 const PartRelationInfo *get_pathman_relation_info(Oid relid);
 const PartRelationInfo *get_pathman_relation_info_after_lock(Oid relid,
@@ -301,6 +302,7 @@ bool is_equal_to_partitioning_expression(Oid relid, char *expression,
 
 /* Global invalidation routines */
 void delay_pathman_shutdown(void);
+void delay_invalidation_whole_cache(void);
 void delay_invalidation_parent_rel(Oid parent);
 void delay_invalidation_vague_rel(Oid vague_rel);
 void finish_delayed_invalidation(void);

src/init.c

@@ -43,6 +43,7 @@
 
 /* Various memory contexts for caches */
 MemoryContext		TopPathmanContext				= NULL;
+MemoryContext		PathmanInvalJobsContext			= NULL;
 MemoryContext		PathmanRelationCacheContext		= NULL;
 MemoryContext		PathmanParentCacheContext		= NULL;
 MemoryContext		PathmanBoundCacheContext		= NULL;
@@ -316,6 +317,7 @@ init_local_cache(void)
 	if (TopPathmanContext)
 	{
 		/* Check that child contexts exist */
+		Assert(MemoryContextIsValid(PathmanInvalJobsContext));
 		Assert(MemoryContextIsValid(PathmanRelationCacheContext));
 		Assert(MemoryContextIsValid(PathmanParentCacheContext));
 		Assert(MemoryContextIsValid(PathmanBoundCacheContext));
@@ -326,6 +328,7 @@ init_local_cache(void)
 	/* Initialize pg_pathman's memory contexts */
 	else
 	{
+		Assert(PathmanInvalJobsContext == NULL);
 		Assert(PathmanRelationCacheContext == NULL);
 		Assert(PathmanParentCacheContext == NULL);
 		Assert(PathmanBoundCacheContext == NULL);
@@ -335,6 +338,11 @@ init_local_cache(void)
 									  CppAsString(TopPathmanContext),
 									  ALLOCSET_DEFAULT_SIZES);
 
+		PathmanInvalJobsContext =
+				AllocSetContextCreate(TopMemoryContext,
+									  CppAsString(PathmanInvalJobsContext),
+									  ALLOCSET_SMALL_SIZES);
+
 		/* For PartRelationInfo */
 		PathmanRelationCacheContext =
 				AllocSetContextCreate(TopPathmanContext,