Merge commit 'd69df585d9a504d1faeb5ba1ddd806ded7c3dbc1' into PGPRO9_6_1C

Author: dmpgpro

contrib/pg_pathman/LICENSE

@@ -1,11 +1,11 @@
-Copyright (c) 2015-2016, Postgres Professional
-
-Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+pg_pathman is released under the PostgreSQL License, a liberal Open Source license, similar to the BSD or MIT licenses.
 
+Copyright (c) 2015-2017, Postgres Professional
+Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
 Portions Copyright (c) 1994, The Regents of the University of California
 
 Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies.
 
-IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF POSTGRES PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

contrib/pg_pathman/META.json

@@ -0,0 +1,40 @@
+{
+   "name": "pg_pathman",
+   "abstract": "Partitioning tool",
+   "description": "The `pg_pathman` module provides optimized partitioning mechanism and functions to manage partitions.",
+   "version": "1.3.1",
+   "maintainer": [
+      "Ildar Musin <i.musin@postgrespro.ru>",
+      "Dmitry Ivanov <d.ivanov@postgrespro.ru>",
+      "Ildus Kurbangaliev <i.kurbangaliev@postgrespro.ru>"
+   ],
+   "license": "postgresql",
+   "resources": {
+      "bugtracker": {
+         "web": "https://github.com/postgrespro/pg_pathman/issues"
+      },
+      "repository": {
+        "url":  "git://github.com:postgrespro/pg_pathman.git",
+        "web":  "https://github.com/postgrespro/pg_pathman",
+        "type": "git"
+      }
+   },
+   "generated_by": "Ildar Musin",
+   "provides": {
+       "pg_pathman": {
+           "file": "pg_pathman--1.3.sql",
+           "docfile": "README.md",
+           "version": "1.3.1",
+           "abstract": "Partitioning tool"
+      }
+   },
+   "meta-spec": {
+      "version": "1.0.0",
+      "url": "http://pgxn.org/meta/spec.txt"
+   },
+   "tags": [
+      "partitioning",
+      "partition",
+      "optimization"
+   ]
+}

contrib/pg_pathman/Makefile

@@ -25,6 +25,7 @@ REGRESS = pathman_basic \
 		  pathman_cte \
 		  pathman_bgw \
 		  pathman_inserts \
+		  pathman_updates \
 		  pathman_domains \
 		  pathman_interval \
 		  pathman_callbacks \
@@ -33,7 +34,8 @@ REGRESS = pathman_basic \
 		  pathman_rowmarks \
 		  pathman_runtime_nodes \
 		  pathman_utility_stmt_hooking \
-		  pathman_calamity
+		  pathman_calamity \
+		  pathman_join_clause
 
 EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add
 

contrib/pg_pathman/README.md

@@ -1,5 +1,7 @@
 [![Build Status](https://travis-ci.org/postgrespro/pg_pathman.svg?branch=master)](https://travis-ci.org/postgrespro/pg_pathman)
 [![PGXN version](https://badge.fury.io/pg/pg_pathman.svg)](https://badge.fury.io/pg/pg_pathman)
+[![codecov](https://codecov.io/gh/postgrespro/pg_pathman/branch/master/graph/badge.svg)](https://codecov.io/gh/postgrespro/pg_pathman)
+[![GitHub license](https://img.shields.io/badge/license-PostgreSQL-blue.svg)](https://raw.githubusercontent.com/postgrespro/pg_pathman/master/LICENSE)
 
 # pg_pathman
 
@@ -66,7 +68,7 @@ Modify the **`shared_preload_libraries`** parameter in `postgresql.conf` as foll
 ```
 shared_preload_libraries = 'pg_pathman'
 ```
-> **Important:** `pg_pathman` may have conflicts with some other extensions which uses the same hook functions. For example, `pg_pathman` uses `ProcessUtility_hook` hook to handle COPY queries for partitioned tables. And it could sometimes interfere with `pg_stat_statements` extension which uses the same hook. In this case try to list libraries in certain order: `shared_preload_libraries = 'pg_pathman, pg_stat_statements'`
+> **Important:** `pg_pathman` may cause conflicts with some other extensions that use the same hook functions. For example, `pg_pathman` uses `ProcessUtility_hook` to handle COPY queries for partitioned tables, which means it may interfere with `pg_stat_statements` from time to time. In this case, try listing libraries in certain order: `shared_preload_libraries = 'pg_stat_statements, pg_pathman'`.
 
 It is essential to restart the PostgreSQL instance. After that, execute the following query in psql:
 ```plpgsql
@@ -98,8 +100,8 @@ create_hash_partitions(relation         REGCLASS,
                        attribute        TEXT,
                        partitions_count INTEGER,
                        partition_data   BOOLEAN DEFAULT TRUE,
-					   partition_names  TEXT[] DEFAULT NULL,
-					   tablespaces      TEXT[] DEFAULT NULL)
+                       partition_names  TEXT[] DEFAULT NULL,
+                       tablespaces      TEXT[] DEFAULT NULL)
 ```
 Performs HASH partitioning for `relation` by integer key `attribute`. The `partitions_count` parameter specifies the number of partitions to create; it cannot be changed afterwards. If `partition_data` is `true` then all the data will be automatically copied from the parent table to partitions. Note that data migration may took a while to finish and the table will be locked until transaction commits. See `partition_table_concurrently()` for a lock-free way to migrate data. Partition creation callback is invoked for each partition if set beforehand (see `set_init_callback()`).
 
@@ -660,11 +662,11 @@ SELECT disable_pathman_for('range_rel');
 ```
 All sections and data will remain unchanged and will be handled by the standard PostgreSQL inheritance mechanism.
 
-##Feedback
+## Feedback
 Do not hesitate to post your issues, questions and new ideas at the [issues](https://github.com/postgrespro/pg_pathman/issues) page.
 
 ## Authors
-Ildar Musin <i.musin@postgrespro.ru> Postgres Professional Ltd., Russia		
-Alexander Korotkov <a.korotkov@postgrespro.ru> Postgres Professional Ltd., Russia		
-Dmitry Ivanov <d.ivanov@postgrespro.ru> Postgres Professional Ltd., Russia		
+Ildar Musin <i.musin@postgrespro.ru> Postgres Professional Ltd., Russia  
+Alexander Korotkov <a.korotkov@postgrespro.ru> Postgres Professional Ltd., Russia  
+Dmitry Ivanov <d.ivanov@postgrespro.ru> Postgres Professional Ltd., Russia  
 

contrib/pg_pathman/README.rus.md

@@ -488,6 +488,6 @@ SELECT disable_pathman_for('range_rel');
 Если у вас есть вопросы или предложения, а также если вы обнаружили ошибки, напишите нам в разделе [issues](https://github.com/postgrespro/pg_pathman/issues).
 
 ## Авторы
-Ильдар Мусин <i.musin@postgrespro.ru> Postgres Professional, Россия		
-Александр Коротков <a.korotkov@postgrespro.ru> Postgres Professional, Россия		
-Дмитрий Иванов <d.ivanov@postgrespro.ru> Postgres Professional, Россия		
+Ильдар Мусин <i.musin@postgrespro.ru> Postgres Professional, Россия  
+Александр Коротков <a.korotkov@postgrespro.ru> Postgres Professional, Россия  
+Дмитрий Иванов <d.ivanov@postgrespro.ru> Postgres Professional, Россия  

contrib/pg_pathman/expected/pathman_join_clause.out

@@ -0,0 +1,97 @@
+\set VERBOSITY terse
+CREATE SCHEMA pathman;
+CREATE EXTENSION pg_pathman SCHEMA pathman;
+CREATE SCHEMA test;
+/*
+ * Test push down a join clause into child nodes of append
+ */
+/* create test tables */
+CREATE TABLE test.fk (
+    id1 INT NOT NULL,
+    id2 INT NOT NULL,
+    start_key INT,
+    end_key INT,
+    PRIMARY KEY (id1, id2));
+CREATE TABLE test.mytbl (
+    id1 INT NOT NULL,
+    id2 INT NOT NULL,
+    key INT NOT NULL,
+    CONSTRAINT fk_fk FOREIGN KEY (id1, id2) REFERENCES test.fk(id1, id2),
+    PRIMARY KEY (id1, key));
+SELECT pathman.create_hash_partitions('test.mytbl', 'id1', 8);
+ create_hash_partitions 
+------------------------
+                      8
+(1 row)
+
+/* ...fill out with test data */
+INSERT INTO test.fk VALUES (1, 1);
+INSERT INTO test.mytbl VALUES (1, 1, 5), (1,1,6);
+/* gather statistics on test tables to have deterministic plans */
+ANALYZE test.fk;
+ANALYZE test.mytbl;
+/* run test queries */
+EXPLAIN (COSTS OFF)     /* test plan */
+SELECT m.tableoid::regclass, id1, id2, key, start_key, end_key
+    FROM test.mytbl m JOIN test.fk USING(id1, id2)
+    WHERE NOT key <@ int4range(6, end_key);
+                                     QUERY PLAN                                     
+------------------------------------------------------------------------------------
+ Nested Loop
+   ->  Seq Scan on fk
+   ->  Custom Scan (RuntimeAppend)
+         ->  Bitmap Heap Scan on mytbl_0 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_0_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_1 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_1_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_2 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_2_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_3 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_3_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_4 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_4_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_5 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_5_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_6 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_6_pkey
+                     Index Cond: (id1 = fk.id1)
+         ->  Bitmap Heap Scan on mytbl_7 m
+               Recheck Cond: (id1 = fk.id1)
+               Filter: ((fk.id2 = id2) AND (NOT (key <@ int4range(6, fk.end_key))))
+               ->  Bitmap Index Scan on mytbl_7_pkey
+                     Index Cond: (id1 = fk.id1)
+(43 rows)
+
+/* test joint data */
+SELECT m.tableoid::regclass, id1, id2, key, start_key, end_key
+    FROM test.mytbl m JOIN test.fk USING(id1, id2)
+    WHERE NOT key <@ int4range(6, end_key);
+   tableoid   | id1 | id2 | key | start_key | end_key 
+--------------+-----+-----+-----+-----------+---------
+ test.mytbl_6 |   1 |   1 |   5 |           |        
+(1 row)
+
+DROP SCHEMA test CASCADE;
+NOTICE:  drop cascades to 10 other objects
+DROP EXTENSION pg_pathman CASCADE;
+DROP SCHEMA pathman CASCADE;

contrib/pg_pathman/expected/pathman_updates.out

@@ -0,0 +1,64 @@
+\set VERBOSITY terse
+SET search_path = 'public';
+CREATE EXTENSION pg_pathman;
+CREATE SCHEMA test_updates;
+/*
+ * Test UPDATEs on a partition with different TupleDescriptor.
+ */
+/* create partitioned table */
+CREATE TABLE test_updates.test(a FLOAT4, val INT4 NOT NULL, b FLOAT8);
+INSERT INTO test_updates.test SELECT i, i, i FROM generate_series(1, 100) AS i;
+SELECT create_range_partitions('test_updates.test', 'val', 1, 10);
+NOTICE:  sequence "test_seq" does not exist, skipping
+ create_range_partitions 
+-------------------------
+                      10
+(1 row)
+
+/* drop column 'a' */
+ALTER TABLE test_updates.test DROP COLUMN a;
+/* append new partition */
+SELECT append_range_partition('test_updates.test');
+ append_range_partition 
+------------------------
+ test_updates.test_11
+(1 row)
+
+INSERT INTO test_updates.test_11 (val, b) VALUES (101, 10);
+/* tuple descs are the same */
+EXPLAIN (COSTS OFF) UPDATE test_updates.test SET b = 0 WHERE val = 1;
+        QUERY PLAN         
+---------------------------
+ Update on test_1
+   ->  Seq Scan on test_1
+         Filter: (val = 1)
+(3 rows)
+
+UPDATE test_updates.test SET b = 0 WHERE val = 1 RETURNING *, tableoid::REGCLASS;
+ val | b |      tableoid       
+-----+---+---------------------
+   1 | 0 | test_updates.test_1
+(1 row)
+
+/* tuple descs are different */
+EXPLAIN (COSTS OFF) UPDATE test_updates.test SET b = 0 WHERE val = 101;
+         QUERY PLAN          
+-----------------------------
+ Update on test
+   Update on test
+   Update on test_11
+   ->  Seq Scan on test
+         Filter: (val = 101)
+   ->  Seq Scan on test_11
+         Filter: (val = 101)
+(7 rows)
+
+UPDATE test_updates.test SET b = 0 WHERE val = 101 RETURNING *, tableoid::REGCLASS;
+ val | b |       tableoid       
+-----+---+----------------------
+ 101 | 0 | test_updates.test_11
+(1 row)
+
+DROP SCHEMA test_updates CASCADE;
+NOTICE:  drop cascades to 13 other objects
+DROP EXTENSION pg_pathman;

contrib/pg_pathman/expected/pathman_utility_stmt_hooking.out

@@ -256,8 +256,33 @@ SELECT * FROM copy_stmt_hooking.test ORDER BY val;
    6 | hash_2  |  0 |  0
 (2 rows)
 
+/* Check dropped colums before partitioning */
+CREATE TABLE copy_stmt_hooking.test2 (
+	a varchar(50),
+	b varchar(50),
+	t timestamp without time zone not null
+);
+ALTER TABLE copy_stmt_hooking.test2 DROP COLUMN a;
+SELECT create_range_partitions('copy_stmt_hooking.test2',
+	't',
+	'2017-01-01 00:00:00'::timestamp,
+	interval '1 hour', 5, false
+);
+NOTICE:  sequence "test2_seq" does not exist, skipping
+ create_range_partitions 
+-------------------------
+                       5
+(1 row)
+
+COPY copy_stmt_hooking.test2(t) FROM stdin;
+SELECT COUNT(*) FROM copy_stmt_hooking.test2;
+ count 
+-------
+     1
+(1 row)
+
 DROP SCHEMA copy_stmt_hooking CASCADE;
-NOTICE:  drop cascades to 7 other objects
+NOTICE:  drop cascades to 798 other objects
 /*
  * Test auto check constraint renaming
  */

contrib/pg_pathman/sql/pathman_join_clause.sql

@@ -0,0 +1,49 @@
+\set VERBOSITY terse
+
+CREATE SCHEMA pathman;
+CREATE EXTENSION pg_pathman SCHEMA pathman;
+CREATE SCHEMA test;
+
+
+/*
+ * Test push down a join clause into child nodes of append
+ */
+
+/* create test tables */
+CREATE TABLE test.fk (
+    id1 INT NOT NULL,
+    id2 INT NOT NULL,
+    start_key INT,
+    end_key INT,
+    PRIMARY KEY (id1, id2));
+CREATE TABLE test.mytbl (
+    id1 INT NOT NULL,
+    id2 INT NOT NULL,
+    key INT NOT NULL,
+    CONSTRAINT fk_fk FOREIGN KEY (id1, id2) REFERENCES test.fk(id1, id2),
+    PRIMARY KEY (id1, key));
+SELECT pathman.create_hash_partitions('test.mytbl', 'id1', 8);
+
+/* ...fill out with test data */
+INSERT INTO test.fk VALUES (1, 1);
+INSERT INTO test.mytbl VALUES (1, 1, 5), (1,1,6);
+
+/* gather statistics on test tables to have deterministic plans */
+ANALYZE test.fk;
+ANALYZE test.mytbl;
+
+/* run test queries */
+EXPLAIN (COSTS OFF)     /* test plan */
+SELECT m.tableoid::regclass, id1, id2, key, start_key, end_key
+    FROM test.mytbl m JOIN test.fk USING(id1, id2)
+    WHERE NOT key <@ int4range(6, end_key);
+/* test joint data */
+SELECT m.tableoid::regclass, id1, id2, key, start_key, end_key
+    FROM test.mytbl m JOIN test.fk USING(id1, id2)
+    WHERE NOT key <@ int4range(6, end_key);
+
+
+DROP SCHEMA test CASCADE;
+DROP EXTENSION pg_pathman CASCADE;
+DROP SCHEMA pathman CASCADE;
+