merge with shardman

Author: kelvich

contrib/Makefile

@@ -49,6 +49,7 @@ SUBDIRS = \
 		pg_pathman \
 		pg_prewarm	\
 		pg_query_state \
+		pg_shardman	\
 		pg_standby	\
 		pg_stat_statements \
 		pg_transfer \

contrib/pg_shardman/.gitignore

@@ -0,0 +1,7 @@
+*.o
+*.so
+.dir-locals.el
+bin/setup.sh
+
+*.pyc
+__pycache__

contrib/pg_shardman/LICENSE

@@ -0,0 +1,22 @@
+pg_shardman is released under the PostgreSQL License, a liberal Open Source
+license, similar to the BSD or MIT licenses.
+
+Copyright (c) 2017, Postgres Professional
+Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
+Portions Copyright (c) 1994, The Regents of the University of California
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose, without fee, and without a written agreement is
+hereby granted, provided that the above copyright notice and this paragraph and
+the following two paragraphs appear in all copies.
+
+IN NO EVENT SHALL POSTGRES PROFESSIONAL BE LIABLE TO ANY PARTY FOR DIRECT,
+INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS,
+ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF POSTGRES
+PROFESSIONAL HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+POSTGRES PROFESSIONAL SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND
+POSTGRES PROFESSIONAL HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT,
+UPDATES, ENHANCEMENTS, OR MODIFICATIONS.

contrib/pg_shardman/Makefile

@@ -0,0 +1,60 @@
+# the extension name
+EXTENSION = pg_shardman
+EXTVERSION = 0.0.2
+# This file will be executed by CREATE EXTENSION, so let pgxs install it.
+DATA = $(EXTENSION)--$(EXTVERSION).sql
+
+REGRESS = shardman_installation
+
+MODULE_big = pg_shardman
+OBJS = pg_shardman.o
+PGFILEDESC = "pg_shardman - sharding for Postgres"
+
+ifndef USE_PGXS # hmm, user didn't requested to use pgxs
+# relative path to this makefile
+mkfile_path := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST))
+# relative path to dir with this makefile
+mkfile_dir := $(dir $(mkfile_path))
+# abs path to dir with this makefile
+mkfile_abspath := $(shell cd $(mkfile_dir) && pwd -P)
+# parent dir name of directory with makefile
+parent_dir_name := $(shell basename $(shell dirname $(mkfile_abspath)))
+ifneq ($(parent_dir_name),contrib) # a-ha, but this shardman is not inside 'contrib' dir
+USE_PGXS := 1 # so use it anyway, most probably that's what the user wants
+endif
+endif
+# $(info) is introduced in 3.81, and PG doesn't support makes older than 3.80
+ifeq ($(MAKE_VERSION),3.80)
+$(warning $$USE_PGXS is [${USE_PGXS}] (we use it automatically if not in contrib dir))
+else
+$(info $$USE_PGXS is [${USE_PGXS}] (we use it automatically if not in contrib dir))
+endif
+
+ifdef USE_PGXS # use pgxs
+# You can specify path to pg_config in PG_CONFIG var
+ifndef PG_CONFIG
+	PG_CONFIG := pg_config
+endif
+PG_CONFIG = pg_config
+INCLUDEDIR := $(shell $(PG_CONFIG) --includedir)
+PG_CPPFLAGS += -I$(INCLUDEDIR) # add server's include directory for libpq-fe.h
+SHLIB_LINK += -lpq # add libpq
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+
+else # assume the extension is in contrib/ dir of pg distribution
+# install pg_pathman and postgres_fdw too
+EXTRA_INSTALL = contrib/pg_pathman contrib/postgres_fdw
+PG_CPPFLAGS = -I$(libpq_srcdir) # include libpq-fe, defined in Makefile.global.in
+SHLIB_LINK = $(libpq) # defined in Makefile.global.in
+SHLIB_PREREQS = submake-libpq
+subdir = contrib/pg_shardman
+top_builddir = ../..
+# Add pathman to shared preload libraries when running regression tests
+EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif
+
+python_tests:
+	$(MAKE) -C tests/python

contrib/pg_shardman/bin/common.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+script_dir=`dirname "$(readlink -f "$0")"`
+source "${script_dir}/setup.sh"
+
+PATH="$PATH:${pgpath}bin/"
+function start_nodes()
+{
+    echo "Starting nodes"
+    for ((i=0; i<${#worker_datadirs[@]}; ++i)); do
+	datadir="${worker_datadirs[i]}"
+	port="${worker_ports[i]}"
+	pg_ctl -o "-p $port" -D $datadir  -l $logfile start
+    done
+    pg_ctl -o "-p $lord_port" -D $lord_datadir -l $logfile start
+}
+
+function stop_nodes()
+{
+    echo "Stopping nodes"
+    for datadir in $lord_datadir "${worker_datadirs[@]}"; do
+	pg_ctl -D $datadir stop || true
+    done
+}
+
+function restart_nodes()
+{
+    echo "Restarting nodes"
+    for ((i=0; i<${#worker_datadirs[@]}; ++i)); do
+	datadir="${worker_datadirs[i]}"
+	port="${worker_ports[i]}"
+	pg_ctl -o "-p $port" -D $datadir -l $logfile restart
+    done
+    pg_ctl -o "-p $lord_port" -D $lord_datadir -l $logfile restart
+}

contrib/pg_shardman/bin/pgbench_history.sql

@@ -0,0 +1,3 @@
+drop table if exists pgbench_history;
+create table pgbench_history (tid int, bid int, aid int, delta int,
+							  mtime timestamp, filler char(22));

contrib/pg_shardman/bin/setup.sh.example

@@ -0,0 +1,69 @@
+pgpath=~/postgres/install/vanilla/
+pathmanpath=~/postgres/pg_pathman
+install_pathman=false
+logfile=/tmp/shmn.log
+
+lord_datadir=/tmp/data1
+lord_port=5432
+
+# declare -a worker_datadirs=()
+# declare -a worker_ports=()
+
+# declare -a worker_datadirs=("${HOME}/postgres/data2")
+# declare -a worker_ports=("5433")
+
+# declare -a worker_datadirs=("${HOME}/postgres/data2" "${HOME}/postgres/data3")
+# declare -a worker_ports=("5433" "5434")
+
+# declare -a worker_datadirs=("${HOME}/postgres/data2" "${HOME}/postgres/data3" "${HOME}/postgres/data4")
+# declare -a worker_ports=("5433" "5434" "5435")
+
+declare -a worker_datadirs=("/tmp/data2" "/tmp/data3" "/tmp/data4" "/tmp/data5")
+declare -a worker_ports=("5433" "5434" "5435" "5436")
+
+function run_demo()
+{
+    :
+    psql -c "drop table if exists t cascade;"
+    psql -c "CREATE TABLE t(i int);"
+
+    psql -c "drop table if exists pt cascade;"
+    psql -c "CREATE TABLE pt(id INT primary key, payload REAL);"
+    psql -c "INSERT INTO pt SELECT generate_series(1, 10), random();"
+
+    # declarative partitioning
+    # psql -p 5433 -c "drop table if exists ppt cascade;"
+    # psql -p 5433 -c "CREATE TABLE ppt(id INT primary key, payload REAL);"
+    # psql -c "create server node_1 foreign data wrapper postgres_fdw options (port '5433');"
+    # psql -c "create user mapping for current_user server node_1;"
+    # psql -c "create foreign table ppt (id int, payload real) server node_1;"
+
+    # psql -c "create table ppt_root (id int, payload real) partition by range (id);"
+    # psql -c "alter table ppt_root attach partition ppt for values from (MINVALUE) TO (MAXVALUE);"
+
+    # with joe
+    # for port in "${worker_ports[@]}" $lord_port; do
+	# psql -p $port -c "set synchronous_commit to local; drop role if exists joe; create role joe superuser login; grant usage on FOREIGN DATA WRAPPER postgres_fdw to joe;"
+    # done
+    # psql -c "select shardman.add_node('port=5433', conn_string => 'user=joe password=12345 dbname=ars port=5433');"
+    # psql -c "select shardman.add_node('port=5434', conn_string => 'user=joe password=12345 dbname=ars port=5434');"
+    # psql -c "select shardman.add_node('port=5435', conn_string => 'user=joe password=12345 dbname=ars port=5435');"
+    # psql -c "select shardman.add_node('port=5436', conn_string => 'user=joe password=12345 dbname=ars port=5436');"
+
+    psql -c "select shardman.add_node('port=5433');"
+    psql -c "select shardman.add_node('port=5434');"
+    psql -c "select shardman.add_node('port=5435');"
+    psql -c "select shardman.add_node('port=5436');"
+
+    psql -c "select shardman.create_hash_partitions('pt', 'id', 4);"
+    psql -c "select shardman.set_redundancy('pt', 2);"
+
+    # for port in "${worker_ports[@]}"; do
+    	# psql -p $port -f /home/ars/postgres/pg_shardman/bin/pgbench_history.sql
+    # done
+    # psql -f devops/pgbench_ddl.sql
+    # psql -c "select shardman.create_hash_partitions('pgbench_accounts', 'aid', 30, 1);"
+    # psql -c "select shardman.create_hash_partitions('pgbench_tellers', 'tid', 30, 1);"
+    # psql -c "select shardman.create_hash_partitions('pgbench_branches', 'bid', 30, 1);"
+    # pgbench -p 5433 -s 10 -i --no-ddl
+}

contrib/pg_shardman/bin/shardman_init.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+script_dir=`dirname "$(readlink -f "$0")"`
+source "${script_dir}/common.sh"
+
+if $install_pathman; then
+    cd $pathmanpath
+    USE_PGXS=1 make clean
+    USE_PGXS=1 make install
+fi
+
+cd "${script_dir}/.."
+make clean
+make install
+
+> $logfile
+
+stop_nodes
+pkill -9 postgres || true
+for datadir in $lord_datadir "${worker_datadirs[@]}"; do
+    rm -rf "$datadir"
+    mkdir -p "$datadir"
+    initdb -D "$datadir"
+done
+
+cat postgresql.conf.common >> ${lord_datadir}/postgresql.conf
+cat postgresql.conf.lord >> ${lord_datadir}/postgresql.conf
+for worker_datadir in "${worker_datadirs[@]}"; do
+    cat postgresql.conf.common >> ${worker_datadir}/postgresql.conf
+    cat postgresql.conf.worker >> ${worker_datadir}/postgresql.conf
+done
+
+for datadir in $lord_datadir "${worker_datadirs[@]}"; do
+    cat pg_hba.conf > ${datadir}/pg_hba.conf
+done
+
+start_nodes
+for port in $lord_port "${worker_ports[@]}"; do
+    createdb -p $port `whoami`
+    echo "creating pg_shardman extension..."
+    psql -p $port -c "create extension pg_shardman cascade;"
+done
+
+restart_nodes
+
+run_demo
+
+# psql

contrib/pg_shardman/bin/shardman_start.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+script_dir=`dirname "$(readlink -f "$0")"`
+source "${script_dir}/common.sh"
+
+cd "${script_dir}/.."
+
+> $logfile
+
+restart_nodes # make sure nodes run
+# first workers, then lord
+for port in "${worker_ports[@]}" $lord_port; do
+    psql -p $port -c "set synchronous_commit to local; drop extension if exists pg_shardman cascade;"
+done
+
+make clean
+make install
+
+restart_nodes
+for port in $lord_port "${worker_ports[@]}"; do
+    psql -p $port -c "set synchronous_commit to local; create extension pg_shardman cascade;"
+done
+
+run_demo

contrib/pg_shardman/bin/shardman_stop.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+script_dir=`dirname "$(readlink -f "$0")"`
+source "${script_dir}/common.sh"
+
+stop_nodes

contrib/pg_shardman/conf.add

@@ -0,0 +1 @@
+shared_preload_libraries='pg_shardman, pg_pathman'

contrib/pg_shardman/devops/.gitignore

@@ -0,0 +1,14 @@
+*.retry
+inventory_manual/manual
+logs/
+res/
+tester_res.csv*
+wal_lag.txt
+
+/postgresql.conf.common
+/postgresql.conf.lord
+/postgresql.conf.worker
+
+# vagrant stuff
+*.log
+.vagrant/

contrib/pg_shardman/devops/Vagrantfile

@@ -0,0 +1,27 @@
+Vagrant.configure("2") do |config|
+
+  config.vm.define "node1" do |node|
+    node.vm.box = "ubuntu/xenial64"
+    node.vm.network "private_network", ip: "10.42.42.10"
+  end
+
+  config.vm.define "node2" do |node|
+    node.vm.box = "ubuntu/xenial64"
+    node.vm.network "private_network", ip: "10.42.42.20"
+  end
+
+  config.vm.define "node3" do |node|
+    node.vm.box = "ubuntu/xenial64"
+    node.vm.network "private_network", ip: "10.42.42.30"
+  end
+
+  # ssh-copy-id
+  config.vm.provision "shell" do |s|
+    ssh_pub_key = File.readlines("#{Dir.home}/.ssh/id_rsa.pub").first.strip
+    s.inline = <<-SHELL
+      echo #{ssh_pub_key} >> /home/ubuntu/.ssh/authorized_keys
+      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
+    SHELL
+  end
+
+end