Docs for broadcast.

Author: arssher

bin/pgbench_history.sql

@@ -0,0 +1,3 @@
+drop table if exists pgbench_history;
+create table pgbench_history (tid int, bid int, aid int, delta int,
+							  mtime timestamp, filler char(22));

bin/setup.sh.example

@@ -24,45 +24,46 @@ declare -a worker_ports=("5433" "5434" "5435" "5436")
 function run_demo()
 {
     :
-
-for port in "${worker_ports[@]}" $lord_port; do
-    psql -p $port -c "set synchronous_commit to local; drop role if exists joe; create role joe login password '12345'; grant usage on FOREIGN DATA WRAPPER postgres_fdw to joe;"
-done
-
     psql -c "drop table if exists t cascade;"
     psql -c "CREATE TABLE t(i int);"
-    psql -c "drop table if exists t cascade;"
-    psql -c "CREATE TABLE t(i int);"
-
 
     psql -c "drop table if exists pt cascade;"
     psql -c "CREATE TABLE pt(id INT primary key, payload REAL);"
     psql -c "INSERT INTO pt SELECT generate_series(1, 10), random();"
 
-    psql -c "select shardman.add_node('port=5433', conn_string => 'user=joe password=12345 dbname=ars port=5433');"
-    psql -c "select shardman.add_node('port=5434', conn_string => 'user=joe password=12345 dbname=ars port=5434');"
-    psql -c "select shardman.add_node('port=5435', conn_string => 'user=joe password=12345 dbname=ars port=5435');"
-    psql -c "select shardman.add_node('port=5436', conn_string => 'user=joe password=12345 dbname=ars port=5436');"
+    # declarative partitioning
+    # psql -p 5433 -c "drop table if exists ppt cascade;"
+    # psql -p 5433 -c "CREATE TABLE ppt(id INT primary key, payload REAL);"
+    # psql -c "create server node_1 foreign data wrapper postgres_fdw options (port '5433');"
+    # psql -c "create user mapping for current_user server node_1;"
+    # psql -c "create foreign table ppt (id int, payload real) server node_1;"
+
+    # psql -c "create table ppt_root (id int, payload real) partition by range (id);"
+    # psql -c "alter table ppt_root attach partition ppt for values from (MINVALUE) TO (MAXVALUE);"
+
+    # with joe
+    # for port in "${worker_ports[@]}" $lord_port; do
+	# psql -p $port -c "set synchronous_commit to local; drop role if exists joe; create role joe superuser login; grant usage on FOREIGN DATA WRAPPER postgres_fdw to joe;"
+    # done
+    # psql -c "select shardman.add_node('port=5433', conn_string => 'user=joe password=12345 dbname=ars port=5433');"
+    # psql -c "select shardman.add_node('port=5434', conn_string => 'user=joe password=12345 dbname=ars port=5434');"
+    # psql -c "select shardman.add_node('port=5435', conn_string => 'user=joe password=12345 dbname=ars port=5435');"
+    # psql -c "select shardman.add_node('port=5436', conn_string => 'user=joe password=12345 dbname=ars port=5436');"
+
+    psql -c "select shardman.add_node('port=5433');"
+    psql -c "select shardman.add_node('port=5434');"
+    psql -c "select shardman.add_node('port=5435');"
+    psql -c "select shardman.add_node('port=5436');"
 
     psql -c "select shardman.create_hash_partitions('pt', 'id', 4);"
     psql -c "select shardman.set_redundancy('pt', 2);"
 
-    # psql -c "select shardman.create_hash_partitions(2, 'pt', 'id', 4, true);"
-    # psql -c "select shardman.set_replevel('pt', 1);"
-
-    # psql -c "drop table if exists pgbench_accounts cascade";
-    # psql -c "drop table if exists pgbench_tellers cascade";
-    # psql -c "drop table if exists pgbench_branches cascade";
-    # psql -c "drop table if exists pgbench_tellers cascade;"
-    # psql -c "drop table if exists pgbench_accounts cascade;"
-    # pgbench -s 5 -i
-    # psql -c "select shardman.create_hash_partitions(2, 'pgbench_accounts', 'aid', 4);"
-    # psql -c "select shardman.set_replevel('pgbench_accounts', 1);"
-    # psql -c "select shardman.create_hash_partitions(2, 'pgbench_tellers', 'tid', 10);"
-    # psql -c "select shardman.set_replevel('pgbench_tellers', 1);"
-    # psql -c "select shardman.create_hash_partitions(2, 'pgbench_branches', 'bid', 10);"
-    # psql -c "select shardman.set_replevel('pgbench_branches', 1);"
     # for port in "${worker_ports[@]}"; do
-    # 	psql -p $port -f /home/ars/postgres/pg_shardman/bin/pgbench_history.sql
+    	# psql -p $port -f /home/ars/postgres/pg_shardman/bin/pgbench_history.sql
     # done
+    # psql -f devops/pgbench_ddl.sql
+    # psql -c "select shardman.create_hash_partitions('pgbench_accounts', 'aid', 30, 1);"
+    # psql -c "select shardman.create_hash_partitions('pgbench_tellers', 'tid', 30, 1);"
+    # psql -c "select shardman.create_hash_partitions('pgbench_branches', 'bid', 30, 1);"
+    # pgbench -p 5433 -s 10 -i --no-ddl
 }

devops/readme.txt

@@ -53,11 +53,9 @@ ansible-playbook -i inventory_ec2 pg_ctl.yml -e "pg_ctl_action=restart"
 
 Read cmd log on shardlord:
 ansible-playbook -i inventory_ec2/ psql.yml --limit 'shardlord' -e "cmd='\'table shardman.cmd_log\''"
-Read nodes table on workers:
-nodes': ansible-playbook -i inventory_ec2/ psql.yml --limit 'workers' -e "cmd='\'table shardman.nodes\''"
 
 Create, fill and shard pgbench tables:
-ansible-playbook -i inventory_ec2/ pgbench_prepare.yml -e "scale=10 nparts=3 repfactor=0 rebalance=true"
+ansible-playbook -i inventory_ec2/ pgbench_prepare.yml -e "scale=10 nparts=3 redundancy=0"
 Run pgbench test:
 ansible-playbook -i inventory_ec2/ pgbench_run.yml -e 'tmstmp=false tname=t pgbench_opts="-c 1 -T 5"'
 Run pgbench only on node:
@@ -75,14 +73,14 @@ ansible-playbook -i inventory_ec2/ ec2.yml --tags "terminate"
 
 Ubuntu images EC2 locator:
 https://cloud-images.ubuntu.com/locator/ec2/
-We need ami-4199282e.
+We use ami-4199282e.
 
 Other hints:
 Currently pgbench exits on first error. postgres_fdw currently supports only
 repeatable read / serializable isolation levels which immediately leads to
 serialization errors, so you should use either patched postgres_fdw or pgbench.
 
-If you don't want to measure the dist performance, keep data dir on tmpfs or
+If you don't want to measure the disk performance, keep data dir on tmpfs or
 turn of fsync, the effect it similar.
 
 Things that made me wonder during writing this:

postgresql.conf.common

@@ -1,59 +1,8 @@
 shared_preload_libraries = 'pg_pathman'
 
-# We support printing node ids in log messages. For that, include '%z' in
-# log_line_prefix. This feature is only supported if there is one shardman
-# installation per database cluster.
-log_line_prefix = '%m [%p]%z'
-
-log_min_messages = DEBUG1
-client_min_messages = NOTICE
-log_replication_commands = on
-# log_statement = 'all'
-
-# We use logical replication for sending metadata from shardlord to workers
-# and for data replication.
-wal_level = logical # necessary for logical replication
-
-# On shardlord, this must be at least max number of worker nodes + some reserved
-# for initial tablesync.
-#
-# On worker node 'A', this must be at least 'mrs':
-# mrs = 1
-# for each sharded table t
-#   for each primary shard of t lying on A
-#     mrs += number of this shard replicas
-#   for each replica shard of t lying on A
-#     mrs++
-#
-# For example, imagine we have one sharded table with 10 partitions, replication
-# factor 3 (1 primary and 2 replicas for each shard), 5 nodes and distribute
-# data evenly so that each node has 6 shards. In the almost worst case, if node
-# A keeps 5 primaries and 1 replica, this node needs 1 + 5*2 + 1 = 12 repslots.
-max_replication_slots = 100
-
-# Similar is true for max_wal_senders. Shardlord should have this at equal
-# max_replication_slots.
-
-# On worker node 'A', this must be at least 'mws':
-# mws = 0
-# for each sharded table t
-#   for each primary shard of t lying on A
-#     mrs += number of this shard replicas
-#
-# So it is 5*2 = 10 walsenders in previous example.
-max_wal_senders = 50
-
-# never set this to 'off' globally while using pg_shardman if you want
-# synchronous replication between shards and its replicas.
-synchronous_commit = on
-
-# performance-related settings
+# a bit usual performance-related settings
 shared_buffers = 512MB
 effective_cache_size = 512MB
 work_mem = 4MB
 max_connections = 1000
 max_wal_size = 5GB
-
-max_prepared_transactions = 1000
-postgres_fdw.use_2pc = on
-postgres_fdw.use_repeatable_read = off

postgresql.conf.lord

@@ -1,3 +1,5 @@
 shardman.shardlord_connstring = 'port=5432' # shardlord's connstring
+# This node is shardlord?
 shardman.shardlord = on
+# While creating replicas, configure sync or async replication?
 shardman.sync_replication = on

postgresql.conf.worker

@@ -1,12 +1,30 @@
-# On worker node 'A', this must be at least 'mrs':
-# mrs = 1
-# for each sharded table t
-#   for each replica shard of t lying on A
-#     mrs++
-#
-# So it is 1 + 1 = 2 for node A from example in postgresql.conf.common
-max_logical_replication_workers = 50
+# We use logical replication for data replication.
+wal_level = logical # necessary for logical replication
+# On worker node from replication group with n nodes, 'max_replication_slots'
+# must be at least 2n + 1 (+ 1 for moving parts/replicas)
+max_replication_slots = 101
+# On worker node from replication group with n nodes, 'max_replication_slots'
+# must be at least n + 1
+max_wal_senders = 51
+# On worker node from replication group with n nodes, 'max_replication_slots'
+# must be at least n + 1
+max_logical_replication_workers = 51
 # At least max_logical_replication_workers + 1
 max_worker_processes = 60
-# Logical worker dies if it hadn't receive anything new during wal_receiver_timeout
+# Logical worker dies if it hadn't receive anything new during
+# wal_receiver_timeout
 wal_receiver_timeout = 60s
+
+# set this to 'on' on for synchronous replication between shards and their
+# replicas, otherwise to 'local'.
+synchronous_commit = on
+
+# for 2pc
+max_prepared_transactions = 1000
+postgres_fdw.use_2pc = on
+# only for testing performace; setting this to 'on' violates visibilitys
+postgres_fdw.use_repeatable_read = off
+
+shardman.shardlord_connstring = 'port=5432' # shardlord's connstring
+# This node is shardlord?
+shardman.shardlord = off