Adapted devops/ for 'broadcast' version.

Author: arssher

bin/shardman_init.sh

@@ -37,6 +37,7 @@ done
 start_nodes
 for port in $lord_port "${worker_ports[@]}"; do
     createdb -p $port `whoami`
+    echo "creating pg_shardman extension..."
     psql -p $port -c "create extension pg_shardman cascade;"
 done
 

devops/ec2.yml

@@ -11,18 +11,19 @@
     vpc_subnet_id: subnet-e0e2d99a
     image: ami-1e339e71 # ubuntu xenial backed by ebs
     region: eu-central-1
+    instance_type: c3.2xlarge
 
   tasks:
 
   # image here is
-  - import_tasks: tasks/ec2.yml instance_type="t2.micro"
+  - import_tasks: tasks/ec2.yml
     tags:
-      - micro
+      - launch
 
-  # image here is ubuntu xenial backed by instance store
-  - import_tasks: tasks/ec2.yml instance_type="c3.2xlarge" image="ami-4199282e"
+  # image ubuntu xenial backed by instance store
+  - import_tasks: tasks/ec2.yml image="ami-4199282e"
     tags:
-      - c3.2xlarge
+      - launch_instance_store
 
   # good for both launch and termination
   - import_tasks: tasks/clean_ec2_cache.yml

devops/init.yml

@@ -1,4 +1,5 @@
-### initdb, send configs and add all workers.
+### initdb, send configs, create extension and add all workers.
+# controlling vars: use_pgbouncer, nodes_in_repgroup
 
 ---
 - hosts: nodes
@@ -11,9 +12,6 @@
     tags:
       - initdb
 
-  - name: kill bouncer
-    shell: 'pkill -SIGINT pgbouncer || true'
-
   - name: remove datadirs on datanodes
     file: path={{pg_datadir}} state=absent
     tags:
@@ -56,22 +54,30 @@
       - create_ext
 
 - hosts: shardlord
+  vars:
+    nodes_in_repgroup: 999999
+
   environment:
     PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
 
   tasks:
-  - name: add ec2 workers
+  - name: add ec2 workers, nodes_in_repgroup is {{ nodes_in_repgroup }}
     command: >
       psql -p {{ pg_port }} -c "select shardman.add_node(
-      'hostaddr={{ hostvars[item]['ec2_private_ip_address'] }} port={{ ( use_pgbouncer | bool ) | ternary(pgbouncer_port, pg_port) }}')"
-    with_items: "{{ groups.ec2_workers }}"
+      'hostaddr={{ hostvars[item.1]['ec2_private_ip_address'] }} port={{ pg_port }}',
+      'hostaddr={{ hostvars[item.1]['ec2_private_ip_address'] }} port={{ ( use_pgbouncer | bool ) | ternary(pgbouncer_port, pg_port) }}',
+      'group_{{ item.0 // (nodes_in_repgroup | int) }}')
+      "
+    with_indexed_items: "{{ groups.ec2_workers }}"
     tags:
       - add_nodes
 
-  - name: add manual workers
+  - name: add manual workers, nodes_in_repgroup is {{ nodes_in_repgroup }}
     command: >
       psql -p {{ pg_port }} -c "select shardman.add_node(
-      'hostaddr={{ item }} port={{ pg_port }}')"
-    with_items: "{{ groups.manual_workers }}"
+      'hostaddr={{ item.1 }} port={{ pg_port }}',
+      'hostaddr={{ item.1 }} port={{ ( use_pgbouncer | bool ) | ternary(pgbouncer_port, pg_port) }}',
+      'group_{{ item.0 // (nodes_in_repgroup | int) }}')"
+    with_indexed_items: "{{ groups.manual_workers }}"
     tags:
       - add_nodes

devops/inventory

@@ -27,7 +27,8 @@ pg_prefix="/tmp/pg"
 pg_src="{{pg_prefix}}/src"
 cflags="-O2"
 pg_dst="{{pg_prefix}}/install"
-pg_datadir="{{pg_prefix}}/data_{{pg_port}}"
+# pg_datadir="{{pg_prefix}}/data_{{pg_port}}"
+pg_datadir="/mnt/data_{{pg_port}}"
 pg_logfile="{{pg_datadir}}/pg.log"
 pg_conffile="{{pg_datadir}}/postgresql.conf"
 pg_port=5432
@@ -37,17 +38,21 @@ pgbouncer_dst = "{{ pg_prefix }}/pgbouncer_install"
 pgbouncer_conf = "/home/{{ ansible_user }}/pgbouncer.conf"
 pgbouncer_port = 6543
 pgbouncer_logfile = /tmp/pgbouncer.log
+pgbouncer_pool_mode = session
+pgbouncer_pool_size = 10
 use_pgbouncer = False
 
-pathman_repo=https://github.com/postgrespro/pg_pathman.git
+pathman_repo=https://github.com/arssher/pg_pathman
 pathman_src="{{pg_prefix}}/pg_pathman"
-pathman_version_tag=master
+pathman_version_tag=foreign_copy_from
 
 shardman_repo=https://github.com/postgrespro/pg_shardman.git
 shardman_src="{{pg_prefix}}/pg_shardman"
-shardman_version_tag=master
+# shardman_version_tag=master
 # trigger based replication
 # shardman_version_tag=tbr-2pc
+# sync version
+shardman_version_tag=broadcast
 
 shardman_shardlord_connstring="host={{ shardlord_ip }} port={{ pg_port }} user={{ ansible_user }}"
 

devops/logs.yml

@@ -17,19 +17,6 @@
     tags:
       fetch
 
-  - name: find out node id from log
-    local_action: shell grep -m 1 -o -P 'SHND \d' logs/{{ inventory_hostname }}.log | sed 's/SHND //'
-    register: node_id
-    tags:
-      node_symlink
-
-  - name: create symlink 'node_id'.log -> 'ip'.log
-    local_action: file
-      src={{ inventory_hostname }}.log
-      dest={{ logs }}/{{ node_id.stdout }}.log
-      force=yes
-      state=link
-
   - name: touch logs/united.log
     local_action: copy content="" dest={{ logs }}/united.log
     run_once: true

devops/monitor_wal_lag.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-app_name=`psql -qtA -c "select application_name from pg_stat_replication where application_name ~ '.*accounts.*' limit 1;"`
+app_name=$(psql -qtA -c "select application_name from pg_stat_replication where application_name ~ '.*sub_[0-9]+_[0-9]+.*' limit 1;")
 
 > wal_lag.txt
 while true; do

devops/pgbench_ddl.sql

@@ -0,0 +1,27 @@
+drop table if exists pgbench_accounts;
+create table pgbench_accounts (
+	aid int not null,
+	bid int,
+	abalance int,
+	filler char(84)
+);
+
+drop table if exists pgbench_tellers;
+create table pgbench_tellers (
+	tid int not null,
+	bid int,
+	tbalance int,
+	filler char(84)
+);
+
+
+drop table if exists pgbench_branches;
+create table pgbench_branches (
+	bid int not null,
+	bbalance int,
+	filler char(88)
+);
+
+alter table pgbench_branches add primary key (bid);
+alter table pgbench_tellers add primary key (tid);
+alter table pgbench_accounts add primary key (aid);

devops/pgbench_prepare.yml

@@ -1,8 +1,8 @@
-### create pgbench_history everywhere, create pgbench tables on init node,
-# shard them. TODO: drop sharded tables.
-# Controlling vars: scale, nparts, repfactor, use_pgbouncer
+### create pgbench_history everywhere, create pgbench tables,
+# shard them.
+# Controlling vars: scale, nparts, redundancy, use_pgbouncer, pgbouncer_pool_mode,
+# nodes_in_repgroup
 
-# We need this since we don't drop tables yet
 - import_playbook: init.yml
   tags:
     - init
@@ -12,93 +12,66 @@
     PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
 
   tasks:
-  - name: recreate pgbench_history on workers
+  - name: recreate pgbench_history on workers, drop other pgbench tables
     command: >
       psql -p {{ pg_port }} -c "drop table if exists pgbench_history;
+      do $$
+      declare
+      rname text;
+      begin
+      for rname in (select relname from pg_class where relname ~ 'pgbench_.*' and relkind = 'r') loop
+      execute format('drop table if exists %I cascade', rname);
+      end loop;
+      end
+      $$ LANGUAGE 'plpgsql';
+      drop table if exists pgbench_accounts cascade;
+      drop table if exists pgbench_tellers cascade;
+      drop table if exists pgbench_branches cascade;
       create table pgbench_history (tid int, bid int, aid int,
       delta int, mtime timestamp, filler char(22));"
     tags:
       - pgbench_history
 
-  - name: initialize pgbench everywhere to avoid count(*) failure, scale {{ scale }}
-    command: >
-      pgbench -p {{ pg_port }} -s {{ scale }} -i
-    tags:
-      - pgbench_init
-
-
-- hosts: init_node
+- hosts: shardlord
   vars:
-    scale: 10
     nparts: 10
-    repfactor: 0
-    rebalance: true
-    tellers_branches: true
+    redundancy: 0
   environment:
     PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
 
   tasks:
-  - name: drop old pgbench tables with deps
-    command: >
-      psql -p {{ pg_port }} -c "
-      drop table if exists pgbench_accounts cascade;
-      drop table if exists pgbench_tellers cascade;
-      drop table if exists pgbench_branches cascade;
-      drop table if exists pgbench_tellers cascade;
-      drop table if exists pgbench_accounts cascade;
-      "
-    tags:
-      - pgbench_init
+  - name: copy pgbench_ddl.sql
+    copy:
+      src: pgbench_ddl.sql
+      dest: "/home/{{ ansible_user }}/pgbench_ddl.sql"
 
-  - name: learn init node id
+  - name: create pgbench tables on shardlord
     command: >
-      psql -p {{ pg_port }} -qtA -c "select shardman.my_id();"
-    register: my_id
-    until: my_id.stdout != ""
-    retries: 19481948 # oh my
-    tags:
-      - pgbench_init
+      psql -p {{ pg_port }} -f pgbench_ddl.sql
 
-  - name: initialize pgbench, scale {{ scale }}
-    command: >
-      pgbench -p {{ pg_port }} -s {{ scale }} -i
-    tags:
-      - pgbench_init
-
-  - name: create_hash_partitions for pgbench_accounts, nparts {{ nparts }}
+  - name: create_hash_partitions for pgbench_accounts, nparts {{ nparts }}, redundancy {{ redundancy }}
     command: >
       psql -p {{ pg_port }} -c "
-      select shardman.create_hash_partitions({{ my_id.stdout }}, 'pgbench_accounts', 'aid', {{ nparts }}, {{ rebalance }});
-      select shardman.set_replevel('pgbench_accounts', {{ repfactor }});
+      select shardman.create_hash_partitions('pgbench_accounts', 'aid', {{ nparts }}, {{ redundancy }});
+      select shardman.create_hash_partitions('pgbench_tellers', 'tid', {{ nparts }}, {{ redundancy }});
+      select shardman.create_hash_partitions('pgbench_branches', 'bid', {{ nparts }}, {{ redundancy }});
       "
     tags:
-      - create_hash_partitions_accounts
+      - create_hash_partitions
 
-  - name: create_hash_partitions for branches & tellers, nparts {{ nparts }}, repfactor {{ repfactor }}
-    command: >
-      psql -p {{ pg_port }} -c "
-      select shardman.create_hash_partitions({{ my_id.stdout }}, 'pgbench_tellers', 'tid', {{ nparts }}, {{ rebalance }});
-      select shardman.create_hash_partitions({{ my_id.stdout }}, 'pgbench_branches', 'bid', {{ nparts }}, {{ rebalance }});
-      select shardman.set_replevel('pgbench_tellers', {{ repfactor }});
-      select shardman.set_replevel('pgbench_branches', {{ repfactor }});
-      "
-    tags:
-      - create_hash_partitions_tellers_branches
-    when: tellers_branches | bool
-
-- hosts: shardlord
+- hosts: init_node
+  vars:
+    scale: 10
+    nparts: 10
+    redundancy: 0
+    rebalance: true
   environment:
     PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
 
   tasks:
-  - name: wait for cmd finishing
+    # requires patched pgbench
+  - name: initialize pgbench, scale {{ scale }}
     command: >
-      psql -p {{ pg_port }} -qtA -c "
-      select count(*) from shardman.cmd_log
-      where status in ('waiting', 'in progress');
-      "
-    register: jobs_count
-    until: jobs_count.stdout == "0"
-    retries: 19481948 # oh my
+      pgbench -p {{ pg_port }} -s {{ scale }} -i --no-ddl
     tags:
-      - wait
+      - pgbench_init

devops/pgbench_run.yml

@@ -1,5 +1,5 @@
 ### Run pgbench benchmark.
-# Controlling vars: tmstmp (true/false), tname, pgbench_opts
+# Controlling vars: tmstmp (true/false), tname, pgbench_opts, active_workers
 
 
 # set global var to use in all plays, oh my

devops/postgresql.conf.common.example

@@ -3,9 +3,11 @@ log_timezone = 'Europe/Moscow'
 
 listen_addresses = '*'
 
-log_min_messages = NOTICE
+log_min_messages = WARNING
 client_min_messages = NOTICE
 log_replication_commands = on
+log_line_prefix = '%m [%r][%p]'
+# log_statement = 'all'
 
 # log_statement = 'all'
 # logging_collector = on
@@ -30,10 +32,7 @@ fsync = on
 autovacuum = off
 synchronous_commit = on
 # synchronous_commit = local
-shardman.sync_replicas = on
-
-shardman.shardlord_dbname = ubuntu
-shardman.poll_interval = 500 # long operations poll frequency in milliseconds
+shardman.sync_replication = on
 
 max_prepared_transactions = 100000
 postgres_fdw.use_2pc = on