A lot of small improvements.

ec2.yml now reuses tasks for launching instances.
We now take by default branch with read committed postgres_fdw.
Running pgbench on single node.
Mounting tmpfs and keeping pg there.

Author: arssher

devops/ec2.yml

@@ -12,20 +12,9 @@
     region: eu-central-1
 
   tasks:
-  - name: start instances
-    ec2:
-      key_name: "{{ key_name }}"
-      instance_type: "{{ instance_type }}"
-      image: "{{ image }}"
-      group: "{{ security_group }}"
-      count: "{{ count }}"
-      vpc_subnet_id: "{{ vpc_subnet_id }}"
-      region: "{{ region }}"
-      wait: yes
-      assign_public_ip: yes
-    vars:
-      instance_type: t2.micro
-      image: ami-1e339e71 # basic ubuntu backed by ebs
+
+  # image here is basic ubuntu backed by ebs
+  - import_tasks: tasks/ec2.yml instance_type="t2.micro" image="ami-1e339e71"
     tags:
       - micro
 

devops/inventory

@@ -16,9 +16,12 @@ ansible_python_interpreter=python3
 
 # make -j
 makejobs=4
-pg_repo=git://git.postgresql.org/git/postgresql.git
-pg_version_tag=REL_10_STABLE
-pg_prefix="{{ansible_env.HOME}}/pg"
+# pg_repo=git://git.postgresql.org/git/postgresql.git
+# pg_version_tag=REL_10_STABLE
+pg_repo=https://github.com/arssher/postgresql
+pg_version_tag=postgres_fdw_read_committed
+# pg_prefix="{{ansible_env.HOME}}/pg"
+pg_prefix="/tmp/pg"
 pg_src="{{pg_prefix}}/src"
 cflags="-O2"
 pg_dst="{{pg_prefix}}/install"

devops/pgbench_prepare.yml

@@ -22,8 +22,8 @@
 
 - hosts: init_node
   vars:
-    scale: 3
-    nparts: 3
+    scale: 10
+    nparts: 10
     repfactor: 0
   environment:
     PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"

devops/pgbench_run.yml

@@ -1,6 +1,7 @@
 ### create pgbench_history everywhere, create pgbench tables on init node,
 # shard them. TODO: drop sharded tables.
-# Controlling vars: tmstmp (true/false), tname, clients, seconds.
+# Controlling vars: tmstmp (true/false), tname, clients, t (true/false),
+# seconds, xacts
 
 
 # set global var to use in all plays, oh my
@@ -22,6 +23,8 @@
   vars:
     clients: 1
     seconds: 5
+    xacts: 100
+    t: False
   environment:
     PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
 
@@ -30,10 +33,21 @@
     command: >
       psql -p {{ pg_port }} -U {{ ansible_user }} -c "VACUUM;"
 
-  - name: run pgbench
+  - name: run pgbench -T {{ seconds }}
     shell: >
-      pgbench -p {{ pg_port }} --report-latencies --progress 1 --no-vacuum
+      pgbench -p {{ pg_port }} --report-latencies --no-vacuum
       -c {{ clients }} -T {{ seconds }} 2>&1 > "{{ test_id }}.out"
+    when: not ( t | bool )
+
+  - name: run pgbench -t {{ xacts }}
+    shell: >
+      pgbench -p {{ pg_port }} --report-latencies --no-vacuum
+      -c {{ clients }} -t {{ xacts }} 2>&1 > "{{ test_id }}.out"
+    when: t | bool
+
+  - name: clear log dir
+    local_action: file path="res/{{ test_id }}/" state=absent
+    run_once: true
 
   - name: fetch pgbench exec times
     fetch:

devops/pgbench_single.yml

@@ -0,0 +1,48 @@
+### Init and run pgbench on random worker. Controlling vars as in
+# pgbench_prepare and pgbench_run, except for tname.
+---
+
+- hosts: workers[0]
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+  vars:
+    scale: 10
+    clients: 1
+    tmstmp: True
+    tname: pgbench
+
+  tasks:
+
+  - name: drop old pgbench tables with deps
+    command: >
+      psql -p {{ pg_port }} -c "
+      drop table if exists pgbench_accounts cascade;
+      drop table if exists pgbench_tellers cascade;
+      drop table if exists pgbench_branches cascade;
+      drop table if exists pgbench_tellers cascade;
+      drop table if exists pgbench_accounts cascade;
+      "
+  - name: initialize pgbench, scale {{ scale }}
+    command: >
+      pgbench -p {{ pg_port }} -s {{ scale }} -i
+
+  - name: run pgbench -T {{ seconds }}
+    shell: >
+      pgbench -p {{ pg_port }} --report-latencies
+      -c {{ clients }} -T {{ seconds }} 2>&1 > "pgbench.out"
+    when: not ( t | bool )
+
+  - name: run pgbench -t {{ xacts }}
+    shell: >
+      pgbench -p {{ pg_port }} --report-latencies
+      -c {{ clients }} -t {{ xacts }} 2>&1 > "pgbench.out"
+    when: t | bool
+
+  - name: fetch pgbench exec time
+    fetch:
+      src: "pgbench.out"
+      dest: res/pgbench_single.out
+      flat: yes
+      fail_on_missing: yes
+    tags:
+      - fetch

devops/postgresql.conf.common.example

@@ -0,0 +1,27 @@
+TimeZone = 'Europe/Moscow'
+log_timezone = 'Europe/Moscow'
+
+listen_addresses = '*'
+
+log_min_messages = LOG
+client_min_messages = NOTICE
+log_replication_commands = on
+
+max_replication_slots = 300
+max_wal_senders = 300
+max_logical_replication_workers = 300
+max_worker_processes = 300
+
+# performance-related settings
+shared_buffers = {{ (ansible_memtotal_mb * 0.5) | int }}MB
+effective_cache_size = 4GB
+work_mem = 32MB
+max_connections = 1000
+# don't do checkpoints
+checkpoint_timeout = 60min
+max_wal_size = 100GB
+fsync = off
+autovacuum = off
+
+shardman.shardlord_dbname = ubuntu
+shardman.poll_interval = 500 # long operations poll frequency in milliseconds

devops/provision.yml

@@ -15,6 +15,14 @@
     tags:
       - tz
 
+  - name: mount tmpfs as /tmp
+    mount: >
+      name=/tmp src='tmpfs' fstype=tmpfs
+      opts='size={{ (ansible_memtotal_mb * 0.75) | int }}m' state=mounted
+    become: yes
+    tags:
+      - tmpfs
+
   - name: update apt cache (Debian)
     apt: update_cache=yes
     become: yes
@@ -60,6 +68,11 @@
     tags:
       - build_pg_fdw
 
+  - name: rm pg sources
+    file: path="{{ pg_src }}" state=absent
+    tags:
+      - rm_pg_src
+
   - name: clone pathman
     git: repo={{pathman_repo}}
       dest={{pathman_src}}
@@ -93,6 +106,11 @@
     tags:
       - build_shardman
 
+  - name: set PATH in .bashrc
+    lineinfile: dest=~/.bashrc line="export PATH={{ pg_dst }}/bin:$PATH"
+    tags:
+      - set_path
+
 - import_playbook: init.yml
   tags:
     - init

devops/readme.txt

@@ -57,7 +57,9 @@ nodes': ansible-playbook -i inventory_ec2/ psql.yml --limit 'workers' -e "cmd='\
 Create, fill and shard pgbench tables:
 ansible-playbook -i inventory_ec2/ pgbench_prepare.yml -e "scale=10 nparts=3 repfactor=0"
 Run pgbench test:
-ansible-playbook -f 20 -i inventory_ec2/ pgbench_run.yml -e "tmstmp=false tname=test seconds=30"
+ansible-playbook -i inventory_ec2/ pgbench_run.yml -e "tmstmp=false tname=test seconds=30"
+Run pgbench on single worker (to estimate shardman overhead):
+ansible-playbook -i inventory_ec2/ pgbench_single.yml -e "scale=10 tmstmp=false tname=test t=false clients=8 seconds=15"
 
 Gather logs to ./logs:
 ansible-playbook -i inventory_ec2/ logs.yml
@@ -71,6 +73,11 @@ Ubuntu images EC2 locator:
 https://cloud-images.ubuntu.com/locator/ec2/
 We need ami-4199282e.
 
+Other hints:
+Currently pgbench exits on first error. postgres_fdw currently supports only
+repeatable read / serializable isolation levels which immediately leads to
+serialization errors, so you should use either patched postgres_fdw or pgbench.
+
 Things that made me wonder during writing this:
 * Reusability of tasks. Playbooks, files with tasks, roles, includes, imports,
   old include API...probably working directly from Python would be easier? There

devops/send_config.yml

@@ -2,6 +2,9 @@
 # Templates in the root of the project are copied automatically. You can append
 # something to them by creating same-titled in this (devops/) directory, they
 # are not tracked by git.
+# BTW, you can refer to ansible vars there.
+# The only guc we manage dynamically by default is shardlord_connstring, it
+# requires inventory knowledge
 
 ---
 - hosts: shardlord

devops/tasks/ec2.yml

@@ -0,0 +1,13 @@
+---
+
+  - name: start {{ count }} {{ instance_type }} instances
+    ec2:
+      key_name: "{{ key_name }}"
+      instance_type: "{{ instance_type }}"
+      image: "{{ image }}"
+      group: "{{ security_group }}"
+      count: "{{ count }}"
+      vpc_subnet_id: "{{ vpc_subnet_id }}"
+      region: "{{ region }}"
+      wait: yes
+      assign_public_ip: yes