Running pgbench.

Author: arssher

devops/.gitignore

@@ -1,6 +1,7 @@
 *.retry
 inventory_manual/manual
 logs/
+res/
 
 /postgresql.conf.common
 /postgresql.conf.lord

devops/inventory

@@ -37,6 +37,9 @@ shardman_version_tag=master
 
 shardman_shardlord_connstring="host={{ shardlord_ip }} port={{ pg_port }} user={{ ansible_user }}"
 
+test_id=""
+logs=logs
+
 [nodes:children]
 ec2
 manual

devops/logs.yml

@@ -1,33 +1,31 @@
-### Collect logs
+### Collect logs to {{ logs }}
 
 ---
 - hosts: nodes
 
   tasks:
   - name: rm -rf logs
-    local_action: file path=logs state=absent
+    local_action: file path={{ logs }} state=absent
     run_once: true
-  - name: fetch logs
+
+  - name: fetch logs to {{ logs }}
     fetch:
       src: "{{ pg_logfile }}"
-      dest: logs/{{ inventory_hostname }}.log
+      dest: "{{ logs }}/{{ inventory_hostname }}.log"
       validate_checksum: no # new logs might constantly arrive
       flat: yes
 
   - name: touch logs/united.log
-    local_action: copy content="" dest=logs/united.log
+    local_action: copy content="" dest={{ logs }}/united.log
     run_once: true
 
   - name: unite logs
-    local_action: shell cat logs/{{ item }}.log >> logs/united.log
+    local_action: shell cat {{ logs }}/{{ item }}.log >> {{ logs }}/united.log
     with_items: "{{ groups.nodes }}"
     run_once: true
 
-- hosts: shardlord
-
-  tasks:
   - name: create symlink to shadlord log
     local_action: file
-      src={{ inventory_hostname }}.log
-      dest=logs/shardlord.log
+      src={{ groups['shardlord'][0] }}.log
+      dest={{ logs }}/shardlord.log
       state=link

devops/pgbench_prepare.yml

@@ -1,5 +1,11 @@
 ### create pgbench_history everywhere, create pgbench tables on init node,
 # shard them. TODO: drop sharded tables.
+# Controlling vars: scale, nparts, repfactor
+
+# This is useful since we don't drop tables yet
+- import_playbook: init.yml
+  tags:
+    - init
 
 - hosts: workers
   environment:
@@ -70,3 +76,21 @@
     tags:
       - set_repfactor
     when: repfactor != 0
+
+
+- hosts: shardlord
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: wait for cmd finishing
+    command: >
+      psql -p {{ pg_port }} -qtA -c "
+      select count(*) from shardman.cmd_log
+      where status in ('waiting', 'in progress');
+      "
+    register: jobs_count
+    until: jobs_count.stdout == "0"
+    retries: 19481948 # oh my
+    tags:
+      - wait

devops/pgbench_run.yml

@@ -0,0 +1,58 @@
+### create pgbench_history everywhere, create pgbench tables on init node,
+# shard them. TODO: drop sharded tables.
+# Controlling vars: tmstmp (true/false), tname, clients, seconds.
+
+
+# set global var to use in all plays, oh my
+- hosts: nodes
+  vars:
+    tmstmp: True
+    tname: pgbench
+  tasks:
+  - name: gen test date
+    local_action: command date "+%Y-%m-%d-%H-%M"
+    run_once: true
+    register: now
+
+  - set_fact: test_id="{{ tname }}"
+  - set_fact: test_id="{{ test_id }}-{{ now.stdout }}"
+    when: tmstmp
+
+- hosts: workers
+  vars:
+    clients: 1
+    seconds: 5
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: vacuum
+    command: >
+      psql -p {{ pg_port }} -U {{ ansible_user }} -c "VACUUM;"
+
+  - name: run pgbench
+    shell: >
+      pgbench -p {{ pg_port }} --report-latencies --progress 1 --no-vacuum
+      -c {{ clients }} -T {{ seconds }} 2>&1 > "{{ test_id }}.out"
+
+  - name: fetch pgbench exec times
+    fetch:
+      src: "{{ test_id }}.out"
+      dest: res/{{ test_id }}/pgbench/{{ inventory_hostname }}.out
+      flat: yes
+
+  - name: grep tps
+    local_action: >
+      shell grep -h -R "tps.*including" res/{{ test_id }}/pgbench/ |
+      awk '{print $3}' > res/{{ test_id }}/tps.txt
+    run_once: true
+
+  - name: calc sum tps
+    local_action: >
+      shell echo "sum: `awk '{ sum += $1 } END { print sum }'
+      res/{{ test_id }}/tps.txt`" >> res/{{test_id}}/tps.txt
+    run_once: true
+
+- import_playbook: logs.yml logs=res/{{ test_id }}/logs
+  tags:
+    - logs

devops/readme.txt

@@ -55,7 +55,9 @@ Read nodes table on workers:
 nodes': ansible-playbook -i inventory_ec2/ psql.yml --limit 'workers' -e "cmd='\'table shardman.nodes\''"
 
 Create, fill and shard pgbench tables:
-ansible-playbook -i inventory_ec2/ pgbench_prepare.yml -e "scale=10 nparts=10 repfactor=1"
+ansible-playbook -i inventory_ec2/ pgbench_prepare.yml -e "scale=10 nparts=3 repfactor=0"
+Run pgbench test:
+ansible-playbook -f 20 -i inventory_ec2/ pgbench_run.yml -e "tmstmp=false tname=test seconds=30"
 
 Gather logs to ./logs:
 ansible-playbook -i inventory_ec2/ logs.yml