Provision and init done.

Now we can provision and init (including create extension) ec2 nodes or manually
typed ones. Also simple ansible psql wrapper added. Collecting logs started.

Author: arssher

devops/.gitignore

@@ -0,0 +1,6 @@
+*.retry
+inventory/manual
+
+/postgresql.conf.common
+/postgresql.conf.lord
+/postgresql.conf.worker

devops/Vagrantfile

@@ -0,0 +1,27 @@
+Vagrant.configure("2") do |config|
+
+  config.vm.define "node1" do |node|
+    node.vm.box = "ubuntu/xenial64"
+    node.vm.network "private_network", ip: "10.42.42.10"
+  end
+
+  config.vm.define "node2" do |node|
+    node.vm.box = "ubuntu/xenial64"
+    node.vm.network "private_network", ip: "10.42.42.20"
+  end
+
+  config.vm.define "node3" do |node|
+    node.vm.box = "ubuntu/xenial64"
+    node.vm.network "private_network", ip: "10.42.42.30"
+  end
+
+  # ssh-copy-id
+  config.vm.provision "shell" do |s|
+    ssh_pub_key = File.readlines("#{Dir.home}/.ssh/id_rsa.pub").first.strip
+    s.inline = <<-SHELL
+      echo #{ssh_pub_key} >> /home/ubuntu/.ssh/authorized_keys
+      echo #{ssh_pub_key} >> /root/.ssh/authorized_keys
+    SHELL
+  end
+
+end

ec2/ec2.py renamed to devops/ec2.py

@@ -0,0 +1,45 @@
+### initdb and send configs
+
+---
+- hosts: nodes
+
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: pkill pg
+    shell: 'pkill -9 postgres || true'
+    tags:
+      - initdb
+
+  - name: remove datadirs on datanodes
+    file: path={{pg_datadir}} state=absent
+    tags:
+      - initdb
+
+  - name: create datadirs on datanodes
+    command: "initdb {{pg_datadir}}"
+    tags:
+      - initdb
+
+  - import_tasks: tasks/pg_ctl.yml pg_ctl_action=start
+    tags:
+      - initdb
+
+  - name: create whoami database
+    command: "createdb {{ ansible_user }}"
+    tags:
+      - initdb
+
+- import_playbook: send_config.yml
+
+- hosts: nodes
+
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: create extension pg_shardman
+    command: psql -p {{ pg_port }} -c "create extension pg_shardman cascade;"
+    tags:
+      - create_ext

devops/init.yml

@@ -1,7 +1,10 @@
-# Put here manual stuff, aws-related inventory will be generated by ec2.py
 localhost
 
 [nodes]
+# nodes consists of 'manual' (e.g. for local testing) and 'ec2' autogenerated
+# by ec2_elect_shardlord.py
+# If use the former variant, you must also choose shardlord, init_node and
+# worker nodes by hand. See manual.example for example
 
 [nodes:vars]
 ansible_user=ubuntu
@@ -17,6 +20,7 @@ pg_src="{{pg_prefix}}/src"
 pg_dst="{{pg_prefix}}/install"
 pg_datadir="{{pg_prefix}}/data_{{pg_port}}"
 pg_logfile="{{pg_datadir}}/pg.log"
+pg_conffile="{{pg_datadir}}/postgresql.conf"
 pg_port=5432
 
 pathman_repo=https://github.com/postgrespro/pg_pathman.git
@@ -31,15 +35,34 @@ cflags="-O0"
 
 [nodes:children]
 ec2
-vagrant
+manual
 
-# generated by ec2.py
+[shardlord]
+[shardlord:children]
+ec2_shardlord
+manual_shardlord
+
+[workers]
+[workers:children]
+ec2_workers
+manual_workers
+
+# designated worker on which we will initially create tables
+[init_node]
+[init_node:children]
+ec2_init_node
+manual_init_node
+
+# empty groups to be able to run without ec2/manual inventories
 [ec2]
+[ec2_shardlord]
+[ec2_workers]
+[ec2_init_node]
+[manual]
+[manual_shardlord]
+[manual_workers]
+[manual_init_node]
 
 [ec2:vars]
 # Your aws key
 ansible_ssh_private_key_file=~/.ssh/aws_rsa
-
-# For local development
-[vagrant]
-# 10.42.42.10

ec2/inventory/inventory renamed to devops/inventory

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
 '''
-Run ec2.py stored in .. directory and appends ec2_shardlord ec2_workers groups
-to it.
+Run ec2.py stored in .. directory and append ec2_shardlord, ec2_workers and
+ec2_init_node groups to it.
 '''
 
 import os
@@ -18,12 +18,12 @@ def ec2_elect_shardlord():
     hosts = list(ec2_inv["_meta"]["hostvars"].keys())
     # sort to choose the same shardlord each time
     hosts.sort(key = socket.inet_aton)
-    shardlord, workers = [hosts[0]], hosts[1:]
+    shardlord, workers, init_node = hosts[0:1], hosts[1:], hosts[1:2]
     ec2_inv["ec2_shardlord"] = shardlord
     ec2_inv["ec2_workers"] = workers
+    ec2_inv["ec2_init_node"] = init_node
     print json.dumps(ec2_inv, sort_keys=True, indent=2)
 
 
 if __name__ == '__main__':
-    # Run the script
     ec2_elect_shardlord()

ec2/inventory/ec2_elect_shardlord.py renamed to devops/inventory_ec2/ec2_elect_shardlord.py

@@ -0,0 +1 @@
+../inventory

devops/inventory_ec2/inventory

@@ -0,0 +1 @@
+../inventory

devops/inventory_manual/inventory

@@ -0,0 +1,16 @@
+# For local development
+
+[manual]
+[manual:children]
+manual_shardlord
+manual_workers
+
+[manual_shardlord]
+10.42.42.10
+
+[manual_workers]
+10.42.42.20
+# 10.42.42.30
+
+[manual_init_node]
+10.42.42.20

devops/inventory_manual/manual

@@ -0,0 +1,16 @@
+# For local development
+
+# [manual]
+# [manual:children]
+# manual_shardlord
+# manual_workers
+
+# [manual_shardlord]
+# 10.42.42.10
+
+# [manual_workers]
+# 10.42.42.20
+# 10.42.42.30
+
+# [manual_init_node]
+# 10.42.42.20

devops/inventory_manual/manual.example

@@ -0,0 +1,14 @@
+### Collect logs
+
+---
+- hosts: nodes
+
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: fetch logs
+    fetch:
+      src: "{{ pg_logfile }}"
+      dest: /tmp
+      validate_checksum: no # new logs might constantly arrive

devops/logs.yml

@@ -0,0 +1,8 @@
+### start, stop, restart all nodes. We call task here because in some places
+# it is more convenient to import task instead of whole role.
+
+---
+- hosts: nodes
+
+  tasks:
+  - import_tasks: tasks/pg_ctl.yml

devops/pg_ctl.yml

@@ -46,6 +46,13 @@
     tags:
       - build_pg
 
+  - name: build and install postgres_fdw
+    shell: make clean && make -j {{makejobs}} && make install
+    args:
+      chdir: "{{pg_src}}/contrib/postgres_fdw"
+    tags:
+      - build_pg_fdw
+
   - name: clone pathman
     git: repo={{pathman_repo}}
       dest={{pathman_src}}

ec2/provision.yml renamed to devops/provision.yml

@@ -0,0 +1,14 @@
+### execute psql cmd and print res
+
+---
+- hosts: nodes
+
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: psql cmd
+    command: psql -p {{ pg_port }} -c {{ cmd }}
+    register: out
+
+  - debug: var=out.stdout_lines

devops/psql.yml

@@ -0,0 +1,65 @@
+I used ansible 2.5.0.
+
+This stuff allows to deploy and configure pg_shardman on ec2 nodes or on
+manually hardcoded ones, inventories are inventory_ec2 and inventory_manual.
+
+For manual setup, type nodes to inventory_manual/manual file, see manual.example
+file there and comments in ./inventory. You can also check out Vagrantfile I
+used for testing this.
+
+EC2 setup:
+Download aws ssh key in needed region
+chmod 0400 ~/.ssh/aws_rsa
+
+install python packages for ec2.py: (venv is recommended):
+python3 -m venv ~/venv/aws
+As of writing this, python 3.5 has a nasty bug, so better use python 2:
+virtualenv -p /usr/bin/python ~/venv/aws_2.7
+source ~/venv/aws_2.7/bin/activate
+pip install boto boto3 six ansible
+
+Download and set up ec2.ini:
+sudo wget https://raw.githubusercontent.com/ansible/ansible/devel/contrib/inventory/ec2.ini
+rds = False # we don't need rds
+regions = eu-central-1 # set region
+set aws_access_key_id = xxx
+set aws_secret_access_key = xxx
+export EC2_INI_PATH=/etc/ansible/ec2.ini
+
+Now launch some instances and check that ec2.py works:
+inventory_ec2/ec2_elect_shardlord.py --list
+
+Check that ansible works:
+ansible -i inventory_ec2 -m ping all
+For manually configured nodes:
+ansible -i inventory_manual -m ping all
+Now you are ready to go.
+
+Ansible cheatsheet:
+
+Build only PG on ec2:
+ansible-playbook -i inventory_ec2 --tags "build_pg"
+Same on hardcoded nodes:
+ansible-playbook -i inventory_manual --tags "build_pg"
+Provision only the second node:
+ansible-playbook -i inventory_ec2 provision.yml --limit nodes[1]
+
+Reload configs:
+ansible-playbook -i inventory_ec2/ send_config.yml
+Restart nodes:
+ansible-playbook -i inventory_ec2 pg_ctl.yml -e "pg_ctl_action=restart"
+
+Read cmd log on shardlord:
+ansible-playbook -i inventory_ec2/ psql.yml --limit 'shardlord' -e "cmd='\'table shardman.cmd_log\''"
+
+Things that made me wonder during writing this:
+* Reusability of tasks. Playbooks, files with tasks, roles, includes, imports,
+  old include API...probably working directly from Python would be easier? There
+  seems to be no way to run only one play from playbook, and the only way to run
+  several tasks from play is to use tags, which is a bit ugly. This is the reason
+  why projects like
+  https://github.com/larsks/ansible-toolbox
+  emerge. Printing cmd output via saved var is pretty horrible too.
+* No way to specify multiple inventory files, only whole directory or one file.
+* No way to append file to file, except for j2 templates, which doesn't work
+  with files outside the template dir, really?

devops/readme.txt

@@ -0,0 +1,21 @@
+### send configs and restart nodes
+# Templates in the root of the project are copied automatically. You can append
+# something to them by creating same-titled in this (devops/) directory, they
+# are not tracked by git.
+
+---
+- hosts: shardlord
+  tasks:
+  - name: copy postgresql.conf to shardlord
+    template:
+      src=templates/postgresql.conf.lord.j2
+      dest="{{ pg_conffile }}"
+
+- hosts: workers
+  tasks:
+  - name: copy postgresql.conf to workers
+    template:
+      src=templates/postgresql.conf.worker.j2
+      dest="{{ pg_conffile }}"
+
+- import_playbook: pg_ctl.yml pg_ctl_action=restart

devops/send_config.yml

@@ -0,0 +1,4 @@
+---
+
+- name: pg_ctl {{pg_ctl_action}}
+  command: "{{pg_dst}}/bin/pg_ctl {{pg_ctl_action}} -D {{pg_datadir}} -l {{pg_logfile}}"

devops/tasks/pg_ctl.yml

@@ -0,0 +1 @@
+../../postgresql.conf.common

devops/templates/postgresql.conf.common

@@ -0,0 +1 @@
+../postgresql.conf.common

devops/templates/postgresql.conf.devops.common

@@ -0,0 +1 @@
+../postgresql.conf.lord

devops/templates/postgresql.conf.devops.lord

@@ -0,0 +1 @@
+../postgresql.conf.worker

devops/templates/postgresql.conf.devops.worker

@@ -0,0 +1 @@
+../../postgresql.conf.lord

devops/templates/postgresql.conf.lord

@@ -0,0 +1,10 @@
+{# jinja2 won't work with outside file, so I've created a bunch of symlinks #}
+{# pointing to them, oh my... #}
+{# see https://github.com/ansible/ansible/issues/7106 #}
+{% include "postgresql.conf.common" ignore missing %}
+
+{% include "postgresql.conf.lord" ignore missing %}
+
+{% include "postgresql.conf.devops.common" ignore missing %}
+
+{% include "postgresql.conf.devops.lord" ignore missing %}

devops/templates/postgresql.conf.lord.j2

@@ -0,0 +1 @@
+../../postgresql.conf.worker

devops/templates/postgresql.conf.worker

@@ -0,0 +1,10 @@
+{# jinja2 won't work with outside file, so I've created a bunch of symlinks #}
+{# pointing to them, oh my... #}
+{# see https://github.com/ansible/ansible/issues/7106 #}
+{% include "postgresql.conf.common" %}
+
+{% include "postgresql.conf.worker" %}
+
+{% include "postgresql.conf.devops.common" ignore missing %}
+
+{% include "postgresql.conf.devops.worker" ignore missing %}