init.yml started, shardlord elections.

Author: arssher

ec2/inventory/ec2.py renamed to ec2/ec2.py

@@ -0,0 +1,17 @@
+### initdb and send configs
+
+---
+- hosts: nodes
+
+  environment:
+    PATH: "{{ pg_dst }}/bin:{{ ansible_env.PATH }}"
+
+  tasks:
+  - name: pkill pg
+    shell: 'pkill -9 postgres || true'
+
+  - name: remove datadirs on datanodes
+    file: path={{pg_datadir}} state=absent
+
+  - name: create datadirs on datanodes
+    command: "initdb {{pg_datadir}}"

ec2/init.yml

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+
+'''
+Run ec2.py stored in .. directory and appends ec2_shardlord ec2_workers groups
+to it.
+'''
+
+import os
+import subprocess
+import json
+import socket
+
+def ec2_elect_shardlord():
+    script_dir = os.path.dirname(os.path.realpath(__file__))
+    ec2_path = os.path.join(os.path.dirname(script_dir), "ec2.py")
+    ec2_inv_txt = subprocess.check_output([ec2_path, "--list"])
+    ec2_inv = json.loads(ec2_inv_txt)
+    hosts = list(ec2_inv["_meta"]["hostvars"].keys())
+    # sort to choose the same shardlord each time
+    hosts.sort(key = socket.inet_aton)
+    shardlord, workers = [hosts[0]], hosts[1:]
+    ec2_inv["ec2_shardlord"] = shardlord
+    ec2_inv["ec2_workers"] = workers
+    print json.dumps(ec2_inv, sort_keys=True, indent=2)
+
+
+if __name__ == '__main__':
+    # Run the script
+    ec2_elect_shardlord()

ec2/inventory/ec2_elect_shardlord.py

@@ -16,6 +16,7 @@ pg_prefix="{{ansible_env.HOME}}/pg"
 pg_src="{{pg_prefix}}/src"
 pg_dst="{{pg_prefix}}/install"
 pg_datadir="{{pg_prefix}}/data_{{pg_port}}"
+pg_logfile="{{pg_datadir}}/pg.log"
 pg_port=5432
 
 pathman_repo=https://github.com/postgrespro/pg_pathman.git
@@ -41,4 +42,4 @@ ansible_ssh_private_key_file=~/.ssh/aws_rsa
 
 # For local development
 [vagrant]
-10.42.42.10
+# 10.42.42.10

ec2/inventory/inventory

@@ -1,3 +1,5 @@
+### Build & install from sources PG, pg_pathman and pg_shardman
+
 ---
 - hosts: nodes
 

ec2/provision.yml

@@ -3,7 +3,9 @@ chmod 0400 ~/.ssh/aws_rsa
 
 install python packages for ec2.py: (venv is recommended):
 python3 -m venv ~/venv/aws
-source ~/venv/aws/bin/activate
+As of writing this, python 3.5 has a nasty bug, so better use python 2:
+virtualenv -p /usr/bin/python ~/venv/aws_2.7
+source ~/venv/aws_2.7/bin/activate
 pip install boto boto3 six ansible
 
 Download and set up ec2.ini: