Merge branch 'master' of https://github.com/postgrespro/postgres_cluster

Author: knizhnik

contrib/mmts/testeaux/Cluster.pm renamed to contrib/mmts/testeaux/lib/Cluster.pm

@@ -80,14 +80,21 @@ sub configure
 			max_wal_senders = 10
 			wal_sender_timeout = 0
 			max_replication_slots = 10
+
+			tcp_keepalives_idle = 2
+			tcp_keepalives_interval = 1
+			tcp_keepalives_count = 2
+
 			shared_preload_libraries = 'raftable,multimaster'
 			multimaster.workers = 10
 			multimaster.queue_size = 10485760 # 10mb
 			multimaster.node_id = $id
 			multimaster.conn_strings = '$connstr'
-			multimaster.use_raftable = true
+			multimaster.use_raftable = false
 			multimaster.ignore_tables_without_pk = true
 			multimaster.twopc_min_timeout = 60000
+			multimaster.twopc_prepare_ratio = 1000 #%
+
 			raftable.id = $id
 			raftable.peers = '$raftpeers'
 		));
@@ -130,9 +137,9 @@ sub psql
 }
 
 # XXX: test
-my $cluster = new RemoteCluster('ssh-config');
-$cluster->init;
-$cluster->configure;
-$cluster->start;
-
+#my $cluster = new RemoteCluster('ssh-config');
+# $cluster->init;
+# $cluster->configure;
+# $cluster->start;
+# 
 1;

contrib/mmts/testeaux/Combineaux.pm renamed to contrib/mmts/testeaux/lib/Combineaux.pm

@@ -3,6 +3,7 @@ package RemoteNode;
 use strict;
 use warnings;
 use Net::OpenSSH;
+use IPC::Run;
 
 sub new
 {
@@ -24,9 +25,8 @@ sub new
 
 	bless $self, $class;
 
-	# kill postgres here to ensure
-	# predictable initial state.
-	$self->execute("pkill -9 postgres || true");
+	$self->execute("sudo iptables -F");
+	$self->execute("sudo iptables -A INPUT -p tcp --dport ssh -j ACCEPT");
 
 	return $self;
 }
@@ -35,7 +35,7 @@ sub connstr
 {
 	my ($self, $dbname) = @_;
 
-	"host=$self->{_host} dbname=$dbname";
+	"host=$self->{_host} dbname=$dbname user=$self->{_user}";
 }
 
 sub execute
@@ -71,6 +71,10 @@ sub init
 	my $pgbin = $self->{_pgbin}; 
 	my $pgdata = $self->{_pgdata};
 
+	# kill postgres here to ensure
+	# predictable initial state.
+	$self->execute("pkill -9 postgres || true");
+
 	$self->execute("rm -rf $pgdata");
 	$self->execute("env LANG=C LC_ALL=C $pgbin/initdb -D $pgdata -A trust -N");
 
@@ -114,4 +118,42 @@ sub append_conf
 	$self->execute($cmd);
 }
 
+sub psql
+{
+	my ($self, $dbname, $sql) = @_;
+	my $stderr;
+	my $stdout;
+
+	my @psql_command =
+	  ('psql', '-XAtq', '-d', $self->connstr($dbname), '-f', '-');
+	
+	my @ipcrun_command = (\@psql_command, '<', \$sql);
+	#push @ipcrun_command, '>', $stdout;
+	#push @ipcrun_command, '2>', $stderr;
+
+	IPC::Run::run @ipcrun_command;
+	my $ret = $?;
+
+	return $ret;
+}
+
+sub net_deny_in
+{
+	my ($self)  = @_;
+	$self->execute("sudo iptables -A INPUT -j DROP");
+}
+
+sub net_deny_out
+{
+	my ($self)  = @_;
+	$self->execute("sudo iptables -A OUTPUT -j DROP");
+}
+
+sub net_allow
+{
+	my ($self)  = @_;
+	$self->execute("sudo iptables -D INPUT -j DROP");
+	$self->execute("sudo iptables -D OUTPUT -j DROP");
+}
+
 1;

contrib/mmts/testeaux/RemoteCluster.pm renamed to contrib/mmts/testeaux/lib/RemoteCluster.pm

@@ -3,66 +3,9 @@
 - hosts: all
 
   roles:
-  - role: kelvich.postgres
+  - role: postgres
     pg_port: 5432
     pg_repo: https://github.com/postgrespro/postgres_cluster.git
     pg_version_tag: master
     pg_destroy_and_init: true
-    pg_config_role:
-    - line: "multimaster.buffer_size = 65536"
-
-  tasks:
-  - name: generate connstrings
-    set_fact:
-      connstr: "host={{item}} user={{ansible_ssh_user}} dbname=postgres"
-    with_items:
-      groups['all'] | reverse | batch(nnodes | d(3) | int) | first
-    register: connstrs
-
-  - name: make a list
-    set_fact:
-      connections: "{{ connstrs.results | map(attribute='ansible_facts.connstr') | join(', ') }}"
-
-  - debug: var=hostvars
-
-  - debug: var=inventory_hostname
-
-  #- debug: var=hostvars[inventory_hostname]
-
-  - name: build raftable
-    shell: "make clean && make -j {{makejobs}} install"
-    args:
-      chdir: "{{pg_src}}/contrib/raftable"
-
-  - name: build multimaster
-    shell: "make clean && make -j {{makejobs}} install"
-    args:
-      chdir: "{{pg_src}}/contrib/mmts"
-
-  - name: enable dtm extension on datanodes
-    lineinfile:
-      dest: "{{pg_datadir}}/postgresql.conf"
-      line: "{{item}}"
-      state: present
-    with_items:
-    - "wal_level = logical"
-    - "max_wal_senders = 10"
-    - "wal_sender_timeout = 0"
-    - "max_replication_slots = 10"
-    - "max_connections = 200"
-    - "max_worker_processes = 100"
-    - "shared_preload_libraries = 'raftable,multimaster'"
-    - "multimaster.conn_strings = '{{connections}}'"
-    - "multimaster.node_id = {{ inventory_hostname | regex_replace('([0-9]+)', '\\1') }}"
-    - "multimaster.buffer_size = 65536"
-    - "multimaster.queue_size = 1073741824"
-    - "multimaster.arbiter_port = 5600"
-    - "multimaster.vacuum_delay = 1"
-    - "multimaster.workers = 32"
-    - "multimaster.use_dtm = 1"
-
-  - name: restart postgrespro
-    command: "{{pg_dst}}/bin/pg_ctl restart -w -D {{pg_datadir}} -l {{pg_datadir}}/pg.log"
-    environment:
-      LD_LIBRARY_PATH: "{{pg_dst}}/lib/"
 

contrib/mmts/testeaux/RemoteNode.pm renamed to contrib/mmts/testeaux/lib/RemoteNode.pm

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

contrib/mmts/testeaux/Starteaux.pm renamed to contrib/mmts/testeaux/lib/Starteaux.pm

@@ -0,0 +1,2 @@
+---
+# defaults file for ansible-postgres

contrib/mmts/testeaux/Stresseaux.pm renamed to contrib/mmts/testeaux/lib/Stresseaux.pm

@@ -0,0 +1,2 @@
+---
+# handlers file for ansible-postgres

contrib/mmts/testeaux/Troubleaux.pm renamed to contrib/mmts/testeaux/lib/Troubleaux.pm

@@ -0,0 +1,20 @@
+---
+galaxy_info:
+  author: Stas Kelvich
+  description: Role to build and install postgres from sources
+  company: PostgresPro
+  license: MIT
+  min_ansible_version: 1.9
+  platforms:
+  - name: Debian
+    versions:
+    - all
+  - name: CentOS
+    versions:
+    - all
+  galaxy_tags:
+  - postgres
+  - installer
+  - db
+
+dependencies: []

contrib/mmts/testeaux/provision.yml

@@ -0,0 +1,112 @@
+# vim: ts=2 sts=2 sw=2 expandtab
+---
+
+- name: ensure dependencies (Debian)
+  apt: pkg={{item}} state=installed
+  with_items:
+  - git
+  - automake
+  - libtool
+  - build-essential
+  - bison
+  - flex
+  - libreadline-dev
+  when: ansible_os_family == "Debian"
+  become: yes
+
+- name: ensure dependencies (RedHat)
+  yum: name="@Development tools" state=present
+  when: ansible_os_family == "RedHat"
+  become: yes
+
+- name: ensure dependencies (RedHat)
+  yum: name={{item}} state=installed
+  with_items:
+  - git
+  - automake
+  - libtool
+  - bison
+  - flex
+  - readline-devel
+  when: ansible_os_family == "RedHat"
+  become: yes
+
+- name: increase semaphores
+  sysctl: name=kernel.sem value="1000 128000 128 512"
+  become: yes
+
+- name: increase open files
+  lineinfile:
+    dest: /etc/security/limits.d/cluster.conf
+    line: "{{ansible_ssh_user}}  soft  nofile  65535"
+    state: present
+    create: yes
+  become: yes
+
+#############################################################################
+
+- name: clone postgres sources
+  git: repo={{pg_repo}}
+    dest={{pg_src}}
+    version={{pg_version_tag}}
+    depth=1
+    accept_hostkey=True
+    key_file={{pg_repo_key}}
+  register: pg_sources
+
+  # XXX: Consider using file module with state=absent rather than running rm
+- name: force rebuild on changed sources
+  #command: "rm -f {{pg_dst}}/bin/postgres"
+  file: path="{{pg_dst}}/bin/postgres" state=absent
+  when: pg_sources.changed 
+
+- name: build and install
+  shell: env CFLAGS="-O0" ./configure --prefix={{pg_dst}} --enable-debug --without-zlib && make clean && make -j {{makejobs}} && make install
+  args:
+    chdir: "{{pg_src}}"
+    creates: "{{pg_dst}}/bin/postgres"
+
+#############################################################################
+
+- name: stop postgres if it was running
+  shell: "pkill -9 postgres || true"
+  when: pg_destroy_and_init
+
+- name: remove datadirs
+  #command: "rm -rf {{pg_datadir}}"
+  file: path="{{pg_datadir}}" state=absent
+  when: pg_destroy_and_init
+
+- name: create datadirs
+  command: "{{pg_dst}}/bin/initdb {{pg_datadir}}"
+  environment:
+    LD_LIBRARY_PATH: "{{pg_dst}}/lib/"
+  args:
+    creates: "{{pg_datadir}}"
+  when: pg_destroy_and_init
+
+- name: configure postgres on datanodes
+  lineinfile:
+    dest: "{{pg_datadir}}/postgresql.conf"
+    line: "{{item.line}}"
+    state: present
+  with_items: "{{pg_config}}"
+  when: pg_destroy_and_init
+
+- name: enable blind trust on datanodes
+  lineinfile:
+    dest: "{{pg_datadir}}/pg_hba.conf"
+    line: "{{item}}"
+    state: present
+  with_items:
+    - "host all all 0.0.0.0/0 trust"
+    - "host replication all 0.0.0.0/0 trust"
+    - "local replication all trust"
+  when: pg_destroy_and_init
+
+- name: start postgrespro
+  shell: "{{pg_dst}}/bin/pg_ctl start -w -D {{pg_datadir}} -l {{pg_datadir}}/pg.log"
+  environment:
+    LD_LIBRARY_PATH: "{{pg_dst}}/lib/"
+  when: pg_destroy_and_init
+

contrib/mmts/testeaux/roles/postgres/README.md

@@ -0,0 +1,23 @@
+# vim: ts=2 sts=2 sw=2 expandtab
+---
+makejobs: 4
+
+pg_repo: git://git.postgresql.org/git/postgresql.git
+pg_repo_key: None
+pg_version_tag: master
+
+pg_destroy_and_init: false
+
+pg_port: 5432
+pg_config:
+  - line: "synchronous_commit = off"
+  - line: "listen_addresses = '*'"
+  - line: "max_connections = 100"
+  - line: "max_prepared_transactions = 100"
+  - line: "port = {{pg_port}}"
+
+pg_prefix: "{{ansible_env.HOME}}/pg_cluster"
+pg_src: "{{pg_prefix}}/src"
+pg_dst: "{{pg_prefix}}/install"
+pg_datadir: "{{pg_prefix}}/data_{{pg_port}}"
+