Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 8cf24ca

Browse files
knizhnikknizhnik
committed
Merge branch 'broadcast' of https://git.postgrespro.ru/a.sher/pg_shardman into broadcast
2 parents 8c97fac + 3014fb9 commit 8cf24ca

File tree

5 files changed

+61
-28
lines changed

5 files changed

+61
-28
lines changed

‎bin/setup.sh.example

Lines changed: 44 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
pgpath=~/postgres/install/vanilla/
22
pathmanpath=~/postgres/pg_pathman
33
install_pathman=false
4-
logfile=$HOME/tmp/tmp/tmp.log
4+
logfile=/tmp/shmn.log
55

6-
lord_datadir=~/postgres/data1
6+
lord_datadir=/tmp/data1
77
lord_port=5432
88

99
# declare -a worker_datadirs=()
@@ -18,23 +18,51 @@ lord_port=5432
1818
# declare -a worker_datadirs=("${HOME}/postgres/data2" "${HOME}/postgres/data3" "${HOME}/postgres/data4")
1919
# declare -a worker_ports=("5433" "5434" "5435")
2020

21-
declare -a worker_datadirs=("${HOME}/postgres/data2" "${HOME}/postgres/data3" "${HOME}/postgres/data4" "${HOME}/postgres/data5")
21+
declare -a worker_datadirs=("/tmp/data2" "/tmp/data3" "/tmp/data4" "/tmp/data5")
2222
declare -a worker_ports=("5433" "5434" "5435" "5436")
2323

2424
function run_demo()
2525
{
2626
:
27-
psql -p 5433 -c "drop table if exists pt cascade;"
28-
psql -p 5433 -c "CREATE TABLE pt(id INT NOT NULL, payload REAL);"
29-
psql -p 5433 -c "INSERT INTO pt SELECT generate_series(1, 10), random();"
30-
psql -c "select shardman.add_node('port=5433');"
31-
psql -c "select shardman.add_node('port=5434');"
32-
psql -c "select shardman.add_node('port=5435');"
33-
psql -c "select shardman.add_node('port=5436');"
34-
sleep 5
35-
psql -c "select shardman.create_hash_partitions(2, 'pt', 'id', 4, true);"
36-
37-
# psql -c "select shardman.create_replica('pt_0', 3);"
38-
# psql -c "select shardman.create_replica('pt_0', 5);"
39-
# psql -c "select shardman.move_part('pt_0', 4, 3);"
27+
28+
for port in "${worker_ports[@]}" $lord_port; do
29+
psql -p $port -c "set synchronous_commit to local; drop role if exists joe; create role joe login password '12345'; grant usage on FOREIGN DATA WRAPPER postgres_fdw to joe;"
30+
done
31+
32+
psql -c "drop table if exists t cascade;"
33+
psql -c "CREATE TABLE t(i int);"
34+
psql -c "drop table if exists t cascade;"
35+
psql -c "CREATE TABLE t(i int);"
36+
37+
38+
psql -c "drop table if exists pt cascade;"
39+
psql -c "CREATE TABLE pt(id INT primary key, payload REAL);"
40+
psql -c "INSERT INTO pt SELECT generate_series(1, 10), random();"
41+
42+
psql -c "select shardman.add_node('port=5433', conn_string => 'user=joe password=12345 dbname=ars port=5433');"
43+
psql -c "select shardman.add_node('port=5434', conn_string => 'user=joe password=12345 dbname=ars port=5434');"
44+
psql -c "select shardman.add_node('port=5435', conn_string => 'user=joe password=12345 dbname=ars port=5435');"
45+
psql -c "select shardman.add_node('port=5436', conn_string => 'user=joe password=12345 dbname=ars port=5436');"
46+
47+
psql -c "select shardman.create_hash_partitions('pt', 'id', 4);"
48+
psql -c "select shardman.set_redundancy('pt', 2);"
49+
50+
# psql -c "select shardman.create_hash_partitions(2, 'pt', 'id', 4, true);"
51+
# psql -c "select shardman.set_replevel('pt', 1);"
52+
53+
# psql -c "drop table if exists pgbench_accounts cascade";
54+
# psql -c "drop table if exists pgbench_tellers cascade";
55+
# psql -c "drop table if exists pgbench_branches cascade";
56+
# psql -c "drop table if exists pgbench_tellers cascade;"
57+
# psql -c "drop table if exists pgbench_accounts cascade;"
58+
# pgbench -s 5 -i
59+
# psql -c "select shardman.create_hash_partitions(2, 'pgbench_accounts', 'aid', 4);"
60+
# psql -c "select shardman.set_replevel('pgbench_accounts', 1);"
61+
# psql -c "select shardman.create_hash_partitions(2, 'pgbench_tellers', 'tid', 10);"
62+
# psql -c "select shardman.set_replevel('pgbench_tellers', 1);"
63+
# psql -c "select shardman.create_hash_partitions(2, 'pgbench_branches', 'bid', 10);"
64+
# psql -c "select shardman.set_replevel('pgbench_branches', 1);"
65+
# for port in "${worker_ports[@]}"; do
66+
# psql -p $port -f /home/ars/postgres/pg_shardman/bin/pgbench_history.sql
67+
# done
4068
}

‎bin/shardman_init.sh

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ for worker_datadir in "${worker_datadirs[@]}"; do
3030
cat postgresql.conf.worker >> ${worker_datadir}/postgresql.conf
3131
done
3232

33+
for datadir in $lord_datadir "${worker_datadirs[@]}"; do
34+
cat pg_hba.conf > ${datadir}/pg_hba.conf
35+
done
36+
3337
start_nodes
3438
for port in $lord_port "${worker_ports[@]}"; do
3539
createdb -p $port `whoami`

‎bin/shardman_start.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,15 @@ cd "${script_dir}/.."
1010
restart_nodes # make sure nodes run
1111
# first workers, then lord
1212
for port in "${worker_ports[@]}" $lord_port; do
13-
psql -p $port -c "drop extension if exists pg_shardman cascade;"
13+
psql -p $port -c "set synchronous_commit to local; drop extension if exists pg_shardman cascade;"
1414
done
1515

1616
make clean
1717
make install
1818

1919
restart_nodes
2020
for port in $lord_port "${worker_ports[@]}"; do
21-
psql -p $port -c "create extension pg_shardman cascade;"
21+
psql -p $port -c "set synchronous_commit to local; create extension pg_shardman cascade;"
2222
done
2323

2424
run_demo

‎pg_shardman.c

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,8 @@ wait_command_completion(PGconn* conn)
120120
Datum
121121
broadcast(PG_FUNCTION_ARGS)
122122
{
123-
char* sql = text_to_cstring(PG_GETARG_TEXT_PP(0));
123+
char *sql_full = text_to_cstring(PG_GETARG_TEXT_PP(0));
124+
char* sql = pstrdup(sql_full);
124125
bool ignore_errors = PG_GETARG_BOOL(1);
125126
bool two_phase = PG_GETARG_BOOL(2);
126127
bool sync_commit_on = PG_GETARG_BOOL(3);
@@ -203,6 +204,7 @@ broadcast(PG_FUNCTION_ARGS)
203204
}
204205
if (!sync_commit_on)
205206
{
207+
/* mem freed with context */
206208
if (two_phase)
207209
{
208210
sql = psprintf("SET SESSION synchronous_commit TO local; BEGIN; %s; PREPARE TRANSACTION 'shardlord';", sql);
@@ -227,8 +229,6 @@ broadcast(PG_FUNCTION_ARGS)
227229
node_id, PQerrorMessage(conn[n_cmds-1]));
228230
goto cleanup;
229231
}
230-
if (!sync_commit_on)
231-
pfree(sql);
232232

233233
sql = sep + 1;
234234
}
@@ -256,10 +256,10 @@ broadcast(PG_FUNCTION_ARGS)
256256
{
257257
if (ignore_errors)
258258
{
259-
errmsg = psprintf("%s%d:Failed to received response for '%s': %s", errmsg ? errmsg : "", node_id, sql, PQerrorMessage(conn[i]));
259+
errmsg = psprintf("%s%d:Failed to received response for '%s': %s", errmsg ? errmsg : "", node_id, sql_full, PQerrorMessage(conn[i]));
260260
continue;
261261
}
262-
errmsg = psprintf("Failed to receive response for query %s from node %d: %s", sql, node_id, PQerrorMessage(conn[i]));
262+
errmsg = psprintf("Failed to receive response for query %s from node %d: %s", sql_full, node_id, PQerrorMessage(conn[i]));
263263
goto cleanup;
264264
}
265265

@@ -269,11 +269,11 @@ broadcast(PG_FUNCTION_ARGS)
269269
{
270270
if (ignore_errors)
271271
{
272-
errmsg = psprintf("%s%d:Command %s failed: %s", errmsg ? errmsg : "", node_id, sql, PQerrorMessage(conn[i]));
272+
errmsg = psprintf("%s%d:Command %s failed: %s", errmsg ? errmsg : "", node_id, sql_full, PQerrorMessage(conn[i]));
273273
PQclear(res);
274274
continue;
275275
}
276-
errmsg = psprintf("Command %s failed at node %d: %s", sql, node_id, PQerrorMessage(conn[i]));
276+
errmsg = psprintf("Command %s failed at node %d: %s", sql_full, node_id, PQerrorMessage(conn[i]));
277277
PQclear(res);
278278
goto cleanup;
279279
}
@@ -284,11 +284,11 @@ broadcast(PG_FUNCTION_ARGS)
284284
if (ignore_errors)
285285
{
286286
appendStringInfoString(&resp, "?;");
287-
elog(WARNING, "SHARDMAN: Query '%s' doesn't return single tuple at node %d", sql, node_id);
287+
elog(WARNING, "SHARDMAN: Query '%s' doesn't return single tuple at node %d", sql_full, node_id);
288288
}
289289
else
290290
{
291-
errmsg = psprintf("Query '%s' doesn't return single tuple at node %d", sql, node_id);
291+
errmsg = psprintf("Query '%s' doesn't return single tuple at node %d", sql_full, node_id);
292292
PQclear(res);
293293
goto cleanup;
294294
}
@@ -391,7 +391,7 @@ gen_create_table_sql(PG_FUNCTION_ARGS)
391391
join_path_components(pg_dump_path, pg_dump_path, "pg_dump");
392392
canonicalize_path(pg_dump_path);
393393

394-
cmd = psprintf("%s -t '%s' --schema-only --dbname='%s' 2>&1",
394+
cmd = psprintf("%s -t '%s' --no-owner --schema-only --dbname='%s' 2>&1",
395395
pg_dump_path, relation, shardlord_connstring);
396396

397397
if ((fp = popen(cmd, "r")) == NULL)

‎postgresql.conf.lord

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
shardman.shardlord_connstring = 'port=5432' # shardlord's connstring
2+
shardman.shardlord = on
23
shardman.sync_replication = on

0 commit comments

Comments
 (0)