Making replicas read-only for all but apply workers.

create_replica in progress.

Author: arssher

postgresql.conf.common.template

@@ -8,7 +8,6 @@ autovacuum = off
 
 log_min_messages = DEBUG1
 # client_min_messages = NOTICE
-# client_min_messages = INFO
-client_min_messages = PANIC
+client_min_messages = WARNING
 
 wal_level = logical

shard.sql

@@ -26,7 +26,6 @@ DECLARE
 	pname text;
 BEGIN
 	IF NEW.initial_node != (SELECT shardman.get_node_id()) THEN
-		RAISE DEBUG '[SHARDMAN] new table trig, pid %', (select pg_backend_pid());
 		EXECUTE format('DROP TABLE IF EXISTS %I CASCADE;', NEW.relation);
 		partition_names :=
 			(SELECT ARRAY(SELECT part_name FROM shardman.gen_part_names(
@@ -210,12 +209,12 @@ END $$ LANGUAGE plpgsql;
 
 -- On adding new partition, create proper foreign server & foreign table and
 -- replace tmp (empty) partition with it.
--- TODO: race condition between this trigger and new_table_worker_side
--- definitely deserves attention.
+-- TODO: There is a race condition between this trigger and
+-- new_table_worker_side trigger during initial tablesync, we should deal with
+-- it.
 CREATE FUNCTION new_primary() RETURNS TRIGGER AS $$
 BEGIN
 	IF NEW.owner != (SELECT shardman.get_node_id()) THEN
-		RAISE DEBUG 'SHARDMAN new prim trigger, pid %', (select pg_backend_pid());
 		PERFORM shardman.replace_usual_part_with_foreign(NEW);
 	END IF;
 	RETURN NULL;
@@ -297,28 +296,86 @@ $$ LANGUAGE plpgsql;
 CREATE FUNCTION readonly_table_on(relation regclass)
 	RETURNS void AS $$
 BEGIN
-	-- Create go away trigger to prevent any new ones
+	-- Create go away trigger to prevent any modifications
 	PERFORM shardman.readonly_table_off(relation);
-	EXECUTE format(
-		'CREATE TRIGGER shardman_readonly BEFORE INSERT OR UPDATE OR DELETE OR
-		TRUNCATE ON %I FOR EACH STATEMENT EXECUTE PROCEDURE shardman.go_away();',
-		relation);
-	EXECUTE format(
-		'ALTER TABLE %I ENABLE ALWAYS TRIGGER shardman_readonly;', relation);
+	PERFORM shardman.create_modification_triggers(relation, 'shardman_readonly',
+												 'shardman.go_away()');
 END
 $$ LANGUAGE plpgsql STRICT;
 CREATE FUNCTION go_away() RETURNS TRIGGER AS $$
 BEGIN
 	RAISE EXCEPTION 'The "%" table is read only.', TG_TABLE_NAME
 		USING HINT = 'Probably table copy is in progress';
-  RETURN NULL;
 END;
 $$ LANGUAGE plpgsql;
 -- And make it writable again
 CREATE FUNCTION readonly_table_off(relation regclass)
 	RETURNS void AS $$
 BEGIN
 	EXECUTE format('DROP TRIGGER IF EXISTS shardman_readonly ON %s', relation);
+	EXECUTE format('DROP TRIGGER IF EXISTS shardman_readonly_stmt ON %s', relation);
+END $$ LANGUAGE plpgsql STRICT;
+
+-- Make replica read-only, i.e. readonly for all but LR apply workers
+CREATE FUNCTION readonly_replica_on(relation regclass)
+	RETURNS void AS $$
+BEGIN
+	RAISE DEBUG '[SHARDMAN] table % made read-only for all but apply workers', relation;
+	PERFORM shardman.readonly_replica_off(relation);
+	PERFORM shardman.create_modification_triggers(
+		relation, 'shardman_readonly_replica', 'shardman.ror_go_away()');
+END $$ LANGUAGE plpgsql STRICT;
+-- This function is impudent because it is used as both stmt and row trigger.
+-- The idea is that we must never reach RETURN NEW after stmt row trigger,
+-- because stmt trigger fires only on TRUNCATE which is impossible in LR.
+-- Besides, I checked that nothing bad happens if we return NEW from stmt
+-- trigger function anyway.
+CREATE FUNCTION ror_go_away() RETURNS TRIGGER AS $$
+BEGIN
+	IF NOT shardman.inside_apply_worker() THEN
+		RAISE EXCEPTION 'The "%" table is read only for non-apply workers', TG_TABLE_NAME
+		USING HINT =
+		'If you see this, most probably node with primary part has failed and' ||
+		' you need to promote replica. Promotion is not yet implemented, sorry :(';
+	END IF;
+	raise warning 'NEW IS %', NEW;
+  RETURN NEW;
+END $$ LANGUAGE plpgsql;
+-- And make replica writable again
+CREATE FUNCTION readonly_replica_off(relation regclass) RETURNS void AS $$
+BEGIN
+	EXECUTE format('DROP TRIGGER IF EXISTS shardman_readonly_replica ON %s',
+				   relation);
+	EXECUTE format('DROP TRIGGER IF EXISTS shardman_readonly_replica_stmt ON %s',
+				   relation);
+END $$ LANGUAGE plpgsql STRICT;
+CREATE FUNCTION inside_apply_worker() RETURNS bool AS 'pg_shardman' LANGUAGE C;
+
+-- Create two triggers firing exec_proc before any modification operation, make
+-- them ALWAYS ENABLE. We need two triggers because TRUNCATE doesn't work with
+-- FOR EACH ROW, while LR doesn't support STATEMENT triggers (well, there is no
+-- statements in WAL) and changes may sneak through it.
+-- If you are curious, we use %I to format any identifiers (e.g. quote identifier
+-- with "" if it contains spaces) and use %s while formatting regclass, because
+-- it quotes everything automatically while casting oid to name.
+CREATE FUNCTION create_modification_triggers(
+	relation regclass, trigname name, exec_proc text) RETURNS void AS $$
+DECLARE
+	stmt_trigname text;
+BEGIN
+	EXECUTE format(
+		'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE
+		ON %s FOR EACH ROW EXECUTE PROCEDURE %s;',
+		trigname, relation, exec_proc);
+	EXECUTE format(
+		'ALTER TABLE %s ENABLE ALWAYS TRIGGER %I;', relation::text, trigname);
+	stmt_trigname := format('%s_stmt', trigname);
+	EXECUTE format(
+		'CREATE TRIGGER %I BEFORE
+		TRUNCATE ON %s FOR EACH STATEMENT EXECUTE PROCEDURE %s;',
+		stmt_trigname, relation, exec_proc);
+	EXECUTE format(
+		'ALTER TABLE %s ENABLE ALWAYS TRIGGER %I;', relation::text, stmt_trigname);
 END $$ LANGUAGE plpgsql STRICT;
 
 CREATE FUNCTION gen_create_table_sql(relation text, connstring text) RETURNS text

src/include/pg_shardman.h

@@ -47,6 +47,8 @@ extern void update_cmd_status(int64 id, const char *new_status);
 extern void cmd_canceled(Cmd *cmd);
 extern char *get_worker_node_connstr(int node_id);
 extern int32 get_primary_owner(const char *part_name);
+extern int32 get_reptail_owner(const char *part_name, int32 *owner,
+							   int32 *partnum);
 extern char *get_partition_relation(const char *part_name);
 
 #endif							/* PG_SHARDMAN_H */

src/include/shard.h

@@ -5,5 +5,6 @@
 
 extern void create_hash_partitions(Cmd *cmd);
 extern void move_primary(Cmd *cmd);
+extern void create_replica(Cmd *cmd);
 
 #endif							/* SHARD_H */

src/pg_shardman.c

@@ -287,7 +287,7 @@ wait_notify()
 	/* eat all notifications at once */
 	while ((notify = PQnotifies(conn)) != NULL)
 	{
-		shmn_elog(LOG, "NOTIFY %s received from backend PID %d",
+		shmn_elog(DEBUG1, "NOTIFY %s received from backend PID %d",
 			 notify->relname, notify->be_pid);
 		PQfreemem(notify);
 	}
@@ -446,7 +446,7 @@ void
 cmd_canceled(Cmd *cmd)
 {
 	got_sigusr1 = false;
-	shmn_elog(LOG, "Command %ld canceled", cmd->id);
+	shmn_elog(INFO, "Command %ld canceled", cmd->id);
 	update_cmd_status(cmd->id, "canceled");
 }
 
@@ -755,14 +755,10 @@ get_primary_owner(const char *part_name)
 		part_name);
 
 	if (SPI_execute(sql, true, 0) < 0)
-	{
 		shmn_elog(FATAL, "Stmt failed : %s", sql);
-	}
 
 	if (SPI_processed == 0)
-	{
 		owner = -1;
-	}
 	else
 	{
 		owner =	DatumGetInt32(SPI_getbinval(SPI_tuptable->vals[0],
@@ -774,6 +770,42 @@ get_primary_owner(const char *part_name)
 	return owner;
 }
 
+/*
+ * Get node id on which the last replica in the 'part_name' replica chain
+ * resides, and its partnum.  -1 is returned if such partition doesn't exist
+ * at all.
+ */
+int32
+get_reptail_owner(const char *part_name, int32 *owner, int32 *partnum)
+{
+	char *sql;
+	bool isnull;
+	int result = 0;
+
+	SPI_PROLOG;
+	sql = psprintf( /* allocated in SPI ctxt, freed with ctxt release */
+		"select owner from shardman.partitions where part_name = '%s'"
+		" and nxt is NULL;", part_name);
+
+	if (SPI_execute(sql, true, 0) < 0)
+		shmn_elog(FATAL, "Stmt failed : %s", sql);
+
+	if (SPI_processed == 0)
+		result = -1;
+	else
+	{
+		*owner = DatumGetInt32(SPI_getbinval(SPI_tuptable->vals[0],
+											 SPI_tuptable->tupdesc,
+											 1, &isnull));
+		*partnum = DatumGetInt32(SPI_getbinval(SPI_tuptable->vals[0],
+											   SPI_tuptable->tupdesc,
+											   2, &isnull));
+	}
+
+	SPI_EPILOG;
+	return result;
+}
+
 /*
  * Get relation name of partition part_name. Memory is palloc'ed.
  * NULL is returned, if there is no such partition.