Add monitor_deadlocks function

Author: knizhnik

pg_shardman--1.0.sql

@@ -1996,34 +1996,49 @@ END;
 $$ LANGUAGE plpgsql;
 
 
--- Detect distributed deadlock and return set of process involed in deadlock. If there is no deadlock then this view ias empty.
---
--- This query is based on the algorithm described by Knuth for detecting a cycle in a linked list. In one column, keep track of the children,
--- the children's children, the children's children's children, etc. In another column, keep track of the grandchildren, the grandchildren's grandchildren,
--- the grandchildren's grandchildren's grandchildren, etc.
---
--- For the initial selection, the distance between Child and Grandchild columns is 1. Every selection from union all increases the depth of Child by 1, and that of Grandchild by 2.
--- The distance between them increases by 1.
---
--- If there is any loop, since the distance only increases by 1 each time, at some point after Child is in the loop, the distance will be a multiple of the cycle length.
--- When that happens, the Child and the Grandchild columns are the same. Use that as an additional condition to stop the recursion, and detect it in the rest of your code as an error.
-CREATE VIEW deadlock AS
+-- Detect distributed deadlock and returns path in the lock graph forming deadlock loop
+CREATE FUNCTION detect_deadlock() RETURNS shardman.process[] AS $$
 	WITH RECURSIVE LinkTable AS (SELECT wait AS Parent, hold AS Child FROM shardman.deserialize_lock_graph(shardman.global_lock_graph())),
 	cte AS (
-    	SELECT lt1.Parent, lt1.Child, lt2.Child AS Grandchild
-    	FROM LinkTable lt1
-    	INNER JOIN LinkTable lt2 on lt2.Parent = lt1.Child
-    	UNION ALL
-    	SELECT cte.Parent, lt1.Child, lt3.Child AS Grandchild
-    	FROM cte
-    	INNER JOIN LinkTable lt1 ON lt1.Parent = cte.Child
-    	INNER JOIN LinkTable lt2 ON lt2.Parent = cte.Grandchild
-    	INNER JOIN LinkTable lt3 ON lt3.Parent = lt2.Child
-    	WHERE cte.Child <> cte.Grandchild
+		SELECT Child, Parent, ARRAY[Child] AS AllParents, false AS Loop
+  	  	FROM LinkTable
+	  	UNION ALL
+	  	SELECT c.Child, c.Parent, p.AllParents||c.Child, c.Child=ANY(p.AllParents)
+	 	FROM LinkTable c JOIN cte p	ON c.Parent = p.Child AND NOT p.Loop
 	)
-	SELECT DISTINCT Parent
-	FROM cte
-	WHERE Child = Grandchild;
+	SELECT AllParents FROM cte WHERE Loop;
+$$ LANGUAGE sql;
+
+-- Monitor cluster for presence of distributed deadlocks and cancel correspondent queries
+CREATE FUNCTION monitor_deadlocks(timeout_sec int = 5) RETURNS void AS $$
+DECLARE
+	prev_deadlock_path shardman.process[];
+	deadlock_path shardman.process[];
+	victim shardman.process;
+	loop_begin int;
+	loop_end int;
+BEGIN
+	IF shardman.redirect_to_shardlord(format('monitor_deadlocks(%s)', timeout_sec))
+	THEN
+		RETURN;
+	END IF;
+
+	LOOP
+		deadlock_path := shardman.detect_deadlock();
+		IF deadlock_path && prev_deadlock_path
+		THEN
+			loop_end := array_upper(deadlock_path, 1);
+			loop_begin := array_position(deadlock_path, deadlock_path[loop_end]);
+			victim := deadlock_path[loop_begin + ((loop_end - loop_begin)*random())::integer];
+			RAISE NOTICE 'Detect deadlock: cancel process % at node %', victim.pid, victim.node;
+			PERFORM shardman.broadcast(format('%s:SELECT pg_cancel_backend(%s);',
+				victim.node, victim.pid));
+		END IF;
+		prev_deadlock_path := deadlock_path;
+		PERFORM pg_sleep(timeout_sec);
+	END LOOP;
+END;
+$$ LANGUAGE plpgsql;
 
 
 -- View for monitoring logical replication lag.