Modify snapshot definition so that lazy vacuums are ignored by other

vacuums.  This allows a OLTP-like system with big tables to continue
regular vacuuming on small-but-frequently-updated tables while the
big tables are being vacuumed.

Original patch from Hannu Krossing, rewritten by Tom Lane and updated
by me.

Author: alvherre

src/backend/access/transam/twophase.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.21 2006/07/14 14:52:17 momjian Exp $
+ *		$PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.22 2006/07/30 02:07:18 alvherre Exp $
  *
  * NOTES
  *		Each global transaction is associated with a global transaction
@@ -279,6 +279,7 @@ MarkAsPreparing(TransactionId xid, const char *gid,
 	gxact->proc.pid = 0;
 	gxact->proc.databaseId = databaseid;
 	gxact->proc.roleId = owner;
+	gxact->proc.inVacuum = false;
 	gxact->proc.lwWaiting = false;
 	gxact->proc.lwExclusive = false;
 	gxact->proc.lwWaitLink = NULL;

src/backend/access/transam/xact.c

@@ -10,7 +10,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/transam/xact.c,v 1.224 2006/07/24 16:32:44 petere Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/transam/xact.c,v 1.225 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1529,6 +1529,7 @@ CommitTransaction(void)
 		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
 		MyProc->xid = InvalidTransactionId;
 		MyProc->xmin = InvalidTransactionId;
+		MyProc->inVacuum = false;	/* must be cleared with xid/xmin */
 
 		/* Clear the subtransaction-XID cache too while holding the lock */
 		MyProc->subxids.nxids = 0;
@@ -1764,6 +1765,7 @@ PrepareTransaction(void)
 	LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
 	MyProc->xid = InvalidTransactionId;
 	MyProc->xmin = InvalidTransactionId;
+	MyProc->inVacuum = false;	/* must be cleared with xid/xmin */
 
 	/* Clear the subtransaction-XID cache too while holding the lock */
 	MyProc->subxids.nxids = 0;
@@ -1927,6 +1929,7 @@ AbortTransaction(void)
 		LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE);
 		MyProc->xid = InvalidTransactionId;
 		MyProc->xmin = InvalidTransactionId;
+		MyProc->inVacuum = false;	/* must be cleared with xid/xmin */
 
 		/* Clear the subtransaction-XID cache too while holding the lock */
 		MyProc->subxids.nxids = 0;

src/backend/access/transam/xlog.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.244 2006/07/14 14:52:17 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.245 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -5413,7 +5413,7 @@ CreateCheckPoint(bool shutdown, bool force)
 	 * StartupSUBTRANS hasn't been called yet.
 	 */
 	if (!InRecovery)
-		TruncateSUBTRANS(GetOldestXmin(true));
+		TruncateSUBTRANS(GetOldestXmin(true, false));
 
 	if (!shutdown)
 		ereport(DEBUG2,

src/backend/catalog/index.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.269 2006/07/13 16:49:13 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/catalog/index.c,v 1.270 2006/07/30 02:07:18 alvherre Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -1367,7 +1367,8 @@ IndexBuildHeapScan(Relation heapRelation,
 	else
 	{
 		snapshot = SnapshotAny;
-		OldestXmin = GetOldestXmin(heapRelation->rd_rel->relisshared);
+		/* okay to ignore lazy VACUUMs here */
+		OldestXmin = GetOldestXmin(heapRelation->rd_rel->relisshared, true);
 	}
 
 	scan = heap_beginscan(heapRelation, /* relation */

src/backend/commands/vacuum.c

@@ -13,7 +13,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.335 2006/07/14 14:52:18 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.336 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -37,6 +37,7 @@
 #include "postmaster/autovacuum.h"
 #include "storage/freespace.h"
 #include "storage/pmsignal.h"
+#include "storage/proc.h"
 #include "storage/procarray.h"
 #include "utils/acl.h"
 #include "utils/builtins.h"
@@ -589,7 +590,16 @@ vacuum_set_xid_limits(VacuumStmt *vacstmt, bool sharedRel,
 {
 	TransactionId limit;
 
-	*oldestXmin = GetOldestXmin(sharedRel);
+	/*
+	 * We can always ignore processes running lazy vacuum.  This is because we
+	 * use these values only for deciding which tuples we must keep in the
+	 * tables.  Since lazy vacuum doesn't write its xid to the table, it's
+	 * safe to ignore it.  In theory it could be problematic to ignore lazy
+	 * vacuums on a full vacuum, but keep in mind that only one vacuum process
+	 * can be working on a particular table at any time, and that each vacuum
+	 * is always an independent transaction.
+	 */
+	*oldestXmin = GetOldestXmin(sharedRel, true);
 
 	Assert(TransactionIdIsNormal(*oldestXmin));
 
@@ -645,6 +655,11 @@ vacuum_set_xid_limits(VacuumStmt *vacstmt, bool sharedRel,
  *		pg_class would've been obsoleted.  Of course, this only works for
  *		fixed-size never-null columns, but these are.
  *
+ *		Another reason for doing it this way is that when we are in a lazy
+ *		VACUUM and have inVacuum set, we mustn't do any updates --- somebody
+ *		vacuuming pg_class might think they could delete a tuple marked with
+ *		xmin = our xid.
+ *
  *		This routine is shared by full VACUUM, lazy VACUUM, and stand-alone
  *		ANALYZE.
  */
@@ -996,8 +1011,35 @@ vacuum_rel(Oid relid, VacuumStmt *vacstmt, char expected_relkind)
 
 	/* Begin a transaction for vacuuming this relation */
 	StartTransactionCommand();
-	/* functions in indexes may want a snapshot set */
-	ActiveSnapshot = CopySnapshot(GetTransactionSnapshot());
+
+	if (vacstmt->full)
+	{
+		/* functions in indexes may want a snapshot set */
+		ActiveSnapshot = CopySnapshot(GetTransactionSnapshot());
+	}
+	else
+	{
+		/*
+		 * During a lazy VACUUM we do not run any user-supplied functions,
+		 * and so it should be safe to not create a transaction snapshot.
+		 *
+		 * We can furthermore set the inVacuum flag, which lets other
+		 * concurrent VACUUMs know that they can ignore this one while
+		 * determining their OldestXmin.  (The reason we don't set inVacuum
+		 * during a full VACUUM is exactly that we may have to run user-
+		 * defined functions for functional indexes, and we want to make
+		 * sure that if they use the snapshot set above, any tuples it
+		 * requires can't get removed from other tables.  An index function
+		 * that depends on the contents of other tables is arguably broken,
+		 * but we won't break it here by violating transaction semantics.)
+		 *
+		 * Note: the inVacuum flag remains set until CommitTransaction or
+		 * AbortTransaction.  We don't want to clear it until we reset
+		 * MyProc->xid/xmin, else OldestXmin might appear to go backwards,
+		 * which is probably Not Good.
+		 */
+		MyProc->inVacuum = true;
+	}
 
 	/*
 	 * Tell the cache replacement strategy that vacuum is causing all

src/backend/storage/ipc/procarray.c

@@ -23,7 +23,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/procarray.c,v 1.14 2006/07/14 14:52:22 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/procarray.c,v 1.15 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -388,20 +388,24 @@ TransactionIdIsActive(TransactionId xid)
  * If allDbs is TRUE then all backends are considered; if allDbs is FALSE
  * then only backends running in my own database are considered.
  *
+ * If ignoreVacuum is TRUE then backends with inVacuum set are ignored.
+ *
  * This is used by VACUUM to decide which deleted tuples must be preserved
  * in a table.	allDbs = TRUE is needed for shared relations, but allDbs =
  * FALSE is sufficient for non-shared relations, since only backends in my
- * own database could ever see the tuples in them.
+ * own database could ever see the tuples in them.  Also, we can ignore
+ * concurrently running lazy VACUUMs because (a) they must be working on other
+ * tables, and (b) they don't need to do snapshot-based lookups.
  *
  * This is also used to determine where to truncate pg_subtrans.  allDbs
- * must be TRUE for that case.
+ * must be TRUE for that case, and ignoreVacuum FALSE.
  *
  * Note: we include the currently running xids in the set of considered xids.
  * This ensures that if a just-started xact has not yet set its snapshot,
  * when it does set the snapshot it cannot set xmin less than what we compute.
  */
 TransactionId
-GetOldestXmin(bool allDbs)
+GetOldestXmin(bool allDbs, bool ignoreVacuum)
 {
 	ProcArrayStruct *arrayP = procArray;
 	TransactionId result;
@@ -425,15 +429,28 @@ GetOldestXmin(bool allDbs)
 	{
 		PGPROC	   *proc = arrayP->procs[index];
 
+		if (ignoreVacuum && proc->inVacuum)
+			continue;
+
 		if (allDbs || proc->databaseId == MyDatabaseId)
 		{
 			/* Fetch xid just once - see GetNewTransactionId */
 			TransactionId xid = proc->xid;
 
 			if (TransactionIdIsNormal(xid))
 			{
+				/* First consider the transaction own's Xid */
 				if (TransactionIdPrecedes(xid, result))
 					result = xid;
+
+				/*
+				 * Also consider the transaction's Xmin, if set.
+				 *
+				 * Note that this Xmin may seem to be guaranteed to be always
+				 * lower than the transaction's Xid, but this is not so because
+				 * there is a time window on which the Xid is already assigned
+				 * but the Xmin has not being calculated yet.
+				 */
 				xid = proc->xmin;
 				if (TransactionIdIsNormal(xid))
 					if (TransactionIdPrecedes(xid, result))
@@ -471,8 +488,8 @@ GetOldestXmin(bool allDbs)
  *		RecentXmin: the xmin computed for the most recent snapshot.  XIDs
  *			older than this are known not running any more.
  *		RecentGlobalXmin: the global xmin (oldest TransactionXmin across all
- *			running transactions).	This is the same computation done by
- *			GetOldestXmin(TRUE).
+ *			running transactions, except those running LAZY VACUUM).  This is
+ *			the same computation done by GetOldestXmin(true, false).
  *----------
  */
 Snapshot
@@ -561,15 +578,17 @@ GetSnapshotData(Snapshot snapshot, bool serializable)
 
 		/*
 		 * Ignore my own proc (dealt with my xid above), procs not running a
-		 * transaction, and xacts started since we read the next transaction
-		 * ID.	There's no need to store XIDs above what we got from
-		 * ReadNewTransactionId, since we'll treat them as running anyway.  We
-		 * also assume that such xacts can't compute an xmin older than ours,
-		 * so they needn't be considered in computing globalxmin.
+		 * transaction, xacts started since we read the next transaction
+		 * ID, and xacts executing LAZY VACUUM.	There's no need to store XIDs
+		 * above what we got from ReadNewTransactionId, since we'll treat them
+		 * as running anyway.  We also assume that such xacts can't compute an
+		 * xmin older than ours, so they needn't be considered in computing
+		 * globalxmin.
 		 */
 		if (proc == MyProc ||
 			!TransactionIdIsNormal(xid) ||
-			TransactionIdFollowsOrEquals(xid, xmax))
+			TransactionIdFollowsOrEquals(xid, xmax) ||
+			proc->inVacuum)
 			continue;
 
 		if (TransactionIdPrecedes(xid, xmin))

src/backend/storage/lmgr/proc.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.178 2006/07/23 23:08:46 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.179 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -257,6 +257,7 @@ InitProcess(void)
 	/* databaseId and roleId will be filled in later */
 	MyProc->databaseId = InvalidOid;
 	MyProc->roleId = InvalidOid;
+	MyProc->inVacuum = false;
 	MyProc->lwWaiting = false;
 	MyProc->lwExclusive = false;
 	MyProc->lwWaitLink = NULL;
@@ -388,6 +389,7 @@ InitDummyProcess(void)
 	MyProc->xmin = InvalidTransactionId;
 	MyProc->databaseId = InvalidOid;
 	MyProc->roleId = InvalidOid;
+	MyProc->inVacuum = false;
 	MyProc->lwWaiting = false;
 	MyProc->lwExclusive = false;
 	MyProc->lwWaitLink = NULL;

src/include/storage/proc.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/storage/proc.h,v 1.89 2006/07/13 16:49:20 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/storage/proc.h,v 1.90 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -66,13 +66,16 @@ struct PGPROC
 								 * this proc */
 
 	TransactionId xmin;			/* minimal running XID as it was when we were
-								 * starting our xact: vacuum must not remove
-								 * tuples deleted by xid >= xmin ! */
+								 * starting our xact, excluding LAZY VACUUM:
+								 * vacuum must not remove tuples deleted by
+								 * xid >= xmin ! */
 
 	int			pid;			/* This backend's process id, or 0 */
 	Oid			databaseId;		/* OID of database this backend is using */
 	Oid			roleId;			/* OID of role using this backend */
 
+	bool		inVacuum;		/* true if current xact is a LAZY VACUUM */
+	
 	/* Info about LWLock the process is currently waiting for, if any. */
 	bool		lwWaiting;		/* true if waiting for an LW lock */
 	bool		lwExclusive;	/* true if waiting for exclusive access */

src/include/storage/procarray.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2006, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/storage/procarray.h,v 1.9 2006/06/19 01:51:22 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/storage/procarray.h,v 1.10 2006/07/30 02:07:18 alvherre Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -24,7 +24,7 @@ extern void ProcArrayRemove(PGPROC *proc);
 
 extern bool TransactionIdIsInProgress(TransactionId xid);
 extern bool TransactionIdIsActive(TransactionId xid);
-extern TransactionId GetOldestXmin(bool allDbs);
+extern TransactionId GetOldestXmin(bool allDbs, bool ignoreVacuum);
 
 extern PGPROC *BackendPidGetProc(int pid);
 extern int	BackendXidGetPid(TransactionId xid);