Keep track of transaction commit timestamps

Transactions can now set their commit timestamp directly as they commit,
or an external transaction commit timestamp can be fed from an outside
system using the new function TransactionTreeSetCommitTsData().  This
data is crash-safe, and truncated at Xid freeze point, same as pg_clog.

This module is disabled by default because it causes a performance hit,
but can be enabled in postgresql.conf requiring only a server restart.

A new test in src/test/modules is included.

Catalog version bumped due to the new subdirectory within PGDATA and a
couple of new SQL functions.

Authors: Álvaro Herrera and Petr Jelínek

Reviewed to varying degrees by Michael Paquier, Andres Freund, Robert
Haas, Amit Kapila, Fujii Masao, Jaime Casanova, Simon Riggs, Steven
Singer, Peter Eisentraut

Author: alvherre

contrib/pg_upgrade/pg_upgrade.c

@@ -430,6 +430,13 @@ copy_clog_xlog_xid(void)
 			  "\"%s/pg_resetxlog\" -f -e %u \"%s\"",
 			  new_cluster.bindir, old_cluster.controldata.chkpnt_nxtepoch,
 			  new_cluster.pgdata);
+	/* must reset commit timestamp limits also */
+	exec_prog(UTILITY_LOG_FILE, NULL, true,
+			  "\"%s/pg_resetxlog\" -f -c %u,%u \"%s\"",
+			  new_cluster.bindir,
+			  old_cluster.controldata.chkpnt_nxtxid,
+			  old_cluster.controldata.chkpnt_nxtxid,
+			  new_cluster.pgdata);
 	check_ok();
 
 	/*

contrib/pg_xlogdump/rmgrdesc.c

@@ -10,6 +10,7 @@
 
 #include "access/brin_xlog.h"
 #include "access/clog.h"
+#include "access/commit_ts.h"
 #include "access/gin.h"
 #include "access/gist_private.h"
 #include "access/hash.h"

doc/src/sgml/config.sgml

@@ -2673,6 +2673,20 @@ include_dir 'conf.d'
       </listitem>
      </varlistentry>
 
+     <varlistentry id="guc-track-commit-timestamp" xreflabel="track_commit_timestamp">
+      <term><varname>track_commit_timestamp</varname> (<type>bool</type>)</term>
+      <indexterm>
+       <primary><varname>track_commit_timestamp</> configuration parameter</primary>
+      </indexterm>
+      <listitem>
+       <para>
+        Record commit time of transactions. This parameter
+        can only be set in <filename>postgresql.conf</> file or on the server
+        command line. The default value is <literal>off</literal>.
+       </para>
+      </listitem>
+     </varlistentry>
+
      </variablelist>
     </sect2>
 

doc/src/sgml/func.sgml

@@ -15938,6 +15938,45 @@ SELECT collation for ('foo' COLLATE "de_DE");
     For example <literal>10:20:10,14,15</literal> means
     <literal>xmin=10, xmax=20, xip_list=10, 14, 15</literal>.
    </para>
+
+   <para>
+    The functions shown in <xref linkend="functions-commit-timestamp">
+    provide information about transactions that have been already committed.
+    These functions mainly provide information about when the transactions
+    were committed. They only provide useful data when
+    <xref linkend="guc-track-commit-timestamp"> configuration option is enabled
+    and only for transactions that were committed after it was enabled.
+   </para>
+
+   <table id="functions-commit-timestamp">
+    <title>Committed transaction information</title>
+    <tgroup cols="3">
+     <thead>
+      <row><entry>Name</entry> <entry>Return Type</entry> <entry>Description</entry></row>
+     </thead>
+
+     <tbody>
+      <row>
+       <entry>
+        <indexterm><primary>pg_xact_commit_timestamp</primary></indexterm>
+        <literal><function>pg_xact_commit_timestamp(<parameter>xid</parameter>)</function></literal>
+       </entry>
+       <entry><type>timestamp with time zone</type></entry>
+       <entry>get commit timestamp of a transaction</entry>
+      </row>
+
+      <row>
+       <entry>
+        <indexterm><primary>pg_last_committed_xact</primary></indexterm>
+        <literal><function>pg_last_committed_xact()</function></literal>
+       </entry>
+       <entry><parameter>xid</> <type>xid</>, <parameter>timestamp</> <type>timestamp with time zone</></entry>
+       <entry>get transaction ID and commit timestamp of latest committed transaction</entry>
+      </row>
+     </tbody>
+    </tgroup>
+   </table>
+
   </sect1>
 
   <sect1 id="functions-admin">

doc/src/sgml/ref/pg_resetxlog.sgml

@@ -22,6 +22,7 @@ PostgreSQL documentation
  <refsynopsisdiv>
   <cmdsynopsis>
    <command>pg_resetxlog</command>
+   <arg choice="opt"><option>-c</option> <replaceable class="parameter">xid</replaceable>,<replaceable class="parameter">xid</replaceable></arg>
    <arg choice="opt"><option>-f</option></arg>
    <arg choice="opt"><option>-n</option></arg>
    <arg choice="opt"><option>-o</option> <replaceable class="parameter">oid</replaceable></arg>
@@ -79,9 +80,12 @@ PostgreSQL documentation
   <para>
    The <option>-o</>, <option>-x</>, <option>-e</>,
    <option>-m</>, <option>-O</>,
+   <option>-c</>
    and <option>-l</>
    options allow the next OID, next transaction ID, next transaction ID's
-   epoch, next and oldest multitransaction ID, next multitransaction offset, and WAL
+   epoch, next and oldest multitransaction ID, next multitransaction offset,
+   oldest and newest transaction IDs for which the commit time can be retrieved,
+   and WAL
    starting address values to be set manually.  These are only needed when
    <command>pg_resetxlog</command> is unable to determine appropriate values
    by reading <filename>pg_control</>.  Safe values can be determined as
@@ -128,6 +132,19 @@ PostgreSQL documentation
      </para>
     </listitem>
 
+    <listitem>
+     <para>
+      A safe value for the oldest transaction ID for which the commit time can
+      be retrieved (first part of <option>-c</>) can be determined by looking
+      for the numerically smallest file name in the directory
+      <filename>pg_committs</> under the data directory.  Conversely, a safe
+      value for the newest transaction ID for which the commit time can be
+      retrieved (second part of <option>-c</>) can be determined by looking for
+      the numerically greatest file name in the same directory.  As above, the
+      file names are in hexadecimal.
+     </para>
+    </listitem>
+
     <listitem>
      <para>
       The WAL starting address (<option>-l</>) should be

doc/src/sgml/storage.sgml

@@ -66,6 +66,11 @@ Item
  <structname>pg_database</></entry>
 </row>
 
+<row>
+ <entry><filename>pg_commit_ts</></entry>
+ <entry>Subdirectory containing transaction commit timestamp data</entry>
+</row>
+
 <row>
  <entry><filename>pg_clog</></entry>
  <entry>Subdirectory containing transaction commit status data</entry>

src/backend/access/rmgrdesc/Makefile

@@ -8,7 +8,7 @@ subdir = src/backend/access/rmgrdesc
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = brindesc.o clogdesc.o dbasedesc.o gindesc.o gistdesc.o \
+OBJS = brindesc.o clogdesc.o committsdesc.o dbasedesc.o gindesc.o gistdesc.o \
 	   hashdesc.o heapdesc.o \
 	   mxactdesc.o nbtdesc.o relmapdesc.o seqdesc.o smgrdesc.o spgdesc.o \
 	   standbydesc.o tblspcdesc.o xactdesc.o xlogdesc.o

src/backend/access/rmgrdesc/committsdesc.c

@@ -0,0 +1,82 @@
+/*-------------------------------------------------------------------------
+ *
+ * committsdesc.c
+ *    rmgr descriptor routines for access/transam/commit_ts.c
+ *
+ * Portions Copyright (c) 1996-2014, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ *    src/backend/access/rmgrdesc/committsdesc.c
+ *
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/commit_ts.h"
+#include "utils/timestamp.h"
+
+
+void
+commit_ts_desc(StringInfo buf, XLogReaderState *record)
+{
+	char	   *rec = XLogRecGetData(record);
+	uint8		info = XLogRecGetInfo(record) & ~XLR_INFO_MASK;
+
+	if (info == COMMIT_TS_ZEROPAGE)
+	{
+		int			pageno;
+
+		memcpy(&pageno, rec, sizeof(int));
+		appendStringInfo(buf, "%d", pageno);
+	}
+	else if (info == COMMIT_TS_TRUNCATE)
+	{
+		int			pageno;
+
+		memcpy(&pageno, rec, sizeof(int));
+		appendStringInfo(buf, "%d", pageno);
+	}
+	else if (info == COMMIT_TS_SETTS)
+	{
+		xl_commit_ts_set *xlrec = (xl_commit_ts_set *) rec;
+		int		nsubxids;
+
+		appendStringInfo(buf, "set %s/%d for: %u",
+						 timestamptz_to_str(xlrec->timestamp),
+						 xlrec->nodeid,
+						 xlrec->mainxid);
+		nsubxids = ((XLogRecGetDataLen(record) - SizeOfCommitTsSet) /
+					sizeof(TransactionId));
+		if (nsubxids > 0)
+		{
+			int		i;
+			TransactionId *subxids;
+
+			subxids = palloc(sizeof(TransactionId) * nsubxids);
+			memcpy(subxids,
+				   XLogRecGetData(record) + SizeOfCommitTsSet,
+				   sizeof(TransactionId) * nsubxids);
+			for (i = 0; i < nsubxids; i++)
+				appendStringInfo(buf, ", %u", subxids[i]);
+			pfree(subxids);
+		}
+	}
+}
+
+const char *
+commit_ts_identify(uint8 info)
+{
+	switch (info)
+	{
+		case COMMIT_TS_ZEROPAGE:
+			return "ZEROPAGE";
+		case COMMIT_TS_TRUNCATE:
+			return "TRUNCATE";
+		case COMMIT_TS_SETTS:
+			return "SETTS";
+		default:
+			return NULL;
+	}
+}

src/backend/access/rmgrdesc/xlogdesc.c

@@ -45,6 +45,7 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 		appendStringInfo(buf, "redo %X/%X; "
 						 "tli %u; prev tli %u; fpw %s; xid %u/%u; oid %u; multi %u; offset %u; "
 						 "oldest xid %u in DB %u; oldest multi %u in DB %u; "
+						 "oldest/newest commit timestamp xid: %u/%u; "
 						 "oldest running xid %u; %s",
 				(uint32) (checkpoint->redo >> 32), (uint32) checkpoint->redo,
 						 checkpoint->ThisTimeLineID,
@@ -58,6 +59,8 @@ xlog_desc(StringInfo buf, XLogReaderState *record)
 						 checkpoint->oldestXidDB,
 						 checkpoint->oldestMulti,
 						 checkpoint->oldestMultiDB,
+						 checkpoint->oldestCommitTs,
+						 checkpoint->newestCommitTs,
 						 checkpoint->oldestActiveXid,
 				 (info == XLOG_CHECKPOINT_SHUTDOWN) ? "shutdown" : "online");
 	}

src/backend/access/transam/Makefile

@@ -12,8 +12,9 @@ subdir = src/backend/access/transam
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = clog.o transam.o varsup.o xact.o rmgr.o slru.o subtrans.o multixact.o \
-	timeline.o twophase.o twophase_rmgr.o xlog.o xlogarchive.o xlogfuncs.o \
+OBJS = clog.o commit_ts.o multixact.o rmgr.o slru.o subtrans.o \
+	timeline.o transam.o twophase.o twophase_rmgr.o varsup.o \
+	xact.o xlog.o xlogarchive.o xlogfuncs.o \
 	xloginsert.o xlogreader.o xlogutils.o
 
 include $(top_srcdir)/src/backend/common.mk

src/backend/access/transam/README

@@ -840,7 +840,7 @@ parent transaction to complete.
 
 Not all transactional behaviour is emulated, for example we do not insert
 a transaction entry into the lock table, nor do we maintain the transaction
-stack in memory. Clog and multixact entries are made normally.
+stack in memory. Clog, multixact and commit_ts entries are made normally.
 Subtrans is maintained during recovery but the details of the transaction
 tree are ignored and all subtransactions reference the top-level TransactionId
 directly. Since commit is atomic this provides correct lock wait behaviour