Merge branch 'PGPROEE10_DEV' into PGPROEE10_pg_shardman

Author: vbwagner

contrib/pageinspect/expected/page.out

@@ -46,7 +46,7 @@ SELECT get_raw_page('test1', 0) = get_raw_page('test1', 'main', 0);
 SELECT pagesize, version FROM page_header(get_raw_page('test1', 0));
  pagesize | version 
 ----------+---------
-     8192 |       4
+     8192 |     254
 (1 row)
 
 SELECT page_checksum(get_raw_page('test1', 0), 0) IS NOT NULL AS silly_checksum_test;

doc/src/sgml/charset.sgml

@@ -580,19 +580,6 @@ SELECT * FROM test1 ORDER BY a || b COLLATE "fr_FR";
     </para>
    </important>
 
-    <important>
-    <para>
-     On Windows, if the collation provider is <literal>icu</literal>,
-     you cannot use long locale names with the <literal>locale</literal>,
-     <literal>lc_collate</literal>, and <literal>lc_ctype</literal> options.
-     Instead, you must use canonical locale names supported on your
-     operating system. For example, <literal>"ru-RU"</literal> or
-     <literal>"en-US"</literal>. Specifying the encoding with these
-     options is not allowed either. If required, you can specify the
-     UTF8 encoding using the <literal>--encoding</literal> option.
-    </para>
-   </important>
-
    <para>
      When you start a cluster server or connect to a database,
      <productname>&productname;</> checks that the selected collation

doc/src/sgml/demodb-bookings-schema.svg

@@ -720,12 +720,30 @@ psql --username=postgres --file=script.sql postgres
   </para>
 
   <para>
-    When upgrading from <productname>Postgres Pro</> 9.6 or lower that
-    uses indexes or constraints with non-default collations,
-    <application>pg_upgrade</> generates separate SQL files with
-    <command>REINDEX</command>/<command>VALIDATE</command> commands,
-    respectively. You need to run these files to rebuild indexes
-    and re-validate constraints.
+    When performing an upgrade from <productname>Postgres Pro</> 9.6 or lower,
+    for databases with a multibyte encoding, <application>pg_upgrade</> may
+    generate SQL files with
+    <command>REINDEX</command>/<command>VALIDATE</command> commands. You must
+    run these files to rebuild indexes and re-validate constraints.
+    This can happen in the following cases:
+    <itemizedlist spacing="compact">
+      <listitem>
+       <para>
+        The old cluster uses indexes or constraints depending on collations
+        other than the default collation of the database, <literal>C</literal>,
+        or <literal>POSIX</literal>.
+       </para>
+      </listitem>
+      <listitem>
+       <para>
+        On Windows, the old cluster uses indexes or constraints depending on the
+        default collation with a verbose name, such as
+        <literal>"Russian_Russia[.<replaceable>encoding</replaceable>]"</literal>
+        or <literal>"English_United
+        States[.<replaceable>encoding</replaceable>]"</literal>.
+       </para>
+      </listitem>
+    </itemizedlist>
   </para>
 
   <para>

doc/src/sgml/demodb-bookings.sgml

@@ -183,11 +183,18 @@
           If the previous <productname>Postgres Pro</productname> installation
           contained any indexes or constraints depending on collations
           other than the default collation of the database, <literal>C</literal>,
-          or <literal>POSIX</literal>, such databases could contain some data
-          that violated the specified constraints and made indexes inconsistent.
+          or <literal>POSIX</literal> in databases with multibyte encodings,
+          such databases could contain some data that violated the specified
+          constraints and made indexes inconsistent. On Windows, this situation
+          can also happen if the database with a multibyte encoding contained
+          any indexes or constraints depending on the default collation with a
+          verbose name, such as
+          <literal>"Russian_Russia[.<replaceable>encoding</replaceable>]"</literal>
+          or <literal>"English_United
+          States[.<replaceable>encoding</replaceable>]"</literal>.
           In such cases, you can only use <application>pg_upgrade</application> to upgrade to
           <productname>Postgres Pro</productname> 10, as a dump/restore scenario may be impossible.
-          To resolve such issues, <application>pg_upgrade</application> declares such indexes and
+          To resolve these issues, <application>pg_upgrade</application> declares such indexes and
           constraints invalid and creates <filename>reindex_text_indexes.sql</filename>
           and <filename>validate_text_contraints.sql</filename>, respectively.
        </para>

doc/src/sgml/ref/pgupgrade.sgml

@@ -1010,7 +1010,6 @@ btree_xlog_unlink_page(uint8 info, XLogReaderState *record)
 	_bt_pageinit(page, BufferGetPageSize(buffer));
 	pageop = (BTPageOpaque) PageGetSpecialPointer(page);
 
-	pageop->btpo1.btpo_prev = leftsib;
 	pageop->btpo_next = rightsib;
 	BTP_SET_XACT(pageop, xlrec->btpo_xact);
 	pageop->btpo_flags = BTP_DELETED;

doc/src/sgml/release-pro-10.sgml

@@ -84,6 +84,7 @@
 #include "access/twophase_rmgr.h"
 #include "access/xact.h"
 #include "access/xlog.h"
+#include "access/xlog_internal.h"
 #include "access/xloginsert.h"
 #include "access/xlogutils.h"
 #include "access/xlogreader.h"
@@ -2414,3 +2415,106 @@ PrepareRedoRemove(TransactionId xid, bool giveWarning)
 
 	return;
 }
+
+Datum
+pg_prepared_xact_status(PG_FUNCTION_ARGS)
+{
+    char const* gid = text_to_cstring(PG_GETARG_TEXT_PP(0));
+	XLogRecord *record;
+	XLogReaderState *xlogreader;
+	char	   *errormsg;
+	XLogRecPtr start_lsn;
+	XLogRecPtr lsn;
+	char const* xact_status = "unknown";
+	bool done = false;
+	TimeLineID timeline;
+	TransactionId xid = InvalidTransactionId;
+	XLogRecPtr end_lsn = GetFlushRecPtr();
+
+	GetOldestRestartPoint(&start_lsn, &timeline);
+
+	xlogreader = XLogReaderAllocate(&read_local_xlog_page, NULL);
+	if (!xlogreader)
+		ereport(ERROR,
+				(errcode(ERRCODE_OUT_OF_MEMORY),
+				 errmsg("out of memory"),
+				 errdetail("Failed while allocating a WAL reading processor.")));
+	while (true)
+	{
+		lsn = start_lsn;
+		do
+		{
+			record = XLogReadRecord(xlogreader, lsn, &errormsg);
+			if (record == NULL)
+				break;
+			lsn = InvalidXLogRecPtr; /* continue after the record */
+			if (XLogRecGetRmid(xlogreader) == RM_XACT_ID)
+			{
+				uint32 info = XLogRecGetInfo(xlogreader);
+				switch (info & XLOG_XACT_OPMASK)
+				{
+				    case XLOG_XACT_PREPARE:
+					{
+						TwoPhaseFileHeader *hdr = (TwoPhaseFileHeader *)XLogRecGetData(xlogreader);
+						char* xact_gid = (char*)hdr + MAXALIGN(sizeof(TwoPhaseFileHeader));
+ 						if (strcmp(xact_gid, gid) == 0)
+						{
+							xid = hdr->xid;
+							xact_status = "prepared";
+						}
+						break;
+					}
+				    case XLOG_XACT_COMMIT_PREPARED:
+					{
+						xl_xact_commit *xlrec;
+						xl_xact_parsed_commit parsed;
+
+						xlrec = (xl_xact_commit *) XLogRecGetData(xlogreader);
+						ParseCommitRecord(info, xlrec, &parsed);
+						if (xid == parsed.twophase_xid)
+						{
+							Assert(TransactionIdIsValid(xid));
+							xact_status = "committed";
+							done = true;
+						}
+						break;
+					}
+				    case XLOG_XACT_ABORT_PREPARED:
+					{
+						xl_xact_abort *xlrec;
+						xl_xact_parsed_abort parsed;
+
+						xlrec = (xl_xact_abort *) XLogRecGetData(xlogreader);
+						ParseAbortRecord(info, xlrec, &parsed);
+						if (xid == parsed.twophase_xid)
+						{
+							Assert(TransactionIdIsValid(xid));
+							xact_status = "aborted";
+							done = true;
+						}
+						break;
+					}
+				    default:
+					    break;
+				}
+			}
+		} while (!done && xlogreader->EndRecPtr < end_lsn);
+
+		if (done)
+			break;
+
+		end_lsn = start_lsn;
+		XLogSegNoOffsetToRecPtr(end_lsn/XLogSegSize, 0, start_lsn);
+		start_lsn = XLogFindNextRecord(xlogreader, start_lsn);
+		if (start_lsn == end_lsn)
+		{
+			if (end_lsn <= XLogSegSize)
+				break;
+			XLogSegNoOffsetToRecPtr(lsn/XLogSegSize-1, 0, start_lsn);
+			start_lsn = XLogFindNextRecord(xlogreader, start_lsn);
+		}
+	}
+
+	XLogReaderFree(xlogreader);
+	PG_RETURN_TEXT_P(cstring_to_text(xact_status));
+}

src/backend/access/nbtree/nbtxlog.c

@@ -1861,6 +1861,7 @@ typedef struct {
 	void *PredicateState;
 	void *StorageState;
 	struct TransInvalidationInfo* InvalidationInfo;
+	List *on_commit_actions;
 } SuspendedTransactionState;
 
 static int suspendedXactNum = 0;
@@ -2019,8 +2020,10 @@ CommitTransaction(void)
 	TransactionState s = CurrentTransactionState;
 	TransactionId latestXid;
 	bool		is_parallel_worker;
+	bool        is_autonomous_transaction;
 
 	is_parallel_worker = (s->blockState == TBLOCK_PARALLEL_INPROGRESS);
+	is_autonomous_transaction = getNestLevelATX() != 0;
 
 	/* Enforce parallel mode restrictions during parallel worker commit. */
 	if (is_parallel_worker)
@@ -2054,7 +2057,7 @@ CommitTransaction(void)
 		 * If there weren't any, we are done ... otherwise loop back to check
 		 * if they queued deferred triggers.  Lather, rinse, repeat.
 		 */
-		if (!PreCommit_Portals(false))
+		if (is_autonomous_transaction || !PreCommit_Portals(false))
 			break;
 	}
 
@@ -2168,7 +2171,7 @@ CommitTransaction(void)
 						 RESOURCE_RELEASE_BEFORE_LOCKS,
 						 true, true);
 
-	if (getNestLevelATX() == 0)
+	if (!is_autonomous_transaction)
 	{
 		/* Check we've released all buffer pins */
 		AtEOXact_Buffers(true);
@@ -2188,7 +2191,10 @@ CommitTransaction(void)
 	xactHasRelcacheInvalidationMessages = HasRelcacheInvalidationMessages();
 	AtEOXact_Inval(true);
 
-	AtEOXact_MultiXact();
+	if (!is_autonomous_transaction)
+	{
+		AtEOXact_MultiXact();
+	}
 
 	ResourceOwnerRelease(TopTransactionResourceOwner,
 						 RESOURCE_RELEASE_LOCKS,
@@ -2197,27 +2203,34 @@ CommitTransaction(void)
 						 RESOURCE_RELEASE_AFTER_LOCKS,
 						 true, true);
 
-	/*
-	 * Likewise, dropping of files deleted during the transaction is best done
-	 * after releasing relcache and buffer pins.  (This is not strictly
-	 * necessary during commit, since such pins should have been released
-	 * already, but this ordering is definitely critical during abort.)  Since
-	 * this may take many seconds, also delay until after releasing locks.
-	 * Other backends will observe the attendant catalog changes and not
-	 * attempt to access affected files.
-	 */
-	smgrDoPendingDeletes(true);
+	if (!is_autonomous_transaction)
+	{
+		/*
+		 * Likewise, dropping of files deleted during the transaction is best done
+		 * after releasing relcache and buffer pins.  (This is not strictly
+		 * necessary during commit, since such pins should have been released
+		 * already, but this ordering is definitely critical during abort.)  Since
+		 * this may take many seconds, also delay until after releasing locks.
+		 * Other backends will observe the attendant catalog changes and not
+		 * attempt to access affected files.
+		 */
+		smgrDoPendingDeletes(true);
+	}
 
 	AtCommit_Notify();
 	AtEOXact_GUC(true, s->gucNestLevel);
-	if (getNestLevelATX() == 0) 
+	if (!is_autonomous_transaction)
+	{
 		AtEOXact_SPI(true);
+	}
 	AtEOXact_on_commit_actions(true);
 	AtEOXact_Namespace(true, is_parallel_worker);
-	AtEOXact_SMgr();
-	if (getNestLevelATX() == 0) 
+	if (!is_autonomous_transaction)
+	{
+		AtEOXact_SMgr();
 		AtEOXact_Files();
-	AtEOXact_ComboCid();
+		AtEOXact_ComboCid();
+	}
 	AtEOXact_HashTables(true);
 	AtEOXact_PgStat(true);
 	AtEOXact_Snapshot(true, false);
@@ -2296,7 +2309,7 @@ PrepareTransaction(void)
 		 * If there weren't any, we are done ... otherwise loop back to check
 		 * if they queued deferred triggers.  Lather, rinse, repeat.
 		 */
-		if (!PreCommit_Portals(true))
+		if (getNestLevelATX() != 0 || !PreCommit_Portals(true))
 			break;
 	}
 
@@ -3594,6 +3607,7 @@ void SuspendTransaction(void)
 
 		MOVELEFT(sus->vxid.backendId, MyProc->backendId, MyBackendId);
 		MOVELEFT(sus->vxid.localTransactionId, MyProc->lxid, GetNextLocalTransactionId());
+		MOVELEFT(sus->on_commit_actions, pg_on_commit_actions, NULL);
 
 		sus->PgStatState = PgStatSuspend();
 		sus->TriggerState = TriggerSuspend();
@@ -3669,6 +3683,8 @@ bool ResumeTransaction(void)
 
 		MyProc->backendId = sus->vxid.backendId;
 		MyProc->lxid = sus->vxid.localTransactionId;
+		
+		pg_on_commit_actions = sus->on_commit_actions;
 	}
 
 	ResumePgXact(MyPgXact);

src/backend/access/transam/twophase.c

@@ -42,6 +42,7 @@
 #include "catalog/pg_database.h"
 #include "commands/tablespace.h"
 #include "commands/waitlsn.h"
+#include "common/edition.h"
 #include "common/controldata_utils.h"
 #include "miscadmin.h"
 #include "pgstat.h"
@@ -4421,6 +4422,8 @@ WriteControlFile(void)
 	Assert(sizeof(ControlFile->icu_version) == sizeof(UVersionInfo));
 	u_getVersion(ControlFile->icu_version);
 #endif
+	CONTROLFILE_SET_OLDVERSION(ControlFile, 0);
+	CONTROLFILE_SET_OLDEDITION(ControlFile, PG_UNKNOWN);
 
 	/* Contents are protected with a CRC */
 	INIT_CRC32C(ControlFile->crc);

src/backend/access/transam/xact.c

@@ -849,7 +849,6 @@ ValidXLogPageHeader(XLogReaderState *state, XLogRecPtr recptr,
 	return true;
 }
 
-#ifdef FRONTEND
 /*
  * Functions that are currently not needed in the backend, but are better
  * implemented inside xlogreader.c because of the internal facilities available
@@ -974,8 +973,6 @@ XLogFindNextRecord(XLogReaderState *state, XLogRecPtr RecPtr)
 	return found;
 }
 
-#endif							/* FRONTEND */
-
 
 /* ----------------------------------------
  * Functions for decoding the data and block references in a record.