Rejigger materializing and fetching a HeapTuple from a slot.

Previously materializing a slot always returned a HeapTuple. As
current work aims to reduce the reliance on HeapTuples (so other
storage systems can work efficiently), that needs to change. Thus
split the tasks of materializing a slot (i.e. making it independent
from the underlying storage / other memory contexts) from fetching a
HeapTuple from the slot.  For brevity, allow to fetch a HeapTuple from
a slot and materializing the slot at the same time, controlled by a
parameter.

For now some callers of ExecFetchSlotHeapTuple, with materialize =
true, expect that changes to the heap tuple will be reflected in the
underlying slot.  Those places will be adapted in due course, so while
not pretty, that's OK for now.

Also rename ExecFetchSlotTuple to ExecFetchSlotHeapTupleDatum and
ExecFetchSlotTupleDatum to ExecFetchSlotHeapTupleDatum, as it's likely
that future storage methods will need similar methods. There already
is ExecFetchSlotMinimalTuple, so the new names make the naming scheme
more coherent.

Author: Ashutosh Bapat and Andres Freund, with changes by Amit Khandekar
Discussion: https://postgr.es/m/20181105210039.hh4vvi4vwoq5ba2q@alap3.anarazel.de

Author: anarazel

contrib/postgres_fdw/postgres_fdw.c

@@ -3947,11 +3947,12 @@ apply_returning_filter(PgFdwDirectModifyState *dmstate,
 	ExecStoreVirtualTuple(resultSlot);
 
 	/*
-	 * If we have any system columns to return, install them.
+	 * If we have any system columns to return, materialize a heap tuple in the
+	 * slot from column values set above and install system columns in that tuple.
 	 */
 	if (dmstate->hasSystemCols)
 	{
-		HeapTuple	resultTup = ExecMaterializeSlot(resultSlot);
+		HeapTuple	resultTup = ExecFetchSlotHeapTuple(resultSlot, true, NULL);
 
 		/* ctid */
 		if (dmstate->ctidAttno)

src/backend/commands/copy.c

@@ -2899,7 +2899,7 @@ CopyFrom(CopyState cstate)
 			if (slot == NULL)	/* "do nothing" */
 				skip_tuple = true;
 			else				/* trigger might have changed tuple */
-				tuple = ExecMaterializeSlot(slot);
+				tuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 		}
 
 		if (!skip_tuple)
@@ -2975,7 +2975,7 @@ CopyFrom(CopyState cstate)
 							continue;	/* next tuple please */
 
 						/* FDW might have changed tuple */
-						tuple = ExecMaterializeSlot(slot);
+						tuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 
 						/*
 						 * AFTER ROW Triggers might reference the tableoid

src/backend/commands/createas.c

@@ -589,7 +589,7 @@ intorel_receive(TupleTableSlot *slot, DestReceiver *self)
 	 * get the heap tuple out of the tuple table slot, making sure we have a
 	 * writable copy
 	 */
-	tuple = ExecMaterializeSlot(slot);
+	tuple = ExecCopySlotTuple(slot);
 
 	/*
 	 * force assignment of new OID (see comments in ExecInsert)

src/backend/commands/matview.c

@@ -484,7 +484,7 @@ transientrel_receive(TupleTableSlot *slot, DestReceiver *self)
 	 * get the heap tuple out of the tuple table slot, making sure we have a
 	 * writable copy
 	 */
-	tuple = ExecMaterializeSlot(slot);
+	tuple = ExecCopySlotTuple(slot);
 
 	heap_insert(myState->transientrel,
 				tuple,
@@ -494,6 +494,9 @@ transientrel_receive(TupleTableSlot *slot, DestReceiver *self)
 
 	/* We know this is a newly created relation, so there are no indexes */
 
+	/* Free the copied tuple. */
+	heap_freetuple(tuple);
+
 	return true;
 }
 

src/backend/commands/trigger.c

@@ -2517,7 +2517,8 @@ ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo,
 					 TupleTableSlot *slot)
 {
 	TriggerDesc *trigdesc = relinfo->ri_TrigDesc;
-	HeapTuple	slottuple = ExecMaterializeSlot(slot);
+	bool		should_free;
+	HeapTuple	slottuple = ExecFetchSlotHeapTuple(slot, true, &should_free);
 	HeapTuple	newtuple = slottuple;
 	HeapTuple	oldtuple;
 	TriggerData LocTriggerData;
@@ -2556,7 +2557,11 @@ ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo,
 		if (oldtuple != newtuple && oldtuple != slottuple)
 			heap_freetuple(oldtuple);
 		if (newtuple == NULL)
+		{
+			if (should_free)
+				heap_freetuple(slottuple);
 			return NULL;		/* "do nothing" */
+		}
 	}
 
 	if (newtuple != slottuple)
@@ -2575,6 +2580,9 @@ ExecBRInsertTriggers(EState *estate, ResultRelInfo *relinfo,
 		ExecStoreHeapTuple(newtuple, newslot, false);
 		slot = newslot;
 	}
+
+	if (should_free)
+		heap_freetuple(slottuple);
 	return slot;
 }
 
@@ -2598,7 +2606,8 @@ ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo,
 					 TupleTableSlot *slot)
 {
 	TriggerDesc *trigdesc = relinfo->ri_TrigDesc;
-	HeapTuple	slottuple = ExecMaterializeSlot(slot);
+	bool		should_free;
+	HeapTuple	slottuple = ExecFetchSlotHeapTuple(slot, true, &should_free);
 	HeapTuple	newtuple = slottuple;
 	HeapTuple	oldtuple;
 	TriggerData LocTriggerData;
@@ -2637,7 +2646,11 @@ ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo,
 		if (oldtuple != newtuple && oldtuple != slottuple)
 			heap_freetuple(oldtuple);
 		if (newtuple == NULL)
+		{
+			if (should_free)
+				heap_freetuple(slottuple);
 			return NULL;		/* "do nothing" */
+		}
 	}
 
 	if (newtuple != slottuple)
@@ -2656,6 +2669,9 @@ ExecIRInsertTriggers(EState *estate, ResultRelInfo *relinfo,
 		ExecStoreHeapTuple(newtuple, newslot, false);
 		slot = newslot;
 	}
+
+	if (should_free)
+		heap_freetuple(slottuple);
 	return slot;
 }
 
@@ -2976,7 +2992,7 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate,
 					 TupleTableSlot *slot)
 {
 	TriggerDesc *trigdesc = relinfo->ri_TrigDesc;
-	HeapTuple	slottuple = ExecMaterializeSlot(slot);
+	HeapTuple	slottuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 	HeapTuple	newtuple = slottuple;
 	TriggerData LocTriggerData;
 	HeapTuple	trigtuple;
@@ -3018,7 +3034,7 @@ ExecBRUpdateTriggers(EState *estate, EPQState *epqstate,
 	if (newSlot != NULL)
 	{
 		slot = ExecFilterJunk(relinfo->ri_junkFilter, newSlot);
-		slottuple = ExecMaterializeSlot(slot);
+		slottuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 		newtuple = slottuple;
 	}
 
@@ -3132,7 +3148,7 @@ ExecIRUpdateTriggers(EState *estate, ResultRelInfo *relinfo,
 					 HeapTuple trigtuple, TupleTableSlot *slot)
 {
 	TriggerDesc *trigdesc = relinfo->ri_TrigDesc;
-	HeapTuple	slottuple = ExecMaterializeSlot(slot);
+	HeapTuple	slottuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 	HeapTuple	newtuple = slottuple;
 	TriggerData LocTriggerData;
 	HeapTuple	oldtuple;
@@ -4262,22 +4278,22 @@ AfterTriggerExecute(AfterTriggerEvent event,
 		case AFTER_TRIGGER_FDW_REUSE:
 
 			/*
-			 * Using ExecMaterializeSlot() rather than ExecFetchSlotTuple()
-			 * ensures that tg_trigtuple does not reference tuplestore memory.
-			 * (It is formally possible for the trigger function to queue
-			 * trigger events that add to the same tuplestore, which can push
-			 * other tuples out of memory.)  The distinction is academic,
-			 * because we start with a minimal tuple that ExecFetchSlotTuple()
-			 * must materialize anyway.
+			 * Materialize tuple in the slot so that tg_trigtuple does not
+			 * reference tuplestore memory.  (It is formally possible for the
+			 * trigger function to queue trigger events that add to the same
+			 * tuplestore, which can push other tuples out of memory.)  The
+			 * distinction is academic, because we start with a minimal tuple
+			 * that is stored as a heap tuple, constructed in different memory
+			 * context, in the slot anyway.
 			 */
-			LocTriggerData.tg_trigtuple =
-				ExecMaterializeSlot(trig_tuple_slot1);
+			LocTriggerData.tg_trigtuple = ExecFetchSlotHeapTuple(trig_tuple_slot1,
+																	true, NULL);
 			LocTriggerData.tg_trigtuplebuf = InvalidBuffer;
 
 			LocTriggerData.tg_newtuple =
 				((evtshared->ats_event & TRIGGER_EVENT_OPMASK) ==
 				 TRIGGER_EVENT_UPDATE) ?
-				ExecMaterializeSlot(trig_tuple_slot2) : NULL;
+				ExecFetchSlotHeapTuple(trig_tuple_slot2, true, NULL) : NULL;
 			LocTriggerData.tg_newtuplebuf = InvalidBuffer;
 
 			break;

src/backend/executor/execMain.c

@@ -2549,7 +2549,7 @@ EvalPlanQual(EState *estate, EPQState *epqstate,
 	 * is to guard against early re-use of the EPQ query.
 	 */
 	if (!TupIsNull(slot))
-		(void) ExecMaterializeSlot(slot);
+		ExecMaterializeSlot(slot);
 
 	/*
 	 * Clear out the test tuple.  This is needed in case the EPQ query is

src/backend/executor/execReplication.c

@@ -418,7 +418,7 @@ ExecSimpleRelationInsert(EState *estate, TupleTableSlot *slot)
 			ExecPartitionCheck(resultRelInfo, slot, estate, true);
 
 		/* Materialize slot into a tuple that we can scribble upon. */
-		tuple = ExecMaterializeSlot(slot);
+		tuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 
 		/* OK, store the tuple and create index entries for it */
 		simple_heap_insert(rel, tuple);
@@ -485,7 +485,7 @@ ExecSimpleRelationUpdate(EState *estate, EPQState *epqstate,
 			ExecPartitionCheck(resultRelInfo, slot, estate, true);
 
 		/* Materialize slot into a tuple that we can scribble upon. */
-		tuple = ExecMaterializeSlot(slot);
+		tuple = ExecFetchSlotHeapTuple(slot, true, NULL);
 
 		/* OK, update the tuple and index entries for it */
 		simple_heap_update(rel, &searchslot->tts_tuple->t_self,

src/backend/executor/execSRF.c

@@ -521,7 +521,7 @@ ExecMakeFunctionResultSet(SetExprState *fcache,
 			{
 				/* We must return the whole tuple as a Datum. */
 				*isNull = false;
-				return ExecFetchSlotTupleDatum(fcache->funcResultSlot);
+				return ExecFetchSlotHeapTupleDatum(fcache->funcResultSlot);
 			}
 			else
 			{

src/backend/executor/execTuples.c

@@ -676,30 +676,38 @@ ExecCopySlotMinimalTuple(TupleTableSlot *slot)
 								   slot->tts_isnull);
 }
 
-/* --------------------------------
- *		ExecFetchSlotTuple
- *			Fetch the slot's regular physical tuple.
- *
- *		If the slot contains a virtual tuple, we convert it to physical
- *		form.  The slot retains ownership of the physical tuple.
- *		If it contains a minimal tuple we convert to regular form and store
- *		that in addition to the minimal tuple (not instead of, because
- *		callers may hold pointers to Datums within the minimal tuple).
- *
- * The main difference between this and ExecMaterializeSlot() is that this
- * does not guarantee that the contained tuple is local storage.
- * Hence, the result must be treated as read-only.
- * --------------------------------
+/*
+ * ExecFetchSlotHeapTuple - fetch HeapTuple representing the slot's content
+ *
+ * The returned HeapTuple represents the slot's content as closely as
+ * possible.
+ *
+ * If materialize is true, the contents of the slots will be made independent
+ * from the underlying storage (i.e. all buffer pins are release, memory is
+ * allocated in the slot's context).
+ *
+ * If shouldFree is not-NULL it'll be set to true if the returned tuple has
+ * been allocated in the calling memory context, and must be freed by the
+ * caller (via explicit pfree() or a memory context reset).
+ *
+ * NB: If materialize is true, modifications of the returned tuple are
+ * allowed. But it depends on the type of the slot whether such modifications
+ * will also affect the slot's contents. While that is not the nicest
+ * behaviour, all such modifcations are in the process of being removed.
  */
 HeapTuple
-ExecFetchSlotTuple(TupleTableSlot *slot)
+ExecFetchSlotHeapTuple(TupleTableSlot *slot, bool materialize, bool *shouldFree)
 {
 	/*
 	 * sanity checks
 	 */
 	Assert(slot != NULL);
 	Assert(!TTS_EMPTY(slot));
 
+	/* will be used in the near future */
+	if (shouldFree)
+		*shouldFree = false;
+
 	/*
 	 * If we have a regular physical tuple then just return it.
 	 */
@@ -722,7 +730,9 @@ ExecFetchSlotTuple(TupleTableSlot *slot)
 	/*
 	 * Otherwise materialize the slot...
 	 */
-	return ExecMaterializeSlot(slot);
+	ExecMaterializeSlot(slot);
+
+	return slot->tts_tuple;
 }
 
 /* --------------------------------
@@ -739,7 +749,7 @@ ExecFetchSlotTuple(TupleTableSlot *slot)
  * --------------------------------
  */
 MinimalTuple
-ExecFetchSlotMinimalTuple(TupleTableSlot *slot)
+ExecFetchSlotMinimalTuple(TupleTableSlot *slot, bool *shouldFree)
 {
 	MemoryContext oldContext;
 
@@ -749,6 +759,9 @@ ExecFetchSlotMinimalTuple(TupleTableSlot *slot)
 	Assert(slot != NULL);
 	Assert(!TTS_EMPTY(slot));
 
+	/* will be used in the near future */
+	if (shouldFree)
+		*shouldFree = false;
 
 	/*
 	 * If we have a minimal physical tuple (local or not) then just return it.
@@ -779,40 +792,44 @@ ExecFetchSlotMinimalTuple(TupleTableSlot *slot)
 }
 
 /* --------------------------------
- *		ExecFetchSlotTupleDatum
+ *		ExecFetchSlotHeapTupleDatum
  *			Fetch the slot's tuple as a composite-type Datum.
  *
  *		The result is always freshly palloc'd in the caller's memory context.
  * --------------------------------
  */
 Datum
-ExecFetchSlotTupleDatum(TupleTableSlot *slot)
+ExecFetchSlotHeapTupleDatum(TupleTableSlot *slot)
 {
 	HeapTuple	tup;
 	TupleDesc	tupdesc;
+	bool		shouldFree;
+	Datum		ret;
 
 	/* Fetch slot's contents in regular-physical-tuple form */
-	tup = ExecFetchSlotTuple(slot);
+	tup = ExecFetchSlotHeapTuple(slot, false, &shouldFree);
 	tupdesc = slot->tts_tupleDescriptor;
 
 	/* Convert to Datum form */
-	return heap_copy_tuple_as_datum(tup, tupdesc);
+	ret = heap_copy_tuple_as_datum(tup, tupdesc);
+
+	if (shouldFree)
+		pfree(tup);
+
+	return ret;
 }
 
-/* --------------------------------
- *		ExecMaterializeSlot
- *			Force a slot into the "materialized" state.
+/* ExecMaterializeSlot - force a slot into the "materialized" state.
  *
- *		This causes the slot's tuple to be a local copy not dependent on
- *		any external storage.  A pointer to the contained tuple is returned.
+ * This causes the slot's tuple to be a local copy not dependent on any
+ * external storage (i.e. pointing into a Buffer, or having allocations in
+ * another memory context).
  *
- *		A typical use for this operation is to prepare a computed tuple
- *		for being stored on disk.  The original data may or may not be
- *		virtual, but in any case we need a private copy for heap_insert
- *		to scribble on.
- * --------------------------------
+ * A typical use for this operation is to prepare a computed tuple for being
+ * stored on disk.  The original data may or may not be virtual, but in any
+ * case we need a private copy for heap_insert to scribble on.
  */
-HeapTuple
+void
 ExecMaterializeSlot(TupleTableSlot *slot)
 {
 	MemoryContext oldContext;
@@ -828,7 +845,7 @@ ExecMaterializeSlot(TupleTableSlot *slot)
 	 * nothing to do.
 	 */
 	if (slot->tts_tuple && TTS_SHOULDFREE(slot))
-		return slot->tts_tuple;
+		return;
 
 	/*
 	 * Otherwise, copy or build a physical tuple, and store it into the slot.
@@ -868,8 +885,6 @@ ExecMaterializeSlot(TupleTableSlot *slot)
 	 */
 	if (!TTS_SHOULDFREEMIN(slot))
 		slot->tts_mintuple = NULL;
-
-	return slot->tts_tuple;
 }
 
 /* --------------------------------

src/backend/executor/functions.c

@@ -969,7 +969,7 @@ postquel_get_single_result(TupleTableSlot *slot,
 	{
 		/* We must return the whole tuple as a Datum. */
 		fcinfo->isnull = false;
-		value = ExecFetchSlotTupleDatum(slot);
+		value = ExecFetchSlotHeapTupleDatum(slot);
 	}
 	else
 	{