Fix planner problems with LATERAL references in PlaceHolderVars.

The planner largely failed to consider the possibility that a
PlaceHolderVar's expression might contain a lateral reference to a Var
coming from somewhere outside the PHV's syntactic scope.  We had a previous
report of a problem in this area, which I tried to fix in a quick-hack way
in commit 4da6439, but Antonin Houska
pointed out that there were still some problems, and investigation turned
up other issues.  This patch largely reverts that commit in favor of a more
thoroughly thought-through solution.  The new theory is that a PHV's
ph_eval_at level cannot be higher than its original syntactic level.  If it
contains lateral references, those don't change the ph_eval_at level, but
rather they create a lateral-reference requirement for the ph_eval_at join
relation.  The code in joinpath.c needs to handle that.

Another issue is that createplan.c wasn't handling nested PlaceHolderVars
properly.

In passing, push knowledge of lateral-reference checks for join clauses
into join_clause_is_movable_to.  This is mainly so that FDWs don't need
to deal with it.

This patch doesn't fix the original join-qual-placement problem reported by
Jeremy Evans (and indeed, one of the new regression test cases shows the
wrong answer because of that).  But the PlaceHolderVar problems need to be
fixed before that issue can be addressed, so committing this separately
seems reasonable.

Author: tglsfdc

contrib/postgres_fdw/postgres_fdw.c

@@ -540,7 +540,6 @@ postgresGetForeignPaths(PlannerInfo *root,
 {
 	PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) baserel->fdw_private;
 	ForeignPath *path;
-	Relids		lateral_referencers;
 	List	   *join_quals;
 	Relids		required_outer;
 	double		rows;
@@ -579,34 +578,13 @@ postgresGetForeignPaths(PlannerInfo *root,
 	 * consider combinations of clauses, probably.
 	 */
 
-	/*
-	 * If there are any rels that have LATERAL references to this one, we
-	 * cannot use join quals referencing them as remote quals for this one,
-	 * since such rels would have to be on the inside not the outside of a
-	 * nestloop join relative to this one.	Create a Relids set listing all
-	 * such rels, for use in checks of potential join clauses.
-	 */
-	lateral_referencers = NULL;
-	foreach(lc, root->lateral_info_list)
-	{
-		LateralJoinInfo *ljinfo = (LateralJoinInfo *) lfirst(lc);
-
-		if (bms_is_member(baserel->relid, ljinfo->lateral_lhs))
-			lateral_referencers = bms_add_member(lateral_referencers,
-												 ljinfo->lateral_rhs);
-	}
-
 	/* Scan the rel's join clauses */
 	foreach(lc, baserel->joininfo)
 	{
 		RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
 
 		/* Check if clause can be moved to this rel */
-		if (!join_clause_is_movable_to(rinfo, baserel->relid))
-			continue;
-
-		/* Not useful if it conflicts with any LATERAL references */
-		if (bms_overlap(rinfo->clause_relids, lateral_referencers))
+		if (!join_clause_is_movable_to(rinfo, baserel))
 			continue;
 
 		/* See if it is safe to send to remote */
@@ -667,7 +645,7 @@ postgresGetForeignPaths(PlannerInfo *root,
 															 baserel,
 												   ec_member_matches_foreign,
 															 (void *) &arg,
-														lateral_referencers);
+											   baserel->lateral_referencers);
 
 			/* Done if there are no more expressions in the foreign rel */
 			if (arg.current == NULL)
@@ -682,12 +660,9 @@ postgresGetForeignPaths(PlannerInfo *root,
 				RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
 
 				/* Check if clause can be moved to this rel */
-				if (!join_clause_is_movable_to(rinfo, baserel->relid))
+				if (!join_clause_is_movable_to(rinfo, baserel))
 					continue;
 
-				/* Shouldn't conflict with any LATERAL references */
-				Assert(!bms_overlap(rinfo->clause_relids, lateral_referencers));
-
 				/* See if it is safe to send to remote */
 				if (!is_foreign_expr(root, baserel, rinfo->clause))
 					continue;

src/backend/nodes/copyfuncs.c

@@ -1921,8 +1921,8 @@ _copyLateralJoinInfo(const LateralJoinInfo *from)
 {
 	LateralJoinInfo *newnode = makeNode(LateralJoinInfo);
 
-	COPY_SCALAR_FIELD(lateral_rhs);
 	COPY_BITMAPSET_FIELD(lateral_lhs);
+	COPY_BITMAPSET_FIELD(lateral_rhs);
 
 	return newnode;
 }
@@ -1956,6 +1956,7 @@ _copyPlaceHolderInfo(const PlaceHolderInfo *from)
 	COPY_SCALAR_FIELD(phid);
 	COPY_NODE_FIELD(ph_var);
 	COPY_BITMAPSET_FIELD(ph_eval_at);
+	COPY_BITMAPSET_FIELD(ph_lateral);
 	COPY_BITMAPSET_FIELD(ph_needed);
 	COPY_SCALAR_FIELD(ph_width);
 

src/backend/nodes/equalfuncs.c

@@ -763,15 +763,19 @@ _equalPlaceHolderVar(const PlaceHolderVar *a, const PlaceHolderVar *b)
 	/*
 	 * We intentionally do not compare phexpr.	Two PlaceHolderVars with the
 	 * same ID and levelsup should be considered equal even if the contained
-	 * expressions have managed to mutate to different states.	One way in
-	 * which that can happen is that initplan sublinks would get replaced by
-	 * differently-numbered Params when sublink folding is done.  (The end
-	 * result of such a situation would be some unreferenced initplans, which
-	 * is annoying but not really a problem.)
+	 * expressions have managed to mutate to different states.	This will
+	 * happen during final plan construction when there are nested PHVs, since
+	 * the inner PHV will get replaced by a Param in some copies of the outer
+	 * PHV.  Another way in which it can happen is that initplan sublinks
+	 * could get replaced by differently-numbered Params when sublink folding
+	 * is done.  (The end result of such a situation would be some
+	 * unreferenced initplans, which is annoying but not really a problem.) On
+	 * the same reasoning, there is no need to examine phrels.
 	 *
 	 * COMPARE_NODE_FIELD(phexpr);
+	 *
+	 * COMPARE_BITMAPSET_FIELD(phrels);
 	 */
-	COMPARE_BITMAPSET_FIELD(phrels);
 	COMPARE_SCALAR_FIELD(phid);
 	COMPARE_SCALAR_FIELD(phlevelsup);
 
@@ -796,8 +800,8 @@ _equalSpecialJoinInfo(const SpecialJoinInfo *a, const SpecialJoinInfo *b)
 static bool
 _equalLateralJoinInfo(const LateralJoinInfo *a, const LateralJoinInfo *b)
 {
-	COMPARE_SCALAR_FIELD(lateral_rhs);
 	COMPARE_BITMAPSET_FIELD(lateral_lhs);
+	COMPARE_BITMAPSET_FIELD(lateral_rhs);
 
 	return true;
 }
@@ -819,8 +823,9 @@ static bool
 _equalPlaceHolderInfo(const PlaceHolderInfo *a, const PlaceHolderInfo *b)
 {
 	COMPARE_SCALAR_FIELD(phid);
-	COMPARE_NODE_FIELD(ph_var);
+	COMPARE_NODE_FIELD(ph_var); /* should be redundant */
 	COMPARE_BITMAPSET_FIELD(ph_eval_at);
+	COMPARE_BITMAPSET_FIELD(ph_lateral);
 	COMPARE_BITMAPSET_FIELD(ph_needed);
 	COMPARE_SCALAR_FIELD(ph_width);
 

src/backend/nodes/outfuncs.c

@@ -1756,6 +1756,7 @@ _outRelOptInfo(StringInfo str, const RelOptInfo *node)
 	WRITE_INT_FIELD(max_attr);
 	WRITE_NODE_FIELD(lateral_vars);
 	WRITE_BITMAPSET_FIELD(lateral_relids);
+	WRITE_BITMAPSET_FIELD(lateral_referencers);
 	WRITE_NODE_FIELD(indexlist);
 	WRITE_UINT_FIELD(pages);
 	WRITE_FLOAT_FIELD(tuples, "%.0f");
@@ -1913,8 +1914,8 @@ _outLateralJoinInfo(StringInfo str, const LateralJoinInfo *node)
 {
 	WRITE_NODE_TYPE("LATERALJOININFO");
 
-	WRITE_UINT_FIELD(lateral_rhs);
 	WRITE_BITMAPSET_FIELD(lateral_lhs);
+	WRITE_BITMAPSET_FIELD(lateral_rhs);
 }
 
 static void
@@ -1938,6 +1939,7 @@ _outPlaceHolderInfo(StringInfo str, const PlaceHolderInfo *node)
 	WRITE_UINT_FIELD(phid);
 	WRITE_NODE_FIELD(ph_var);
 	WRITE_BITMAPSET_FIELD(ph_eval_at);
+	WRITE_BITMAPSET_FIELD(ph_lateral);
 	WRITE_BITMAPSET_FIELD(ph_needed);
 	WRITE_INT_FIELD(ph_width);
 }

src/backend/optimizer/README

@@ -802,5 +802,19 @@ parameterized paths still apply, though; in particular, each such path is
 still expected to enforce any join clauses that can be pushed down to it,
 so that all paths of the same parameterization have the same rowcount.
 
+We also allow LATERAL subqueries to be flattened (pulled up into the parent
+query) by the optimizer, but only when they don't contain any lateral
+references to relations outside the lowest outer join that can null the
+LATERAL subquery.  This restriction prevents lateral references from being
+introduced into outer-join qualifications, which would create semantic
+confusion.  Note that even with this restriction, pullup of a LATERAL
+subquery can result in creating PlaceHolderVars that contain lateral
+references to relations outside their syntactic scope.  We still evaluate
+such PHVs at their syntactic location or lower, but the presence of such a
+PHV in the quals or targetlist of a plan node requires that node to appear
+on the inside of a nestloop join relative to the rel(s) supplying the
+lateral reference.  (Perhaps now that that stuff works, we could relax the
+pullup restriction?)
+
 
 -- bjm & tgl

src/backend/optimizer/path/allpaths.c

@@ -386,8 +386,7 @@ set_plain_rel_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
 	/*
 	 * We don't support pushing join clauses into the quals of a seqscan, but
 	 * it could still have required parameterization due to LATERAL refs in
-	 * its tlist.  (That can only happen if the seqscan is on a relation
-	 * pulled up out of a UNION ALL appendrel.)
+	 * its tlist.
 	 */
 	required_outer = rel->lateral_relids;
 
@@ -550,8 +549,8 @@ set_append_rel_size(PlannerInfo *root, RelOptInfo *rel,
 		 * Note: the resulting childrel->reltargetlist may contain arbitrary
 		 * expressions, which otherwise would not occur in a reltargetlist.
 		 * Code that might be looking at an appendrel child must cope with
-		 * such.  Note in particular that "arbitrary expression" can include
-		 * "Var belonging to another relation", due to LATERAL references.
+		 * such.  (Normally, a reltargetlist would only include Vars and
+		 * PlaceHolderVars.)
 		 */
 		childrel->joininfo = (List *)
 			adjust_appendrel_attrs(root,
@@ -1355,8 +1354,7 @@ set_cte_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
 	/*
 	 * We don't support pushing join clauses into the quals of a CTE scan, but
 	 * it could still have required parameterization due to LATERAL refs in
-	 * its tlist.  (That can only happen if the CTE scan is on a relation
-	 * pulled up out of a UNION ALL appendrel.)
+	 * its tlist.
 	 */
 	required_outer = rel->lateral_relids;
 
@@ -1408,10 +1406,8 @@ set_worktable_pathlist(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte)
 	/*
 	 * We don't support pushing join clauses into the quals of a worktable
 	 * scan, but it could still have required parameterization due to LATERAL
-	 * refs in its tlist.  (That can only happen if the worktable scan is on a
-	 * relation pulled up out of a UNION ALL appendrel.  I'm not sure this is
-	 * actually possible given the restrictions on recursive references, but
-	 * it's easy enough to support.)
+	 * refs in its tlist.  (I'm not sure this is actually possible given the
+	 * restrictions on recursive references, but it's easy enough to support.)
 	 */
 	required_outer = rel->lateral_relids;
 

src/backend/optimizer/path/costsize.c

@@ -3943,10 +3943,9 @@ set_rel_width(PlannerInfo *root, RelOptInfo *rel)
 
 		/*
 		 * Ordinarily, a Var in a rel's reltargetlist must belong to that rel;
-		 * but there are corner cases involving LATERAL references in
-		 * appendrel members where that isn't so (see set_append_rel_size()).
-		 * If the Var has the wrong varno, fall through to the generic case
-		 * (it doesn't seem worth the trouble to be any smarter).
+		 * but there are corner cases involving LATERAL references where that
+		 * isn't so.  If the Var has the wrong varno, fall through to the
+		 * generic case (it doesn't seem worth the trouble to be any smarter).
 		 */
 		if (IsA(node, Var) &&
 			((Var *) node)->varno == rel->relid)

src/backend/optimizer/path/indxpath.c

@@ -141,12 +141,10 @@ static void match_restriction_clauses_to_index(RelOptInfo *rel,
 								   IndexClauseSet *clauseset);
 static void match_join_clauses_to_index(PlannerInfo *root,
 							RelOptInfo *rel, IndexOptInfo *index,
-							Relids lateral_referencers,
 							IndexClauseSet *clauseset,
 							List **joinorclauses);
 static void match_eclass_clauses_to_index(PlannerInfo *root,
 							  IndexOptInfo *index,
-							  Relids lateral_referencers,
 							  IndexClauseSet *clauseset);
 static void match_clauses_to_index(IndexOptInfo *index,
 					   List *clauses,
@@ -220,14 +218,14 @@ static Const *string_to_const(const char *str, Oid datatype);
  *
  * Note: check_partial_indexes() must have been run previously for this rel.
  *
- * Note: in corner cases involving LATERAL appendrel children, it's possible
- * that rel->lateral_relids is nonempty.  Currently, we include lateral_relids
- * into the parameterization reported for each path, but don't take it into
- * account otherwise.  The fact that any such rels *must* be available as
- * parameter sources perhaps should influence our choices of index quals ...
- * but for now, it doesn't seem worth troubling over.  In particular, comments
- * below about "unparameterized" paths should be read as meaning
- * "unparameterized so far as the indexquals are concerned".
+ * Note: in cases involving LATERAL references in the relation's tlist, it's
+ * possible that rel->lateral_relids is nonempty.  Currently, we include
+ * lateral_relids into the parameterization reported for each path, but don't
+ * take it into account otherwise.	The fact that any such rels *must* be
+ * available as parameter sources perhaps should influence our choices of
+ * index quals ... but for now, it doesn't seem worth troubling over.
+ * In particular, comments below about "unparameterized" paths should be read
+ * as meaning "unparameterized so far as the indexquals are concerned".
  */
 void
 create_index_paths(PlannerInfo *root, RelOptInfo *rel)
@@ -236,7 +234,6 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel)
 	List	   *bitindexpaths;
 	List	   *bitjoinpaths;
 	List	   *joinorclauses;
-	Relids		lateral_referencers;
 	IndexClauseSet rclauseset;
 	IndexClauseSet jclauseset;
 	IndexClauseSet eclauseset;
@@ -246,23 +243,6 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel)
 	if (rel->indexlist == NIL)
 		return;
 
-	/*
-	 * If there are any rels that have LATERAL references to this one, we
-	 * cannot use join quals referencing them as index quals for this one,
-	 * since such rels would have to be on the inside not the outside of a
-	 * nestloop join relative to this one.	Create a Relids set listing all
-	 * such rels, for use in checks of potential join clauses.
-	 */
-	lateral_referencers = NULL;
-	foreach(lc, root->lateral_info_list)
-	{
-		LateralJoinInfo *ljinfo = (LateralJoinInfo *) lfirst(lc);
-
-		if (bms_is_member(rel->relid, ljinfo->lateral_lhs))
-			lateral_referencers = bms_add_member(lateral_referencers,
-												 ljinfo->lateral_rhs);
-	}
-
 	/* Bitmap paths are collected and then dealt with at the end */
 	bitindexpaths = bitjoinpaths = joinorclauses = NIL;
 
@@ -303,15 +283,15 @@ create_index_paths(PlannerInfo *root, RelOptInfo *rel)
 		 * EquivalenceClasses.	Also, collect join OR clauses for later.
 		 */
 		MemSet(&jclauseset, 0, sizeof(jclauseset));
-		match_join_clauses_to_index(root, rel, index, lateral_referencers,
+		match_join_clauses_to_index(root, rel, index,
 									&jclauseset, &joinorclauses);
 
 		/*
 		 * Look for EquivalenceClasses that can generate joinclauses matching
 		 * the index.
 		 */
 		MemSet(&eclauseset, 0, sizeof(eclauseset));
-		match_eclass_clauses_to_index(root, index, lateral_referencers,
+		match_eclass_clauses_to_index(root, index,
 									  &eclauseset);
 
 		/*
@@ -1957,7 +1937,6 @@ match_restriction_clauses_to_index(RelOptInfo *rel, IndexOptInfo *index,
 static void
 match_join_clauses_to_index(PlannerInfo *root,
 							RelOptInfo *rel, IndexOptInfo *index,
-							Relids lateral_referencers,
 							IndexClauseSet *clauseset,
 							List **joinorclauses)
 {
@@ -1969,11 +1948,7 @@ match_join_clauses_to_index(PlannerInfo *root,
 		RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
 
 		/* Check if clause can be moved to this rel */
-		if (!join_clause_is_movable_to(rinfo, rel->relid))
-			continue;
-
-		/* Not useful if it conflicts with any LATERAL references */
-		if (bms_overlap(rinfo->clause_relids, lateral_referencers))
+		if (!join_clause_is_movable_to(rinfo, rel))
 			continue;
 
 		/* Potentially usable, so see if it matches the index or is an OR */
@@ -1991,7 +1966,6 @@ match_join_clauses_to_index(PlannerInfo *root,
  */
 static void
 match_eclass_clauses_to_index(PlannerInfo *root, IndexOptInfo *index,
-							  Relids lateral_referencers,
 							  IndexClauseSet *clauseset)
 {
 	int			indexcol;
@@ -2012,7 +1986,7 @@ match_eclass_clauses_to_index(PlannerInfo *root, IndexOptInfo *index,
 														 index->rel,
 												  ec_member_matches_indexcol,
 														 (void *) &arg,
-														 lateral_referencers);
+											index->rel->lateral_referencers);
 
 		/*
 		 * We have to check whether the results actually do match the index,
@@ -2644,7 +2618,7 @@ check_partial_indexes(PlannerInfo *root, RelOptInfo *rel)
 		RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
 
 		/* Check if clause can be moved to this rel */
-		if (!join_clause_is_movable_to(rinfo, rel->relid))
+		if (!join_clause_is_movable_to(rinfo, rel))
 			continue;
 
 		clauselist = lappend(clauselist, rinfo);