Planner speedup hacking. Avoid saving useless pathkeys, so that path

comparison does not consider paths different when they differ only in
uninteresting aspects of sort order.  (We had a special case of this
consideration for indexscans already, but generalize it to apply to
ordered join paths too.)  Be stricter about what is a canonical pathkey
to allow faster pathkey comparison.  Cache canonical pathkeys and
dispersion stats for left and right sides of a RestrictInfo's clause,
to avoid repeated computation.  Total speedup will depend on number of
tables in a query, but I see about 4x speedup of planning phase for
a sample seven-table query.

Author: tglsfdc

src/backend/nodes/copyfuncs.c

@@ -15,7 +15,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/nodes/copyfuncs.c,v 1.134 2000/12/12 23:33:32 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/nodes/copyfuncs.c,v 1.135 2000/12/14 22:30:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1424,7 +1424,12 @@ _copyRestrictInfo(RestrictInfo *from)
 	newnode->mergejoinoperator = from->mergejoinoperator;
 	newnode->left_sortop = from->left_sortop;
 	newnode->right_sortop = from->right_sortop;
+	/* Do not copy pathkeys, since they'd not be canonical in a copied query */
+	newnode->left_pathkey = NIL;
+	newnode->right_pathkey = NIL;
 	newnode->hashjoinoperator = from->hashjoinoperator;
+	newnode->left_dispersion = from->left_dispersion;
+	newnode->right_dispersion = from->right_dispersion;
 
 	return newnode;
 }

src/backend/nodes/equalfuncs.c

@@ -20,7 +20,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/nodes/equalfuncs.c,v 1.84 2000/12/12 23:33:33 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/nodes/equalfuncs.c,v 1.85 2000/12/14 22:30:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -515,8 +515,9 @@ _equalRestrictInfo(RestrictInfo *a, RestrictInfo *b)
 	if (!equal(a->clause, b->clause))
 		return false;
 	/*
-	 * ignore eval_cost, since it may not be set yet, and should be
-	 * derivable from the clause anyway
+	 * ignore eval_cost, left/right_pathkey, and left/right_dispersion,
+	 * since they may not be set yet, and should be derivable from the
+	 * clause anyway
 	 */
 	if (a->ispusheddown != b->ispusheddown)
 		return false;

src/backend/nodes/readfuncs.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/nodes/readfuncs.c,v 1.101 2000/12/12 23:33:33 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/nodes/readfuncs.c,v 1.102 2000/12/14 22:30:42 tgl Exp $
  *
  * NOTES
  *	  Most of the read functions for plan nodes are tested. (In fact, they
@@ -1848,6 +1848,11 @@ _readRestrictInfo(void)
 
 	/* eval_cost is not part of saved representation; compute on first use */
 	local_node->eval_cost = -1;
+	/* ditto for cached pathkeys and dispersion */
+	local_node->left_pathkey = NIL;
+	local_node->right_pathkey = NIL;
+	local_node->left_dispersion = -1;
+	local_node->right_dispersion = -1;
 
 	return local_node;
 }

src/backend/optimizer/README

@@ -356,9 +356,13 @@ relation the pathkey is for, *no matter how we formed the join*.  It works
 as long as the clause has been applied at some point while forming the
 join relation.  (In the current implementation, we always apply qual
 clauses as soon as possible, ie, as far down in the plan tree as possible.
-So we can always make this deduction.  If we postponed filtering by qual
-clauses then we'd not be able to assume pathkey equivalence until after
-the equality check(s) had been applied.)
+So we can treat the pathkeys as equivalent everywhere.  The exception is
+when the relations A and B are joined inside the nullable side of an
+OUTER JOIN and the equijoin clause comes from above the OUTER JOIN.  In this
+case we cannot apply the qual as soon as A and B are joined, so we do not
+consider the pathkeys to be equivalent.  This could be improved if we wanted
+to go to the trouble of making pathkey equivalence be context-dependent,
+but that seems much more complex than it's worth.)
 
 In short, then: when producing the pathkeys for a merge or nestloop join,
 we can keep all of the keys of the outer path, since the ordering of the
@@ -415,18 +419,14 @@ whenever we make a pathkey sublist that mentions any var appearing in an
 equivalence set, we instantly add all the other vars equivalenced to it,
 whether they appear yet in the pathkey's relation or not.  And we also
 mandate that the pathkey sublist appear in the same order as the
-equivalence set it comes from.  (In practice, we simply return a pointer
-to the relevant equivalence set without building any new sublist at all.
-Each equivalence set becomes a "canonical pathkey" for all its members.)
-This makes comparing pathkeys very simple and fast, and saves a lot of
-work and memory space for pathkey construction as well.
-
-Note that pathkey sublists having just one item still exist, and are
-not expected to be equal() to any equivalence set.  This occurs when
-we describe a sort order that involves a var that's not mentioned in
-any equijoin clause of the WHERE.  We could add singleton sets containing
-such vars to the query's list of equivalence sets, but there's little
-point in doing so.
+equivalence set it comes from.
+
+In fact, we can go even further, and say that the canonical representation
+of a pathkey sublist is a pointer directly to the relevant equivalence set,
+which is kept in a list of pathkey equivalence sets for the query.  Then
+pathkey sublist comparison reduces to pointer-equality checking!  To do this
+we also have to add single-element pathkey sublists to the query's list of
+equivalence sets, but that's a small price to pay.
 
 By the way, it's OK and even useful for us to build equivalence sets
 that mention multiple vars from the same relation.  For example, if
@@ -469,4 +469,15 @@ we're given WHERE int2var = int4var AND int4var = int8var, we'll fail
 while trying to create a representation of the implied clause
 int2var = int8var.
 
+An additional refinement we can make is to insist that canonical pathkey
+lists (sort orderings) do not mention the same pathkey set more than once.
+For example, a pathkey list ((A) (B) (A)) is redundant --- the second
+occurrence of (A) does not change the ordering, since the data must already
+be sorted by A.  Although a user probably wouldn't write ORDER BY A,B,A
+directly, such redundancies are more probable once equijoin equivalences
+have been considered.  Also, the system is likely to generate redundant
+pathkey lists when computing the sort ordering needed for a mergejoin.  By
+eliminating the redundancy, we save time and improve planning, since the
+planner will more easily recognize equivalent orderings as being equivalent.
+
 -- bjm & tgl

src/backend/optimizer/path/allpaths.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/optimizer/path/allpaths.c,v 1.67 2000/11/12 00:36:58 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/optimizer/path/allpaths.c,v 1.68 2000/12/14 22:30:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -162,9 +162,7 @@ set_plain_rel_pathlist(Query *root, RelOptInfo *rel, RangeTblEntry *rte)
 	create_tidscan_paths(root, rel);
 
 	/* Consider index paths for both simple and OR index clauses */
-	create_index_paths(root, rel, indices,
-					   rel->baserestrictinfo,
-					   rel->joininfo);
+	create_index_paths(root, rel, indices);
 
 	/*
 	 * Note: create_or_index_paths depends on create_index_paths to

src/backend/optimizer/path/indxpath.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/optimizer/path/indxpath.c,v 1.99 2000/11/25 20:33:51 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/optimizer/path/indxpath.c,v 1.100 2000/12/14 22:30:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -87,11 +87,6 @@ static void indexable_joinclauses(RelOptInfo *rel, IndexOptInfo *index,
 					  List **clausegroups, List **outerrelids);
 static List *index_innerjoin(Query *root, RelOptInfo *rel, IndexOptInfo *index,
 				List *clausegroup_list, List *outerrelids_list);
-static bool useful_for_mergejoin(RelOptInfo *rel, IndexOptInfo *index,
-					 List *joininfo_list);
-static bool useful_for_ordering(Query *root, RelOptInfo *rel,
-					IndexOptInfo *index,
-					ScanDirection scandir);
 static bool match_index_to_operand(int indexkey, Var *operand,
 					   RelOptInfo *rel, IndexOptInfo *index);
 static bool function_index_operand(Expr *funcOpnd, RelOptInfo *rel,
@@ -125,31 +120,31 @@ static Const *string_to_const(const char *str, Oid datatype);
  * attributes are available and fixed during any one scan of the indexpath.
  *
  * An IndexPath is generated and submitted to add_path() for each index
- * this routine deems potentially interesting for the current query
- * (at most one IndexPath per index on the given relation).  An innerjoin
- * path is also generated for each interesting combination of outer join
- * relations.  The innerjoin paths are *not* passed to add_path(), but are
- * appended to the "innerjoin" list of the relation for later consideration
- * in nested-loop joins.
+ * this routine deems potentially interesting for the current query.
+ * An innerjoin path is also generated for each interesting combination of
+ * outer join relations.  The innerjoin paths are *not* passed to add_path(),
+ * but are appended to the "innerjoin" list of the relation for later
+ * consideration in nested-loop joins.
  *
  * 'rel' is the relation for which we want to generate index paths
  * 'indices' is a list of available indexes for 'rel'
- * 'restrictinfo_list' is a list of restrictinfo nodes for 'rel'
- * 'joininfo_list' is a list of joininfo nodes for 'rel'
  */
 void
 create_index_paths(Query *root,
 				   RelOptInfo *rel,
-				   List *indices,
-				   List *restrictinfo_list,
-				   List *joininfo_list)
+				   List *indices)
 {
+	List	   *restrictinfo_list = rel->baserestrictinfo;
+	List	   *joininfo_list = rel->joininfo;
 	List	   *ilist;
 
 	foreach(ilist, indices)
 	{
 		IndexOptInfo *index = (IndexOptInfo *) lfirst(ilist);
 		List	   *restrictclauses;
+		List	   *index_pathkeys;
+		List	   *useful_pathkeys;
+		bool		index_is_ordered;
 		List	   *joinclausegroups;
 		List	   *joinouterrelids;
 
@@ -179,53 +174,58 @@ create_index_paths(Query *root,
 		match_index_orclauses(rel, index, restrictinfo_list);
 
 		/*
-		 * 2. If the keys of this index match any of the available
-		 * non-'or' restriction clauses, then create a path using those
-		 * clauses as indexquals.
+		 * 2. Match the index against non-'or' restriction clauses.
 		 */
 		restrictclauses = group_clauses_by_indexkey(rel,
 													index,
 													index->indexkeys,
 													index->classlist,
 													restrictinfo_list);
 
-		if (restrictclauses != NIL)
-			add_path(rel, (Path *) create_index_path(root, rel, index,
-													 restrictclauses,
-											   NoMovementScanDirection));
-
 		/*
-		 * 3. If this index can be used for a mergejoin, then create an
-		 * index path for it even if there were no restriction clauses.
-		 * (If there were, there is no need to make another index path.)
-		 * This will allow the index to be considered as a base for a
-		 * mergejoin in later processing.  Similarly, if the index matches
-		 * the ordering that is needed for the overall query result, make
-		 * an index path for it even if there is no other reason to do so.
+		 * 3. Compute pathkeys describing index's ordering, if any,
+		 * then see how many of them are actually useful for this query.
 		 */
-		if (restrictclauses == NIL)
-		{
-			if (useful_for_mergejoin(rel, index, joininfo_list) ||
-			 useful_for_ordering(root, rel, index, ForwardScanDirection))
-				add_path(rel, (Path *)
-						 create_index_path(root, rel, index,
-										   restrictclauses,
-										   ForwardScanDirection));
-		}
+		index_pathkeys = build_index_pathkeys(root, rel, index,
+											  ForwardScanDirection);
+		index_is_ordered = (index_pathkeys != NIL);
+		useful_pathkeys = truncate_useless_pathkeys(root, rel,
+													index_pathkeys);
 
 		/*
-		 * Currently, backwards scan is never considered except for the
-		 * case of matching a query result ordering.  Possibly should
-		 * consider it in other places?
+		 * 4. Generate an indexscan path if there are relevant restriction
+		 * clauses OR the index ordering is potentially useful for later
+		 * merging or final output ordering.
 		 */
-		if (useful_for_ordering(root, rel, index, BackwardScanDirection))
+		if (restrictclauses != NIL || useful_pathkeys != NIL)
 			add_path(rel, (Path *)
 					 create_index_path(root, rel, index,
 									   restrictclauses,
-									   BackwardScanDirection));
+									   useful_pathkeys,
+									   index_is_ordered ?
+									   ForwardScanDirection :
+									   NoMovementScanDirection));
+
+		/*
+		 * 5. If the index is ordered, a backwards scan might be interesting.
+		 * Currently this is only possible for a DESC query result ordering.
+		 */
+		if (index_is_ordered)
+		{
+			index_pathkeys = build_index_pathkeys(root, rel, index,
+												  BackwardScanDirection);
+			useful_pathkeys = truncate_useless_pathkeys(root, rel,
+														index_pathkeys);
+			if (useful_pathkeys != NIL)
+				add_path(rel, (Path *)
+						 create_index_path(root, rel, index,
+										   restrictclauses,
+										   useful_pathkeys,
+										   BackwardScanDirection));
+		}
 
 		/*
-		 * 4. Create an innerjoin index path for each combination of other
+		 * 6. Create an innerjoin index path for each combination of other
 		 * rels used in available join clauses.  These paths will be
 		 * considered as the inner side of nestloop joins against those
 		 * sets of other rels.	indexable_joinclauses() finds sets of
@@ -904,88 +904,6 @@ indexable_operator(Expr *clause, Oid opclass, Oid relam,
 	return InvalidOid;
 }
 
-/*
- * useful_for_mergejoin
- *	  Determine whether the given index can support a mergejoin based
- *	  on any available join clause.
- *
- *	  We look to see whether the first indexkey of the index matches the
- *	  left or right sides of any of the mergejoinable clauses and provides
- *	  the ordering needed for that side.  If so, the index is useful.
- *	  Matching a second or later indexkey is not useful unless there is
- *	  also a mergeclause for the first indexkey, so we need not consider
- *	  secondary indexkeys at this stage.
- *
- * 'rel' is the relation for which 'index' is defined
- * 'joininfo_list' is the list of JoinInfo nodes for 'rel'
- */
-static bool
-useful_for_mergejoin(RelOptInfo *rel,
-					 IndexOptInfo *index,
-					 List *joininfo_list)
-{
-	int		   *indexkeys = index->indexkeys;
-	Oid		   *ordering = index->ordering;
-	List	   *i;
-
-	if (!indexkeys || indexkeys[0] == 0 ||
-		!ordering || ordering[0] == InvalidOid)
-		return false;			/* unordered index is not useful */
-
-	foreach(i, joininfo_list)
-	{
-		JoinInfo   *joininfo = (JoinInfo *) lfirst(i);
-		List	   *j;
-
-		foreach(j, joininfo->jinfo_restrictinfo)
-		{
-			RestrictInfo *restrictinfo = (RestrictInfo *) lfirst(j);
-
-			if (restrictinfo->mergejoinoperator)
-			{
-				if (restrictinfo->left_sortop == ordering[0] &&
-					match_index_to_operand(indexkeys[0],
-										get_leftop(restrictinfo->clause),
-										   rel, index))
-					return true;
-				if (restrictinfo->right_sortop == ordering[0] &&
-					match_index_to_operand(indexkeys[0],
-									   get_rightop(restrictinfo->clause),
-										   rel, index))
-					return true;
-			}
-		}
-	}
-	return false;
-}
-
-/*
- * useful_for_ordering
- *	  Determine whether the given index can produce an ordering matching
- *	  the order that is wanted for the query result.
- *
- * 'rel' is the relation for which 'index' is defined
- * 'scandir' is the contemplated scan direction
- */
-static bool
-useful_for_ordering(Query *root,
-					RelOptInfo *rel,
-					IndexOptInfo *index,
-					ScanDirection scandir)
-{
-	List	   *index_pathkeys;
-
-	if (root->query_pathkeys == NIL)
-		return false;			/* no special ordering requested */
-
-	index_pathkeys = build_index_pathkeys(root, rel, index, scandir);
-
-	if (index_pathkeys == NIL)
-		return false;			/* unordered index */
-
-	return pathkeys_contained_in(root->query_pathkeys, index_pathkeys);
-}
-
 /****************************************************************************
  *				----  ROUTINES TO DO PARTIAL INDEX PREDICATE TESTS	----
  ****************************************************************************/