Consider explicit sort of the MergeAppend subpaths.

Expand the optimiser search scope a little: fetching optimal subpath matching
pathkeys of the planning MergeAppend, consider the extra case of
overall-optimal path plus explicit Sort node at the top.

It may provide a more effective plan in both full and fractional scan cases:
1. The Sort node may be pushed down to subpaths under a parallel or
async Append.
2. The case when a minor set of subpaths doesn't have a proper index, and it
is profitable to sort them instead of switching to plain Append.

In general, analysing the regression tests changed by this code, it seems that
the cost model prefers a series of small sortings instead of a single massive
one. This feature increases a little the number of such paths.

Overhead:
It seems multiple subpaths may be encountered, as well as many pathkeys.
So, to be as careful as possible here, only cost estimation is performed.

Author: danolivo

contrib/postgres_fdw/expected/postgres_fdw.out

@@ -10352,13 +10352,15 @@ SELECT t1.a, t2.b FROM fprt1 t1 INNER JOIN fprt2 t2 ON (t1.a = t2.b) WHERE t1.a
    ->  Nested Loop
          Join Filter: (t1.a = t2.b)
          ->  Append
-               ->  Foreign Scan on ftprt1_p1 t1_1
+               ->  Sort
+                     Sort Key: t1_1.a
+                     ->  Foreign Scan on ftprt1_p1 t1_1
                ->  Foreign Scan on ftprt1_p2 t1_2
          ->  Materialize
                ->  Append
                      ->  Foreign Scan on ftprt2_p1 t2_1
                      ->  Foreign Scan on ftprt2_p2 t2_2
-(10 rows)
+(12 rows)
 
 SELECT t1.a, t2.b FROM fprt1 t1 INNER JOIN fprt2 t2 ON (t1.a = t2.b) WHERE t1.a % 25 = 0 ORDER BY 1,2 FOR UPDATE OF t1;
   a  |  b  

src/backend/optimizer/path/allpaths.c

@@ -1855,29 +1855,18 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel,
 
 			/* Locate the right paths, if they are available. */
 			cheapest_startup =
-				get_cheapest_path_for_pathkeys(childrel->pathlist,
-											   pathkeys,
-											   NULL,
-											   STARTUP_COST,
-											   false);
+				get_cheapest_path_for_pathkeys_ext(root, childrel, pathkeys,
+												   NULL, STARTUP_COST, false);
 			cheapest_total =
-				get_cheapest_path_for_pathkeys(childrel->pathlist,
-											   pathkeys,
-											   NULL,
-											   TOTAL_COST,
-											   false);
+				get_cheapest_path_for_pathkeys_ext(root, childrel, pathkeys,
+												   NULL, TOTAL_COST, false);
 
 			/*
-			 * If we can't find any paths with the right order just use the
-			 * cheapest-total path; we'll have to sort it later.
+			 * In accordance to current planning logic there are no
+			 * parameterised paths under a merge append.
 			 */
-			if (cheapest_startup == NULL || cheapest_total == NULL)
-			{
-				cheapest_startup = cheapest_total =
-					childrel->cheapest_total_path;
-				/* Assert we do have an unparameterized path for this child */
-				Assert(cheapest_total->param_info == NULL);
-			}
+			Assert(cheapest_startup != NULL && cheapest_total != NULL);
+			Assert(cheapest_total->param_info == NULL);
 
 			/*
 			 * When building a fractional path, determine a cheapest
@@ -1904,21 +1893,17 @@ generate_orderedappend_paths(PlannerInfo *root, RelOptInfo *rel,
 					path_fraction /= childrel->rows;
 
 				cheapest_fractional =
-					get_cheapest_fractional_path_for_pathkeys(childrel->pathlist,
-															  pathkeys,
-															  NULL,
-															  path_fraction);
+					get_cheapest_fractional_path_for_pathkeys_ext(root,
+																  childrel,
+																  pathkeys,
+																  NULL,
+																  path_fraction);
 
 				/*
-				 * If we found no path with matching pathkeys, use the
-				 * cheapest total path instead.
-				 *
-				 * XXX We might consider partially sorted paths too (with an
-				 * incremental sort on top). But we'd have to build all the
-				 * incremental paths, do the costing etc.
+				 * In accordance to current planning logic there are no
+				 * parameterised fractional paths under a merge append.
 				 */
-				if (!cheapest_fractional)
-					cheapest_fractional = cheapest_total;
+				Assert(cheapest_fractional != NULL);
 			}
 
 			/*

src/backend/optimizer/path/pathkeys.c

@@ -19,11 +19,13 @@
 
 #include "access/stratnum.h"
 #include "catalog/pg_opfamily.h"
+#include "miscadmin.h"
 #include "nodes/nodeFuncs.h"
 #include "optimizer/cost.h"
 #include "optimizer/optimizer.h"
 #include "optimizer/pathnode.h"
 #include "optimizer/paths.h"
+#include "optimizer/planner.h"
 #include "partitioning/partbounds.h"
 #include "rewrite/rewriteManip.h"
 #include "utils/lsyscache.h"
@@ -648,6 +650,64 @@ get_cheapest_path_for_pathkeys(List *paths, List *pathkeys,
 	return matched_path;
 }
 
+/*
+ * get_cheapest_path_for_pathkeys_ext
+ *	  Calls get_cheapest_path_for_pathkeys to obtain cheapest path that
+ *	  satisfies defined criterias and сonsiders one more option: choose
+ *	  overall-optimal path (according the criterion) and explicitly sort its
+ *	  output to satisfy the pathkeys.
+ *
+ *	  Caller is responsible to insert corresponding sort path at the top of
+ *	  returned path if it will be chosen to be used.
+ *
+ *	  Return NULL if no such path.
+ */
+Path *
+get_cheapest_path_for_pathkeys_ext(PlannerInfo *root, RelOptInfo *rel,
+								   List *pathkeys, Relids required_outer,
+								   CostSelector cost_criterion,
+								   bool require_parallel_safe)
+{
+	Path		sort_path;
+	Path	   *base_path = rel->cheapest_total_path;
+	Path	   *path;
+
+	/* In generate_orderedappend_paths() all childrels do have some paths */
+	Assert(base_path);
+
+	path = get_cheapest_path_for_pathkeys(rel->pathlist, pathkeys,
+										  required_outer, cost_criterion,
+										  require_parallel_safe);
+
+	/*
+	 * Stop here if the cheapest total path doesn't satisfy necessary
+	 * conditions
+	 */
+	if ((require_parallel_safe && !base_path->parallel_safe) ||
+		!bms_is_subset(PATH_REQ_OUTER(base_path), required_outer))
+		return path;
+
+	if (path == NULL)
+
+		/*
+		 * Current pathlist doesn't fit the pathkeys. No need to check extra
+		 * sort path ways.
+		 */
+		return base_path;
+
+	/* Consider the cheapest total path with extra sort */
+	if (path != base_path)
+	{
+		cost_sort(&sort_path, root, pathkeys, base_path->disabled_nodes,
+				  base_path->total_cost, base_path->rows,
+				  base_path->pathtarget->width, 0.0, work_mem, -1.0);
+
+		if (compare_path_costs(&sort_path, path, cost_criterion) < 0)
+			return base_path;
+	}
+	return path;
+}
+
 /*
  * get_cheapest_fractional_path_for_pathkeys
  *	  Find the cheapest path (for retrieving a specified fraction of all
@@ -690,6 +750,61 @@ get_cheapest_fractional_path_for_pathkeys(List *paths,
 	return matched_path;
 }
 
+/*
+ * get_cheapest_fractional_path_for_pathkeys_ext
+ *	  obtain cheapest fractional path that satisfies defined criterias excluding
+ *	  pathkeys and explicitly sort its output to satisfy the pathkeys.
+ *
+ *	  Caller is responsible to insert corresponding sort path at the top of
+ *	  returned path if it will be chosen to be used.
+ *
+ *	  Return NULL if no such path.
+ */
+Path *
+get_cheapest_fractional_path_for_pathkeys_ext(PlannerInfo *root,
+											  RelOptInfo *rel,
+											  List *pathkeys,
+											  Relids required_outer,
+											  double fraction)
+{
+	Path		sort_path;
+	Path	   *base_path = rel->cheapest_total_path;
+	Path	   *path;
+
+	/* In generate_orderedappend_paths() all childrels do have some paths */
+	Assert(base_path);
+
+	path = get_cheapest_fractional_path_for_pathkeys(rel->pathlist, pathkeys,
+													 required_outer, fraction);
+
+	/*
+	 * Stop here if the cheapest total path doesn't satisfy necessary
+	 * conditions
+	 */
+	if (!bms_is_subset(PATH_REQ_OUTER(base_path), required_outer))
+		return path;
+
+	if (path == NULL)
+
+		/*
+		 * Current pathlist doesn't fit the pathkeys. No need to check extra
+		 * sort path ways.
+		 */
+		return base_path;
+
+	/* Consider the cheapest total path with extra sort */
+	if (path != base_path)
+	{
+		cost_sort(&sort_path, root, pathkeys, base_path->disabled_nodes,
+				  base_path->total_cost, base_path->rows,
+				  base_path->pathtarget->width, 0.0, work_mem, -1.0);
+
+		if (compare_fractional_path_costs(&sort_path, path, fraction) <= 0)
+			return base_path;
+	}
+
+	return path;
+}
 
 /*
  * get_cheapest_parallel_safe_total_inner

src/include/optimizer/paths.h

@@ -224,10 +224,20 @@ extern Path *get_cheapest_path_for_pathkeys(List *paths, List *pathkeys,
 											Relids required_outer,
 											CostSelector cost_criterion,
 											bool require_parallel_safe);
+extern Path *get_cheapest_path_for_pathkeys_ext(PlannerInfo *root,
+												RelOptInfo *rel, List *pathkeys,
+												Relids required_outer,
+												CostSelector cost_criterion,
+												bool require_parallel_safe);
 extern Path *get_cheapest_fractional_path_for_pathkeys(List *paths,
 													   List *pathkeys,
 													   Relids required_outer,
 													   double fraction);
+extern Path *get_cheapest_fractional_path_for_pathkeys_ext(PlannerInfo *root,
+														   RelOptInfo *rel,
+														   List *pathkeys,
+														   Relids required_outer,
+														   double fraction);
 extern Path *get_cheapest_parallel_safe_total_inner(List *paths);
 extern List *build_index_pathkeys(PlannerInfo *root, IndexOptInfo *index,
 								  ScanDirection scandir);

src/test/regress/expected/collate.icu.utf8.out

@@ -2412,16 +2412,19 @@ EXPLAIN (COSTS OFF)
 SELECT c collate "C", count(c) FROM pagg_tab3 GROUP BY c collate "C" ORDER BY 1;
                        QUERY PLAN                       
 --------------------------------------------------------
- Sort
+ Merge Append
    Sort Key: ((pagg_tab3.c)::text) COLLATE "C"
-   ->  Append
+   ->  Sort
+         Sort Key: ((pagg_tab3.c)::text) COLLATE "C"
          ->  HashAggregate
                Group Key: (pagg_tab3.c)::text
                ->  Seq Scan on pagg_tab3_p2 pagg_tab3
+   ->  Sort
+         Sort Key: ((pagg_tab3_1.c)::text) COLLATE "C"
          ->  HashAggregate
                Group Key: (pagg_tab3_1.c)::text
                ->  Seq Scan on pagg_tab3_p1 pagg_tab3_1
-(9 rows)
+(12 rows)
 
 SELECT c collate "C", count(c) FROM pagg_tab3 GROUP BY c collate "C" ORDER BY 1;
  c | count 

src/test/regress/expected/inherit.out

@@ -1665,11 +1665,13 @@ insert into matest2 (name) values ('Test 3');
 insert into matest2 (name) values ('Test 4');
 insert into matest3 (name) values ('Test 5');
 insert into matest3 (name) values ('Test 6');
-set enable_indexscan = off;  -- force use of seqscan/sort, so no merge
+set enable_indexscan = off;  -- force use of seqscan/sort
+set enable_sort = off; -- since merge append may employ sort in children we need to disable sort
 explain (verbose, costs off) select * from matest0 order by 1-id;
                          QUERY PLAN                         
 ------------------------------------------------------------
  Sort
+   Disabled: true
    Output: matest0.id, matest0.name, ((1 - matest0.id))
    Sort Key: ((1 - matest0.id))
    ->  Result
@@ -1683,7 +1685,7 @@ explain (verbose, costs off) select * from matest0 order by 1-id;
                      Output: matest0_3.id, matest0_3.name
                ->  Seq Scan on public.matest3 matest0_4
                      Output: matest0_4.id, matest0_4.name
-(14 rows)
+(15 rows)
 
 select * from matest0 order by 1-id;
  id |  name  
@@ -1719,6 +1721,7 @@ select min(1-id) from matest0;
 (1 row)
 
 reset enable_indexscan;
+reset enable_sort;
 set enable_seqscan = off;  -- plan with fewest seqscans should be merge
 set enable_parallel_append = off; -- Don't let parallel-append interfere
 explain (verbose, costs off) select * from matest0 order by 1-id;
@@ -1844,16 +1847,20 @@ order by t1.b limit 10;
          Merge Cond: (t1.b = t2.b)
          ->  Merge Append
                Sort Key: t1.b
-               ->  Index Scan using matest0i on matest0 t1_1
+               ->  Sort
+                     Sort Key: t1_1.b
+                     ->  Seq Scan on matest0 t1_1
                ->  Index Scan using matest1i on matest1 t1_2
          ->  Materialize
                ->  Merge Append
                      Sort Key: t2.b
-                     ->  Index Scan using matest0i on matest0 t2_1
-                           Filter: (c = d)
+                     ->  Sort
+                           Sort Key: t2_1.b
+                           ->  Seq Scan on matest0 t2_1
+                                 Filter: (c = d)
                      ->  Index Scan using matest1i on matest1 t2_2
                            Filter: (c = d)
-(14 rows)
+(18 rows)
 
 reset enable_nestloop;
 drop table matest0 cascade;