generate distributed grouping paths with a combine phase

Generate grouping paths with an extra aggregate step before the
remote subplan, looking like this:

    Finalize GroupAggregate
        ->  Remote Subquery Scan
            ->  Combine Aggregate
                ->  Gather
                    ->  Partial GroupAggregate
                        -> ...

The idea is that the combine phase may significantly reduce the
number of rows transferred to the coordinators (and processed on
them), possibly up to 1/N where N is the number of workers.

The plans don't seem to be really chosen at this point, most likely
because of incorrect costing that makes them look more expensive
than the paths without the combine step.

diff --git a/src/backend/optimizer/plan/planner.c b/src/backend/optimizer/plan/planner.c
index 70c7d51e2264c9077d57932e1b0ea7cf0d232424..210e7971cf19e0b1f3e4dbcd6fce22ea6afae688 100644 (file)
--- a/src/backend/optimizer/plan/planner.c
+++ b/src/backend/optimizer/plan/planner.c
@@ -4309,8 +4309,8 @@ create_grouping_paths(PlannerInfo *root,
                }
 
                /*
-                * TODO So far we've only constructed simple paths combining partial
-                * and distributed aggregate paths, i.e.
+                * So far we've only constructed simple paths combining partial and
+                * distributed aggregate paths, i.e.
                 *
                 *     Finalize -> RemoteSubplan -> Gather -> Partial
                 *
@@ -4325,7 +4325,101 @@ create_grouping_paths(PlannerInfo *root,
                 *
                 * We only want to do this for partial paths with RemoteSubplan on
                 * top of them, i.e. when the whole aggregate was not pushed down.
+                *
+                * XXX Gather output is never sorted, so we can only bother with the
+                * cheapest partial path here (just like above).
+                *
+                * XXX This only generates paths with both the combine and finalize
+                * steps using the same implementation (sort+sort or hash+hash). Maybe
+                * we should relax that, and allow hash+sort or sort+hash?
+                *
+                * XXX grouped_rel->partial_pathlist may be empty here, if the planner
+                * did not consider parallel paths (try_parallel_aggregation=false).
+                * But that's OK - we only want to put the combine on top of a Gather,
+                * so if there's none we're done.
+                *
+                * XXX The "combine" paths seem not to be picked up, most likely
+                * because of bad costing, not reflecting the reduction in number of
+                * rows transferred over the network.
                 */
+               if (grouped_rel->partial_pathlist)
+               {
+                       Path       *path = (Path *) linitial(grouped_rel->partial_pathlist);
+                       double          total_groups = path->rows * path->parallel_workers;
+
+                       /* We don't care about paths that were fully pushed down. */
+                       if (! can_push_down_grouping(root, parse, path))
+                       {
+                               path = (Path *) create_gather_path(root,
+                                                                                                  grouped_rel,
+                                                                                                  path,
+                                                                                                  partial_grouping_target,
+                                                                                                  NULL,
+                                                                                                  &total_groups);
+
+                               /*
+                                * Gather is always unsorted, so we'll need to sort, unless
+                                * there's no GROUP BY clause, in which case there will only be a
+                                * single group.
+                                */
+                               if (parse->groupClause)
+                                       path = (Path *) create_sort_path(root,
+                                                                                                        grouped_rel,
+                                                                                                        path,
+                                                                                                        root->group_pathkeys,
+                                                                                                        -1.0);
+
+                               /* Intermediate combine phase. */
+                               if (parse->hasAggs)
+                               {
+                                       path = (Path *) create_agg_path(root,
+                                                                                                       grouped_rel,
+                                                                                                       path,
+                                                                                                       target,
+                                                                       parse->groupClause ? AGG_SORTED : AGG_PLAIN,
+                                                                                                       AGGSPLIT_COMBINE,
+                                                                                                       parse->groupClause,
+                                                                                                       (List *) parse->havingQual,
+                                                                                                       &agg_final_costs,
+                                                                                                       dNumGroups);
+
+                                       path = create_remotesubplan_path(root, path, NULL);
+
+                                       add_path(grouped_rel, (Path *)
+                                                        create_agg_path(root,
+                                                                                        grouped_rel,
+                                                                                        path,
+                                                                                        target,
+                                                                        parse->groupClause ? AGG_SORTED : AGG_PLAIN,
+                                                                                        AGGSPLIT_FINAL_DESERIAL,
+                                                                                        parse->groupClause,
+                                                                                        (List *) parse->havingQual,
+                                                                                        &agg_final_costs,
+                                                                                        dNumGroups));
+                               }
+                               else
+                               {
+                                       path = (Path *) create_group_path(root,
+                                                                                                         grouped_rel,
+                                                                                                         path,
+                                                                                                         target,
+                                                                                                         parse->groupClause,
+                                                                                                         (List *) parse->havingQual,
+                                                                                                         dNumGroups);
+
+                                       path = create_remotesubplan_path(root, path, NULL);
+
+                                       add_path(grouped_rel, (Path *)
+                                                        create_group_path(root,
+                                                                                          grouped_rel,
+                                                                                          path,
+                                                                                          target,
+                                                                                          parse->groupClause,
+                                                                                          (List *) parse->havingQual,
+                                                                                          dNumGroups));
+                               }
+                       }
+               }
        }
 
        if (can_hash && !(agg_costs->hasNonPartial || agg_costs->hasNonSerial))
@@ -4390,7 +4484,63 @@ create_grouping_paths(PlannerInfo *root,
                        }
                }
 
-               /* TODO Generate the additional paths with extra combine phase here. */
+               /*
+                * Generate a path with the extra combine phase.
+                *
+                * XXX See the comments in the block generating combine paths for
+                * the sorted case.
+                */
+               if (grouped_rel->partial_pathlist)
+               {
+                       Path       *path = (Path *) linitial(grouped_rel->partial_pathlist);
+
+                       hashaggtablesize = estimate_hashagg_tablesize(path,
+                                                                                                                 &agg_final_costs,
+                                                                                                                 dNumGroups);
+
+                       /*
+                        * Ignore the path if the hash table won't fit into memory, or
+                        * if we managed to push dowh the whole aggregation.
+                        */
+                       if ((hashaggtablesize < work_mem * 1024L) &&
+                               (! can_push_down_grouping(root, parse, path)))
+                       {
+                               double          total_groups = path->rows * path->parallel_workers;
+
+                               path = (Path *) create_gather_path(root,
+                                                                                                  grouped_rel,
+                                                                                                  path,
+                                                                                                  partial_grouping_target,
+                                                                                                  NULL,
+                                                                                                  &total_groups);
+
+                               path = (Path *) create_agg_path(root,
+                                                                                               grouped_rel,
+                                                                                               path,
+                                                                                               target,
+                                                                                               AGG_HASHED,
+                                                                                               AGGSPLIT_COMBINE,
+                                                                                               parse->groupClause,
+                                                                                               (List *) parse->havingQual,
+                                                                                               &agg_final_costs,
+                                                                                               dNumGroups);
+
+                               /* We know the full push down can't happen, so redistribute. */
+                               path = create_remotesubplan_path(root, path, NULL);
+
+                               add_path(grouped_rel, (Path *)
+                                                create_agg_path(root,
+                                                                                grouped_rel,
+                                                                                path,
+                                                                                target,
+                                                                                AGG_HASHED,
+                                                                                AGGSPLIT_FINAL_DESERIAL,
+                                                                                parse->groupClause,
+                                                                                (List *) parse->havingQual,
+                                                                                &agg_final_costs,
+                                                                                dNumGroups));
+                       }
+               }
        }
 
        /* Give a helpful error if we failed to find any implementation */

diff --git a/src/backend/optimizer/util/pathnode.c b/src/backend/optimizer/util/pathnode.c
index 53d571bd4a040c65339684ec663ccf5bf06840a8..8a93138b3c65bd8f51c5b31c840309d1464ff8ba 100644 (file)
--- a/src/backend/optimizer/util/pathnode.c
+++ b/src/backend/optimizer/util/pathnode.c
@@ -3498,6 +3498,8 @@ create_projection_path(PlannerInfo *root,
        /* Projection does not change the sort order */
        pathnode->path.pathkeys = subpath->pathkeys;
 
+       pathnode->path.distribution = (Distribution *) copyObject(subpath->distribution);
+
        pathnode->subpath = subpath;
 
        /*

diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h
index b8bd0d99d2c9580d864c82a6616f15746e9acac3..88d615d6fd8812cdc95b0ad4d251306b1868a9bf 100644 (file)
--- a/src/include/nodes/nodes.h
+++ b/src/include/nodes/nodes.h
@@ -766,7 +766,10 @@ typedef enum AggSplit
        /* Initial phase of partial aggregation, with serialization: */
        AGGSPLIT_INITIAL_SERIAL = AGGSPLITOP_SKIPFINAL | AGGSPLITOP_SERIALIZE,
        /* Final phase of partial aggregation, with deserialization: */
-       AGGSPLIT_FINAL_DESERIAL = AGGSPLITOP_COMBINE | AGGSPLITOP_DESERIALIZE
+       AGGSPLIT_FINAL_DESERIAL = AGGSPLITOP_COMBINE | AGGSPLITOP_DESERIALIZE,
+       /* Combine phase of partial aggregation, with both steps: */
+       AGGSPLIT_COMBINE = AGGSPLITOP_COMBINE | AGGSPLITOP_DESERIALIZE | \
+                                          AGGSPLITOP_SERIALIZE | AGGSPLITOP_SKIPFINAL
 } AggSplit;
 
 /* Test whether an AggSplit value selects each primitive option: */