remove AggDistribution and Agg->aggdistribution
authorTomas Vondra <[email protected]>
Thu, 10 Nov 2016 22:34:40 +0000 (23:34 +0100)
committerTomas Vondra <[email protected]>
Thu, 10 Nov 2016 22:47:29 +0000 (23:47 +0100)
As the plan is to rework the aggregate to use the infrastructure
from 9.6, the best idea seems to be to use the partial aggregate
stuff. Which means we can use the aggsplit field in Agg plan node,
and we don't need the custom aggdistribution.

Due to the upper-planner pathification, this should be at AggPath
anyway (but we already have the aggsplit there).

Furthermore, remove the custom plan construction from make_agg().
This was appropriate when the upper planner was not using paths,
but that's not true anymore. The idea is to construct the usual
Finalize+Partial aggregate nodes, possibly with a RemoteSubPath
node in between them (when pushdown is not possible).

I wonder how this will interact with the parallel query, as that
is also using the Partial node, but I suppose that should work
just fine. In the worst case we'll need to introduce something
like GatherCombine, collecting (and combining) partial results
on the data node before sending them to the coordinator.

Thanks to using Finalize/Partial Agg nodes, we can also remove
the changes from fix_upper_expr, or at least that's how it looks
to me.

At some point, bits of the removed code probably will need to be
restored, although most likely different place. That applies in
particular to:

(a) find_referenced_cols_walker(), which checks what part of the
aggregate we can push to the remote node (everything, partial
or nothing)

(b) the removed bits of make_agg() constructing the remote plan
(although most of this will hopefully be handled by the partial
aggregate infrastructure)

src/backend/commands/explain.c
src/backend/nodes/copyfuncs.c
src/backend/nodes/outfuncs.c
src/backend/optimizer/plan/createplan.c
src/backend/optimizer/plan/setrefs.c
src/include/nodes/plannodes.h

index c91960ac3123372ca40beff6b86972bbb698e57f..fb02c165e2e812b90523f86db64c1ee8766cb68d 100644 (file)
@@ -1020,21 +1020,6 @@ ExplainNode(PlanState *planstate, List *ancestors,
                                else
                                        partialmode = "Simple";
                        }
-#ifdef XCP
-                       switch (((Agg *) plan)->aggstrategy)
-                       {
-                               case AGG_SLAVE:
-                                       operation = "Transition";
-                                       break;
-                               case AGG_MASTER:
-                                       operation = "Collection";
-                                       break;
-                               default:
-                                       operation = NULL;
-                                       break;
-                       }
-#endif
-
                        break;
                case T_WindowAgg:
                        pname = sname = "WindowAgg";
index cb657a8a4b369ca0aafb80371fc48260790d93c8..0cdd6559d0f8f695eebea57908ab74395e55b6d8 100644 (file)
@@ -888,9 +888,6 @@ _copyAgg(const Agg *from)
        CopyPlanFields((const Plan *) from, (Plan *) newnode);
 
        COPY_SCALAR_FIELD(aggstrategy);
-#ifdef XCP
-       COPY_SCALAR_FIELD(aggdistribution);
-#endif
        COPY_SCALAR_FIELD(aggsplit);
        COPY_SCALAR_FIELD(numCols);
        if (from->numCols > 0)
index 6c83bbbe07fcdeb69aaf9a2316f18c0d9b2fbc22..21f47c0d84072b4c01fbf55a94e8d255d690cdbe 100644 (file)
@@ -1054,9 +1054,6 @@ _outAgg(StringInfo str, const Agg *node)
        _outPlanInfo(str, (const Plan *) node);
 
        WRITE_ENUM_FIELD(aggstrategy, AggStrategy);
-#ifdef XCP
-       WRITE_ENUM_FIELD(aggdistribution, AggDistribution);
-#endif
        WRITE_ENUM_FIELD(aggsplit, AggSplit);
        WRITE_INT_FIELD(numCols);
 
index 618b218756629cb6ceae9016dab0856f2c95ae2e..7d2a348b0ff85ace4d0000ec11585c83b1223a2c 100644 (file)
@@ -6431,141 +6431,6 @@ materialize_finished_plan(Plan *subplan)
        return matplan;
 }
 
-
-#ifdef XCP
-typedef struct
-{
-       List       *subtlist;
-       List       *newtlist;
-} find_referenced_cols_context;
-
-static bool
-find_referenced_cols_walker(Node *node, find_referenced_cols_context *context)
-{
-       TargetEntry *tle;
-
-       if (node == NULL)
-               return false;
-       if (IsA(node, Aggref))
-       {
-               /*
-                * We can not push down aggregates with DISTINCT.
-                */
-               if (((Aggref *) node)->aggdistinct)
-                       return true;
-
-               /*
-                * We can not push down aggregates with ORDER BY.
-                */
-               if (((Aggref *) node)->aggorder)
-                       return true;
-
-               /*
-                * We need to add aggregate reference to the new tlist if it
-                * is not already there. Phase 1 aggregate is actually returns values
-                * of transition data type, so we should change the data type of the
-                * expression.
-                */
-               if (!tlist_member(node, context->newtlist))
-               {
-                       Aggref *aggref = (Aggref *) node;
-                       Aggref *newagg;
-                       TargetEntry *newtle;
-                       HeapTuple       aggTuple;
-                       Form_pg_aggregate aggform;
-                       Oid     aggtranstype;
-
-                       aggTuple = SearchSysCache1(AGGFNOID,
-                                                                          ObjectIdGetDatum(aggref->aggfnoid));
-                       if (!HeapTupleIsValid(aggTuple))
-                               elog(ERROR, "cache lookup failed for aggregate %u",
-                                        aggref->aggfnoid);
-                       aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple);
-                       aggtranstype = aggform->aggtranstype;
-                       ReleaseSysCache(aggTuple);
-
-                       /* Can not split two-phase aggregate */
-                       if (!OidIsValid(InvalidOid))
-                               return true;
-
-                       if (IsPolymorphicType(aggtranstype))
-                       {
-                               Oid        *inputTypes;
-                               Oid                *declaredArgTypes;
-                               int                     agg_nargs;
-                               int                     numArgs;
-                               ListCell   *l;
-
-                               inputTypes = (Oid *) palloc(sizeof(Oid) * list_length(aggref->args));
-                               numArgs = 0;
-                               foreach(l, aggref->args)
-                               {
-                                       TargetEntry *tle = (TargetEntry *) lfirst(l);
-
-                                       if (!tle->resjunk)
-                                               inputTypes[numArgs++] = exprType((Node *) tle->expr);
-                               }
-
-                               /* have to fetch the agg's declared input types... */
-                               (void) get_func_signature(aggref->aggfnoid,
-                                                                                 &declaredArgTypes, &agg_nargs);
-                               Assert(agg_nargs == numArgs);
-
-
-                               aggtranstype = enforce_generic_type_consistency(inputTypes,
-                                                                                                                               declaredArgTypes,
-                                                                                                                               agg_nargs,
-                                                                                                                               aggtranstype,
-                                                                                                                               false);
-                               pfree(inputTypes);
-                               pfree(declaredArgTypes);
-                       }
-                       newagg = copyObject(aggref);
-                       newagg->aggtype = aggtranstype;
-
-                       newtle = makeTargetEntry((Expr *) newagg,
-                                                                        list_length(context->newtlist) + 1,
-                                                                        NULL,
-                                                                        false);
-                       context->newtlist = lappend(context->newtlist, newtle);
-               }
-
-               return false;
-       }
-       /*
-        * If expression is in the subtlist copy it into new tlist
-        */
-       tle = tlist_member(node, context->subtlist);
-       if (tle && !tlist_member((Node *) tle->expr, context->newtlist))
-       {
-               TargetEntry *newtle;
-               newtle = makeTargetEntry((Expr *) copyObject(node),
-                                                                list_length(context->newtlist) + 1,
-                                                                tle->resname,
-                                                                false);
-               context->newtlist = lappend(context->newtlist, newtle);
-               return false;
-       }
-       if (IsA(node, Var))
-       {
-               /*
-                * Referenced Var is not a member of subtlist.
-                * Go ahead and add junk one.
-                */
-               TargetEntry *newtle;
-               newtle = makeTargetEntry((Expr *) copyObject(node),
-                                                                list_length(context->newtlist) + 1,
-                                                                NULL,
-                                                                true);
-               context->newtlist = lappend(context->newtlist, newtle);
-               return false;
-       }
-       return expression_tree_walker(node, find_referenced_cols_walker,
-                                                                 (void *) context);
-}
-#endif
-
-
 Agg *
 make_agg(List *tlist, List *qual,
                 AggStrategy aggstrategy, AggSplit aggsplit,
@@ -6575,9 +6440,6 @@ make_agg(List *tlist, List *qual,
 {
        Agg                *node = makeNode(Agg);
        Plan       *plan = &node->plan;
-#ifdef XCP
-       RemoteSubplan *pushdown;
-#endif
        long            numGroups;
 
        /* Reduce to long, but 'ware overflow! */
@@ -6597,141 +6459,6 @@ make_agg(List *tlist, List *qual,
        plan->lefttree = lefttree;
        plan->righttree = NULL;
 
-#ifdef XCP
-       /*
-        * If lefttree is a distributed subplan we may optimize aggregates by
-        * pushing down transition phase to remote data notes, and therefore reduce
-        * traffic and distribute evaluation load.
-        * We need to find all Var and Aggref expressions in tlist and qual and make
-        * up a new tlist from these expressions. Update original Vars.
-        * Create new Agg node with the new tlist and aggdistribution AGG_SLAVE.
-        * Set new Agg node as a lefttree of the distributed subplan, moving
-        * existing lefttree down under the new Agg node. Set new tlist to the
-        * distributed subplan - it should be matching to the subquery.
-        * Set node's aggdistribution to AGG_MASTER and continue node initialization
-        */
-       pushdown = find_push_down_plan(lefttree, true);
-       if (pushdown)
-       {
-               find_referenced_cols_context context;
-
-               context.subtlist = pushdown->scan.plan.targetlist;
-               context.newtlist = NIL;
-               if (find_referenced_cols_walker((Node *) tlist, &context) ||
-                               find_referenced_cols_walker((Node *) qual, &context))
-               {
-                       /*
-                        * We found we can not push down this aggregate, clean up and
-                        * fallback to default procedure
-                        */
-                       node->aggdistribution = AGG_ONENODE;
-               }
-               else
-               {
-                       Agg                *phase1 = makeNode(Agg);
-                       Plan       *plan1 = &phase1->plan;
-                       int                     i;
-
-                       phase1->aggdistribution = AGG_SLAVE;
-                       phase1->aggstrategy = aggstrategy;
-                       phase1->numCols = numGroupCols;
-                       phase1->grpColIdx = grpColIdx;
-                       phase1->grpOperators = grpOperators;
-                       phase1->numGroups = numGroups;
-
-                       /*
-                        * If we perform grouping we should make sure the grouping
-                        * expressions are in the new tlist, and we should update indexes
-                        * for the Phase2 aggregation node
-                        */
-                       if (numGroupCols > 0)
-                       {
-                               AttrNumber *newGrpColIdx;
-                               newGrpColIdx = (AttrNumber *) palloc(sizeof(AttrNumber)
-                                                                                                                * numGroupCols);
-                               for (i = 0; i < numGroupCols; i++)
-                               {
-                                       TargetEntry *tle;
-                                       TargetEntry *newtle;
-
-                                       tle = (TargetEntry *) list_nth(context.subtlist,
-                                                                                                  grpColIdx[i] - 1);
-                                       newtle = tlist_member((Node *) tle->expr, context.newtlist);
-                                       if (newtle == NULL)
-                                       {
-                                               newtle = makeTargetEntry((Expr *) copyObject(tle->expr),
-                                                                                        list_length(context.newtlist) + 1,
-                                                                                                tle->resname,
-                                                                                                false);
-                                               context.newtlist = lappend(context.newtlist, newtle);
-                                       }
-                                       newGrpColIdx[i] = newtle->resno;
-                               }
-                               node->grpColIdx = newGrpColIdx;
-                       }
-
-                       /*
-                        * If the pushdown plan is sorting update sort column indexes
-                        */
-                       if (pushdown->sort)
-                       {
-                               SimpleSort *ssort = pushdown->sort;
-                               for (i = 0; i < ssort->numCols; i++)
-                               {
-                                       TargetEntry *tle;
-                                       TargetEntry *newtle;
-
-                                       tle = (TargetEntry *) list_nth(context.subtlist,
-                                                                                                  grpColIdx[i] - 1);
-                                       newtle = tlist_member((Node *) tle->expr, context.newtlist);
-                                       if (newtle == NULL)
-                                       {
-                                               /* XXX maybe we should just remove the sort key ? */
-                                               newtle = makeTargetEntry((Expr *) copyObject(tle->expr),
-                                                                                        list_length(context.newtlist) + 1,
-                                                                                                tle->resname,
-                                                                                                false);
-                                               context.newtlist = lappend(context.newtlist, newtle);
-                                       }
-                                       ssort->sortColIdx[i] = newtle->resno;
-                               }
-                       }
-
-                       copy_plan_costsize(plan1, (Plan *) pushdown); // ???
-
-                       /*
-                        * We will produce a single output tuple if not grouping, and a tuple per
-                        * group otherwise.
-                        */
-                       if (aggstrategy == AGG_PLAIN)
-                               plan1->plan_rows = 1;
-                       else
-                               plan1->plan_rows = numGroups;
-
-                       plan1->targetlist = context.newtlist;
-                       plan1->qual = NIL;
-                       plan1->lefttree = pushdown->scan.plan.lefttree;
-                       pushdown->scan.plan.lefttree = plan1;
-                       plan1->righttree = NULL;
-
-                       /*
-                        * Update target lists of all plans from lefttree till phase1.
-                        * All they should be the same if the tree is transparent for push
-                        * down modification.
-                        */
-                       while (lefttree != plan1)
-                       {
-                               lefttree->targetlist = context.newtlist;
-                               lefttree = lefttree->lefttree;
-                       }
-
-                       node->aggdistribution = AGG_MASTER;
-               }
-       }
-       else
-               node->aggdistribution = AGG_ONENODE;
-#endif
-
        return node;
 }
 
index 344ebdc015d2c8edee42ca78e3fc45823969be8b..d5bc9e07602bb99de6fdf64ef57bf496be86c5e0 100644 (file)
@@ -69,9 +69,6 @@ typedef struct
        indexed_tlist *subplan_itlist;
        Index           newvarno;
        int                     rtoffset;
-#ifdef XCP
-       bool            agg_master;
-#endif
 } fix_upper_expr_context;
 
 typedef struct
@@ -144,8 +141,7 @@ static Node *fix_upper_expr(PlannerInfo *root,
                           Node *node,
                           indexed_tlist *subplan_itlist,
                           Index newvarno,
-                          int rtoffset,
-                          bool agg_master);
+                          int rtoffset);
 static Node *fix_upper_expr_mutator(Node *node,
                                           fix_upper_expr_context *context);
 static List *set_returning_clause_references(PlannerInfo *root,
@@ -1015,15 +1011,13 @@ set_indexonlyscan_references(PlannerInfo *root,
                                           (Node *) plan->scan.plan.targetlist,
                                           index_itlist,
                                           INDEX_VAR,
-                                          rtoffset,
-                                          false);
+                                          rtoffset);
        plan->scan.plan.qual = (List *)
                fix_upper_expr(root,
                                           (Node *) plan->scan.plan.qual,
                                           index_itlist,
                                           INDEX_VAR,
-                                          rtoffset,
-                                          false);
+                                          rtoffset);
 
        /* indexqual is already transformed to reference index columns */
        plan->indexqual = fix_scan_list(root, plan->indexqual, rtoffset);
@@ -1193,29 +1187,25 @@ set_foreignscan_references(PlannerInfo *root,
                                                   (Node *) fscan->scan.plan.targetlist,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
                fscan->scan.plan.qual = (List *)
                        fix_upper_expr(root,
                                                   (Node *) fscan->scan.plan.qual,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
                fscan->fdw_exprs = (List *)
                        fix_upper_expr(root,
                                                   (Node *) fscan->fdw_exprs,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
                fscan->fdw_recheck_quals = (List *)
                        fix_upper_expr(root,
                                                   (Node *) fscan->fdw_recheck_quals,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
                pfree(itlist);
                /* fdw_scan_tlist itself just needs fix_scan_list() adjustments */
                fscan->fdw_scan_tlist =
@@ -1274,22 +1264,19 @@ set_customscan_references(PlannerInfo *root,
                                                   (Node *) cscan->scan.plan.targetlist,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
                cscan->scan.plan.qual = (List *)
                        fix_upper_expr(root,
                                                   (Node *) cscan->scan.plan.qual,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
                cscan->custom_exprs = (List *)
                        fix_upper_expr(root,
                                                   (Node *) cscan->custom_exprs,
                                                   itlist,
                                                   INDEX_VAR,
-                                                  rtoffset,
-                                                  false);
+                                                  rtoffset);
 
                pfree(itlist);
                /* custom_scan_tlist itself just needs fix_scan_list() adjustments */
@@ -1637,8 +1624,7 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset)
                                                                                                   (Node *) nlp->paramval,
                                                                                                   outer_itlist,
                                                                                                   OUTER_VAR,
-                                                                                                  rtoffset,
-                                                                                                  false);
+                                                                                                  rtoffset);
 
                        /* Check we replaced any PlaceHolderVar with simple Var */
                        if (!(IsA(nlp->paramval, Var) &&
@@ -1741,12 +1727,6 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset)
        indexed_tlist *subplan_itlist;
        List       *output_targetlist;
        ListCell   *l;
-#ifdef XCP
-       bool            agg_master;
-
-       agg_master = (IsA(plan, Agg) &&
-                                         ((Agg *) plan)->aggdistribution == AGG_MASTER);
-#endif
 
        subplan_itlist = build_tlist_index(subplan->targetlist);
 
@@ -1769,16 +1749,14 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset)
                                                                                 (Node *) tle->expr,
                                                                                 subplan_itlist,
                                                                                 OUTER_VAR,
-                                                                                rtoffset,
-                                                                                agg_master);
+                                                                                rtoffset);
                }
                else
                        newexpr = fix_upper_expr(root,
                                                                         (Node *) tle->expr,
                                                                         subplan_itlist,
                                                                         OUTER_VAR,
-                                                                        rtoffset,
-                                                                        agg_master);
+                                                                        rtoffset);
 
                tle = flatCopyTargetEntry(tle);
                tle->expr = (Expr *) newexpr;
@@ -1791,8 +1769,7 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset)
                                           (Node *) plan->qual,
                                           subplan_itlist,
                                           OUTER_VAR,
-                                          rtoffset,
-                                          agg_master);
+                                          rtoffset);
        pfree(subplan_itlist);
 }
 
@@ -2320,8 +2297,7 @@ fix_upper_expr(PlannerInfo *root,
                           Node *node,
                           indexed_tlist *subplan_itlist,
                           Index newvarno,
-                          int rtoffset,
-                          bool agg_master)
+                          int rtoffset)
 {
        fix_upper_expr_context context;
 
@@ -2329,7 +2305,6 @@ fix_upper_expr(PlannerInfo *root,
        context.subplan_itlist = subplan_itlist;
        context.newvarno = newvarno;
        context.rtoffset = rtoffset;
-       context.agg_master = agg_master;
        return fix_upper_expr_mutator(node, &context);
 }
 
@@ -2398,16 +2373,6 @@ fix_upper_expr_mutator(Node *node, fix_upper_expr_context *context)
                newvar = search_indexed_tlist_for_non_var(node,
                                                                                                  context->subplan_itlist,
                                                                                                  context->newvarno);
-#ifdef XCP
-               if (newvar && context->agg_master && IsA(node, Aggref))
-               {
-                       TargetEntry *newtle;
-                       Aggref *newnode = copyObject(node);
-                       newtle = makeTargetEntry((Expr *) newvar, 1, NULL, false);
-                       newnode->args = list_make1(newtle);
-                       return (Node *) newnode;
-               }
-#endif
                if (newvar)
                        return (Node *) newvar;
        }
index fbdaa2cce66cbcad5a872a723ab91ca222b4df6c..d811d09cca61cd32f7bbf0e8f6fddcc120b6612f 100644 (file)
@@ -706,16 +706,6 @@ typedef struct Group
        Oid                *grpOperators;       /* equality operators to compare with */
 } Group;
 
-#ifdef XCP
-typedef enum AggDistribution
-{
-       AGG_ONENODE,                            /* not distributed aggregation */
-       AGG_SLAVE,                                      /* execute only transient function */
-       AGG_MASTER                                      /* execute collection function as transient
-                                                                * and final finction */
-} AggDistribution;
-#endif
-
 /* ---------------
  *             aggregate node
  *
@@ -734,9 +724,6 @@ typedef struct Agg
 {
        Plan            plan;
        AggStrategy aggstrategy;        /* basic strategy, see nodes.h */
-#ifdef XCP
-       AggDistribution aggdistribution;
-#endif
        AggSplit        aggsplit;               /* agg-splitting mode, see nodes.h */
        int                     numCols;                /* number of grouping columns */
        AttrNumber *grpColIdx;          /* their indexes in the target list */