From: Tomas Vondra Date: Thu, 10 Nov 2016 22:34:40 +0000 (+0100) Subject: remove AggDistribution and Agg->aggdistribution X-Git-Tag: XL_10_R1BETA1~498 X-Git-Url: http://git.postgresql.org/gitweb/static/gitweb.js?a=commitdiff_plain;h=0124efc3634c5789ee713e4ffef4f3ed95be79c4;p=postgres-xl.git remove AggDistribution and Agg->aggdistribution As the plan is to rework the aggregate to use the infrastructure from 9.6, the best idea seems to be to use the partial aggregate stuff. Which means we can use the aggsplit field in Agg plan node, and we don't need the custom aggdistribution. Due to the upper-planner pathification, this should be at AggPath anyway (but we already have the aggsplit there). Furthermore, remove the custom plan construction from make_agg(). This was appropriate when the upper planner was not using paths, but that's not true anymore. The idea is to construct the usual Finalize+Partial aggregate nodes, possibly with a RemoteSubPath node in between them (when pushdown is not possible). I wonder how this will interact with the parallel query, as that is also using the Partial node, but I suppose that should work just fine. In the worst case we'll need to introduce something like GatherCombine, collecting (and combining) partial results on the data node before sending them to the coordinator. Thanks to using Finalize/Partial Agg nodes, we can also remove the changes from fix_upper_expr, or at least that's how it looks to me. At some point, bits of the removed code probably will need to be restored, although most likely different place. That applies in particular to: (a) find_referenced_cols_walker(), which checks what part of the aggregate we can push to the remote node (everything, partial or nothing) (b) the removed bits of make_agg() constructing the remote plan (although most of this will hopefully be handled by the partial aggregate infrastructure) --- diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index c91960ac31..fb02c165e2 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -1020,21 +1020,6 @@ ExplainNode(PlanState *planstate, List *ancestors, else partialmode = "Simple"; } -#ifdef XCP - switch (((Agg *) plan)->aggstrategy) - { - case AGG_SLAVE: - operation = "Transition"; - break; - case AGG_MASTER: - operation = "Collection"; - break; - default: - operation = NULL; - break; - } -#endif - break; case T_WindowAgg: pname = sname = "WindowAgg"; diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index cb657a8a4b..0cdd6559d0 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -888,9 +888,6 @@ _copyAgg(const Agg *from) CopyPlanFields((const Plan *) from, (Plan *) newnode); COPY_SCALAR_FIELD(aggstrategy); -#ifdef XCP - COPY_SCALAR_FIELD(aggdistribution); -#endif COPY_SCALAR_FIELD(aggsplit); COPY_SCALAR_FIELD(numCols); if (from->numCols > 0) diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 6c83bbbe07..21f47c0d84 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -1054,9 +1054,6 @@ _outAgg(StringInfo str, const Agg *node) _outPlanInfo(str, (const Plan *) node); WRITE_ENUM_FIELD(aggstrategy, AggStrategy); -#ifdef XCP - WRITE_ENUM_FIELD(aggdistribution, AggDistribution); -#endif WRITE_ENUM_FIELD(aggsplit, AggSplit); WRITE_INT_FIELD(numCols); diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c index 618b218756..7d2a348b0f 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -6431,141 +6431,6 @@ materialize_finished_plan(Plan *subplan) return matplan; } - -#ifdef XCP -typedef struct -{ - List *subtlist; - List *newtlist; -} find_referenced_cols_context; - -static bool -find_referenced_cols_walker(Node *node, find_referenced_cols_context *context) -{ - TargetEntry *tle; - - if (node == NULL) - return false; - if (IsA(node, Aggref)) - { - /* - * We can not push down aggregates with DISTINCT. - */ - if (((Aggref *) node)->aggdistinct) - return true; - - /* - * We can not push down aggregates with ORDER BY. - */ - if (((Aggref *) node)->aggorder) - return true; - - /* - * We need to add aggregate reference to the new tlist if it - * is not already there. Phase 1 aggregate is actually returns values - * of transition data type, so we should change the data type of the - * expression. - */ - if (!tlist_member(node, context->newtlist)) - { - Aggref *aggref = (Aggref *) node; - Aggref *newagg; - TargetEntry *newtle; - HeapTuple aggTuple; - Form_pg_aggregate aggform; - Oid aggtranstype; - - aggTuple = SearchSysCache1(AGGFNOID, - ObjectIdGetDatum(aggref->aggfnoid)); - if (!HeapTupleIsValid(aggTuple)) - elog(ERROR, "cache lookup failed for aggregate %u", - aggref->aggfnoid); - aggform = (Form_pg_aggregate) GETSTRUCT(aggTuple); - aggtranstype = aggform->aggtranstype; - ReleaseSysCache(aggTuple); - - /* Can not split two-phase aggregate */ - if (!OidIsValid(InvalidOid)) - return true; - - if (IsPolymorphicType(aggtranstype)) - { - Oid *inputTypes; - Oid *declaredArgTypes; - int agg_nargs; - int numArgs; - ListCell *l; - - inputTypes = (Oid *) palloc(sizeof(Oid) * list_length(aggref->args)); - numArgs = 0; - foreach(l, aggref->args) - { - TargetEntry *tle = (TargetEntry *) lfirst(l); - - if (!tle->resjunk) - inputTypes[numArgs++] = exprType((Node *) tle->expr); - } - - /* have to fetch the agg's declared input types... */ - (void) get_func_signature(aggref->aggfnoid, - &declaredArgTypes, &agg_nargs); - Assert(agg_nargs == numArgs); - - - aggtranstype = enforce_generic_type_consistency(inputTypes, - declaredArgTypes, - agg_nargs, - aggtranstype, - false); - pfree(inputTypes); - pfree(declaredArgTypes); - } - newagg = copyObject(aggref); - newagg->aggtype = aggtranstype; - - newtle = makeTargetEntry((Expr *) newagg, - list_length(context->newtlist) + 1, - NULL, - false); - context->newtlist = lappend(context->newtlist, newtle); - } - - return false; - } - /* - * If expression is in the subtlist copy it into new tlist - */ - tle = tlist_member(node, context->subtlist); - if (tle && !tlist_member((Node *) tle->expr, context->newtlist)) - { - TargetEntry *newtle; - newtle = makeTargetEntry((Expr *) copyObject(node), - list_length(context->newtlist) + 1, - tle->resname, - false); - context->newtlist = lappend(context->newtlist, newtle); - return false; - } - if (IsA(node, Var)) - { - /* - * Referenced Var is not a member of subtlist. - * Go ahead and add junk one. - */ - TargetEntry *newtle; - newtle = makeTargetEntry((Expr *) copyObject(node), - list_length(context->newtlist) + 1, - NULL, - true); - context->newtlist = lappend(context->newtlist, newtle); - return false; - } - return expression_tree_walker(node, find_referenced_cols_walker, - (void *) context); -} -#endif - - Agg * make_agg(List *tlist, List *qual, AggStrategy aggstrategy, AggSplit aggsplit, @@ -6575,9 +6440,6 @@ make_agg(List *tlist, List *qual, { Agg *node = makeNode(Agg); Plan *plan = &node->plan; -#ifdef XCP - RemoteSubplan *pushdown; -#endif long numGroups; /* Reduce to long, but 'ware overflow! */ @@ -6597,141 +6459,6 @@ make_agg(List *tlist, List *qual, plan->lefttree = lefttree; plan->righttree = NULL; -#ifdef XCP - /* - * If lefttree is a distributed subplan we may optimize aggregates by - * pushing down transition phase to remote data notes, and therefore reduce - * traffic and distribute evaluation load. - * We need to find all Var and Aggref expressions in tlist and qual and make - * up a new tlist from these expressions. Update original Vars. - * Create new Agg node with the new tlist and aggdistribution AGG_SLAVE. - * Set new Agg node as a lefttree of the distributed subplan, moving - * existing lefttree down under the new Agg node. Set new tlist to the - * distributed subplan - it should be matching to the subquery. - * Set node's aggdistribution to AGG_MASTER and continue node initialization - */ - pushdown = find_push_down_plan(lefttree, true); - if (pushdown) - { - find_referenced_cols_context context; - - context.subtlist = pushdown->scan.plan.targetlist; - context.newtlist = NIL; - if (find_referenced_cols_walker((Node *) tlist, &context) || - find_referenced_cols_walker((Node *) qual, &context)) - { - /* - * We found we can not push down this aggregate, clean up and - * fallback to default procedure - */ - node->aggdistribution = AGG_ONENODE; - } - else - { - Agg *phase1 = makeNode(Agg); - Plan *plan1 = &phase1->plan; - int i; - - phase1->aggdistribution = AGG_SLAVE; - phase1->aggstrategy = aggstrategy; - phase1->numCols = numGroupCols; - phase1->grpColIdx = grpColIdx; - phase1->grpOperators = grpOperators; - phase1->numGroups = numGroups; - - /* - * If we perform grouping we should make sure the grouping - * expressions are in the new tlist, and we should update indexes - * for the Phase2 aggregation node - */ - if (numGroupCols > 0) - { - AttrNumber *newGrpColIdx; - newGrpColIdx = (AttrNumber *) palloc(sizeof(AttrNumber) - * numGroupCols); - for (i = 0; i < numGroupCols; i++) - { - TargetEntry *tle; - TargetEntry *newtle; - - tle = (TargetEntry *) list_nth(context.subtlist, - grpColIdx[i] - 1); - newtle = tlist_member((Node *) tle->expr, context.newtlist); - if (newtle == NULL) - { - newtle = makeTargetEntry((Expr *) copyObject(tle->expr), - list_length(context.newtlist) + 1, - tle->resname, - false); - context.newtlist = lappend(context.newtlist, newtle); - } - newGrpColIdx[i] = newtle->resno; - } - node->grpColIdx = newGrpColIdx; - } - - /* - * If the pushdown plan is sorting update sort column indexes - */ - if (pushdown->sort) - { - SimpleSort *ssort = pushdown->sort; - for (i = 0; i < ssort->numCols; i++) - { - TargetEntry *tle; - TargetEntry *newtle; - - tle = (TargetEntry *) list_nth(context.subtlist, - grpColIdx[i] - 1); - newtle = tlist_member((Node *) tle->expr, context.newtlist); - if (newtle == NULL) - { - /* XXX maybe we should just remove the sort key ? */ - newtle = makeTargetEntry((Expr *) copyObject(tle->expr), - list_length(context.newtlist) + 1, - tle->resname, - false); - context.newtlist = lappend(context.newtlist, newtle); - } - ssort->sortColIdx[i] = newtle->resno; - } - } - - copy_plan_costsize(plan1, (Plan *) pushdown); // ??? - - /* - * We will produce a single output tuple if not grouping, and a tuple per - * group otherwise. - */ - if (aggstrategy == AGG_PLAIN) - plan1->plan_rows = 1; - else - plan1->plan_rows = numGroups; - - plan1->targetlist = context.newtlist; - plan1->qual = NIL; - plan1->lefttree = pushdown->scan.plan.lefttree; - pushdown->scan.plan.lefttree = plan1; - plan1->righttree = NULL; - - /* - * Update target lists of all plans from lefttree till phase1. - * All they should be the same if the tree is transparent for push - * down modification. - */ - while (lefttree != plan1) - { - lefttree->targetlist = context.newtlist; - lefttree = lefttree->lefttree; - } - - node->aggdistribution = AGG_MASTER; - } - } - else - node->aggdistribution = AGG_ONENODE; -#endif - return node; } diff --git a/src/backend/optimizer/plan/setrefs.c b/src/backend/optimizer/plan/setrefs.c index 344ebdc015..d5bc9e0760 100644 --- a/src/backend/optimizer/plan/setrefs.c +++ b/src/backend/optimizer/plan/setrefs.c @@ -69,9 +69,6 @@ typedef struct indexed_tlist *subplan_itlist; Index newvarno; int rtoffset; -#ifdef XCP - bool agg_master; -#endif } fix_upper_expr_context; typedef struct @@ -144,8 +141,7 @@ static Node *fix_upper_expr(PlannerInfo *root, Node *node, indexed_tlist *subplan_itlist, Index newvarno, - int rtoffset, - bool agg_master); + int rtoffset); static Node *fix_upper_expr_mutator(Node *node, fix_upper_expr_context *context); static List *set_returning_clause_references(PlannerInfo *root, @@ -1015,15 +1011,13 @@ set_indexonlyscan_references(PlannerInfo *root, (Node *) plan->scan.plan.targetlist, index_itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); plan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) plan->scan.plan.qual, index_itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); /* indexqual is already transformed to reference index columns */ plan->indexqual = fix_scan_list(root, plan->indexqual, rtoffset); @@ -1193,29 +1187,25 @@ set_foreignscan_references(PlannerInfo *root, (Node *) fscan->scan.plan.targetlist, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); fscan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) fscan->scan.plan.qual, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); fscan->fdw_exprs = (List *) fix_upper_expr(root, (Node *) fscan->fdw_exprs, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); fscan->fdw_recheck_quals = (List *) fix_upper_expr(root, (Node *) fscan->fdw_recheck_quals, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); pfree(itlist); /* fdw_scan_tlist itself just needs fix_scan_list() adjustments */ fscan->fdw_scan_tlist = @@ -1274,22 +1264,19 @@ set_customscan_references(PlannerInfo *root, (Node *) cscan->scan.plan.targetlist, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); cscan->scan.plan.qual = (List *) fix_upper_expr(root, (Node *) cscan->scan.plan.qual, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); cscan->custom_exprs = (List *) fix_upper_expr(root, (Node *) cscan->custom_exprs, itlist, INDEX_VAR, - rtoffset, - false); + rtoffset); pfree(itlist); /* custom_scan_tlist itself just needs fix_scan_list() adjustments */ @@ -1637,8 +1624,7 @@ set_join_references(PlannerInfo *root, Join *join, int rtoffset) (Node *) nlp->paramval, outer_itlist, OUTER_VAR, - rtoffset, - false); + rtoffset); /* Check we replaced any PlaceHolderVar with simple Var */ if (!(IsA(nlp->paramval, Var) && @@ -1741,12 +1727,6 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) indexed_tlist *subplan_itlist; List *output_targetlist; ListCell *l; -#ifdef XCP - bool agg_master; - - agg_master = (IsA(plan, Agg) && - ((Agg *) plan)->aggdistribution == AGG_MASTER); -#endif subplan_itlist = build_tlist_index(subplan->targetlist); @@ -1769,16 +1749,14 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) (Node *) tle->expr, subplan_itlist, OUTER_VAR, - rtoffset, - agg_master); + rtoffset); } else newexpr = fix_upper_expr(root, (Node *) tle->expr, subplan_itlist, OUTER_VAR, - rtoffset, - agg_master); + rtoffset); tle = flatCopyTargetEntry(tle); tle->expr = (Expr *) newexpr; @@ -1791,8 +1769,7 @@ set_upper_references(PlannerInfo *root, Plan *plan, int rtoffset) (Node *) plan->qual, subplan_itlist, OUTER_VAR, - rtoffset, - agg_master); + rtoffset); pfree(subplan_itlist); } @@ -2320,8 +2297,7 @@ fix_upper_expr(PlannerInfo *root, Node *node, indexed_tlist *subplan_itlist, Index newvarno, - int rtoffset, - bool agg_master) + int rtoffset) { fix_upper_expr_context context; @@ -2329,7 +2305,6 @@ fix_upper_expr(PlannerInfo *root, context.subplan_itlist = subplan_itlist; context.newvarno = newvarno; context.rtoffset = rtoffset; - context.agg_master = agg_master; return fix_upper_expr_mutator(node, &context); } @@ -2398,16 +2373,6 @@ fix_upper_expr_mutator(Node *node, fix_upper_expr_context *context) newvar = search_indexed_tlist_for_non_var(node, context->subplan_itlist, context->newvarno); -#ifdef XCP - if (newvar && context->agg_master && IsA(node, Aggref)) - { - TargetEntry *newtle; - Aggref *newnode = copyObject(node); - newtle = makeTargetEntry((Expr *) newvar, 1, NULL, false); - newnode->args = list_make1(newtle); - return (Node *) newnode; - } -#endif if (newvar) return (Node *) newvar; } diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h index fbdaa2cce6..d811d09cca 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -706,16 +706,6 @@ typedef struct Group Oid *grpOperators; /* equality operators to compare with */ } Group; -#ifdef XCP -typedef enum AggDistribution -{ - AGG_ONENODE, /* not distributed aggregation */ - AGG_SLAVE, /* execute only transient function */ - AGG_MASTER /* execute collection function as transient - * and final finction */ -} AggDistribution; -#endif - /* --------------- * aggregate node * @@ -734,9 +724,6 @@ typedef struct Agg { Plan plan; AggStrategy aggstrategy; /* basic strategy, see nodes.h */ -#ifdef XCP - AggDistribution aggdistribution; -#endif AggSplit aggsplit; /* agg-splitting mode, see nodes.h */ int numCols; /* number of grouping columns */ AttrNumber *grpColIdx; /* their indexes in the target list */