Run ANALYZE (COORDINATOR) on remote coordinators iff running outside a txn

block

We'd seen some distributed deadlocks when ANALYZE (COORDINATOR) is run on
remote coordinators, in parallel with VACUUM FULL on catalog tables. The
investigations so far indicate that the auxilliary datanode connections from
the other coordinators can lead to a distributed deadlock as the connections
from the originating coordinator still have the transaction open.

We now restrict remote analyze only when we are not running inside a user
transaction block. This allows us to commit the first transaction and then
start a new transaction to run ANALYZE (COORDINATOR) on the remote
coordinators, thus avoiding the distributed deadlock.

Per report from Pallavi Sontakke and lots of analysis by me, which may not
still be sufficient.

diff --git a/src/backend/commands/analyze.c b/src/backend/commands/analyze.c
index 44adb3494eede4a68b9cd2a7e427eac79b7ffe1f..ec4888e9e2dc82df98716f1f27267918d9fd2de3 100644 (file)
--- a/src/backend/commands/analyze.c
+++ b/src/backend/commands/analyze.c
@@ -117,7 +117,6 @@ static Datum ind_fetch_func(VacAttrStatsP stats, int rownum, bool *isNull);
 static void analyze_rel_coordinator(Relation onerel, bool inh, int attr_cnt,
                                                VacAttrStats **vacattrstats, int nindexes,
                                                Relation *indexes, AnlIndexData *indexdata);
-static void analyze_remote_coordinators(Relation onerel);
 #endif
 
 /*
@@ -130,6 +129,7 @@ static void analyze_remote_coordinators(Relation onerel);
 void
 analyze_rel(Oid relid, RangeVar *relation, int options,
                        VacuumParams *params, List *va_cols, bool in_outer_xact,
+                       bool *need_remote_analyze,
                        BufferAccessStrategy bstrategy)
 {
        Relation        onerel;
@@ -284,6 +284,13 @@ analyze_rel(Oid relid, RangeVar *relation, int options,
        MyPgXact->vacuumFlags |= PROC_IN_ANALYZE;
        LWLockRelease(ProcArrayLock);
 
+       /*
+        * If distribited relation, then ANALYZE (COORDINATOR) on the other
+        * coordinators is required.
+        */
+       if (onerel->rd_locator_info)
+               *need_remote_analyze = true;
+
        /*
         * Do the normal non-recursive ANALYZE.  We can skip this for partitioned
         * tables, which don't contain any rows.
@@ -513,11 +520,6 @@ do_analyze_rel(Relation onerel, int options, VacuumParams *params,
                analyze_rel_coordinator(onerel, inh, attr_cnt, vacattrstats,
                                                                nindexes, Irel, indexdata);
 
-               /*
-                * Run ANALYZE on remote coordinators as well.
-                */
-               analyze_remote_coordinators(onerel);
-
                /*
                 * Skip acquiring local stats. Coordinator does not store data of
                 * distributed tables.
@@ -3618,20 +3620,37 @@ analyze_rel_coordinator(Relation onerel, bool inh, int attr_cnt,
        coord_collect_extended_stats(onerel, attr_cnt);
 }
 
-static void
-analyze_remote_coordinators(Relation onerel)
+void
+analyze_remote_coordinators(Oid relid, RangeVar *relation, int options,
+               VacuumParams *params)
 {
        RemoteQuery             *step;
        StringInfoData  sql;
+       Relation                onerel;
 
        if (IsConnFromCoord())
                return;
 
+       if (!(options & VACOPT_NOWAIT))
+               onerel = try_relation_open(relid, ShareUpdateExclusiveLock);
+       else if (ConditionalLockRelationOid(relid, ShareUpdateExclusiveLock))
+               onerel = try_relation_open(relid, NoLock);
+       else
+       {
+               onerel = NULL;
+               if (relation &&
+                               IsAutoVacuumWorkerProcess() && params->log_min_duration >= 0)
+                       ereport(LOG,
+                                       (errcode(ERRCODE_LOCK_NOT_AVAILABLE),
+                                        errmsg("skipping remote analyze of \"%s\" --- lock not available",
+                                                relation->relname)));
+               return;
+       }
        /*
         * Temporary tables are not available on remote coordinators.
         */
        if (onerel->rd_rel->relpersistence == RELPERSISTENCE_TEMP)
-               return;
+               goto cleanup;
 
        initStringInfo(&sql);
        appendStringInfo(&sql, "ANALYZE (COORDINATOR) %s.%s",
@@ -3650,5 +3669,8 @@ analyze_remote_coordinators(Relation onerel)
 
        /* Be sure to advance the command counter after the last command */
        CommandCounterIncrement();
+
+cleanup:
+       relation_close(onerel, NoLock);
 }
 #endif

diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c
index e0d8901d60739d371b32473159d05a6e45bbedb6..fc1b53eb3fff42402458eab0e78bf58f98d8a113 100644 (file)
--- a/src/backend/commands/vacuum.c
+++ b/src/backend/commands/vacuum.c
@@ -169,6 +169,7 @@ vacuum(int options, RangeVar *relation, Oid relid, VacuumParams *params,
                                use_own_xacts;
        List       *relations;
        static bool in_vacuum = false;
+       bool            need_remote_analyze = false;
 
        Assert(params != NULL);
 
@@ -334,12 +335,42 @@ vacuum(int options, RangeVar *relation, Oid relid, VacuumParams *params,
                                }
 
                                analyze_rel(relid, relation, options, params,
-                                                       va_cols, in_outer_xact, vac_strategy);
+                                                       va_cols, in_outer_xact, &need_remote_analyze,
+                                                       vac_strategy);
 
                                if (use_own_xacts)
                                {
                                        PopActiveSnapshot();
                                        CommitTransactionCommand();
+
+                                       if (IS_PGXC_COORDINATOR && need_remote_analyze)
+                                       {
+                                               StartTransactionCommand();
+                                               PushActiveSnapshot(GetTransactionSnapshot());
+
+                                               /*
+                                                * Run ANALYZE on remote coordinators as well, if we're
+                                                * running in our own transaction. We have this
+                                                * restriction because the remote coordinator will
+                                                * connect to the datanodes, which may have the current
+                                                * transaction open originating from us and not
+                                                * prepared to handle a auxilliary connection from
+                                                * other coordinators.
+                                                *
+                                                * This implies that ANALYZE executed inside a user
+                                                * transaction block won't be sent down other
+                                                * coordinators and the stats may remain stale on those
+                                                * coordinators. But at the least, this takes care of
+                                                * cases of auto-analyzing and explicit ANALYZE outside
+                                                * a transaction block.
+                                                */
+                                               analyze_remote_coordinators(relid, relation, options,
+                                                               params);
+
+                                               PopActiveSnapshot();
+                                               CommitTransactionCommand();
+
+                                       }
                                }
                        }
                }

diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h
index 307ec96cc64cca4e95ef62cec85633222a1fbb6a..3bcf2b20268e01ffeec7e0a536b02b8ec536d3c6 100644 (file)
--- a/src/include/commands/vacuum.h
+++ b/src/include/commands/vacuum.h
@@ -199,8 +199,11 @@ extern void lazy_vacuum_rel(Relation onerel, int options,
 /* in commands/analyze.c */
 extern void analyze_rel(Oid relid, RangeVar *relation, int options,
                        VacuumParams *params, List *va_cols, bool in_outer_xact,
+                       bool *need_remote_analyze,
                        BufferAccessStrategy bstrategy);
 extern bool std_typanalyze(VacAttrStats *stats);
+extern void analyze_remote_coordinators(Oid relid, RangeVar *relation,
+                       int options, VacuumParams *params);
 
 /* in utils/misc/sampling.c --- duplicate of declarations in utils/sampling.h */
 extern double anl_random_fract(void);