Add debug facility to GTM_RWLock

This can be turned on by #define GTM_LOCK_DEBUG and should be useful for
deadlock detections and other such causes

diff --git a/src/gtm/common/gtm_lock.c b/src/gtm/common/gtm_lock.c
index 9cc368aafdf16eb036d98d9f557729c3fc547c0f..53ccd0a463d8a3a6ea06904ab85c677fba1f3a46 100644 (file)
--- a/src/gtm/common/gtm_lock.c
+++ b/src/gtm/common/gtm_lock.c
@@ -29,15 +29,101 @@ bool
 GTM_RWLockAcquire(GTM_RWLock *lock, GTM_LockMode mode)
 {
        int status = EINVAL;
+#ifdef GTM_LOCK_DEBUG
+       int indx;
+       int ii;
+#endif
 
        switch (mode)
        {
                case GTM_LOCKMODE_WRITE:
+#ifdef GTM_LOCK_DEBUG
+                       pthread_mutex_lock(&lock->lk_debug_mutex);
+                       for (ii = 0; ii < lock->rd_holders_count; ii++)
+                       {
+                               if (pthread_equal(lock->rd_holders[ii], pthread_self()))
+                                       elog(WARNING, "Thread %p already owns a read-lock and may deadlock",
+                                                       (void *) pthread_self());
+                       }
+                       if (pthread_equal(lock->wr_owner, pthread_self()))
+                               elog(WARNING, "Thread %p already owns a write-lock and may deadlock",
+                                               (void *) pthread_self());
+                       indx = lock->wr_waiters_count;
+                       if (indx < GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+                               lock->wr_waiters[lock->wr_waiters_count++] = pthread_self();
+                       else
+                               indx = -1;
+                       pthread_mutex_unlock(&lock->lk_debug_mutex);
+#endif
                        status = pthread_rwlock_wrlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+                       if (!status)
+                       {
+                               pthread_mutex_lock(&lock->lk_debug_mutex);
+                               lock->wr_granted = true;
+                               lock->wr_owner = pthread_self();
+                               lock->rd_holders_count = 0;
+                               lock->rd_holders_overflow = false;
+                               if (indx != -1)
+                               {
+                                       lock->wr_waiters[indx] = 0;
+                                       lock->wr_waiters_count--;
+                               }
+                               pthread_mutex_unlock(&lock->lk_debug_mutex);
+                       }
+                       else
+                               elog(ERROR, "pthread_rwlock_wrlock returned %d", status);
+#endif
                        break;
 
                case GTM_LOCKMODE_READ:
+#ifdef GTM_LOCK_DEBUG
+                       pthread_mutex_lock(&lock->lk_debug_mutex);
+                       if (lock->wr_waiters_count > 0)
+                       {
+                               for (ii = 0; ii < lock->rd_holders_count; ii++)
+                               {
+                                       if (pthread_equal(lock->rd_holders[ii], pthread_self()))
+                                               elog(WARNING, "Thread %p already owns a read-lock and "
+                                                               "there are blocked writers - this may deadlock",
+                                                                       (void *) pthread_self());
+                               }
+                       }
+                       if (pthread_equal(lock->wr_owner, pthread_self()))
+                               elog(WARNING, "Thread %p already owns a write-lock and may deadlock",
+                                               (void *) pthread_self());
+                       indx = lock->rd_waiters_count;
+                       if (indx < GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+                               lock->rd_waiters[lock->rd_waiters_count++] = pthread_self();
+                       else
+                               indx = -1;
+                       pthread_mutex_unlock(&lock->lk_debug_mutex);
+#endif
+                       /* Now acquire the lock */
                        status = pthread_rwlock_rdlock(&lock->lk_lock);
+
+#ifdef GTM_LOCK_DEBUG
+                       if (!status)
+                       {
+                               pthread_mutex_lock(&lock->lk_debug_mutex);
+                               lock->wr_granted = false;
+                               if (lock->rd_holders_count == GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+                                       lock->rd_holders_overflow = true;
+                               else
+                               {
+                                       lock->rd_holders[lock->rd_holders_count++] = pthread_self();
+                                       lock->rd_holders_overflow = false;
+                                       if (indx != -1)
+                                       {
+                                               lock->rd_waiters[indx] = 0;
+                                               lock->rd_waiters_count--;
+                                       }
+                               }
+                               pthread_mutex_unlock(&lock->lk_debug_mutex);
+                       }
+                       else
+                               elog(ERROR, "pthread_rwlock_rdlock returned %d", status);
+#endif
                        break;
 
                default:
@@ -56,6 +142,42 @@ GTM_RWLockRelease(GTM_RWLock *lock)
 {
        int status;
        status = pthread_rwlock_unlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+       if (status)
+               elog(PANIC, "pthread_rwlock_unlock returned %d", status);
+       else
+       {
+               pthread_mutex_lock(&lock->lk_debug_mutex);
+               if (lock->wr_granted)
+               {
+                       Assert(pthread_equal(lock->wr_owner, pthread_self()));
+                       lock->wr_granted = false;
+                       lock->wr_owner = 0;
+               }
+               else
+               {
+                       int ii;
+                       bool found = false;
+                       for (ii = 0; ii < lock->rd_holders_count; ii++)
+                       {
+                               if (pthread_equal(lock->rd_holders[ii], pthread_self()))
+                               {
+                                       found = true;
+                                       lock->rd_holders[ii] =
+                                               lock->rd_holders[lock->rd_holders_count - 1];
+                                       lock->rd_holders_count--;
+                                       lock->rd_holders[lock->rd_holders_count] = 0;
+                                       break;
+                               }
+                       }
+
+                       if (!found && !lock->rd_holders_overflow)
+                               elog(PANIC, "Thread %p does not own a read-lock",
+                                               (void *)pthread_self());
+               }
+               pthread_mutex_unlock(&lock->lk_debug_mutex);
+       }
+#endif
        return status ? false : true;
 }
 
@@ -65,6 +187,10 @@ GTM_RWLockRelease(GTM_RWLock *lock)
 int
 GTM_RWLockInit(GTM_RWLock *lock)
 {
+#ifdef GTM_LOCK_DEBUG
+       memset(lock, 0, sizeof (GTM_RWLock));
+       pthread_mutex_init(&lock->lk_debug_mutex, NULL);
+#endif
        return pthread_rwlock_init(&lock->lk_lock, NULL);
 }
 
@@ -92,10 +218,38 @@ GTM_RWLockConditionalAcquire(GTM_RWLock *lock, GTM_LockMode mode)
        {
                case GTM_LOCKMODE_WRITE:
                        status = pthread_rwlock_trywrlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+                       if (!status)
+                       {
+                               pthread_mutex_lock(&lock->lk_debug_mutex);
+                               lock->wr_granted = true;
+                               lock->wr_owner = pthread_self();
+                               lock->rd_holders_count = 0;
+                               lock->rd_holders_overflow = false;
+                               pthread_mutex_unlock(&lock->lk_debug_mutex);
+                       }
+#endif
                        break;
 
                case GTM_LOCKMODE_READ:
                        status = pthread_rwlock_tryrdlock(&lock->lk_lock);
+#ifdef GTM_LOCK_DEBUG
+                       if (!status)
+                       {
+                               pthread_mutex_lock(&lock->lk_debug_mutex);
+                               if (lock->rd_holders_count == GTM_LOCK_DEBUG_MAX_READ_TRACKERS)
+                               {
+                                       elog(WARNING, "Too many threads waiting for a read-lock");
+                                       lock->rd_holders_overflow = true;
+                               }
+                               else
+                               {
+                                       lock->rd_holders[lock->rd_holders_count++] = pthread_self();
+                                       lock->rd_holders_overflow = false;
+                               }
+                               pthread_mutex_unlock(&lock->lk_debug_mutex);
+                       }
+#endif
                        break;
 
                default:

diff --git a/src/gtm/main/gtm_standby.c b/src/gtm/main/gtm_standby.c
index 045545532393aba6d0e460aa66f0fa2bcbe07660..45b545b1072eb30c170562ef93d94c13a7c4cfc6 100644 (file)
--- a/src/gtm/main/gtm_standby.c
+++ b/src/gtm/main/gtm_standby.c
@@ -47,9 +47,6 @@ gtm_standby_start_startup(void)
        }
        elog(LOG, "Connection established to the GTM active.");
 
-       /* Initialize standby lock */
-       Recovery_InitStandbyLock();
-
        return 1;
 }
 

diff --git a/src/gtm/main/main.c b/src/gtm/main/main.c
index 0f3d89f08ceb669c1edc7cb3f633f29d14ae1a15..d122591f32988cf460d0e1be702094b8fc40c838 100644 (file)
--- a/src/gtm/main/main.c
+++ b/src/gtm/main/main.c
@@ -213,6 +213,9 @@ InitGTMProcess()
 static void
 BaseInit()
 {
+       /* Initialize standby lock before doing anything else */
+       Recovery_InitStandbyLock();
+
        checkDataDir();
        SetDataDir();
        ChangeToDataDir();
@@ -547,18 +550,6 @@ main(int argc, char *argv[])
                free(dest_port);
                dest_port = NULL;
        }
-       /*
-        * Check options for the standby mode.
-        */
-       if (Recovery_IsStandby())
-       {
-               if (active_addr == NULL || active_port < 1)
-               {
-                       help(argv[0]);
-                       exit(1);
-               }
-               Recovery_StandbySetConnInfo(active_addr, active_port);
-       }
 
        if (GTMDataDir == NULL)
        {
@@ -602,6 +593,20 @@ main(int argc, char *argv[])
         */
        BaseInit();
 
+       /*
+        * Check options for the standby mode. Do it after StandbyLock has been
+        * initialised in BaseInit()
+        */
+       if (Recovery_IsStandby())
+       {
+               if (active_addr == NULL || active_port < 1)
+               {
+                       help(argv[0]);
+                       exit(1);
+               }
+               Recovery_StandbySetConnInfo(active_addr, active_port);
+       }
+
        /*
         * Establish a connection between the active and standby.
         */

diff --git a/src/include/gtm/gtm_lock.h b/src/include/gtm/gtm_lock.h
index c7f79dd959414635f9c46a5feaf4bd2284206450..26e8faee49a19db33f082f4b3bc0695b63891842 100644 (file)
--- a/src/include/gtm/gtm_lock.h
+++ b/src/include/gtm/gtm_lock.h
@@ -16,10 +16,23 @@
 #define GTM_LOCK_H
 
 #include <pthread.h>
-
 typedef struct GTM_RWLock
 {
        pthread_rwlock_t lk_lock;
+#ifdef GTM_LOCK_DEBUG
+#define GTM_LOCK_DEBUG_MAX_READ_TRACKERS       1024
+       pthread_mutex_t lk_debug_mutex;
+       int                             wr_waiters_count;
+       pthread_t               wr_waiters[GTM_LOCK_DEBUG_MAX_READ_TRACKERS];
+       bool                    wr_granted;
+       pthread_t               wr_owner;
+       int                             rd_holders_count;
+       bool                    rd_holders_overflow;
+       pthread_t               rd_holders[GTM_LOCK_DEBUG_MAX_READ_TRACKERS];
+       int                             rd_waiters_count;
+       bool                    rd_waiters_overflow;
+       pthread_t               rd_waiters[GTM_LOCK_DEBUG_MAX_READ_TRACKERS];
+#endif
 } GTM_RWLock;
 
 typedef struct GTM_MutexLock