]> git.sur5r.net Git - openldap/blobdiff - libraries/libldap_r/tpool.c
ITS#7270 Protect accesses to ldap_int_hostname with a mutex.
[openldap] / libraries / libldap_r / tpool.c
index 6884f32fc3d4d1cb9900dc1981c60f421a823e9f..140e1af55a4de75211b14abd535d253bafbd62cb 100644 (file)
@@ -1,7 +1,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 1998-2007 The OpenLDAP Foundation.
+ * Copyright 1998-2012 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -17,6 +17,7 @@
 
 #include <stdio.h>
 
+#include <ac/signal.h>
 #include <ac/stdarg.h>
 #include <ac/stdlib.h>
 #include <ac/string.h>
 
 #ifndef LDAP_THREAD_HAVE_TPOOL
 
-typedef enum ldap_int_thread_pool_state_e {
-       LDAP_INT_THREAD_POOL_RUNNING,
-       LDAP_INT_THREAD_POOL_FINISHING,
-       LDAP_INT_THREAD_POOL_STOPPING
-} ldap_int_thread_pool_state_t;
-
 /* Thread-specific key with data and optional free function */
-typedef struct ldap_int_thread_key_s {
+typedef struct ldap_int_tpool_key_s {
        void *ltk_key;
        void *ltk_data;
        ldap_pvt_thread_pool_keyfree_t *ltk_free;
-} ldap_int_thread_key_t;
+} ldap_int_tpool_key_t;
 
 /* Max number of thread-specific keys we store per thread.
  * We don't expect to use many...
@@ -52,10 +47,16 @@ typedef struct ldap_int_thread_key_s {
 /* Max number of threads */
 #define        LDAP_MAXTHR     1024    /* must be a power of 2 */
 
+/* (Theoretical) max number of pending requests */
+#define MAX_PENDING (INT_MAX/2)        /* INT_MAX - (room to avoid overflow) */
+
+/* pool->ltp_pause values */
+enum { NOT_PAUSED = 0, WANT_PAUSE = 1, PAUSED = 2 };
+
 /* Context: thread ID and thread-specific key/data pairs */
 typedef struct ldap_int_thread_userctx_s {
        ldap_pvt_thread_t ltu_id;
-       ldap_int_thread_key_t ltu_key[MAXKEYS];
+       ldap_int_tpool_key_t ltu_key[MAXKEYS];
 } ldap_int_thread_userctx_t;
 
 
@@ -88,6 +89,8 @@ typedef struct ldap_int_thread_task_s {
        void *ltt_arg;
 } ldap_int_thread_task_t;
 
+typedef LDAP_STAILQ_HEAD(tcq, ldap_int_thread_task_s) ldap_int_tpool_plist_t;
+
 struct ldap_int_thread_pool_s {
        LDAP_STAILQ_ENTRY(ldap_int_thread_pool_s) ltp_next;
 
@@ -100,23 +103,56 @@ struct ldap_int_thread_pool_s {
        /* ltp_active_count <= 1 && ltp_pause */
        ldap_pvt_thread_cond_t ltp_pcond;
 
+       /* ltp_pause == 0 ? &ltp_pending_list : &empty_pending_list,
+        * maintaned to reduce work for pool_wrapper()
+        */
+       ldap_int_tpool_plist_t *ltp_work_list;
+
        /* pending tasks, and unused task objects */
-       LDAP_STAILQ_HEAD(tcq, ldap_int_thread_task_s) ltp_pending_list;
+       ldap_int_tpool_plist_t ltp_pending_list;
        LDAP_SLIST_HEAD(tcl, ldap_int_thread_task_s) ltp_free_list;
 
-       ldap_int_thread_pool_state_t ltp_state;
+       /* The pool is finishing, waiting for its threads to close.
+        * They close when ltp_pending_list is done.  pool_submit()
+        * rejects new tasks.  ltp_max_pending = -(its old value).
+        */
+       int ltp_finishing;
+
+       /* Some active task needs to be the sole active task.
+        * Atomic variable so ldap_pvt_thread_pool_pausing() can read it.
+        * Note: Pauses adjust ltp_<open_count/vary_open_count/work_list>,
+        * so pool_<submit/wrapper>() mostly can avoid testing ltp_pause.
+        */
+       volatile sig_atomic_t ltp_pause;
+
+       /* Max number of threads in pool, or 0 for default (LDAP_MAXTHR) */
+       int ltp_max_count;
+
+       /* Max pending + paused + idle tasks, negated when ltp_finishing */
+       int ltp_max_pending;
 
-       /* some active request needs to be the sole active request */
-       int ltp_pause;
+       int ltp_pending_count;          /* Pending + paused + idle tasks */
+       int ltp_active_count;           /* Active, not paused/idle tasks */
+       int ltp_open_count;                     /* Number of threads, negated when ltp_pause */
+       int ltp_starting;                       /* Currenlty starting threads */
 
-       long ltp_max_count;                     /* max number of threads in pool, or 0 */
-       long ltp_max_pending;           /* max pending or paused requests, or 0 */
-       long ltp_pending_count;         /* pending or paused requests */
-       long ltp_active_count;          /* active, not paused requests */
-       long ltp_open_count;            /* number of threads */
-       long ltp_starting;                      /* currenlty starting threads */
+       /* >0 if paused or we may open a thread, <0 if we should close a thread.
+        * Updated when ltp_<finishing/pause/max_count/open_count> change.
+        * Maintained to reduce the time ltp_mutex must be locked in
+        * ldap_pvt_thread_pool_<submit/wrapper>().
+        */
+       int ltp_vary_open_count;
+#      define SET_VARY_OPEN_COUNT(pool)        \
+               ((pool)->ltp_vary_open_count =  \
+                (pool)->ltp_pause      ?  1 :  \
+                (pool)->ltp_finishing  ? -1 :  \
+                ((pool)->ltp_max_count ? (pool)->ltp_max_count : LDAP_MAXTHR) \
+                - (pool)->ltp_open_count)
 };
 
+static ldap_int_tpool_plist_t empty_pending_list =
+       LDAP_STAILQ_HEAD_INITIALIZER(empty_pending_list);
+
 static int ldap_int_has_thread_pool = 0;
 static LDAP_STAILQ_HEAD(tpq, ldap_int_thread_pool_s)
        ldap_int_thread_pool_list =
@@ -126,6 +162,8 @@ static ldap_pvt_thread_mutex_t ldap_pvt_thread_pool_mutex;
 
 static void *ldap_int_thread_pool_wrapper( void *pool );
 
+static ldap_pvt_thread_key_t   ldap_tpool_key;
+
 /* Context of the main thread */
 static ldap_int_thread_userctx_t ldap_int_main_thrctx;
 
@@ -133,6 +171,7 @@ int
 ldap_int_thread_pool_startup ( void )
 {
        ldap_int_main_thrctx.ltu_id = ldap_pvt_thread_self();
+       ldap_pvt_thread_key_create( &ldap_tpool_key );
        return ldap_pvt_thread_mutex_init(&ldap_pvt_thread_pool_mutex);
 }
 
@@ -145,6 +184,7 @@ ldap_int_thread_pool_shutdown ( void )
                (ldap_pvt_thread_pool_destroy)(&pool, 0); /* ignore thr_debug macro */
        }
        ldap_pvt_thread_mutex_destroy(&ldap_pvt_thread_pool_mutex);
+       ldap_pvt_thread_key_destroy( ldap_tpool_key );
        return(0);
 }
 
@@ -164,8 +204,8 @@ ldap_pvt_thread_pool_init (
 
        if (! (0 <= max_threads && max_threads <= LDAP_MAXTHR))
                max_threads = 0;
-       if (max_pending < 0)
-               max_pending = 0;
+       if (! (1 <= max_pending && max_pending <= MAX_PENDING))
+               max_pending = MAX_PENDING;
 
        *tpool = NULL;
        pool = (ldap_pvt_thread_pool_t) LDAP_CALLOC(1,
@@ -184,49 +224,25 @@ ldap_pvt_thread_pool_init (
                return(rc);
 
        ldap_int_has_thread_pool = 1;
-       pool->ltp_state = LDAP_INT_THREAD_POOL_RUNNING;
+
        pool->ltp_max_count = max_threads;
+       SET_VARY_OPEN_COUNT(pool);
        pool->ltp_max_pending = max_pending;
+
        LDAP_STAILQ_INIT(&pool->ltp_pending_list);
+       pool->ltp_work_list = &pool->ltp_pending_list;
        LDAP_SLIST_INIT(&pool->ltp_free_list);
+
        ldap_pvt_thread_mutex_lock(&ldap_pvt_thread_pool_mutex);
        LDAP_STAILQ_INSERT_TAIL(&ldap_int_thread_pool_list, pool, ltp_next);
        ldap_pvt_thread_mutex_unlock(&ldap_pvt_thread_pool_mutex);
 
-#if 0
-       /* THIS WILL NOT WORK on some systems.  If the process
-        * forks after starting a thread, there is no guarantee
-        * that the thread will survive the fork.  For example,
-        * slapd forks in order to daemonize, and does so after
-        * calling ldap_pvt_thread_pool_init.  On some systems,
-        * this initial thread does not run in the child process,
-        * but ltp_open_count == 1, so two things happen: 
-        * 1) the first client connection fails, and 2) when
-        * slapd is kill'ed, it never terminates since it waits
-        * for all worker threads to exit. */
-
-       /* start up one thread, just so there is one. no need to
-        * lock the mutex right now, since no threads are running.
+       /* Start no threads just yet.  That can break if the process forks
+        * later, as slapd does in order to daemonize.  On at least POSIX,
+        * only the forking thread would survive in the child.  Yet fork()
+        * can't unlock/clean up other threads' locks and data structures,
+        * unless pthread_atfork() handlers have been set up to do so.
         */
-       pool->ltp_open_count++;
-
-       ldap_pvt_thread_t thr;
-       rc = ldap_pvt_thread_create( &thr, 1, ldap_int_thread_pool_wrapper, pool );
-
-       if( rc != 0) {
-               /* couldn't start one?  then don't start any */
-               ldap_pvt_thread_mutex_lock(&ldap_pvt_thread_pool_mutex);
-               LDAP_STAILQ_REMOVE(ldap_int_thread_pool_list, pool, 
-                       ldap_int_thread_pool_s, ltp_next);
-               ldap_int_has_thread_pool = 0;
-               ldap_pvt_thread_mutex_unlock(&ldap_pvt_thread_pool_mutex);
-               ldap_pvt_thread_cond_destroy(&pool->ltp_pcond);
-               ldap_pvt_thread_cond_destroy(&pool->ltp_cond);
-               ldap_pvt_thread_mutex_destroy(&pool->ltp_mutex);
-               LDAP_FREE(pool);
-               return(-1);
-       }
-#endif
 
        *tpool = pool;
        return(0);
@@ -252,23 +268,17 @@ ldap_pvt_thread_pool_submit (
                return(-1);
 
        ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
-       if (pool->ltp_state != LDAP_INT_THREAD_POOL_RUNNING
-               || (pool->ltp_max_pending
-                       && pool->ltp_pending_count >= pool->ltp_max_pending))
-       {
-               ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
-               return(-1);
-       }
+
+       if (pool->ltp_pending_count >= pool->ltp_max_pending)
+               goto failed;
 
        task = LDAP_SLIST_FIRST(&pool->ltp_free_list);
        if (task) {
                LDAP_SLIST_REMOVE_HEAD(&pool->ltp_free_list, ltt_next.l);
        } else {
                task = (ldap_int_thread_task_t *) LDAP_MALLOC(sizeof(*task));
-               if (task == NULL) {
-                       ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
-                       return(-1);
-               }
+               if (task == NULL)
+                       goto failed;
        }
 
        task->ltt_start_routine = start_routine;
@@ -276,17 +286,18 @@ ldap_pvt_thread_pool_submit (
 
        pool->ltp_pending_count++;
        LDAP_STAILQ_INSERT_TAIL(&pool->ltp_pending_list, task, ltt_next.q);
-       if (pool->ltp_pause) {
-               ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
-               return(0);
-       }
-       ldap_pvt_thread_cond_signal(&pool->ltp_cond);
-       if (pool->ltp_open_count < pool->ltp_active_count + pool->ltp_pending_count
-               && (pool->ltp_open_count <
-                       (pool->ltp_max_count ? pool->ltp_max_count : LDAP_MAXTHR)))
+
+       /* true if ltp_pause != 0 or we should open (create) a thread */
+       if (pool->ltp_vary_open_count > 0 &&
+               pool->ltp_open_count < pool->ltp_active_count+pool->ltp_pending_count)
        {
-               pool->ltp_open_count++;
+               if (pool->ltp_pause)
+                       goto done;
+
                pool->ltp_starting++;
+               pool->ltp_open_count++;
+               SET_VARY_OPEN_COUNT(pool);
+
                if (0 != ldap_pvt_thread_create(
                        &thr, 1, ldap_int_thread_pool_wrapper, pool))
                {
@@ -295,6 +306,8 @@ ldap_pvt_thread_pool_submit (
                         */
                        pool->ltp_starting--;
                        pool->ltp_open_count--;
+                       SET_VARY_OPEN_COUNT(pool);
+
                        if (pool->ltp_open_count == 0) {
                                /* no open threads at all?!?
                                 */
@@ -310,24 +323,69 @@ ldap_pvt_thread_pool_submit (
                                         * back out of ltp_pending_count, free the task,
                                         * report the error.
                                         */
+                                       pool->ltp_pending_count--;
                                        LDAP_STAILQ_REMOVE(&pool->ltp_pending_list, task,
                                                ldap_int_thread_task_s, ltt_next.q);
-                                       pool->ltp_pending_count--;
-                                       ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
-                                       LDAP_FREE(task);
-                                       return(-1);
+                                       LDAP_SLIST_INSERT_HEAD(&pool->ltp_free_list, task,
+                                               ltt_next.l);
+                                       goto failed;
                                }
                        }
                        /* there is another open thread, so this
                         * task will be handled eventually.
-                        * continue on, we have signalled that
-                        * the task is waiting.
                         */
                }
        }
+       ldap_pvt_thread_cond_signal(&pool->ltp_cond);
 
+ done:
        ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
        return(0);
+
+ failed:
+       ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
+       return(-1);
+}
+
+static void *
+no_task( void *ctx, void *arg )
+{
+       return NULL;
+}
+
+/* Cancel a pending task that was previously submitted.
+ * Return 1 if the task was successfully cancelled, 0 if
+ * not found, -1 for invalid parameters
+ */
+int
+ldap_pvt_thread_pool_retract (
+       ldap_pvt_thread_pool_t *tpool,
+       ldap_pvt_thread_start_t *start_routine, void *arg )
+{
+       struct ldap_int_thread_pool_s *pool;
+       ldap_int_thread_task_t *task;
+
+       if (tpool == NULL)
+               return(-1);
+
+       pool = *tpool;
+
+       if (pool == NULL)
+               return(-1);
+
+       ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
+       LDAP_STAILQ_FOREACH(task, &pool->ltp_pending_list, ltt_next.q)
+               if (task->ltt_start_routine == start_routine &&
+                       task->ltt_arg == arg) {
+                       /* Could LDAP_STAILQ_REMOVE the task, but that
+                        * walks ltp_pending_list again to find it.
+                        */
+                       task->ltt_start_routine = no_task;
+                       task->ltt_arg = NULL;
+                       break;
+               }
+       ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
+       return task != NULL;
 }
 
 /* Set max #threads.  value <= 0 means max supported #threads (LDAP_MAXTHR) */
@@ -350,7 +408,10 @@ ldap_pvt_thread_pool_maxthreads(
                return(-1);
 
        ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
+
        pool->ltp_max_count = max_threads;
+       SET_VARY_OPEN_COUNT(pool);
+
        ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
        return(0);
 }
@@ -383,10 +444,16 @@ ldap_pvt_thread_pool_query(
 
        case LDAP_PVT_THREAD_POOL_PARAM_MAX_PENDING:
                count = pool->ltp_max_pending;
+               if (count < 0)
+                       count = -count;
+               if (count == MAX_PENDING)
+                       count = 0;
                break;
 
        case LDAP_PVT_THREAD_POOL_PARAM_OPEN:
                count = pool->ltp_open_count;
+               if (count < 0)
+                       count = -count;
                break;
 
        case LDAP_PVT_THREAD_POOL_PARAM_STARTING:
@@ -398,7 +465,7 @@ ldap_pvt_thread_pool_query(
                break;
 
        case LDAP_PVT_THREAD_POOL_PARAM_PAUSING:
-               count = pool->ltp_pause;
+               count = (pool->ltp_pause != 0);
                break;
 
        case LDAP_PVT_THREAD_POOL_PARAM_PENDING:
@@ -418,32 +485,12 @@ ldap_pvt_thread_pool_query(
        case LDAP_PVT_THREAD_POOL_PARAM_BACKLOAD_MAX:
                break;
 
-       case LDAP_PVT_THREAD_POOL_PARAM_STATE: {
-               static struct {
-                       char                            *name;
-                       ldap_int_thread_pool_state_t    state;
-               }               str2state[] = {
-                       { "running",    LDAP_INT_THREAD_POOL_RUNNING },
-                       { "finishing",  LDAP_INT_THREAD_POOL_FINISHING },
-                       { "stopping",   LDAP_INT_THREAD_POOL_STOPPING },
-                       { NULL }
-               };
-               int             i;
-
-               if ( pool->ltp_pause ) {
-                       *((char **)value) = "pausing";
-               } else {
-                       for ( i = 0; str2state[ i ].name != NULL; i++ ) {
-                               if ( str2state[ i ].state == pool->ltp_state ) {
-                                       break;
-                               }
-                       }
-                       *((char **)value) = str2state[ i ].name;
-               }
-               if ( *((char **)value) != NULL ) {
-                       count = -2;
-               }
-               } break;
+       case LDAP_PVT_THREAD_POOL_PARAM_STATE:
+               *((char **)value) =
+                       pool->ltp_pause ? "pausing" :
+                       !pool->ltp_finishing ? "running" :
+                       pool->ltp_pending_count ? "finishing" : "stopping";
+               break;
 
        case LDAP_PVT_THREAD_POOL_PARAM_UNKNOWN:
                break;
@@ -457,6 +504,23 @@ ldap_pvt_thread_pool_query(
        return ( count == -1 ? -1 : 0 );
 }
 
+/*
+ * true if pool is pausing; does not lock any mutex to check.
+ * 0 if not pause, 1 if pause, -1 if error or no pool.
+ */
+int
+ldap_pvt_thread_pool_pausing( ldap_pvt_thread_pool_t *tpool )
+{
+       int rc = -1;
+       struct ldap_int_thread_pool_s *pool;
+
+       if ( tpool != NULL && (pool = *tpool) != NULL ) {
+               rc = (pool->ltp_pause != 0);
+       }
+
+       return rc;
+}
+
 /*
  * wrapper for ldap_pvt_thread_pool_query(), left around
  * for backwards compatibility
@@ -501,22 +565,25 @@ ldap_pvt_thread_pool_destroy ( ldap_pvt_thread_pool_t *tpool, int run_pending )
        if (pool != pptr) return(-1);
 
        ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
-       pool->ltp_state = run_pending
-               ? LDAP_INT_THREAD_POOL_FINISHING
-               : LDAP_INT_THREAD_POOL_STOPPING;
+
+       pool->ltp_finishing = 1;
+       SET_VARY_OPEN_COUNT(pool);
+       if (pool->ltp_max_pending > 0)
+               pool->ltp_max_pending = -pool->ltp_max_pending;
+
+       if (!run_pending) {
+               while ((task = LDAP_STAILQ_FIRST(&pool->ltp_pending_list)) != NULL) {
+                       LDAP_STAILQ_REMOVE_HEAD(&pool->ltp_pending_list, ltt_next.q);
+                       LDAP_FREE(task);
+               }
+               pool->ltp_pending_count = 0;
+       }
 
        while (pool->ltp_open_count) {
                if (!pool->ltp_pause)
                        ldap_pvt_thread_cond_broadcast(&pool->ltp_cond);
                ldap_pvt_thread_cond_wait(&pool->ltp_cond, &pool->ltp_mutex);
        }
-       ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
-
-       while ((task = LDAP_STAILQ_FIRST(&pool->ltp_pending_list)) != NULL)
-       {
-               LDAP_STAILQ_REMOVE_HEAD(&pool->ltp_pending_list, ltt_next.q);
-               LDAP_FREE(task);
-       }
 
        while ((task = LDAP_SLIST_FIRST(&pool->ltp_free_list)) != NULL)
        {
@@ -524,10 +591,12 @@ ldap_pvt_thread_pool_destroy ( ldap_pvt_thread_pool_t *tpool, int run_pending )
                LDAP_FREE(task);
        }
 
+       ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
        ldap_pvt_thread_cond_destroy(&pool->ltp_pcond);
        ldap_pvt_thread_cond_destroy(&pool->ltp_cond);
        ldap_pvt_thread_mutex_destroy(&pool->ltp_mutex);
        LDAP_FREE(pool);
+       *tpool = NULL;
        ldap_int_has_thread_pool = 0;
        return(0);
 }
@@ -539,6 +608,7 @@ ldap_int_thread_pool_wrapper (
 {
        struct ldap_int_thread_pool_s *pool = xpool;
        ldap_int_thread_task_t *task;
+       ldap_int_tpool_plist_t *work_list;
        ldap_int_thread_userctx_t ctx, *kctx;
        unsigned i, keyslot, hash;
 
@@ -551,6 +621,8 @@ ldap_int_thread_pool_wrapper (
        ctx.ltu_id = ldap_pvt_thread_self();
        TID_HASH(ctx.ltu_id, hash);
 
+       ldap_pvt_thread_key_setdata( ldap_tpool_key, &ctx );
+
        ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
 
        /* thread_keys[] is read-only when paused */
@@ -569,61 +641,58 @@ ldap_int_thread_pool_wrapper (
        ldap_pvt_thread_mutex_unlock(&ldap_pvt_thread_pool_mutex);
 
        pool->ltp_starting--;
+       pool->ltp_active_count++;
 
        for (;;) {
-               while (pool->ltp_pause)
-                       ldap_pvt_thread_cond_wait(&pool->ltp_cond, &pool->ltp_mutex);
+               work_list = pool->ltp_work_list; /* help the compiler a bit */
+               task = LDAP_STAILQ_FIRST(work_list);
+               if (task == NULL) {     /* paused or no pending tasks */
+                       if (--(pool->ltp_active_count) < 2) {
+                               /* Notify pool_pause it is the sole active thread. */
+                               ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
+                       }
 
-               if (pool->ltp_state == LDAP_INT_THREAD_POOL_STOPPING)
-                       break;
+                       do {
+                               if (pool->ltp_vary_open_count < 0) {
+                                       /* Not paused, and either finishing or too many
+                                        * threads running (can happen if ltp_max_count
+                                        * was reduced).  Let this thread die.
+                                        */
+                                       goto done;
+                               }
 
-               task = LDAP_STAILQ_FIRST(&pool->ltp_pending_list);
-               if (task == NULL) {
-                       if (pool->ltp_state == LDAP_INT_THREAD_POOL_FINISHING)
-                               break;
-
-                       if (pool->ltp_open_count >
-                               (pool->ltp_max_count ? pool->ltp_max_count : LDAP_MAXTHR))
-                       {
-                               /* too many threads running (can happen if the
-                                * maximum threads value is set during ongoing
-                                * operation using ldap_pvt_thread_pool_maxthreads)
-                                * so let this thread die.
+                               /* We could check an idle timer here, and let the
+                                * thread die if it has been inactive for a while.
+                                * Only die if there are other open threads (i.e.,
+                                * always have at least one thread open).
+                                * The check should be like this:
+                                *   if (pool->ltp_open_count>1 && pool->ltp_starting==0)
+                                *       check timer, wait if ltp_pause, leave thread;
+                                *
+                                * Just use pthread_cond_timedwait() if we want to
+                                * check idle time.
                                 */
-                               break;
-                       }
+                               ldap_pvt_thread_cond_wait(&pool->ltp_cond, &pool->ltp_mutex);
 
-                       /* we could check an idle timer here, and let the
-                        * thread die if it has been inactive for a while.
-                        * only die if there are other open threads (i.e.,
-                        * always have at least one thread open).  the check
-                        * should be like this:
-                        *   if (pool->ltp_open_count > 1 && pool->ltp_starting == 0)
-                        *       check timer, wait if ltp_pause, leave thread (break;)
-                        *
-                        * Just use pthread_cond_timedwait if we want to
-                        * check idle time.
-                        */
+                               work_list = pool->ltp_work_list;
+                               task = LDAP_STAILQ_FIRST(work_list);
+                       } while (task == NULL);
 
-                       assert(pool->ltp_state == LDAP_INT_THREAD_POOL_RUNNING);
-                       ldap_pvt_thread_cond_wait(&pool->ltp_cond, &pool->ltp_mutex);
-                       continue;
+                       pool->ltp_active_count++;
                }
 
-               LDAP_STAILQ_REMOVE_HEAD(&pool->ltp_pending_list, ltt_next.q);
+               LDAP_STAILQ_REMOVE_HEAD(work_list, ltt_next.q);
                pool->ltp_pending_count--;
-               pool->ltp_active_count++;
                ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
 
                task->ltt_start_routine(&ctx, task->ltt_arg);
 
                ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
                LDAP_SLIST_INSERT_HEAD(&pool->ltp_free_list, task, ltt_next.l);
-               pool->ltp_active_count--;
-               /* let pool_pause know when it is the sole active thread */
-               if (pool->ltp_active_count < 2)
-                       ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
        }
+ done:
+
+       assert(!pool->ltp_pause); /* thread_keys writable, ltp_open_count >= 0 */
 
        /* The ltp_mutex lock protects ctx->ltu_key from pool_purgekey()
         * during this call, since it prevents new pauses. */
@@ -634,8 +703,9 @@ ldap_int_thread_pool_wrapper (
        ldap_pvt_thread_mutex_unlock(&ldap_pvt_thread_pool_mutex);
 
        pool->ltp_open_count--;
+       SET_VARY_OPEN_COUNT(pool);
        /* let pool_destroy know we're all done */
-       if (pool->ltp_open_count < 1)
+       if (pool->ltp_open_count == 0)
                ldap_pvt_thread_cond_signal(&pool->ltp_cond);
 
        ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
@@ -644,12 +714,21 @@ ldap_int_thread_pool_wrapper (
        return(NULL);
 }
 
-/* Pause the pool.  Return when all other threads are paused. */
-int
-ldap_pvt_thread_pool_pause ( 
-       ldap_pvt_thread_pool_t *tpool )
+/* Arguments > ltp_pause to handle_pause(,PAUSE_ARG()).  arg=PAUSE_ARG
+ * ensures (arg-ltp_pause) sets GO_* at need and keeps DO_PAUSE/GO_*.
+ */
+#define GO_IDLE                8
+#define GO_UNIDLE      16
+#define CHECK_PAUSE    32      /* if ltp_pause: GO_IDLE; wait; GO_UNIDLE */
+#define DO_PAUSE       64      /* CHECK_PAUSE; pause the pool */
+#define PAUSE_ARG(a) \
+               ((a) | ((a) & (GO_IDLE|GO_UNIDLE) ? GO_IDLE-1 : CHECK_PAUSE))
+
+static int
+handle_pause( ldap_pvt_thread_pool_t *tpool, int pause_type )
 {
        struct ldap_int_thread_pool_s *pool;
+       int ret = 0, pause, max_ltp_pause;
 
        if (tpool == NULL)
                return(-1);
@@ -659,30 +738,97 @@ ldap_pvt_thread_pool_pause (
        if (pool == NULL)
                return(0);
 
+       if (pause_type == CHECK_PAUSE && !pool->ltp_pause)
+               return(0);
+
+       /* Let pool_unidle() ignore requests for new pauses */
+       max_ltp_pause = pause_type==PAUSE_ARG(GO_UNIDLE) ? WANT_PAUSE : NOT_PAUSED;
+
        ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
 
-       /* If someone else has already requested a pause, we have to wait */
-       if (pool->ltp_pause) {
+       pause = pool->ltp_pause;        /* NOT_PAUSED, WANT_PAUSE or PAUSED */
+
+       /* If ltp_pause and not GO_IDLE|GO_UNIDLE: Set GO_IDLE,GO_UNIDLE */
+       pause_type -= pause;
+
+       if (pause_type & GO_IDLE) {
                pool->ltp_pending_count++;
                pool->ltp_active_count--;
-               /* let the other pool_pause() know when it can proceed */
-               if (pool->ltp_active_count < 2)
+               if (pause && pool->ltp_active_count < 2) {
+                       /* Tell the task waiting to DO_PAUSE it can proceed */
                        ldap_pvt_thread_cond_signal(&pool->ltp_pcond);
-               do {
-                       ldap_pvt_thread_cond_wait(&pool->ltp_cond, &pool->ltp_mutex);
-               } while (pool->ltp_pause);
+               }
+       }
+
+       if (pause_type & GO_UNIDLE) {
+               /* Wait out pause if any, then cancel GO_IDLE */
+               if (pause > max_ltp_pause) {
+                       ret = 1;
+                       do {
+                               ldap_pvt_thread_cond_wait(&pool->ltp_cond, &pool->ltp_mutex);
+                       } while (pool->ltp_pause > max_ltp_pause);
+               }
                pool->ltp_pending_count--;
                pool->ltp_active_count++;
        }
 
-       /* Wait for everyone else to pause or finish */
-       pool->ltp_pause = 1;
-       while (pool->ltp_active_count > 1) {
-               ldap_pvt_thread_cond_wait(&pool->ltp_pcond, &pool->ltp_mutex);
+       if (pause_type & DO_PAUSE) {
+               /* Tell everyone else to pause or finish, then await that */
+               ret = 0;
+               assert(!pool->ltp_pause);
+               pool->ltp_pause = WANT_PAUSE;
+               /* Let ldap_pvt_thread_pool_submit() through to its ltp_pause test,
+                * and do not finish threads in ldap_pvt_thread_pool_wrapper() */
+               pool->ltp_open_count = -pool->ltp_open_count;
+               SET_VARY_OPEN_COUNT(pool);
+               /* Hide pending tasks from ldap_pvt_thread_pool_wrapper() */
+               pool->ltp_work_list = &empty_pending_list;
+               /* Wait for this task to become the sole active task */
+               while (pool->ltp_active_count > 1) {
+                       ldap_pvt_thread_cond_wait(&pool->ltp_pcond, &pool->ltp_mutex);
+               }
+               assert(pool->ltp_pause == WANT_PAUSE);
+               pool->ltp_pause = PAUSED;
        }
 
        ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
-       return(0);
+       return(ret);
+}
+
+/* Consider this task idle: It will not block pool_pause() in other tasks. */
+void
+ldap_pvt_thread_pool_idle( ldap_pvt_thread_pool_t *tpool )
+{
+       handle_pause(tpool, PAUSE_ARG(GO_IDLE));
+}
+
+/* Cancel pool_idle(). If the pool is paused, wait it out first. */
+void
+ldap_pvt_thread_pool_unidle( ldap_pvt_thread_pool_t *tpool )
+{
+       handle_pause(tpool, PAUSE_ARG(GO_UNIDLE));
+}
+
+/*
+ * If a pause was requested, wait for it.  If several threads
+ * are waiting to pause, let through one or more pauses.
+ * The calling task must be active, not idle.
+ * Return 1 if we waited, 0 if not, -1 at parameter error.
+ */
+int
+ldap_pvt_thread_pool_pausecheck( ldap_pvt_thread_pool_t *tpool )
+{
+       return handle_pause(tpool, PAUSE_ARG(CHECK_PAUSE));
+}
+
+/*
+ * Pause the pool.  The calling task must be active, not idle.
+ * Return when all other tasks are paused or idle.
+ */
+int
+ldap_pvt_thread_pool_pause( ldap_pvt_thread_pool_t *tpool )
+{
+       return handle_pause(tpool, PAUSE_ARG(DO_PAUSE));
 }
 
 /* End a pause */
@@ -701,8 +847,16 @@ ldap_pvt_thread_pool_resume (
                return(0);
 
        ldap_pvt_thread_mutex_lock(&pool->ltp_mutex);
+
+       assert(pool->ltp_pause == PAUSED);
        pool->ltp_pause = 0;
+       if (pool->ltp_open_count <= 0) /* true when paused, but be paranoid */
+               pool->ltp_open_count = -pool->ltp_open_count;
+       SET_VARY_OPEN_COUNT(pool);
+       pool->ltp_work_list = &pool->ltp_pending_list;
+
        ldap_pvt_thread_cond_broadcast(&pool->ltp_cond);
+
        ldap_pvt_thread_mutex_unlock(&pool->ltp_mutex);
        return(0);
 }
@@ -752,7 +906,9 @@ int ldap_pvt_thread_pool_setkey(
        void *xctx,
        void *key,
        void *data,
-       ldap_pvt_thread_pool_keyfree_t *kfree )
+       ldap_pvt_thread_pool_keyfree_t *kfree,
+       void **olddatap,
+       ldap_pvt_thread_pool_keyfree_t **oldkfreep )
 {
        ldap_int_thread_userctx_t *ctx = xctx;
        int i, found;
@@ -768,6 +924,22 @@ int ldap_pvt_thread_pool_setkey(
                }
        }
 
+       if ( olddatap ) {
+               if ( found ) {
+                       *olddatap = ctx->ltu_key[i].ltk_data;
+               } else {
+                       *olddatap = NULL;
+               }
+       }
+
+       if ( oldkfreep ) {
+               if ( found ) {
+                       *oldkfreep = ctx->ltu_key[i].ltk_free;
+               } else {
+                       *oldkfreep = 0;
+               }
+       }
+
        if ( data || kfree ) {
                if ( i>=MAXKEYS )
                        return ENOMEM;
@@ -817,29 +989,10 @@ void ldap_pvt_thread_pool_purgekey( void *key )
  */
 void *ldap_pvt_thread_pool_context( )
 {
-       ldap_pvt_thread_t tid;
-       unsigned i, hash;
-       ldap_int_thread_userctx_t *ctx;
-
-       tid = ldap_pvt_thread_self();
-       if ( ldap_pvt_thread_equal( tid, ldap_int_main_thrctx.ltu_id ))
-               return &ldap_int_main_thrctx;
-
-       TID_HASH( tid, hash );
-       i = hash &= (LDAP_MAXTHR-1);
-       ldap_pvt_thread_mutex_lock(&ldap_pvt_thread_pool_mutex);
-       do {
-               ctx = thread_keys[i].ctx;
-               if ( ctx == DELETED_THREAD_CTX )
-                       continue;
-               if ( !ctx || ldap_pvt_thread_equal(thread_keys[i].ctx->ltu_id, tid) )
-                       goto done;
-       } while ( (i = (i+1) & (LDAP_MAXTHR-1)) != hash );
-       ctx = NULL;
- done:
-       ldap_pvt_thread_mutex_unlock(&ldap_pvt_thread_pool_mutex);
+       void *ctx = NULL;
 
-       return ctx;
+       ldap_pvt_thread_key_getdata( ldap_tpool_key, &ctx );
+       return ctx ? ctx : (void *) &ldap_int_main_thrctx;
 }
 
 /*