From 98928d02d2473ceb9f81b1c3bc527f8b0a0039e6 Mon Sep 17 00:00:00 2001
From: Graham Leggett <minfrin@apache.org>
Date: Fri, 21 Sep 2018 13:30:15 +0000
Subject: [PATCH 188/504] MPMs: Initialize all runtime/asynchronous objects on
 a dedicated pool and before signals handling to avoid lifetime issues on
 restart or shutdown. PR 62658. trunk patch: http://svn.apache.org/r1835845   
           http://svn.apache.org/r1835846             
 http://svn.apache.org/r1837354              http://svn.apache.org/r1837356   
           http://svn.apache.org/r1839571             
 http://svn.apache.org/r1839583 2.4.x patch:
 http://home.apache.org/~ylavic/patches/2.4.x-mpms_async_objects_lifetime.patch
 +1: ylavic, jim (but not for 2.4.35), minfrin

git-svn-id: https://svn.apache.org/repos/asf/httpd/httpd/branches/2.4.x@1841586 13f79535-47bb-0310-9956-ffa450edef68
---
 CHANGES                              |   4 +
 STATUS                               |  11 --
 server/mpm/event/event.c             | 203 +++++++++++++++------------
 server/mpm/mpmt_os2/mpmt_os2_child.c |   1 +
 server/mpm/netware/mpm_netware.c     |   1 +
 server/mpm/prefork/prefork.c         |   6 +-
 server/mpm/winnt/child.c             |   3 +
 server/mpm/winnt/mpm_winnt.c         |   1 +
 server/mpm/worker/worker.c           | 148 +++++++++++--------
 9 files changed, 215 insertions(+), 163 deletions(-)

diff --git a/server/mpm/event/event.c b/server/mpm/event/event.c
index f07b757ab9..ffe8a23cbd 100644
--- a/server/mpm/event/event.c
+++ b/server/mpm/event/event.c
@@ -436,6 +436,7 @@ int raise_sigstop_flags;
 
 static apr_pool_t *pconf;       /* Pool for config stuff */
 static apr_pool_t *pchild;      /* Pool for httpd child stuff */
+static apr_pool_t *pruntime;    /* Pool for MPM threads stuff */
 
 static pid_t ap_my_pid;         /* Linux getpid() doesn't work except in main
                                    thread. Use this instead */
@@ -709,10 +710,10 @@ static void event_note_child_killed(int childnum, pid_t pid, ap_generation_t gen
 
 static void event_note_child_started(int slot, pid_t pid)
 {
+    ap_generation_t gen = retained->mpm->my_generation;
     ap_scoreboard_image->parent[slot].pid = pid;
-    ap_run_child_status(ap_server_conf,
-                        ap_scoreboard_image->parent[slot].pid,
-                        retained->mpm->my_generation, slot, MPM_CHILD_STARTED);
+    ap_scoreboard_image->parent[slot].generation = gen;
+    ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
 }
 
 static const char *event_get_name(void)
@@ -1270,36 +1271,6 @@ static void dummy_signal_handler(int sig)
 }
 
 
-static apr_status_t init_pollset(apr_pool_t *p)
-{
-    ap_listen_rec *lr;
-    listener_poll_type *pt;
-    int i = 0;
-
-    listener_pollfd = apr_palloc(p, sizeof(apr_pollfd_t) * num_listensocks);
-    for (lr = my_bucket->listeners; lr != NULL; lr = lr->next, i++) {
-        apr_pollfd_t *pfd;
-        AP_DEBUG_ASSERT(i < num_listensocks);
-        pfd = &listener_pollfd[i];
-        pt = apr_pcalloc(p, sizeof(*pt));
-        pfd->desc_type = APR_POLL_SOCKET;
-        pfd->desc.s = lr->sd;
-        pfd->reqevents = APR_POLLIN;
-
-        pt->type = PT_ACCEPT;
-        pt->baton = lr;
-
-        pfd->client_data = pt;
-
-        apr_socket_opt_set(pfd->desc.s, APR_SO_NONBLOCK, 1);
-        apr_pollset_add(event_pollset, pfd);
-
-        lr->accept_func = ap_unixd_accept;
-    }
-
-    return APR_SUCCESS;
-}
-
 static apr_status_t push_timer2worker(timer_event_t* te)
 {
     return ap_queue_push_timer(worker_queue, te);
@@ -1611,7 +1582,6 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
     proc_info *ti = dummy;
     int process_slot = ti->pslot;
     struct process_score *ps = ap_get_scoreboard_process(process_slot);
-    apr_pool_t *tpool = apr_thread_pool_get(thd);
     int closed = 0;
     int have_idle_worker = 0;
     apr_time_t last_log;
@@ -1619,16 +1589,6 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t * thd, void *dummy)
     last_log = apr_time_now();
     free(ti);
 
-    rc = init_pollset(tpool);
-    if (rc != APR_SUCCESS) {
-        ap_log_error(APLOG_MARK, APLOG_ERR, rc, ap_server_conf,
-                     "failed to initialize pollset, "
-                     "shutdown process now");
-        resource_shortage = 1;
-        signal_threads(ST_UNGRACEFUL);
-        return NULL;
-    }
-
     /* Unblock the signal used to wake this thread up, and set a handler for
      * it.
      */
@@ -2168,8 +2128,6 @@ static int check_signal(int signum)
     return 0;
 }
 
-
-
 static void create_listener_thread(thread_starter * ts)
 {
     int my_child_num = ts->child_num_arg;
@@ -2181,7 +2139,7 @@ static void create_listener_thread(thread_starter * ts)
     my_info->pslot = my_child_num;
     my_info->tslot = -1;      /* listener thread doesn't have a thread slot */
     rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,
-                           my_info, pchild);
+                           my_info, pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00474)
                      "apr_thread_create: unable to create listener thread");
@@ -2191,25 +2149,12 @@ static void create_listener_thread(thread_starter * ts)
     apr_os_thread_get(&listener_os_thread, ts->listener);
 }
 
-/* XXX under some circumstances not understood, children can get stuck
- *     in start_threads forever trying to take over slots which will
- *     never be cleaned up; for now there is an APLOG_DEBUG message issued
- *     every so often when this condition occurs
- */
-static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
+static void setup_threads_runtime(void)
 {
-    thread_starter *ts = dummy;
-    apr_thread_t **threads = ts->threads;
-    apr_threadattr_t *thread_attr = ts->threadattr;
-    int my_child_num = ts->child_num_arg;
-    proc_info *my_info;
     apr_status_t rv;
-    int i;
-    int threads_created = 0;
-    int listener_started = 0;
-    int loops;
-    int prev_threads_created;
-    int max_recycled_pools = -1;
+    ap_listen_rec *lr;
+    apr_pool_t *pskip = NULL;
+    int max_recycled_pools = -1, i;
     const int good_methods[] = { APR_POLLSET_KQUEUE,
                                  APR_POLLSET_PORT,
                                  APR_POLLSET_EPOLL };
@@ -2218,10 +2163,39 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
     const apr_uint32_t pollset_size = (apr_uint32_t)num_listensocks +
                                       (apr_uint32_t)threads_per_child *
                                       (async_factor > 2 ? async_factor : 2);
+    int pollset_flags;
+
+    /* Event's skiplist operations will happen concurrently with other modules'
+     * runtime so they need their own pool for allocations, and its lifetime
+     * should be at least the one of the connections (ptrans). Thus pskip is
+     * created as a subpool of pconf like/before ptrans (before so that it's
+     * destroyed after). In forked mode pconf is never destroyed so we are good
+     * anyway, but in ONE_PROCESS mode this ensures that the skiplist works
+     * from connection/ptrans cleanups (even after pchild is destroyed).
+     */
+    apr_pool_create(&pskip, pconf);
+    apr_pool_tag(pskip, "mpm_skiplist");
+    apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pskip);
+    APR_RING_INIT(&timer_free_ring, timer_event_t, link);
+    apr_skiplist_init(&timer_skiplist, pskip);
+    apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp);
+
+    /* All threads (listener, workers) and synchronization objects (queues,
+     * pollset, mutexes...) created here should have at least the lifetime of
+     * the connections they handle (i.e. ptrans). We can't use this thread's
+     * self pool because all these objects survive it, nor use pchild or pconf
+     * directly because this starter thread races with other modules' runtime,
+     * nor finally pchild (or subpool thereof) because it is killed explicitely
+     * before pconf (thus connections/ptrans can live longer, which matters in
+     * ONE_PROCESS mode). So this leaves us with a subpool of pconf, created
+     * before any ptrans hence destroyed after.
+     */
+    apr_pool_create(&pruntime, pconf);
+    apr_pool_tag(pruntime, "mpm_runtime");
 
     /* We must create the fd queues before we start up the listener
      * and worker threads. */
-    rv = ap_queue_create(&worker_queue, threads_per_child, pchild);
+    rv = ap_queue_create(&worker_queue, threads_per_child, pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03100)
                      "ap_queue_create() failed");
@@ -2235,7 +2209,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
          */
         max_recycled_pools = threads_per_child * 3 / 4 ;
     }
-    rv = ap_queue_info_create(&worker_queue_info, pchild,
+    rv = ap_queue_info_create(&worker_queue_info, pruntime,
                               threads_per_child, max_recycled_pools);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03101)
@@ -2247,7 +2221,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
      * thread starts.
      */
     rv = apr_thread_mutex_create(&timeout_mutex, APR_THREAD_MUTEX_DEFAULT,
-                                 pchild);
+                                 pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03102)
                      "creation of the timeout mutex failed.");
@@ -2255,25 +2229,30 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
     }
 
     /* Create the main pollset */
+    pollset_flags = APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY |
+                    APR_POLLSET_NODEFAULT | APR_POLLSET_WAKEABLE;
     for (i = 0; i < sizeof(good_methods) / sizeof(good_methods[0]); i++) {
-        apr_uint32_t flags = APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY |
-                             APR_POLLSET_NODEFAULT | APR_POLLSET_WAKEABLE;
-        rv = apr_pollset_create_ex(&event_pollset, pollset_size, pchild, flags,
-                                   good_methods[i]);
+        rv = apr_pollset_create_ex(&event_pollset, pollset_size, pruntime,
+                                   pollset_flags, good_methods[i]);
         if (rv == APR_SUCCESS) {
             listener_is_wakeable = 1;
             break;
         }
-        flags &= ~APR_POLLSET_WAKEABLE;
-        rv = apr_pollset_create_ex(&event_pollset, pollset_size, pchild, flags,
-                                   good_methods[i]);
-        if (rv == APR_SUCCESS) {
-            break;
+    }
+    if (rv != APR_SUCCESS) {
+        pollset_flags &= ~APR_POLLSET_WAKEABLE;
+        for (i = 0; i < sizeof(good_methods) / sizeof(good_methods[0]); i++) {
+            rv = apr_pollset_create_ex(&event_pollset, pollset_size, pruntime,
+                                       pollset_flags, good_methods[i]);
+            if (rv == APR_SUCCESS) {
+                break;
+            }
         }
     }
     if (rv != APR_SUCCESS) {
-        rv = apr_pollset_create(&event_pollset, pollset_size, pchild,
-                                APR_POLLSET_THREADSAFE | APR_POLLSET_NOCOPY);
+        pollset_flags &= ~APR_POLLSET_NODEFAULT;
+        rv = apr_pollset_create(&event_pollset, pollset_size, pruntime,
+                                pollset_flags);
     }
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ERR, rv, ap_server_conf, APLOGNO(03103)
@@ -2281,12 +2260,57 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
         clean_child_exit(APEXIT_CHILDFATAL);
     }
 
+    /* Add listeners to the main pollset */
+    listener_pollfd = apr_pcalloc(pruntime, num_listensocks *
+                                            sizeof(apr_pollfd_t));
+    for (i = 0, lr = my_bucket->listeners; lr; lr = lr->next, i++) {
+        apr_pollfd_t *pfd;
+        listener_poll_type *pt;
+
+        AP_DEBUG_ASSERT(i < num_listensocks);
+        pfd = &listener_pollfd[i];
+
+        pfd->reqevents = APR_POLLIN;
+        pfd->desc_type = APR_POLL_SOCKET;
+        pfd->desc.s = lr->sd;
+
+        pt = apr_pcalloc(pruntime, sizeof(*pt));
+        pfd->client_data = pt;
+        pt->type = PT_ACCEPT;
+        pt->baton = lr;
+
+        apr_socket_opt_set(pfd->desc.s, APR_SO_NONBLOCK, 1);
+        apr_pollset_add(event_pollset, pfd);
+
+        lr->accept_func = ap_unixd_accept;
+    }
+
+    worker_sockets = apr_pcalloc(pruntime, threads_per_child *
+                                           sizeof(apr_socket_t *));
+}
+
+/* XXX under some circumstances not understood, children can get stuck
+ *     in start_threads forever trying to take over slots which will
+ *     never be cleaned up; for now there is an APLOG_DEBUG message issued
+ *     every so often when this condition occurs
+ */
+static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
+{
+    thread_starter *ts = dummy;
+    apr_thread_t **threads = ts->threads;
+    apr_threadattr_t *thread_attr = ts->threadattr;
+    int my_child_num = ts->child_num_arg;
+    proc_info *my_info;
+    apr_status_t rv;
+    int threads_created = 0;
+    int listener_started = 0;
+    int prev_threads_created;
+    int loops, i;
+
     ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, ap_server_conf, APLOGNO(02471)
                  "start_threads: Using %s (%swakeable)",
                  apr_pollset_method_name(event_pollset),
                  listener_is_wakeable ? "" : "not ");
-    worker_sockets = apr_pcalloc(pchild, threads_per_child
-                                 * sizeof(apr_socket_t *));
 
     loops = prev_threads_created = 0;
     while (1) {
@@ -2310,7 +2334,7 @@ static void *APR_THREAD_FUNC start_threads(apr_thread_t * thd, void *dummy)
              * done because it lets us deal with tid better.
              */
             rv = apr_thread_create(&threads[i], thread_attr,
-                                   worker_thread, my_info, pchild);
+                                   worker_thread, my_info, pruntime);
             if (rv != APR_SUCCESS) {
                 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf,
                              APLOGNO(03104)
@@ -2431,7 +2455,6 @@ static void child_main(int child_num_arg, int child_bucket)
     apr_threadattr_t *thread_attr;
     apr_thread_t *start_thread_id;
     int i;
-    apr_pool_t *pskip;
 
     /* for benefit of any hooks that run as this child initializes */
     retained->mpm->mpm_state = AP_MPMQ_STARTING;
@@ -2439,7 +2462,12 @@ static void child_main(int child_num_arg, int child_bucket)
     ap_my_pid = getpid();
     ap_child_slot = child_num_arg;
     ap_fatal_signal_child_setup(ap_server_conf);
+
+    /* Get a sub context for global allocations in this child, so that
+     * we can have cleanups occur when the child exits.
+     */
     apr_pool_create(&pchild, pconf);
+    apr_pool_tag(pchild, "pchild");
 
     /* close unused listeners and pods */
     for (i = 0; i < retained->mpm->num_buckets; i++) {
@@ -2457,12 +2485,6 @@ static void child_main(int child_num_arg, int child_bucket)
         clean_child_exit(APEXIT_CHILDFATAL);
     }
 
-    apr_thread_mutex_create(&g_timer_skiplist_mtx, APR_THREAD_MUTEX_DEFAULT, pchild);
-    APR_RING_INIT(&timer_free_ring, timer_event_t, link);
-    apr_pool_create(&pskip, pchild);
-    apr_skiplist_init(&timer_skiplist, pskip);
-    apr_skiplist_set_compare(timer_skiplist, timer_comp, timer_comp);
-
     /* Just use the standard apr_setup_signal_thread to block all signals
      * from being received.  The child processes no longer use signals for
      * any communication with the parent process. Let's also do this before
@@ -2486,7 +2508,10 @@ static void child_main(int child_num_arg, int child_bucket)
         conns_this_child = APR_INT32_MAX;
     }
 
-    /* Setup worker threads */
+    /* Setup threads */
+
+    /* Globals used by signal_threads() so to be initialized before */
+    setup_threads_runtime();
 
     /* clear the storage; we may not create all our threads immediately,
      * and we want a 0 entry to indicate a thread which was not created
diff --git a/server/mpm/mpmt_os2/mpmt_os2_child.c b/server/mpm/mpmt_os2/mpmt_os2_child.c
index ca9f594754..bb7e1369ea 100644
--- a/server/mpm/mpmt_os2/mpmt_os2_child.c
+++ b/server/mpm/mpmt_os2/mpmt_os2_child.c
@@ -110,6 +110,7 @@ void ap_mpm_child_main(apr_pool_t *pconf)
 
     /* Create pool for child */
     apr_pool_create(&pchild, pconf);
+    apr_pool_tag(pchild, "pchild");
 
     ap_run_child_init(pchild, ap_server_conf);
 
diff --git a/server/mpm/netware/mpm_netware.c b/server/mpm/netware/mpm_netware.c
index 2fab52f598..82480334b8 100644
--- a/server/mpm/netware/mpm_netware.c
+++ b/server/mpm/netware/mpm_netware.c
@@ -886,6 +886,7 @@ static int netware_run(apr_pool_t *_pconf, apr_pool_t *plog, server_rec *s)
 
     /* Only set slot 0 since that is all NetWare will ever have. */
     ap_scoreboard_image->parent[0].pid = getpid();
+    ap_scoreboard_image->parent[0].generation = ap_my_generation;
     ap_run_child_status(ap_server_conf,
                         ap_scoreboard_image->parent[0].pid,
                         ap_my_generation,
diff --git a/server/mpm/prefork/prefork.c b/server/mpm/prefork/prefork.c
index 3fb328467d..8efda72ee1 100644
--- a/server/mpm/prefork/prefork.c
+++ b/server/mpm/prefork/prefork.c
@@ -208,10 +208,10 @@ static void prefork_note_child_killed(int childnum, pid_t pid,
 
 static void prefork_note_child_started(int slot, pid_t pid)
 {
+    ap_generation_t gen = retained->mpm->my_generation;
     ap_scoreboard_image->parent[slot].pid = pid;
-    ap_run_child_status(ap_server_conf,
-                        ap_scoreboard_image->parent[slot].pid,
-                        retained->mpm->my_generation, slot, MPM_CHILD_STARTED);
+    ap_scoreboard_image->parent[slot].generation = gen;
+    ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
 }
 
 /* a clean exit from a child with proper cleanup */
diff --git a/server/mpm/winnt/child.c b/server/mpm/winnt/child.c
index 9ddba18f80..21755f398c 100644
--- a/server/mpm/winnt/child.c
+++ b/server/mpm/winnt/child.c
@@ -917,6 +917,9 @@ void child_main(apr_pool_t *pconf, DWORD parent_pid)
     int i;
     int num_events;
 
+    /* Get a sub context for global allocations in this child, so that
+     * we can have cleanups occur when the child exits.
+     */
     apr_pool_create(&pchild, pconf);
     apr_pool_tag(pchild, "pchild");
 
diff --git a/server/mpm/winnt/mpm_winnt.c b/server/mpm/winnt/mpm_winnt.c
index 2487e45be2..3d71f80e72 100644
--- a/server/mpm/winnt/mpm_winnt.c
+++ b/server/mpm/winnt/mpm_winnt.c
@@ -139,6 +139,7 @@ AP_INIT_TAKE1("ThreadLimit", set_thread_limit, NULL, RSRC_CONF,
 static void winnt_note_child_started(int slot, pid_t pid)
 {
     ap_scoreboard_image->parent[slot].pid = pid;
+    ap_scoreboard_image->parent[slot].generation = my_generation;
     ap_run_child_status(ap_server_conf,
                         ap_scoreboard_image->parent[slot].pid,
                         my_generation, slot, MPM_CHILD_STARTED);
diff --git a/server/mpm/worker/worker.c b/server/mpm/worker/worker.c
index 7804efc457..8012fe29d8 100644
--- a/server/mpm/worker/worker.c
+++ b/server/mpm/worker/worker.c
@@ -134,6 +134,8 @@ static int num_listensocks = 0;
 static int resource_shortage = 0;
 static fd_queue_t *worker_queue;
 static fd_queue_info_t *worker_queue_info;
+static apr_pollset_t *worker_pollset;
+
 
 /* data retained by worker across load/unload of the module
  * allocated on first call to pre-config hook; located on
@@ -218,6 +220,7 @@ int raise_sigstop_flags;
 
 static apr_pool_t *pconf;                 /* Pool for config stuff */
 static apr_pool_t *pchild;                /* Pool for httpd child stuff */
+static apr_pool_t *pruntime;              /* Pool for MPM threads stuff */
 
 static pid_t ap_my_pid; /* Linux getpid() doesn't work except in main
                            thread. Use this instead */
@@ -392,10 +395,10 @@ static void worker_note_child_killed(int childnum, pid_t pid, ap_generation_t ge
 
 static void worker_note_child_started(int slot, pid_t pid)
 {
+    ap_generation_t gen = retained->mpm->my_generation;
     ap_scoreboard_image->parent[slot].pid = pid;
-    ap_run_child_status(ap_server_conf,
-                        ap_scoreboard_image->parent[slot].pid,
-                        retained->mpm->my_generation, slot, MPM_CHILD_STARTED);
+    ap_scoreboard_image->parent[slot].generation = gen;
+    ap_run_child_status(ap_server_conf, pid, gen, slot, MPM_CHILD_STARTED);
 }
 
 static void worker_note_child_lost_slot(int slot, pid_t newpid)
@@ -538,47 +541,15 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy)
 {
     proc_info * ti = dummy;
     int process_slot = ti->pid;
-    apr_pool_t *tpool = apr_thread_pool_get(thd);
     void *csd = NULL;
     apr_pool_t *ptrans = NULL;            /* Pool for per-transaction stuff */
-    apr_pollset_t *pollset;
     apr_status_t rv;
-    ap_listen_rec *lr;
+    ap_listen_rec *lr = NULL;
     int have_idle_worker = 0;
     int last_poll_idx = 0;
 
     free(ti);
 
-    rv = apr_pollset_create(&pollset, num_listensocks, tpool,
-                            APR_POLLSET_NOCOPY);
-    if (rv != APR_SUCCESS) {
-        ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
-                     "Couldn't create pollset in thread;"
-                     " check system or user limits");
-        /* let the parent decide how bad this really is */
-        clean_child_exit(APEXIT_CHILDSICK);
-    }
-
-    for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) {
-        apr_pollfd_t *pfd = apr_pcalloc(tpool, sizeof *pfd);
-
-        pfd->desc_type = APR_POLL_SOCKET;
-        pfd->desc.s = lr->sd;
-        pfd->reqevents = APR_POLLIN;
-        pfd->client_data = lr;
-
-        rv = apr_pollset_add(pollset, pfd);
-        if (rv != APR_SUCCESS) {
-            ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf,
-                         "Couldn't create add listener to pollset;"
-                         " check system or user limits");
-            /* let the parent decide how bad this really is */
-            clean_child_exit(APEXIT_CHILDSICK);
-        }
-
-        lr->accept_func = ap_unixd_accept;
-    }
-
     /* Unblock the signal used to wake this thread up, and set a handler for
      * it.
      */
@@ -630,7 +601,7 @@ static void * APR_THREAD_FUNC listener_thread(apr_thread_t *thd, void * dummy)
                 apr_int32_t numdesc;
                 const apr_pollfd_t *pdesc;
 
-                rv = apr_pollset_poll(pollset, -1, &numdesc, &pdesc);
+                rv = apr_pollset_poll(worker_pollset, -1, &numdesc, &pdesc);
                 if (rv != APR_SUCCESS) {
                     if (APR_STATUS_IS_EINTR(rv)) {
                         continue;
@@ -871,7 +842,7 @@ static void create_listener_thread(thread_starter *ts)
     my_info->tid = -1; /* listener thread doesn't have a thread slot */
     my_info->sd = 0;
     rv = apr_thread_create(&ts->listener, thread_attr, listener_thread,
-                           my_info, pchild);
+                           my_info, pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(00275)
                      "apr_thread_create: unable to create listener thread");
@@ -881,35 +852,34 @@ static void create_listener_thread(thread_starter *ts)
     apr_os_thread_get(&listener_os_thread, ts->listener);
 }
 
-/* XXX under some circumstances not understood, children can get stuck
- *     in start_threads forever trying to take over slots which will
- *     never be cleaned up; for now there is an APLOG_DEBUG message issued
- *     every so often when this condition occurs
- */
-static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
+static void setup_threads_runtime(void)
 {
-    thread_starter *ts = dummy;
-    apr_thread_t **threads = ts->threads;
-    apr_threadattr_t *thread_attr = ts->threadattr;
-    int my_child_num = ts->child_num_arg;
-    proc_info *my_info;
+    ap_listen_rec *lr;
     apr_status_t rv;
-    int i;
-    int threads_created = 0;
-    int listener_started = 0;
-    int loops;
-    int prev_threads_created;
+
+    /* All threads (listener, workers) and synchronization objects (queues,
+     * pollset, mutexes...) created here should have at least the lifetime of
+     * the connections they handle (i.e. ptrans). We can't use this thread's
+     * self pool because all these objects survive it, nor use pchild or pconf
+     * directly because this starter thread races with other modules' runtime,
+     * nor finally pchild (or subpool thereof) because it is killed explicitely
+     * before pconf (thus connections/ptrans can live longer, which matters in
+     * ONE_PROCESS mode). So this leaves us with a subpool of pconf, created
+     * before any ptrans hence destroyed after.
+     */
+    apr_pool_create(&pruntime, pconf);
+    apr_pool_tag(pruntime, "mpm_runtime");
 
     /* We must create the fd queues before we start up the listener
      * and worker threads. */
-    rv = ap_queue_create(&worker_queue, threads_per_child, pchild);
+    rv = ap_queue_create(&worker_queue, threads_per_child, pruntime);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03140)
                      "ap_queue_create() failed");
         clean_child_exit(APEXIT_CHILDFATAL);
     }
 
-    rv = ap_queue_info_create(&worker_queue_info, pchild,
+    rv = ap_queue_info_create(&worker_queue_info, pruntime,
                               threads_per_child, -1);
     if (rv != APR_SUCCESS) {
         ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03141)
@@ -917,8 +887,58 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
         clean_child_exit(APEXIT_CHILDFATAL);
     }
 
-    worker_sockets = apr_pcalloc(pchild, threads_per_child
-                                        * sizeof(apr_socket_t *));
+    /* Create the main pollset */
+    rv = apr_pollset_create(&worker_pollset, num_listensocks, pruntime,
+                            APR_POLLSET_NOCOPY);
+    if (rv != APR_SUCCESS) {
+        ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03285)
+                     "Couldn't create pollset in thread;"
+                     " check system or user limits");
+        /* let the parent decide how bad this really is */
+        clean_child_exit(APEXIT_CHILDSICK);
+    }
+
+    for (lr = my_bucket->listeners; lr != NULL; lr = lr->next) {
+        apr_pollfd_t *pfd = apr_pcalloc(pruntime, sizeof *pfd);
+
+        pfd->desc_type = APR_POLL_SOCKET;
+        pfd->desc.s = lr->sd;
+        pfd->reqevents = APR_POLLIN;
+        pfd->client_data = lr;
+
+        rv = apr_pollset_add(worker_pollset, pfd);
+        if (rv != APR_SUCCESS) {
+            ap_log_error(APLOG_MARK, APLOG_EMERG, rv, ap_server_conf, APLOGNO(03286)
+                         "Couldn't create add listener to pollset;"
+                         " check system or user limits");
+            /* let the parent decide how bad this really is */
+            clean_child_exit(APEXIT_CHILDSICK);
+        }
+
+        lr->accept_func = ap_unixd_accept;
+    }
+
+    worker_sockets = apr_pcalloc(pruntime, threads_per_child *
+                                           sizeof(apr_socket_t *));
+}
+
+/* XXX under some circumstances not understood, children can get stuck
+ *     in start_threads forever trying to take over slots which will
+ *     never be cleaned up; for now there is an APLOG_DEBUG message issued
+ *     every so often when this condition occurs
+ */
+static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
+{
+    thread_starter *ts = dummy;
+    apr_thread_t **threads = ts->threads;
+    apr_threadattr_t *thread_attr = ts->threadattr;
+    int my_child_num = ts->child_num_arg;
+    proc_info *my_info;
+    apr_status_t rv;
+    int threads_created = 0;
+    int listener_started = 0;
+    int prev_threads_created;
+    int loops, i;
 
     loops = prev_threads_created = 0;
     while (1) {
@@ -942,7 +962,7 @@ static void * APR_THREAD_FUNC start_threads(apr_thread_t *thd, void *dummy)
              * done because it lets us deal with tid better.
              */
             rv = apr_thread_create(&threads[i], thread_attr,
-                                   worker_thread, my_info, pchild);
+                                   worker_thread, my_info, pruntime);
             if (rv != APR_SUCCESS) {
                 ap_log_error(APLOG_MARK, APLOG_ALERT, rv, ap_server_conf, APLOGNO(03142)
                              "apr_thread_create: unable to create worker thread");
@@ -1082,7 +1102,12 @@ static void child_main(int child_num_arg, int child_bucket)
 
     ap_my_pid = getpid();
     ap_fatal_signal_child_setup(ap_server_conf);
+
+    /* Get a sub context for global allocations in this child, so that
+     * we can have cleanups occur when the child exits.
+     */
     apr_pool_create(&pchild, pconf);
+    apr_pool_tag(pchild, "pchild");
 
     /* close unused listeners and pods */
     for (i = 0; i < retained->mpm->num_buckets; i++) {
@@ -1132,7 +1157,10 @@ static void child_main(int child_num_arg, int child_bucket)
         requests_this_child = INT_MAX;
     }
 
-    /* Setup worker threads */
+    /* Setup threads */
+
+    /* Globals used by signal_threads() so to be initialized before */
+    setup_threads_runtime();
 
     /* clear the storage; we may not create all our threads immediately,
      * and we want a 0 entry to indicate a thread which was not created
-- 
2.19.1

