packages/python3
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backport upstream to fix memory leak in socketserver.ThreadingMixIn

Browse Source
This commit is contained in:
BruceGW 2021-05-29 11:05:31 +08:00
parent 387c552974
commit 4f3ad12fa5
2 changed files with 159 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
backport-Remove-thread-objects-which-finished-process-its-request.patch Normal file
View File

@ -0,0 +1,150 @@
From 32d11960b513667d1afe392c9ba66c64c9b04779 Mon Sep 17 00:00:00 2001
From: "Jason R. Coombs" <jaraco@jaraco.com>
Date: Thu, 31 Dec 2020 15:19:30 -0500
Subject: [PATCH] bpo-37193: Remove thread objects which finished process its
request (GH-23127)
This reverts commit aca67da4fe68d5420401ac1782203d302875eb27.
---
Lib/socketserver.py | 51 +++++++++++++++++-----
Lib/test/test_socketserver.py | 23 ++++++++++
.../2020-06-12-21-23-20.bpo-37193.wJximU.rst | 2 +
3 files changed, 64 insertions(+), 12 deletions(-)
create mode 100644 Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst
diff --git a/Lib/socketserver.py b/Lib/socketserver.py
index 1ad028f..9e94c76 100644
--- a/Lib/socketserver.py
+++ b/Lib/socketserver.py
@@ -628,6 +628,39 @@ if hasattr(os, "fork"):
self.collect_children(blocking=self.block_on_close)
+class _Threads(list):
+ """
+ Joinable list of all non-daemon threads.
+ """
+ def append(self, thread):
+ self.reap()
+ if thread.daemon:
+ return
+ super().append(thread)
+
+ def pop_all(self):
+ self[:], result = [], self[:]
+ return result
+
+ def join(self):
+ for thread in self.pop_all():
+ thread.join()
+
+ def reap(self):
+ self[:] = (thread for thread in self if thread.is_alive())
+
+
+class _NoThreads:
+ """
+ Degenerate version of _Threads.
+ """
+ def append(self, thread):
+ pass
+
+ def join(self):
+ pass
+
+
class ThreadingMixIn:
"""Mix-in class to handle each request in a new thread."""
@@ -636,9 +669,9 @@ class ThreadingMixIn:
daemon_threads = False
# If true, server_close() waits until all non-daemonic threads terminate.
block_on_close = True
- # For non-daemonic threads, list of threading.Threading objects
+ # Threads object
# used by server_close() to wait for all threads completion.
- _threads = None
+ _threads = _NoThreads()
def process_request_thread(self, request, client_address):
"""Same as in BaseServer but as a thread.
@@ -655,23 +688,17 @@ class ThreadingMixIn:
def process_request(self, request, client_address):
"""Start a new thread to process the request."""
+ if self.block_on_close:
+ vars(self).setdefault('_threads', _Threads())
t = threading.Thread(target = self.process_request_thread,
args = (request, client_address))
t.daemon = self.daemon_threads
- if not t.daemon and self.block_on_close:
- if self._threads is None:
- self._threads = []
- self._threads.append(t)
+ self._threads.append(t)
t.start()
def server_close(self):
super().server_close()
- if self.block_on_close:
- threads = self._threads
- self._threads = None
- if threads:
- for thread in threads:
- thread.join()
+ self._threads.join()
if hasattr(os, "fork"):
diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py
index 6584ba5..78f00d1 100644
--- a/Lib/test/test_socketserver.py
+++ b/Lib/test/test_socketserver.py
@@ -278,6 +278,13 @@ class SocketServerTest(unittest.TestCase):
t.join()
s.server_close()
+ def test_close_immediately(self):
+ class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+ pass
+
+ server = MyServer((HOST, 0), lambda: None)
+ server.server_close()
+
def test_tcpserver_bind_leak(self):
# Issue #22435: the server socket wouldn't be closed if bind()/listen()
# failed.
@@ -492,6 +499,22 @@ class MiscTestCase(unittest.TestCase):
self.assertEqual(server.shutdown_called, 1)
server.server_close()
+ def test_threads_reaped(self):
+ """
+ In #37193, users reported a memory leak
+ due to the saving of every request thread. Ensure that
+ not all threads are kept forever.
+ """
+ class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+ pass
+
+ server = MyServer((HOST, 0), socketserver.StreamRequestHandler)
+ for n in range(10):
+ with socket.create_connection(server.server_address):
+ server.handle_request()
+ self.assertLess(len(server._threads), 10)
+ server.server_close()
+
if __name__ == "__main__":
unittest.main()
diff --git a/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst b/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst
new file mode 100644
index 0000000..fbf56d3
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst
@@ -0,0 +1,2 @@
+Fixed memory leak in ``socketserver.ThreadingMixIn`` introduced in Python
+3.7.
--
1.8.3.1

10
python3.spec
View File

@ -3,7 +3,7 @@ Summary: Interpreter of the Python3 programming language
URL: https://www.python.org/
Version: 3.7.9
Release: 12
Release: 13
License: Python
%global branchversion 3.7
@ -140,6 +140,7 @@ Patch6031: backport-35823-subprocess-Fix-handling-of-pthread_sigmask.patch
Patch6032: backport-35823-Allow-setsid-after-vfork-on-Linux.-GH-2294.patch
Patch6033: backport-42146-Fix-memory-leak-in-subprocess.Popen-in-cas.patch
Patch6034: backport-42146-Unify-cleanup-in-subprocess_fork_exec-GH-2.patch
patch6035: backport-Remove-thread-objects-which-finished-process-its-request.patch
Recommends: %{name}-help = %{version}-%{release}
Provides: python%{branchversion} = %{version}-%{release}
@ -266,6 +267,7 @@ rm Lib/ensurepip/_bundled/*.whl
%patch6032 -p1
%patch6033 -p1
%patch6034 -p1
%patch6035 -p1
sed -i "s/generic_os/%{_vendor}/g" Lib/platform.py
rm configure pyconfig.h.in
@ -867,6 +869,12 @@ export BEP_GTDLIST="$BEP_GTDLIST_TMP"
%{_mandir}/*/*
%changelog
* Sat May 29 2021 BruceGW<gyl93216@163.com> -3.7.9-13
- Type:bugfix
- ID:NA
- SUG:NA
- DESC: fix memory leak in socketserver.ThreadingMixIn
* Tue May 25 2021 hanxinke<hanxinke@huawei.com> - 3.7.9-12
- Type:enhancement
- CVE:NA