Skip to content

Commit 5d236ca

Browse files
authored
bpo-19675: Terminate processes if construction of a pool is failing. (GH-5614)
1 parent b4db249 commit 5d236ca

File tree

3 files changed

+54
-2
lines changed

3 files changed

+54
-2
lines changed

Lib/multiprocessing/pool.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,15 @@ def __init__(self, processes=None, initializer=None, initargs=(),
174174

175175
self._processes = processes
176176
self._pool = []
177-
self._repopulate_pool()
177+
try:
178+
self._repopulate_pool()
179+
except Exception:
180+
for p in self._pool:
181+
if p.exitcode is None:
182+
p.terminate()
183+
for p in self._pool:
184+
p.join()
185+
raise
178186

179187
self._worker_handler = threading.Thread(
180188
target=Pool._handle_workers,
@@ -251,10 +259,10 @@ def _repopulate_pool_static(ctx, Process, processes, pool, inqueue,
251259
initargs, maxtasksperchild,
252260
wrap_exception)
253261
)
254-
pool.append(w)
255262
w.name = w.name.replace('Process', 'PoolWorker')
256263
w.daemon = True
257264
w.start()
265+
pool.append(w)
258266
util.debug('added worker')
259267

260268
@staticmethod

Lib/test/_test_multiprocessing.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#
44

55
import unittest
6+
import unittest.mock
67
import queue as pyqueue
78
import contextlib
89
import time
@@ -4635,6 +4636,48 @@ def test_empty(self):
46354636
proc.join()
46364637

46374638

4639+
class TestPoolNotLeakOnFailure(unittest.TestCase):
4640+
4641+
def test_release_unused_processes(self):
4642+
# Issue #19675: During pool creation, if we can't create a process,
4643+
# don't leak already created ones.
4644+
will_fail_in = 3
4645+
forked_processes = []
4646+
4647+
class FailingForkProcess:
4648+
def __init__(self, **kwargs):
4649+
self.name = 'Fake Process'
4650+
self.exitcode = None
4651+
self.state = None
4652+
forked_processes.append(self)
4653+
4654+
def start(self):
4655+
nonlocal will_fail_in
4656+
if will_fail_in <= 0:
4657+
raise OSError("Manually induced OSError")
4658+
will_fail_in -= 1
4659+
self.state = 'started'
4660+
4661+
def terminate(self):
4662+
self.state = 'stopping'
4663+
4664+
def join(self):
4665+
if self.state == 'stopping':
4666+
self.state = 'stopped'
4667+
4668+
def is_alive(self):
4669+
return self.state == 'started' or self.state == 'stopping'
4670+
4671+
with self.assertRaisesRegex(OSError, 'Manually induced OSError'):
4672+
p = multiprocessing.pool.Pool(5, context=unittest.mock.MagicMock(
4673+
Process=FailingForkProcess))
4674+
p.close()
4675+
p.join()
4676+
self.assertFalse(
4677+
any(process.is_alive() for process in forked_processes))
4678+
4679+
4680+
46384681
class MiscTestCase(unittest.TestCase):
46394682
def test__all__(self):
46404683
# Just make sure names in blacklist are excluded
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
``multiprocessing.Pool`` no longer leaks processes if its initialization fails.

0 commit comments

Comments
 (0)