Visitar URL original
Raise exception for processes that terminate with non-zero exit code · TUCAN-nest/SDF-pipeline@0b867a0 · GitHub
Skip to content
This repository was archived by the owner on May 21, 2024. It is now read-only.

Commit 0b867a0

Browse files
committed
Raise exception for processes that terminate with non-zero exit code
Prior to this commit, `core.run` would wait for results indefinitely when a child process terminated with a non-zero exit code (e.g., segfault). This was due to the fact that `multiprocessing.Pool` does not raise an exception ( python/cpython#66587 ). This commit replaces `multiprocessing.Pool` with `concurrent.futures.ProcessPoolExecutor`. In contrast to the former, the latter raises an exception in case a child process terminates with a non-zero exit code.
1 parent 1b387df commit 0b867a0

3 files changed

Lines changed: 127 additions & 64 deletions

File tree

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "sdf_pipeline"
3-
version = "0.2.0"
3+
version = "1.0.0"
44

55
[project.optional-dependencies]
66
dev = ["pytest"]

‎src/sdf_pipeline/core.py‎

Lines changed: 41 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
import multiprocessing
21
import gzip
2+
from concurrent.futures import ProcessPoolExecutor, as_completed, process
33
from typing import Callable, TYPE_CHECKING
44
from collections.abc import Generator
5+
from sdf_pipeline import logger
56

67
if TYPE_CHECKING:
78
# https://adamj.eu/tech/2021/05/13/python-type-hints-how-to-fix-circular-imports/
@@ -13,75 +14,58 @@ def read_records_from_gzipped_sdf(sdf_path: str) -> Generator[str, None, None]:
1314
current_record = ""
1415
# TODO: guard file opening.
1516
with gzip.open(sdf_path, "rb") as gzipped_sdf:
16-
# decompress SDF line-by-line to avoid loading entire SDF into memory
17+
# Decompress SDF line-by-line to avoid loading entire SDF into memory.
1718
for decompressed_line in gzipped_sdf:
1819
decoded_line = decompressed_line.decode("utf-8", "backslashreplace")
1920
current_record += decoded_line
2021
if decoded_line.strip() == "$$$$":
2122
# TODO: harden SDF parsing according to
22-
# http://www.dalkescientific.com/writings/diary/archive/2020/09/18/handling_the_sdf_record_delimiter.html
23+
# http://www.dalkescientific.com/writings/diary/archive/2020/09/18/handling_the_sdf_record_delimiter.html.
2324
yield current_record
2425
current_record = ""
2526

26-
27-
def _produce_molfiles(
28-
molfile_queue: multiprocessing.Queue, sdf_path: str, n_poison_pills: int
29-
) -> None:
30-
for molfile in read_records_from_gzipped_sdf(sdf_path):
31-
molfile_queue.put(molfile)
32-
33-
for _ in range(n_poison_pills):
34-
molfile_queue.put("DONE") # poison pill: tell consumer processes we're done
35-
36-
37-
def _consume_molfiles(
38-
molfile_queue: multiprocessing.Queue,
39-
result_queue: multiprocessing.Queue,
40-
consumer_function: Callable,
41-
) -> None:
42-
for molfile in iter(molfile_queue.get, "DONE"):
43-
result_queue.put(consumer_function(molfile))
44-
45-
result_queue.put(f"DONE")
27+
return None
4628

4729

4830
def run(
49-
sdf_path: str,
50-
consumer_function: Callable,
51-
number_of_consumer_processes: int,
31+
sdf_path: str, consumer_function: Callable, number_of_consumer_processes: int
5232
) -> Generator["ConsumerResult", None, None]:
53-
molfile_queue: multiprocessing.Queue = multiprocessing.Queue() # TODO: limit size?
54-
result_queue: multiprocessing.Queue = multiprocessing.Queue()
5533

56-
producer_process = multiprocessing.Process(
57-
target=_produce_molfiles,
58-
args=(molfile_queue, sdf_path, number_of_consumer_processes),
59-
)
60-
producer_process.start()
34+
try:
35+
with ProcessPoolExecutor(
36+
max_workers=number_of_consumer_processes
37+
) as process_pool:
38+
futures = (
39+
process_pool.submit(consumer_function, molfile)
40+
for molfile in read_records_from_gzipped_sdf(sdf_path)
41+
)
42+
"""
43+
`as_completed` isn't iterating `futures` lazily. It instantiates a set of futures:
44+
https://github.com/python/cpython/blob/f383ca1a6fa1a2a83c8c1a0e56cf997c77fa2893/Lib/concurrent/futures/_base.py#L220.
45+
In case of memory constraints, consider chunking `futures`: https://bugs.python.org/issue34168.
46+
"""
47+
for future in as_completed(futures):
48+
exception = future.exception()
49+
if exception is not None:
50+
logger.error(f"could not process {sdf_path}: {exception}")
51+
52+
raise exception # Dead programs don't tell lies.
53+
54+
yield future.result()
6155

62-
consumer_processes = [
63-
multiprocessing.Process(
64-
target=_consume_molfiles,
65-
args=(
66-
molfile_queue,
67-
result_queue,
68-
consumer_function,
69-
),
70-
)
71-
for process_id in range(number_of_consumer_processes)
72-
]
73-
for consumer_process in consumer_processes:
74-
consumer_process.start()
56+
except process.BrokenProcessPool as exception:
57+
"""
58+
`exception` can be thrown in two ways with (slightly) different messages:
59+
1) While processing a future in the `as_completed` loop above,
60+
in which case `exception` is re-raised below.
61+
See https://github.com/python/cpython/blob/7f074a771bc4e3e299799fabf9b054a03f6693d2/Lib/concurrent/futures/process.py#L489
62+
and https://discuss.python.org/t/as-completed-not-yielding-futures-that-raised-until-pool-exits/39920.
63+
2) While submitting a task to the `process_pool`.
64+
See https://github.com/python/cpython/blob/7f074a771bc4e3e299799fabf9b054a03f6693d2/Lib/concurrent/futures/process.py#L793.
65+
We need this `except` block in order to catch 2). For 1), the inner `raise` would be sufficient.
66+
"""
67+
logger.error(f"could not process {sdf_path}: {exception}")
7568

76-
number_of_finished_consumer_processes = 0
77-
while number_of_finished_consumer_processes < number_of_consumer_processes:
78-
result = result_queue.get() # blocks until result is available
79-
if result == "DONE":
80-
number_of_finished_consumer_processes += 1
81-
continue
82-
yield result
69+
raise exception
8370

84-
# processes won't join before all queues their interacting with are empty
85-
producer_process.join()
86-
for consumer_process in consumer_processes:
87-
consumer_process.join()
71+
return None

‎tests/test_drivers.py‎

Lines changed: 85 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,46 @@
33
import pytest
44
import logging
55
import json
6+
import ctypes
7+
import timeit
68
from operator import add
79
from functools import reduce
810
from pathlib import Path
911
from typing import Callable
10-
from sdf_pipeline import drivers
12+
from functools import partial
13+
from concurrent.futures.process import BrokenProcessPool
14+
from sdf_pipeline import drivers, core
1115

1216

13-
def regression_consumer(
14-
molfile: str, get_molfile_id: Callable
15-
) -> drivers.ConsumerResult:
17+
def consumer(molfile: str, get_molfile_id: Callable) -> drivers.ConsumerResult:
1618
return drivers.ConsumerResult(
1719
get_molfile_id(molfile),
1820
"regression",
1921
str(len(molfile)),
2022
)
2123

2224

25+
def busy_consumer(molfile: str, get_molfile_id: Callable) -> drivers.ConsumerResult:
26+
n = 0
27+
for i in range(100):
28+
n += sum([len(line) ** i for line in molfile.split("\n")])
29+
return drivers.ConsumerResult(
30+
get_molfile_id(molfile),
31+
"regression",
32+
n,
33+
)
34+
35+
36+
def segfaulting_consumer(
37+
molfile: str, get_molfile_id: Callable
38+
) -> drivers.ConsumerResult:
39+
ctypes.string_at(0)
40+
41+
42+
def raising_consumer(molfile: str, get_molfile_id: Callable) -> drivers.ConsumerResult:
43+
1 / 0
44+
45+
2346
@pytest.fixture
2447
def sdf_path():
2548
return Path(__file__).parent.absolute().joinpath("data/mcule_20000.sdf.gz")
@@ -45,7 +68,7 @@ def test_regression_reference_driver(sdf_path, tmp_path):
4568
exit_code = drivers.regression_reference(
4669
sdf_path=sdf_path,
4770
reference_path=tmp_path / "regression_reference.sqlite",
48-
consumer_function=regression_consumer,
71+
consumer_function=consumer,
4972
get_molfile_id=_get_mcule_id,
5073
number_of_consumer_processes=2,
5174
)
@@ -63,7 +86,7 @@ def test_regression_driver(sdf_path, reference_path, caplog):
6386
exit_code = drivers.regression(
6487
sdf_path=sdf_path,
6588
reference_path=reference_path,
66-
consumer_function=regression_consumer,
89+
consumer_function=consumer,
6790
get_molfile_id=_get_mcule_id,
6891
number_of_consumer_processes=2,
6992
)
@@ -77,3 +100,59 @@ def test_regression_driver(sdf_path, reference_path, caplog):
77100
"info": "regression",
78101
"diff": '{"current": "926", "reference": "42"}',
79102
}
103+
104+
105+
def test_core_raises_on_exception(sdf_path, caplog):
106+
caplog.set_level(logging.ERROR, logger="sdf_pipeline")
107+
with pytest.raises(ZeroDivisionError):
108+
for _ in core.run(
109+
sdf_path=str(sdf_path),
110+
consumer_function=partial(raising_consumer, get_molfile_id=_get_mcule_id),
111+
number_of_consumer_processes=2,
112+
):
113+
pass
114+
assert (
115+
caplog.records[-1].message == f"could not process {sdf_path}: division by zero"
116+
)
117+
118+
119+
def test_core_raises_on_segfault(sdf_path, caplog):
120+
caplog.set_level(logging.ERROR, logger="sdf_pipeline")
121+
with pytest.raises(BrokenProcessPool):
122+
for _ in core.run(
123+
sdf_path=str(sdf_path),
124+
consumer_function=partial(
125+
segfaulting_consumer, get_molfile_id=_get_mcule_id
126+
),
127+
number_of_consumer_processes=2,
128+
):
129+
pass
130+
# See comment in `core.run` for why there's two alternative messages.
131+
assert caplog.records[-1].message in [
132+
f"could not process {sdf_path}: A process in the process pool was terminated abruptly while the future was running or pending.",
133+
f"could not process {sdf_path}: A child process terminated abruptly, the process pool is not usable anymore",
134+
]
135+
136+
137+
@pytest.mark.skip
138+
def test_performance(sdf_path):
139+
def run_core(n_processes):
140+
def _run_core():
141+
for _ in core.run(
142+
sdf_path=str(sdf_path),
143+
consumer_function=partial(busy_consumer, get_molfile_id=_get_mcule_id),
144+
number_of_consumer_processes=n_processes,
145+
):
146+
pass
147+
148+
return _run_core
149+
150+
previous_execution_time = 0
151+
for n in [1, 2, 4, 8, 16]:
152+
execution_time = timeit.Timer(run_core(n)).timeit(1)
153+
print(f"{n} processes finished in {execution_time} seconds.")
154+
155+
if previous_execution_time:
156+
assert execution_time < previous_execution_time
157+
158+
previous_execution_time = execution_time

0 commit comments

Comments
 (0)