1- import multiprocessing
21import gzip
2+ from concurrent .futures import ProcessPoolExecutor , as_completed , process
33from typing import Callable , TYPE_CHECKING
44from collections .abc import Generator
5+ from sdf_pipeline import logger
56
67if TYPE_CHECKING :
78 # https://adamj.eu/tech/2021/05/13/python-type-hints-how-to-fix-circular-imports/
@@ -13,75 +14,58 @@ def read_records_from_gzipped_sdf(sdf_path: str) -> Generator[str, None, None]:
1314 current_record = ""
1415 # TODO: guard file opening.
1516 with gzip .open (sdf_path , "rb" ) as gzipped_sdf :
16- # decompress SDF line-by-line to avoid loading entire SDF into memory
17+ # Decompress SDF line-by-line to avoid loading entire SDF into memory.
1718 for decompressed_line in gzipped_sdf :
1819 decoded_line = decompressed_line .decode ("utf-8" , "backslashreplace" )
1920 current_record += decoded_line
2021 if decoded_line .strip () == "$$$$" :
2122 # TODO: harden SDF parsing according to
22- # http://www.dalkescientific.com/writings/diary/archive/2020/09/18/handling_the_sdf_record_delimiter.html
23+ # http://www.dalkescientific.com/writings/diary/archive/2020/09/18/handling_the_sdf_record_delimiter.html.
2324 yield current_record
2425 current_record = ""
2526
26-
27- def _produce_molfiles (
28- molfile_queue : multiprocessing .Queue , sdf_path : str , n_poison_pills : int
29- ) -> None :
30- for molfile in read_records_from_gzipped_sdf (sdf_path ):
31- molfile_queue .put (molfile )
32-
33- for _ in range (n_poison_pills ):
34- molfile_queue .put ("DONE" ) # poison pill: tell consumer processes we're done
35-
36-
37- def _consume_molfiles (
38- molfile_queue : multiprocessing .Queue ,
39- result_queue : multiprocessing .Queue ,
40- consumer_function : Callable ,
41- ) -> None :
42- for molfile in iter (molfile_queue .get , "DONE" ):
43- result_queue .put (consumer_function (molfile ))
44-
45- result_queue .put (f"DONE" )
27+ return None
4628
4729
4830def run (
49- sdf_path : str ,
50- consumer_function : Callable ,
51- number_of_consumer_processes : int ,
31+ sdf_path : str , consumer_function : Callable , number_of_consumer_processes : int
5232) -> Generator ["ConsumerResult" , None , None ]:
53- molfile_queue : multiprocessing .Queue = multiprocessing .Queue () # TODO: limit size?
54- result_queue : multiprocessing .Queue = multiprocessing .Queue ()
5533
56- producer_process = multiprocessing .Process (
57- target = _produce_molfiles ,
58- args = (molfile_queue , sdf_path , number_of_consumer_processes ),
59- )
60- producer_process .start ()
34+ try :
35+ with ProcessPoolExecutor (
36+ max_workers = number_of_consumer_processes
37+ ) as process_pool :
38+ futures = (
39+ process_pool .submit (consumer_function , molfile )
40+ for molfile in read_records_from_gzipped_sdf (sdf_path )
41+ )
42+ """
43+ `as_completed` isn't iterating `futures` lazily. It instantiates a set of futures:
44+ https://github.com/python/cpython/blob/f383ca1a6fa1a2a83c8c1a0e56cf997c77fa2893/Lib/concurrent/futures/_base.py#L220.
45+ In case of memory constraints, consider chunking `futures`: https://bugs.python.org/issue34168.
46+ """
47+ for future in as_completed (futures ):
48+ exception = future .exception ()
49+ if exception is not None :
50+ logger .error (f"could not process { sdf_path } : { exception } " )
51+
52+ raise exception # Dead programs don't tell lies.
53+
54+ yield future .result ()
6155
62- consumer_processes = [
63- multiprocessing .Process (
64- target = _consume_molfiles ,
65- args = (
66- molfile_queue ,
67- result_queue ,
68- consumer_function ,
69- ),
70- )
71- for process_id in range (number_of_consumer_processes )
72- ]
73- for consumer_process in consumer_processes :
74- consumer_process .start ()
56+ except process .BrokenProcessPool as exception :
57+ """
58+ `exception` can be thrown in two ways with (slightly) different messages:
59+ 1) While processing a future in the `as_completed` loop above,
60+ in which case `exception` is re-raised below.
61+ See https://github.com/python/cpython/blob/7f074a771bc4e3e299799fabf9b054a03f6693d2/Lib/concurrent/futures/process.py#L489
62+ and https://discuss.python.org/t/as-completed-not-yielding-futures-that-raised-until-pool-exits/39920.
63+ 2) While submitting a task to the `process_pool`.
64+ See https://github.com/python/cpython/blob/7f074a771bc4e3e299799fabf9b054a03f6693d2/Lib/concurrent/futures/process.py#L793.
65+ We need this `except` block in order to catch 2). For 1), the inner `raise` would be sufficient.
66+ """
67+ logger .error (f"could not process { sdf_path } : { exception } " )
7568
76- number_of_finished_consumer_processes = 0
77- while number_of_finished_consumer_processes < number_of_consumer_processes :
78- result = result_queue .get () # blocks until result is available
79- if result == "DONE" :
80- number_of_finished_consumer_processes += 1
81- continue
82- yield result
69+ raise exception
8370
84- # processes won't join before all queues their interacting with are empty
85- producer_process .join ()
86- for consumer_process in consumer_processes :
87- consumer_process .join ()
71+ return None
0 commit comments