From 33dfef256a3e4b0d9b2ed93f5170160f823abd9e Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 16 Jul 2026 16:31:25 -0400 Subject: [PATCH] fix: don't run nested ProcessPoolExecutors This is kindof a regression from the introduction of the ProcessPoolExecutor for taskgraph generation. It's not _necessarily_ wrong or broken to nest these...but in graphs where memory usage is high during generation it can sometimes cause OOMs. There's really no advantage to using two levels of multiprocessing, so we should avoid doing so. --- src/taskgraph/main.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index e74da8a7d..f2fdd4f45 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -179,6 +179,16 @@ def format_taskgraph(options, parameters, overrides, logfile=None): return format_method(tg) +def format_taskgraph_multiple(*args, **kwargs): + """Thin wrapper around format_taskgraph to avoid the taskgraph generator + that it runs using multiple processes. (This function is intended to be + called by something that is already using a ProcessPoolExecutor, so + there's no advantage to spawning additional processes in the subprocesses.)""" + + os.environ["TASKGRAPH_SERIAL"] = "1" + return format_taskgraph(*args, **kwargs) + + def dump_output(out, path=None, params_spec=None): from taskgraph.parameters import Parameters # noqa: PLC0415 @@ -226,7 +236,7 @@ def logfile(spec): with ProcessPoolExecutor(max_workers=options["max_workers"]) as executor: for spec in parameters: f = executor.submit( - format_taskgraph, options, spec, overrides, logfile(spec) + format_taskgraph_multiple, options, spec, overrides, logfile(spec) ) futures[f] = spec