From faa0a6e66003dee9172b6388841b0f19c25e4050 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Tue, 11 Aug 2026 11:08:02 +0300 Subject: [PATCH] Apply the capped memory overhead to a job that requests exactly the cap set_cpu_and_mem clamps an over-large request to max_mem * job_max_server_node_memory_allocation and then, because that value is exactly the threshold it was just compared against with a strict `>`, falls through to the uncapped branch on every subsequent call. The clamped job is therefore given DEFAULT_JOB_MEMORY_OVERHEAD (1.10) instead of CAPPED_JOB_MEMORY_OVERHEAD (1.05), so the submit script asks the scheduler for 0.95 * 1.10 = 104.5% of the memory the server is configured to have, and the 'max_total_job_memory' keyword is never appended - making that keyword unreachable for any job that arrived at the cap through the troubleshooting loop, which is the only way a job gets there. trsh_ess_job clamps Gaussian memory to exactly this value, so the boundary is hit by ordinary troubleshooting rather than by an unusual input. On the 256 GB server2 fixture the request drops from 273941 MiB (267.5 GiB, more than the node has) to 261489 MiB (255.4 GiB). Only the Orca branch of trsh_ess_job reads 'max_total_job_memory'; the Gaussian branch carries it without acting on it, so newly emitting it changes no troubleshooting decision. --- arc/job/adapter.py | 4 ++-- arc/job/adapter_test.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/arc/job/adapter.py b/arc/job/adapter.py index 1b673a2057..ebe9674613 100644 --- a/arc/job/adapter.py +++ b/arc/job/adapter.py @@ -591,9 +591,9 @@ def set_cpu_and_mem(self): self.cpu_cores = self.cpu_cores or job_cpu_cores max_mem = servers[self.server].get('memory', None) if self.server is not None else 32.0 # Max memory per node in GB. job_max_server_node_memory_allocation = default_job_settings.get('job_max_server_node_memory_allocation', 0.95) - if max_mem is not None and self.job_memory_gb > max_mem * job_max_server_node_memory_allocation: + if max_mem is not None and self.job_memory_gb >= max_mem * job_max_server_node_memory_allocation: logger.warning(f'The memory for job {self.job_name} using {self.job_adapter} ({self.job_memory_gb} GB) ' - f'exceeds {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' + f'reaches {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' f'{self.server}. Setting it to {job_max_server_node_memory_allocation * max_mem:.2f} GB.') self.job_memory_gb = job_max_server_node_memory_allocation * max_mem total_submit_script_memory_mib = math.ceil(self.job_memory_gb * MEMORY_GB_TO_MIB * CAPPED_JOB_MEMORY_OVERHEAD) diff --git a/arc/job/adapter_test.py b/arc/job/adapter_test.py index 2691aa129a..0c31a548d5 100644 --- a/arc/job/adapter_test.py +++ b/arc/job/adapter_test.py @@ -22,7 +22,8 @@ from arc.level import Level from arc.species import ARCSpecies -servers, submit_filenames = settings['servers'], settings['submit_filenames'] +default_job_settings, servers, submit_filenames = \ + settings['default_job_settings'], settings['servers'], settings['submit_filenames'] class TestEnumerationClasses(unittest.TestCase): @@ -250,6 +251,23 @@ def test_set_cpu_and_mem(self): self.assertEqual(self.job_4.submit_script_memory, expected_memory) self.job_4.server = 'local' + capped_memory_gb = servers['server2']['memory'] \ + * default_job_settings['job_max_server_node_memory_allocation'] + original_job_memory_gb = self.job_4.job_memory_gb + self.addCleanup(setattr, self.job_4, 'job_memory_gb', original_job_memory_gb) + self.addCleanup(setattr, self.job_4, 'server', 'local') + self.job_4.server = 'server2' + self.job_4.cpu_cores = None + self.job_4.job_memory_gb = capped_memory_gb + self.job_4.set_cpu_and_mem() + self.assertEqual(self.job_4.job_memory_gb, capped_memory_gb) + self.assertEqual(self.job_4.submit_script_memory_mib, math.ceil(capped_memory_gb * 1024 * 1.05)) + self.assertLess(self.job_4.submit_script_memory_mib, servers['server2']['memory'] * 1024) + self.assertIn('max_total_job_memory', self.job_4.job_status[1]['keywords']) + self.job_4.job_status[1]['keywords'].remove('max_total_job_memory') + self.job_4.job_memory_gb = original_job_memory_gb + self.job_4.server = 'local' + def test_set_file_paths(self): """Test setting up the job's paths""" self.assertEqual(self.job_1.local_path, os.path.join(self.job_1.project_directory, 'calcs', 'Species',