diff --git a/arc/job/adapter.py b/arc/job/adapter.py index 1b673a2057..ebe9674613 100644 --- a/arc/job/adapter.py +++ b/arc/job/adapter.py @@ -591,9 +591,9 @@ def set_cpu_and_mem(self): self.cpu_cores = self.cpu_cores or job_cpu_cores max_mem = servers[self.server].get('memory', None) if self.server is not None else 32.0 # Max memory per node in GB. job_max_server_node_memory_allocation = default_job_settings.get('job_max_server_node_memory_allocation', 0.95) - if max_mem is not None and self.job_memory_gb > max_mem * job_max_server_node_memory_allocation: + if max_mem is not None and self.job_memory_gb >= max_mem * job_max_server_node_memory_allocation: logger.warning(f'The memory for job {self.job_name} using {self.job_adapter} ({self.job_memory_gb} GB) ' - f'exceeds {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' + f'reaches {100 * job_max_server_node_memory_allocation}% of the the maximum node memory on ' f'{self.server}. Setting it to {job_max_server_node_memory_allocation * max_mem:.2f} GB.') self.job_memory_gb = job_max_server_node_memory_allocation * max_mem total_submit_script_memory_mib = math.ceil(self.job_memory_gb * MEMORY_GB_TO_MIB * CAPPED_JOB_MEMORY_OVERHEAD) diff --git a/arc/job/adapter_test.py b/arc/job/adapter_test.py index 2691aa129a..0c31a548d5 100644 --- a/arc/job/adapter_test.py +++ b/arc/job/adapter_test.py @@ -22,7 +22,8 @@ from arc.level import Level from arc.species import ARCSpecies -servers, submit_filenames = settings['servers'], settings['submit_filenames'] +default_job_settings, servers, submit_filenames = \ + settings['default_job_settings'], settings['servers'], settings['submit_filenames'] class TestEnumerationClasses(unittest.TestCase): @@ -250,6 +251,23 @@ def test_set_cpu_and_mem(self): self.assertEqual(self.job_4.submit_script_memory, expected_memory) self.job_4.server = 'local' + capped_memory_gb = servers['server2']['memory'] \ + * default_job_settings['job_max_server_node_memory_allocation'] + original_job_memory_gb = self.job_4.job_memory_gb + self.addCleanup(setattr, self.job_4, 'job_memory_gb', original_job_memory_gb) + self.addCleanup(setattr, self.job_4, 'server', 'local') + self.job_4.server = 'server2' + self.job_4.cpu_cores = None + self.job_4.job_memory_gb = capped_memory_gb + self.job_4.set_cpu_and_mem() + self.assertEqual(self.job_4.job_memory_gb, capped_memory_gb) + self.assertEqual(self.job_4.submit_script_memory_mib, math.ceil(capped_memory_gb * 1024 * 1.05)) + self.assertLess(self.job_4.submit_script_memory_mib, servers['server2']['memory'] * 1024) + self.assertIn('max_total_job_memory', self.job_4.job_status[1]['keywords']) + self.job_4.job_status[1]['keywords'].remove('max_total_job_memory') + self.job_4.job_memory_gb = original_job_memory_gb + self.job_4.server = 'local' + def test_set_file_paths(self): """Test setting up the job's paths""" self.assertEqual(self.job_1.local_path, os.path.join(self.job_1.project_directory, 'calcs', 'Species',