From 998fe055b16a66bc8bc5494fc1f6d32d14e4acc4 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Fri, 28 Aug 2026 22:09:40 -0700 Subject: [PATCH 1/3] Stop reading a backend config field that no longer exists Exporting llama for Core ML fails before it starts: AttributeError: 'BackendConfig' object has no attribute 'mps' `BackendConfig` lost its `mps` field when the MPS backend was removed, but the Core ML branch in `export_llama_lib` still reads `llm_config.backend.mps.enabled`. That read sits inside the condition guarding the branch, so it is evaluated on every Core ML export rather than only when MPS was requested, which is why the whole path is dead rather than just the MPS part of it. This is what fails `test-llama-runner-mac (fp32, coreml)` and `test-huggingface-transformers-macos (llama3.2-1b|coreml_fp32_gpu)`. Test plan: The removed term could only ever be False, so no outcome changes. Enumerated all eight combinations of the three remaining flags against the old expression with the field forced False: identical results, zero mismatches. Against an installed wheel, the old expression raises the AttributeError above and the new one returns True for a Core ML export, which is the branch it should take. Confirmed no other source file reads `backend.mps` or `MpsConfig`. --- examples/models/llama/export_llama_lib.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/models/llama/export_llama_lib.py b/examples/models/llama/export_llama_lib.py index c22bdac4ed2..5cdc9a40940 100644 --- a/examples/models/llama/export_llama_lib.py +++ b/examples/models/llama/export_llama_lib.py @@ -1805,9 +1805,7 @@ def _export_llama(llm_config: LlmConfig) -> LLMEdgeManager: # noqa: C901 verbose=llm_config.debug.verbose, ) elif llm_config.backend.coreml.enabled and not ( - llm_config.backend.vulkan.enabled - or llm_config.backend.mps.enabled - or llm_config.backend.qnn.enabled + llm_config.backend.vulkan.enabled or llm_config.backend.qnn.enabled ): builder = _to_edge_and_lower_llama_coreml( builder_exported, From 91c4a974272248faaf64496cde6eaab79f3dd0e4 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sat, 29 Aug 2026 04:17:46 -0700 Subject: [PATCH 2/3] Add a test that fails when the stale read comes back Review pointed out the fix had no test, and that the macOS Core ML llama job which would catch this does not run on a pull request touching only this file: it lives in the trunk workflow, whose pull_request paths cover the CI scripts, the pytorch pin and the zephyr tree, none of which this change touches. So nothing here would notice the line returning. The test enables Core ML, patches the Core ML lowering to raise a marker, and asserts control reaches it. No macOS and no coremltools: what matters is that the guard no longer stops the branch before it starts. `enable_dynamic_shape` is False because the Core ML recipe rejects dynamic shapes and that validation runs first. Test plan: Against the installed wheel, swapping only `export_llama_lib.py`: base FAILED, AttributeError: 'BackendConfig' object has no attribute 'mps', line 1809 head 1 passed in 6.66s It runs in the `unittest` jobs, which do run on this pull request. --- .../llama/tests/test_export_llama_lib.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/examples/models/llama/tests/test_export_llama_lib.py b/examples/models/llama/tests/test_export_llama_lib.py index 40ba701f84e..73a0b66f1af 100644 --- a/examples/models/llama/tests/test_export_llama_lib.py +++ b/examples/models/llama/tests/test_export_llama_lib.py @@ -6,6 +6,7 @@ # LICENSE file in the root directory of this source tree. import unittest +from unittest.mock import patch from executorch.devtools.backend_debug import get_delegation_info @@ -41,6 +42,36 @@ class ExportLlamaLibTest(unittest.TestCase): + def test_core_ml_export_reaches_its_lowering(self): + """The Core ML branch must be reachable, not stopped by its own guard. + + The guard used to read a backend config field that no longer existed, so any Core ML + export raised AttributeError before reaching the lowering. Patching the lowering to raise + a marker keeps this off macOS and away from coremltools: what is asserted is that control + arrives there at all. + + The macOS Core ML llama job that would otherwise catch this lives in the trunk workflow, + which does not start for a change to this file, so without this test nothing on a pull + request notices the field coming back. + """ + llm_config = LlmConfig() + llm_config.backend.coreml.enabled = True + # The Core ML recipe rejects dynamic shapes, and that validation runs first. + llm_config.model.enable_dynamic_shape = False + + class _ReachedCoreMLLowering(Exception): + pass + + def _marker(*args, **kwargs): + raise _ReachedCoreMLLowering + + with patch( + "executorch.examples.models.llama.export_llama_lib._to_edge_and_lower_llama_coreml", + _marker, + ): + with self.assertRaises(_ReachedCoreMLLowering): + _export_llama(llm_config) + def test_has_expected_ops_and_op_counts(self): """ Checks the presence of unwanted expensive ops. From 7d3ee154015e8ef50cd906509c4c993886b96d86 Mon Sep 17 00:00:00 2001 From: Anthony Shoumikhin Date: Sun, 30 Aug 2026 10:11:18 -0700 Subject: [PATCH 3/3] Cover both repaired routes, and drop a stale claim from the test The first test pinned only one of the two cases the fix repaired, and it stayed green when the whole exclusion clause was deleted, which silently drops the Vulkan and QNN partitioners. So it guarded the removed field read and not the routing the clause exists for. Core ML with QNN is now covered too. It must fall through to the combined lowering, which still lowers Core ML but keeps the QNN partitioner. That case raised before the fix and it fails on the mutant, so between them the two tests pin both halves. Both now use `side_effect` with `assert_called_once`, which is how the rest of the tree writes this, instead of a local marker exception and a stub. Also removed the paragraph claiming the macOS Core ML job does not run for a change to this file. A trunk tag ran it on this branch and it passed, so the claim was wrong, and a comment inside a test that describes when a workflow starts cannot be checked from the test and goes stale on its own. Test plan: both tests, fixed pass stale field restored the Core ML case fails with the AttributeError exclusion clause deleted the QNN case fails --- .../llama/tests/test_export_llama_lib.py | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/examples/models/llama/tests/test_export_llama_lib.py b/examples/models/llama/tests/test_export_llama_lib.py index 73a0b66f1af..a24c3ec1c47 100644 --- a/examples/models/llama/tests/test_export_llama_lib.py +++ b/examples/models/llama/tests/test_export_llama_lib.py @@ -42,35 +42,35 @@ class ExportLlamaLibTest(unittest.TestCase): - def test_core_ml_export_reaches_its_lowering(self): - """The Core ML branch must be reachable, not stopped by its own guard. - - The guard used to read a backend config field that no longer existed, so any Core ML - export raised AttributeError before reaching the lowering. Patching the lowering to raise - a marker keeps this off macOS and away from coremltools: what is asserted is that control - arrives there at all. - - The macOS Core ML llama job that would otherwise catch this lives in the trunk workflow, - which does not start for a change to this file, so without this test nothing on a pull - request notices the field coming back. - """ + def _assert_export_reaches(self, target, **backends): + """Run an export and assert which lowering it routes to.""" llm_config = LlmConfig() llm_config.backend.coreml.enabled = True - # The Core ML recipe rejects dynamic shapes, and that validation runs first. + for name, enabled in backends.items(): + getattr(llm_config.backend, name).enabled = enabled + # Core ML and QNN reject dynamic shapes, and _validate_args runs before the lowering. llm_config.model.enable_dynamic_shape = False - class _ReachedCoreMLLowering(Exception): - pass - - def _marker(*args, **kwargs): - raise _ReachedCoreMLLowering - with patch( - "executorch.examples.models.llama.export_llama_lib._to_edge_and_lower_llama_coreml", - _marker, - ): - with self.assertRaises(_ReachedCoreMLLowering): + f"executorch.examples.models.llama.export_llama_lib.{target}", + side_effect=RuntimeError("reached"), + ) as lowering: + with self.assertRaises(RuntimeError): _export_llama(llm_config) + lowering.assert_called_once() + + def test_core_ml_alone_reaches_the_core_ml_lowering(self): + """The guard used to read a backend config field that no longer existed, so this raised + AttributeError before reaching any lowering.""" + self._assert_export_reaches("_to_edge_and_lower_llama_coreml") + + def test_core_ml_with_qnn_reaches_the_combined_lowering(self): + """The other case the removed read broke, and the one that pins the exclusion clause. + + Core ML with QNN must fall through to the combined lowering, which still lowers Core ML but + keeps the QNN partitioner. Without this, deleting the whole clause passes. + """ + self._assert_export_reaches("_to_edge_and_lower_llama", qnn=True) def test_has_expected_ops_and_op_counts(self): """