From 7d6ef805e9774db1874a0795b9037bfa3130230c Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Tue, 11 Aug 2026 16:12:01 -0400 Subject: [PATCH 1/3] test: fail the build when the six handlers drift apart again Every handler package tests its own spans, and each was correct on its own terms while a single run emitted `chat` spans that disagreed about what a finish reason or a cached token was. Nothing tested that the six agree, which is the property that actually matters and the one that broke. These tests own that property. They live outside the packages because no package can own an invariant about all six. The shape checks call each package's span constructors directly with a recording tracer, so they need no provider mocks and cannot be fooled by a handler that never reaches its own span code. They pin the three span names, the root's operation attribute, and the rule that the launchdarkly.* identity and the feature_flag event appear on the root and nowhere else. Two of them exist because of specific mistakes this port nearly shipped. One pins `gen_ai.system` to the literal `langchain` on the two LangChain handlers, where Python had been writing the configured provider name. The other pins `gen_ai.provider.name` to a binary anthropic-or-openai choice, because it names who served the model and anything that is not Anthropic is served by the OpenAI client; a passthrough of the configured name reads as correct and reports `bedrock` for a request an OpenAI client made. The vocabulary lock reads every attribute key, event name and naming template out of the sources and compares it to a committed set of 42. It fails when a key is added, removed or renamed anywhere. That is deliberate: an attribute is a public contract with whatever reads the traces, so changing one should mean editing the list and saying why. The set was verified to match the TypeScript SDK exactly. The second half of the lock is the one that earns its keep: it fails when a key stops being emitted, which is how a dashboard goes blank without anything failing. It caught its own regex being wrong while I wrote it, because the feature_flag event's attributes are built as a plain dict and never appear inside a set_attribute call. Verified by mutation rather than by passing. Renaming invoke_agent, adding an unlisted key, turning the LangChain provider into a passthrough, and leaking the LD identity onto a tool span each fail exactly one test and nothing else. --- tests/test_cross_handler_parity.py | 396 +++++++++++++++++++++++++++++ 1 file changed, 396 insertions(+) create mode 100644 tests/test_cross_handler_parity.py diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py new file mode 100644 index 0000000..8a60844 --- /dev/null +++ b/tests/test_cross_handler_parity.py @@ -0,0 +1,396 @@ +"""Cross-handler invariants: the six handlers must agree with each other. + +Every handler package tests its own spans. Nothing tested that the six agree, and that is exactly +how they drifted apart: each was correct on its own terms while a single run emitted `chat` spans +that disagreed about what a finish reason or a cached token was. + +These tests are the oracle for that. They live outside the packages because no package can own an +invariant about all six. + +There are two kinds of check here. + +The shape checks call each package's span constructors directly, with a recording tracer, so they +need no provider mocks and cannot be fooled by a handler that never reaches its own span code. + +The vocabulary lock reads every span attribute literal out of the source and compares it to a +committed set. It fails whenever a key is added, removed or renamed anywhere in the SDK. That is +deliberate: an attribute is a public contract with whatever reads the traces, and changing one +should require editing this list and saying why in the commit. + +See TELEMETRY-CONTRACT.md. +""" + +from __future__ import annotations + +import importlib +import re +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent + +#: Package directory to the module that holds its span construction. +HANDLERS: dict[str, str] = { + "claude-messages": "launchdarkly_ai_claude_messages.spans", + "claude-agents": "launchdarkly_ai_claude_agents.spans", + "openai-messages": "launchdarkly_ai_openai_messages.spans", + "openai-agents": "launchdarkly_ai_openai_agents.spans", + "langchain-messages": "launchdarkly_ai_langchain_messages.spans", + "langchain-agents": "launchdarkly_ai_langchain_agents.spans", +} + +#: `claude-agents` builds its `chat` span inside an inference tracker rather than in a standalone +#: function, because the Claude Agent SDK reports each inference as it streams rather than returning +#: one response per turn. The span it produces is still `chat {model}`; only the call site differs. +NO_STANDALONE_MODEL_SPAN = {"claude-agents"} + +CONFIG: dict[str, Any] = { + "model": {"name": "test-model-1"}, + "provider": {"name": "Anthropic"}, + "instructions": "Be helpful.", +} + +LD_VARIABLES: dict[str, Any] = { + "__ld": { + "configKey": "cfg", + "variationKey": "var", + "runId": "run-1", + "graphKey": "graph-1", + "environmentId": "env-1", + } +} + + +class RecordedSpan: + def __init__(self, name: str, context: Any = None) -> None: + self.name = name + self.context = context + self.attributes: dict[str, Any] = {} + self.events: list[str] = [] + + def set_attribute(self, key: str, value: Any) -> None: + self.attributes[key] = value + + def add_event(self, name: str, attributes: dict[str, Any] | None = None) -> None: + self.events.append(name) + + def set_status(self, code: Any, description: str | None = None) -> None: + pass + + def record_exception(self, exc: BaseException) -> None: + pass + + def end(self) -> None: + pass + + +class RecordingTracer: + """Stands in for the `trace` module inside a package's spans module.""" + + def __init__(self) -> None: + self.spans: list[RecordedSpan] = [] + + def get_tracer(self, name: str) -> RecordingTracer: + return self + + def start_span(self, name: str, context: Any = None) -> RecordedSpan: + span = RecordedSpan(name, context) + self.spans.append(span) + return span + + def set_span_in_context(self, span: RecordedSpan) -> Any: + return ("context-of", span) + + +@pytest.fixture(params=sorted(HANDLERS), ids=sorted(HANDLERS)) +def handler_spans( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Any: + """Each package's spans module, with its tracer replaced by a recorder.""" + package = request.param + module = importlib.import_module(HANDLERS[package]) + tracer = RecordingTracer() + monkeypatch.setattr(module, "trace", tracer) + return package, module, tracer + + +# ─── The surface every handler must expose ─────────────────────────────────── + +REQUIRED_FUNCTIONS = ( + "model_name", + "start_root_span", + "parent_context_of", + "start_tool_span", + "finish_root_span", + "succeed_span", + "mark_ok", + "fail_span", +) + + +class TestSharedSurface: + def test_every_package_exposes_the_same_span_functions( + self, handler_spans: Any + ) -> None: + package, module, _ = handler_spans + missing = [name for name in REQUIRED_FUNCTIONS if not hasattr(module, name)] + assert missing == [], f"{package} is missing {missing}" + + def test_every_package_has_a_model_span_constructor( + self, handler_spans: Any + ) -> None: + package, module, _ = handler_spans + if package in NO_STANDALONE_MODEL_SPAN: + pytest.skip(f"{package} builds its chat span inside an inference tracker") + assert hasattr(module, "start_model_span") + + +# ─── Span names ────────────────────────────────────────────────────────────── + + +class TestSpanNames: + def test_the_root_span_is_always_named_invoke_agent( + self, handler_spans: Any + ) -> None: + _, module, tracer = handler_spans + module.start_root_span(CONFIG, {}) + assert tracer.spans[0].name == "invoke_agent" + + def test_the_root_span_always_declares_its_operation( + self, handler_spans: Any + ) -> None: + _, module, tracer = handler_spans + module.start_root_span(CONFIG, {}) + assert tracer.spans[0].attributes["gen_ai.operation.name"] == "invoke_agent" + + def test_the_model_span_is_always_chat_plus_the_model( + self, handler_spans: Any + ) -> None: + # The semantic conventions name an inference span `{operation} {model}`. A handler that + # emitted a bare `chat` would aggregate more neatly and tell a reader nothing. + package, module, tracer = handler_spans + if package in NO_STANDALONE_MODEL_SPAN: + pytest.skip(f"{package} builds its chat span inside an inference tracker") + module.start_model_span(CONFIG, None) + assert tracer.spans[0].name == "chat test-model-1" + assert tracer.spans[0].attributes["gen_ai.operation.name"] == "chat" + + def test_the_tool_span_is_always_execute_tool_plus_the_name( + self, handler_spans: Any + ) -> None: + _, module, tracer = handler_spans + module.start_tool_span("get_weather", "call-1", None) + span = tracer.spans[0] + assert span.name == "execute_tool get_weather" + assert span.attributes["gen_ai.operation.name"] == "execute_tool" + assert span.attributes["gen_ai.tool.name"] == "get_weather" + assert span.attributes["gen_ai.tool.call.id"] == "call-1" + + +# ─── Where the LaunchDarkly identity lives ─────────────────────────────────── + +LD_ROOT_ATTRIBUTES = ( + "launchdarkly.operation.type", + "launchdarkly.config.key", + "launchdarkly.variation.key", + "launchdarkly.run.id", + "launchdarkly.graph.key", +) + + +class TestLaunchDarklyIdentity: + def test_the_root_carries_the_full_launchdarkly_identity( + self, handler_spans: Any + ) -> None: + _, module, tracer = handler_spans + module.start_root_span(CONFIG, LD_VARIABLES) + attrs = tracer.spans[0].attributes + missing = [k for k in LD_ROOT_ATTRIBUTES if k not in attrs] + assert missing == [] + + def test_the_root_emits_the_feature_flag_event(self, handler_spans: Any) -> None: + # The AI Config Monitoring traces tab finds a run through this event, not an attribute. + _, module, tracer = handler_spans + module.start_root_span(CONFIG, LD_VARIABLES) + assert "feature_flag" in tracer.spans[0].events + + def test_a_tool_span_carries_no_launchdarkly_identity( + self, handler_spans: Any + ) -> None: + # The root is the only span a config-scoped query finds. Duplicating the identity onto + # children makes one run look like several. + _, module, tracer = handler_spans + module.start_tool_span("get_weather", "call-1", None) + span = tracer.spans[0] + assert [k for k in span.attributes if k.startswith("launchdarkly.")] == [] + assert "feature_flag" not in span.events + + def test_a_model_span_carries_no_launchdarkly_identity( + self, handler_spans: Any + ) -> None: + package, module, tracer = handler_spans + if package in NO_STANDALONE_MODEL_SPAN: + pytest.skip(f"{package} builds its chat span inside an inference tracker") + module.start_model_span(CONFIG, None) + span = tracer.spans[0] + assert [k for k in span.attributes if k.startswith("launchdarkly.")] == [] + assert "feature_flag" not in span.events + + +# ─── Model identity ────────────────────────────────────────────────────────── + + +class TestModelIdentity: + def test_the_root_writes_both_provider_keys_and_the_request_model( + self, handler_spans: Any + ) -> None: + # `gen_ai.system` is the pre-1.37 name and `gen_ai.provider.name` the current one. Emitting + # only one of them either breaks old dashboards or leaves the SDK off-spec. + _, module, tracer = handler_spans + module.start_root_span(CONFIG, {}) + attrs = tracer.spans[0].attributes + assert "gen_ai.system" in attrs + assert "gen_ai.provider.name" in attrs + assert attrs["gen_ai.request.model"] == "test-model-1" + + def test_the_langchain_handlers_keep_the_framework_on_the_legacy_key( + self, handler_spans: Any + ) -> None: + # `gen_ai.provider.name` names who served the model, and its enum has no `langchain` + # member, so the framework name stays on the older key. + package, module, tracer = handler_spans + if not package.startswith("langchain-"): + pytest.skip("only the LangChain handlers split the two keys") + module.start_root_span(CONFIG, {}) + attrs = tracer.spans[0].attributes + assert attrs["gen_ai.system"] == "langchain" + assert attrs["gen_ai.provider.name"] == "anthropic" + + def test_the_langchain_provider_name_is_binary_not_a_passthrough( + self, handler_spans: Any + ) -> None: + # Anything that is not Anthropic is served by the OpenAI client, so the attribute follows the + # client actually instantiated rather than whatever the config happens to name. + package, module, _ = handler_spans + if not package.startswith("langchain-"): + pytest.skip("only the LangChain handlers make this choice") + for configured, expected in ( + ("Anthropic", "anthropic"), + ("OpenAI", "openai"), + ("Bedrock", "openai"), + ("Azure", "openai"), + ("", "openai"), + ): + config = {**CONFIG, "provider": {"name": configured}} + assert module.serving_provider(config) == expected, configured + + +# ─── The vocabulary lock ───────────────────────────────────────────────────── + +#: Every span attribute key, event name and naming template the SDK emits. +#: +#: Locked on purpose. An attribute is a public contract with whatever reads the traces, so adding, +#: removing or renaming one should mean editing this list and saying why in the commit message. +#: +#: Derived from the TypeScript SDK at 5178db1 and verified to match it exactly. +EXPECTED_VOCABULARY = { + # Operation and identity + "gen_ai.operation.name", + "gen_ai.system", + "gen_ai.provider.name", + "gen_ai.request.model", + "gen_ai.response.model", + "gen_ai.response.id", + "gen_ai.response.finish_reasons", + "gen_ai.agent.name", + "gen_ai.conversation.id", + # Usage + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.usage.cache_read.input_tokens", + "gen_ai.usage.cache_creation.input_tokens", + "gen_ai.usage.prompt_tokens", + "gen_ai.usage.completion_tokens", + # Content, canonical + "gen_ai.system_instructions", + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.tool.definitions", + # Content, OpenLLMetry + "gen_ai.prompt", + "gen_ai.completion", + "gen_ai.completion.0.role", + "gen_ai.completion.0.content", + # Tool calls + "gen_ai.tool.name", + "gen_ai.tool.call.id", + "gen_ai.tool.call.arguments", + "gen_ai.tool.call.result", + # Content events + "gen_ai.content.prompt", + "gen_ai.content.completion", + # LaunchDarkly + "launchdarkly.operation.type", + "launchdarkly.config.key", + "launchdarkly.variation.key", + "launchdarkly.run.id", + "launchdarkly.graph.key", + "launchdarkly.stream.abandoned", + "feature_flag", + "feature_flag.key", + "feature_flag.provider.name", + "feature_flag.set.id", + # Graph spans, unchanged from before the span work + "ld.ai.graph", + "ld.ai.graph.key", + "ld.ai.graph.path", +} + +_KEY_PATTERN = re.compile( + r'set_attribute\(\s*f?"([^"{]+)"' + r'|add_event\(\s*"([^"]+)"' + r'|start_span\(\s*"(ld\.ai\.graph)"' + r'|"(gen_ai\.[a-z_.0-9]+)"' + r'|f"(gen_ai\.[a-z_.]+)\.\{' + # The feature_flag event's own attributes are built as a plain dict before being handed to + # add_event, so they never appear inside a set_attribute call. + r'|"(feature_flag\.[a-z_.]+)"' +) + + +def _emitted_vocabulary() -> set[str]: + """Every attribute key, event name and template found in the package sources.""" + found: set[str] = set() + for path in (REPO_ROOT / "packages").glob("*/src/*/*.py"): + for match in _KEY_PATTERN.finditer(path.read_text()): + key = next((g for g in match.groups() if g), None) + if key and key.split(".")[0] in ( + "gen_ai", + "launchdarkly", + "feature_flag", + "ld", + ): + found.add(key) + return found + + +class TestVocabularyLock: + def test_the_sdk_emits_no_key_this_list_does_not_know_about(self) -> None: + unexpected = _emitted_vocabulary() - EXPECTED_VOCABULARY + assert unexpected == set(), ( + "New span attribute keys found. If this is deliberate, add them to " + "EXPECTED_VOCABULARY and say why in the commit message. If the two SDKs should agree, " + "add the key to the TypeScript SDK in the same change." + ) + + def test_every_key_this_list_names_is_still_emitted(self) -> None: + # Catches a key silently disappearing, which is how a dashboard goes blank without anything + # failing. + emitted = _emitted_vocabulary() + # `gen_ai.prompt.{i}` and `gen_ai.completion.{i}` are written by template, so the base names + # appear rather than the indexed forms. + missing = EXPECTED_VOCABULARY - emitted + assert missing == set(), f"keys no longer emitted anywhere: {sorted(missing)}" From 04b80ddbd3a1be6d2d00459a58b0adb308b237f4 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Tue, 11 Aug 2026 16:38:53 -0400 Subject: [PATCH 2/3] test: stop the vocabulary lock pinning dead code, and cover the live carrier Two faults in the lock I added, both pointed out by Bugbot on #36. It pinned gen_ai.completion.0.role and .content, which appear only inside set_openllmetry_completion. That helper has no call sites, so those literals describe nothing the SDK emits. Deleting the dead helper would have failed the lock for no reason, and the entries also gave cover to a genuine drop of the live carrier. They now sit in a separate SUPERSEDED_VOCABULARY set that says what it is and when to delete it. The live OpenLLMetry keys were never verified at all. They are written as f{prefix}.{index}.role, so there is no literal for a static scan to find, and the lock was only ever seeing the two prefix arguments. That carrier is the one LaunchDarkly's LLM trace view reads today, so dropping it renders an empty transcript while every canonical attribute is still present and every static check still passes. Adds three runtime tests for it, including the capture gate. Checked both directions by mutation: deleting the dead helper no longer fails anything, and removing the live writes now fails exactly those tests. --- tests/test_cross_handler_parity.py | 61 +++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index 8a60844..c69ebfb 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -319,11 +319,11 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "gen_ai.input.messages", "gen_ai.output.messages", "gen_ai.tool.definitions", - # Content, OpenLLMetry + # Content, OpenLLMetry. These two are the prefixes handed to the indexed writer; the keys it + # actually emits are `gen_ai.prompt.{i}.role` and friends, built by f-string and therefore + # invisible to any static scan. TestOpenLLMetryCarrier below covers those at runtime. "gen_ai.prompt", "gen_ai.completion", - "gen_ai.completion.0.role", - "gen_ai.completion.0.content", # Tool calls "gen_ai.tool.name", "gen_ai.tool.call.id", @@ -349,6 +349,17 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "ld.ai.graph.path", } +#: Keys that appear only inside superseded code, kept exported for one release. +#: +#: Held apart from the live vocabulary on purpose. `set_openllmetry_completion` has no call sites, +#: so these two literals describe nothing the SDK emits. Folding them into the set above would mean +#: deleting that dead helper fails the lock, and would also let a genuine drop of the live indexed +#: carrier hide behind them. Delete this set and its two names together when the helper goes. +SUPERSEDED_VOCABULARY = { + "gen_ai.completion.0.role", + "gen_ai.completion.0.content", +} + _KEY_PATTERN = re.compile( r'set_attribute\(\s*f?"([^"{]+)"' r'|add_event\(\s*"([^"]+)"' @@ -379,7 +390,7 @@ def _emitted_vocabulary() -> set[str]: class TestVocabularyLock: def test_the_sdk_emits_no_key_this_list_does_not_know_about(self) -> None: - unexpected = _emitted_vocabulary() - EXPECTED_VOCABULARY + unexpected = _emitted_vocabulary() - EXPECTED_VOCABULARY - SUPERSEDED_VOCABULARY assert unexpected == set(), ( "New span attribute keys found. If this is deliberate, add them to " "EXPECTED_VOCABULARY and say why in the commit message. If the two SDKs should agree, " @@ -390,7 +401,45 @@ def test_every_key_this_list_names_is_still_emitted(self) -> None: # Catches a key silently disappearing, which is how a dashboard goes blank without anything # failing. emitted = _emitted_vocabulary() - # `gen_ai.prompt.{i}` and `gen_ai.completion.{i}` are written by template, so the base names - # appear rather than the indexed forms. missing = EXPECTED_VOCABULARY - emitted assert missing == set(), f"keys no longer emitted anywhere: {sorted(missing)}" + + +class TestOpenLLMetryCarrier: + """The indexed carrier is written by f-string, so only a runtime check can see it. + + This is the one LaunchDarkly's LLM trace view reads today, so dropping it renders an empty + transcript while every canonical attribute is still present and every static check still passes. + The vocabulary lock above cannot cover it: `f"{prefix}.{index}.role"` has no literal key to scan + for. + """ + + def test_the_input_side_writes_indexed_role_and_content(self) -> None: + from launchdarkly_ai_server import set_input_content_attributes, text_message + + span = RecordedSpan("chat test-model-1") + set_input_content_attributes( + span, + True, + system_instructions="Be brief.", + messages=[text_message("user", "hi")], + ) + assert span.attributes["gen_ai.prompt.0.role"] == "system" + assert span.attributes["gen_ai.prompt.0.content"] == "Be brief." + assert span.attributes["gen_ai.prompt.1.role"] == "user" + assert span.attributes["gen_ai.prompt.1.content"] == "hi" + + def test_the_output_side_writes_indexed_role_and_content(self) -> None: + from launchdarkly_ai_server import set_output_content_attributes, text_message + + span = RecordedSpan("chat test-model-1") + set_output_content_attributes(span, True, [text_message("assistant", "hello")]) + assert span.attributes["gen_ai.completion.0.role"] == "assistant" + assert span.attributes["gen_ai.completion.0.content"] == "hello" + + def test_the_carrier_stays_behind_the_capture_gate(self) -> None: + from launchdarkly_ai_server import set_input_content_attributes, text_message + + span = RecordedSpan("chat test-model-1") + set_input_content_attributes(span, False, messages=[text_message("user", "hi")]) + assert span.attributes == {} From 8eee53ff4a01800a52eff116ae5a985f5b82ef19 Mon Sep 17 00:00:00 2001 From: Alexis Georges Date: Tue, 11 Aug 2026 16:54:18 -0400 Subject: [PATCH 3/3] fix(ci): run the tests at the repo root, and stop dead code satisfying the lock CI ran `pytest packages/*/tests`, a glob that silently skips the repo-root tests/ directory. The cross-handler oracle added in this layer therefore never ran in CI at all: 72 tests, including every invariant no single package can own, invisible to the build the moment they were written. Handler drift would not have failed anything, which is the one thing the layer exists to do. Now runs bare pytest, the same command make test uses, which collects both. The vocabulary lock also let dead code satisfy it. Quarantining the superseded helpers' keys was not enough, because gen_ai.prompt is written by the live content writer AND by dead set_openllmetry_prompt, so naming it as expected let the dead copy keep the lock green after the live write was removed. The scan now cuts the superseded function bodies out of the source before looking, so only a live write can satisfy anything, and the quarantine set is gone. Checked by mutation: removing the live gen_ai.prompt writes now fails the lock and the runtime carrier tests, where before it failed nothing. Both found by Bugbot on #36, the CI one at High severity. --- .github/workflows/ci.yml | 4 +++- tests/test_cross_handler_parity.py | 37 +++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ebe501..cd077e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,7 +61,9 @@ jobs: python-version: "3.12" - run: uv lock - run: uv sync --all-packages - - run: uv run pytest packages/*/tests + # Not `packages/*/tests`: that glob silently skips the repo-root `tests/` directory, which + # holds the invariants no single package can own. `make test` runs this same bare command. + - run: uv run pytest # ─── Build ──────────────────────────────────────────────────────────────── build: diff --git a/tests/test_cross_handler_parity.py b/tests/test_cross_handler_parity.py index c69ebfb..09d7ca4 100644 --- a/tests/test_cross_handler_parity.py +++ b/tests/test_cross_handler_parity.py @@ -349,16 +349,31 @@ def test_the_langchain_provider_name_is_binary_not_a_passthrough( "ld.ai.graph.path", } -#: Keys that appear only inside superseded code, kept exported for one release. +#: Functions kept exported for one release that nothing calls any more. #: -#: Held apart from the live vocabulary on purpose. `set_openllmetry_completion` has no call sites, -#: so these two literals describe nothing the SDK emits. Folding them into the set above would mean -#: deleting that dead helper fails the lock, and would also let a genuine drop of the live indexed -#: carrier hide behind them. Delete this set and its two names together when the helper goes. -SUPERSEDED_VOCABULARY = { - "gen_ai.completion.0.role", - "gen_ai.completion.0.content", -} +#: Their bodies are cut out before the scan below, rather than their keys being listed as expected. +#: Listing the keys does not work: `gen_ai.prompt` is written by the live content writer *and* by +#: dead `set_openllmetry_prompt`, so naming it as expected lets the dead copy satisfy the lock after +#: the live one is removed. Cutting the dead code out means only live writes can satisfy anything. +#: +#: Delete these names when the functions go. The lock will tell you if you miss one. +SUPERSEDED_FUNCTIONS = ( + "set_openllmetry_prompt", + "set_openllmetry_completion", +) + + +def _without_superseded(source: str) -> str: + """Drops the body of every superseded function, so a dead write cannot satisfy the lock.""" + for name in SUPERSEDED_FUNCTIONS: + start = source.find(f"def {name}(") + if start == -1: + continue + nxt = source.find("\ndef ", start + 1) + end = len(source) if nxt == -1 else nxt + source = source[:start] + source[end:] + return source + _KEY_PATTERN = re.compile( r'set_attribute\(\s*f?"([^"{]+)"' @@ -376,7 +391,7 @@ def _emitted_vocabulary() -> set[str]: """Every attribute key, event name and template found in the package sources.""" found: set[str] = set() for path in (REPO_ROOT / "packages").glob("*/src/*/*.py"): - for match in _KEY_PATTERN.finditer(path.read_text()): + for match in _KEY_PATTERN.finditer(_without_superseded(path.read_text())): key = next((g for g in match.groups() if g), None) if key and key.split(".")[0] in ( "gen_ai", @@ -390,7 +405,7 @@ def _emitted_vocabulary() -> set[str]: class TestVocabularyLock: def test_the_sdk_emits_no_key_this_list_does_not_know_about(self) -> None: - unexpected = _emitted_vocabulary() - EXPECTED_VOCABULARY - SUPERSEDED_VOCABULARY + unexpected = _emitted_vocabulary() - EXPECTED_VOCABULARY assert unexpected == set(), ( "New span attribute keys found. If this is deliberate, add them to " "EXPECTED_VOCABULARY and say why in the commit message. If the two SDKs should agree, "