From cd23be505b7df7cef39c0663787ec0960b12a4ab Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 18 Aug 2026 10:43:36 -0700 Subject: [PATCH 1/6] update Luna client to support additional metrics --- engine/src/agent_control_engine/core.py | 2 +- engine/tests/test_core.py | 86 ++++++++++++++++++- .../src/agent_control_evaluators/_base.py | 19 +++- .../__init__.py | 2 + .../luna/__init__.py | 2 + .../luna/client.py | 35 +++++++- .../luna/evaluator.py | 20 ++++- .../galileo/tests/test_luna_evaluator.py | 61 ++++++++++++- models/src/agent_control_models/agent.py | 9 ++ models/tests/test_step_runtime_context.py | 51 +++++++++++ sdks/python/src/agent_control/evaluation.py | 7 ++ .../src/agent_control/integrations/_core.py | 6 +- .../src/agent_control/integrations/_tools.py | 52 +++++++++++ .../integrations/google_adk/plugin.py | 32 +++++++ .../integrations/strands/plugin.py | 21 +++++ sdks/python/tests/test_evaluation.py | 34 +++++++- sdks/python/tests/test_google_adk_plugin.py | 28 ++++++ sdks/python/tests/test_strands_plugin.py | 25 ++++++ 18 files changed, 481 insertions(+), 11 deletions(-) create mode 100644 models/tests/test_step_runtime_context.py create mode 100644 sdks/python/src/agent_control/integrations/_tools.py diff --git a/engine/src/agent_control_engine/core.py b/engine/src/agent_control_engine/core.py index b2cd81b3..7dbba3c7 100644 --- a/engine/src/agent_control_engine/core.py +++ b/engine/src/agent_control_engine/core.py @@ -320,7 +320,7 @@ async def _evaluate_leaf( timeout = DEFAULT_EVALUATOR_TIMEOUT result = await asyncio.wait_for( - evaluator.evaluate(data), + evaluator.evaluate_with_context(data, request.step), timeout=timeout, ) except TimeoutError: diff --git a/engine/tests/test_core.py b/engine/tests/test_core.py index baa46bab..aae3f8fc 100644 --- a/engine/tests/test_core.py +++ b/engine/tests/test_core.py @@ -39,13 +39,15 @@ class SimpleConfig(BaseModel): # Shared state for coordination between test evaluators _execution_log: list[str] = [] _blocker_event: asyncio.Event | None = None +_context_calls: list[tuple[Any, Step]] = [] def reset_test_state() -> None: """Reset shared test state.""" - global _execution_log, _blocker_event + global _execution_log, _blocker_event, _context_calls _execution_log = [] _blocker_event = asyncio.Event() + _context_calls = [] class AllowEvaluator(Evaluator[SimpleConfig]): @@ -163,6 +165,24 @@ async def evaluate(self, data: Any) -> EvaluatorResult: return result +class ContextEvaluator(Evaluator[SimpleConfig]): + """Evaluator that records selector data and full request context.""" + + metadata = EvaluatorMetadata( + name="test-context", + version="1.0.0", + description="Records contextual calls", + ) + config_model = SimpleConfig + + async def evaluate(self, data: Any) -> EvaluatorResult: + raise AssertionError("engine should call evaluate_with_context") + + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + _context_calls.append((data, step)) + return EvaluatorResult(matched=False, confidence=1.0, message="context received") + + @dataclass class MockControlWithIdentity: """Mock control for testing.""" @@ -185,6 +205,7 @@ def setup_test_evaluators(): BlockerEvaluator, SlowEvaluator, MetadataEvaluator, + ContextEvaluator, ]: try: register_evaluator(evaluator_cls) @@ -253,6 +274,69 @@ def make_control( ) +@pytest.mark.asyncio +async def test_context_evaluator_receives_selected_data_and_complete_step() -> None: + # Given: a selector targeting output and a step with structured context + engine = ControlEngine( + [make_control(1, "context", "test-context", action="observe", path="output")] + ) + step = Step( + type="llm", + name="test-step", + input="question", + output="answer", + context={"conversation_id": "c-1"}, + tools=[{"name": "search", "description": "Search", "input_schema": {}}], + ground_truth="expected answer", + ) + + # When: evaluating the request through the engine + await engine.process( + EvaluationRequest( + agent_name="00000000-0000-0000-0000-000000000001", + step=step, + stage="pre", + ) + ) + + # Then: selector behavior is unchanged and the complete Step is separate + assert _context_calls == [("answer", step)] + + +@pytest.mark.asyncio +async def test_cached_context_evaluator_handles_concurrent_steps_without_retaining_state() -> None: + # Given: one cached evaluator configuration and two independent requests + engine = ControlEngine( + [make_control(1, "context", "test-context", action="observe", path="input")] + ) + first = Step(type="llm", name="test-step", input="first", ground_truth="one") + second = Step(type="llm", name="test-step", input="second", ground_truth="two") + + # When: the requests are evaluated concurrently + await asyncio.gather( + engine.process( + EvaluationRequest( + agent_name="00000000-0000-0000-0000-000000000001", + step=first, + stage="pre", + ) + ), + engine.process( + EvaluationRequest( + agent_name="00000000-0000-0000-0000-000000000001", + step=second, + stage="pre", + ) + ), + ) + + # Then: each selected value remains paired with its own full Step + assert {(data, step.ground_truth) for data, step in _context_calls} == { + ("first", "one"), + ("second", "two"), + } + + # ============================================================================= # Test: Parallel Execution # ============================================================================= diff --git a/evaluators/builtin/src/agent_control_evaluators/_base.py b/evaluators/builtin/src/agent_control_evaluators/_base.py index c32b92a5..d83c0c3a 100644 --- a/evaluators/builtin/src/agent_control_evaluators/_base.py +++ b/evaluators/builtin/src/agent_control_evaluators/_base.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypeVar -from agent_control_models import EvaluatorResult +from agent_control_models import EvaluatorResult, Step from agent_control_models.base import BaseModel if TYPE_CHECKING: @@ -161,6 +161,23 @@ async def evaluate(self, data: Any) -> EvaluatorResult: """ pass + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + """Evaluate selected data with access to the complete runtime step. + + The default implementation preserves compatibility with evaluators that + implement only :meth:`evaluate`. Evaluators must treat ``step`` as + immutable request-scoped context because evaluator instances are cached + and may be invoked concurrently. + + Args: + data: Data extracted by the configured selector. + step: Complete runtime step for the current request. + + Returns: + EvaluatorResult produced by this evaluator. + """ + return await self.evaluate(data) + def get_timeout_seconds(self) -> float: """Get timeout in seconds from config or metadata default.""" timeout_ms: int = getattr(self.config, "timeout_ms", self.metadata.timeout_ms) diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py index 5606bf5d..93eb8567 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py @@ -25,6 +25,7 @@ LunaEvaluator, LunaEvaluatorConfig, LunaOperator, + ScorerInvokeRecord, ScorerInvokeRequest, ScorerInvokeResponse, ) @@ -32,6 +33,7 @@ __all__ = [ "GalileoLunaClient", "ScorerInvokeRequest", + "ScorerInvokeRecord", "ScorerInvokeResponse", "LunaEvaluator", "LunaEvaluatorConfig", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py index b26feaac..39f95016 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py @@ -3,6 +3,7 @@ from agent_control_evaluator_galileo.luna.client import ( GalileoLunaClient, ScorerInvokeInputs, + ScorerInvokeRecord, ScorerInvokeRequest, ScorerInvokeResponse, ) @@ -12,6 +13,7 @@ __all__ = [ "GalileoLunaClient", "ScorerInvokeInputs", + "ScorerInvokeRecord", "ScorerInvokeRequest", "ScorerInvokeResponse", "LunaEvaluatorConfig", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index a1fc4d71..e0b759b7 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -14,7 +14,7 @@ from urllib.parse import urlsplit import httpx -from agent_control_models import JSONObject, JSONValue +from agent_control_models import JSONObject, JSONValue, Step from pydantic import BaseModel, Field, PrivateAttr, model_validator logger = logging.getLogger(__name__) @@ -154,7 +154,18 @@ class ScorerInvokeInputs(BaseModel): query: JSONValue = "" response: JSONValue = "" ground_truth: JSONValue = None - tools: JSONValue = None + tools: list[JSONObject] | None = None + + +class ScorerInvokeRecord(BaseModel): + """Structured runtime record sent alongside legacy scorer inputs.""" + + type: str = Field(min_length=1) + input: JSONValue = None + output: JSONValue = None + context: JSONObject | None = None + tools: list[JSONObject] | None = None + dataset_output: JSONValue = None class ScorerInvokeRequest(BaseModel): @@ -172,6 +183,7 @@ class ScorerInvokeRequest(BaseModel): scorer_version_id: str | None = Field(default=None, min_length=1) scorer_label: str | None = Field(default=None, min_length=1) inputs: ScorerInvokeInputs + record: ScorerInvokeRecord | None = None config: JSONObject = Field(default_factory=dict) @model_validator(mode="after") @@ -357,6 +369,7 @@ async def invoke( scorer_label: str | None = None, input: JSONValue = None, output: JSONValue = None, + step: Step | None = None, config: JSONObject | None = None, timeout: float = DEFAULT_TIMEOUT_SECS, headers: dict[str, str] | None = None, @@ -369,6 +382,7 @@ async def invoke( scorer_label: Optional display/metadata label. input: Optional user/system prompt text. output: Optional model response text. + step: Optional complete runtime step used for structured dual-write. config: Optional scorer-specific configuration. timeout: Request timeout in seconds. headers: Additional request headers. @@ -390,7 +404,22 @@ async def invoke( scorer_version_id=scorer_version_id, scorer_label=scorer_label, inputs=ScorerInvokeInputs( - query="" if input is None else input, response="" if output is None else output + query="" if input is None else input, + response="" if output is None else output, + ground_truth=step.ground_truth if step is not None else None, + tools=step.tools if step is not None else None, + ), + record=( + ScorerInvokeRecord( + type=step.type, + input=step.input, + output=step.output, + context=step.context, + tools=step.tools, + dataset_output=step.ground_truth, + ) + if step is not None + else None ), config=config if config is not None else {}, ).to_dict() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 777c22cf..7a837aaa 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -10,7 +10,7 @@ import httpx from agent_control_evaluators import Evaluator, EvaluatorMetadata, register_evaluator -from agent_control_models import EvaluatorResult, JSONValue +from agent_control_models import EvaluatorResult, JSONValue, Step from .client import GalileoLunaClient, ScorerInvokeResponse from .config import LunaEvaluatorConfig, coerce_number @@ -198,6 +198,22 @@ async def evaluate(self, data: Any) -> EvaluatorResult: Returns: EvaluatorResult with local threshold decision and scorer metadata. """ + return await self._evaluate(data, step=None) + + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + """Evaluate selected data while dual-writing the complete runtime step. + + Args: + data: Data selected by the configured control selector. + step: Complete runtime step for structured scorer context. + + Returns: + EvaluatorResult with local threshold decision and scorer metadata. + """ + return await self._evaluate(data, step=step) + + async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: + """Run a Luna evaluation with optional structured runtime context.""" input_text, output_text = self._prepare_payload(data) if not (_has_text(input_text) or _has_text(output_text)): return EvaluatorResult( @@ -209,6 +225,8 @@ async def evaluate(self, data: Any) -> EvaluatorResult: try: scorer_kwargs = self._scorer_kwargs() + if step is not None: + scorer_kwargs["step"] = step response = await self._get_client().invoke( **scorer_kwargs, input=input_text if _has_text(input_text) else None, diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 6c605c9c..37b17c27 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -10,7 +10,7 @@ import httpx import pytest -from agent_control_models import EvaluatorResult +from agent_control_models import EvaluatorResult, Step from pydantic import ValidationError LUNA_ENV = { @@ -515,6 +515,65 @@ def handler(request: httpx.Request) -> httpx.Response: assert payload["internal"] is True assert payload["scope"] == "scorers.invoke" + @pytest.mark.asyncio + async def test_client_dual_writes_legacy_inputs_and_structured_record(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response( + 200, + json={"score": 0.9, "status": "success", "additive_field": "ignored"}, + ) + + # Given: selected legacy values and a complete structured runtime Step + step = Step( + type="llm", + name="answer", + input={"messages": [{"role": "user", "content": "question"}]}, + output={"text": "answer"}, + context={"session": "s-1"}, + tools=[{"name": "search", "description": "Search", "input_schema": {}}], + ground_truth={"text": "expected"}, + ) + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + # When: invoking the rollout-compatible Runners endpoint + try: + response = await client.invoke( + scorer_id="scorer-123", + input="selected question", + output="selected answer", + step=step, + ) + finally: + await client.close() + + # Then: old inputs and the expanded record are sent together + assert response.score == 0.9 + assert captured["body"] == { + "scorer_id": "scorer-123", + "inputs": { + "query": "selected question", + "response": "selected answer", + "ground_truth": {"text": "expected"}, + "tools": [{"name": "search", "description": "Search", "input_schema": {}}], + }, + "record": { + "type": "llm", + "input": {"messages": [{"role": "user", "content": "question"}]}, + "output": {"text": "answer"}, + "context": {"session": "s-1"}, + "tools": [{"name": "search", "description": "Search", "input_schema": {}}], + "dataset_output": {"text": "expected"}, + }, + "config": {}, + } + @pytest.mark.asyncio async def test_client_forwards_scorer_version_id_when_configured(self) -> None: from agent_control_evaluator_galileo.luna import GalileoLunaClient diff --git a/models/src/agent_control_models/agent.py b/models/src/agent_control_models/agent.py index 6a0eedba..e60573c9 100644 --- a/models/src/agent_control_models/agent.py +++ b/models/src/agent_control_models/agent.py @@ -142,6 +142,8 @@ def validate_type(cls, v: str) -> str: class Step(BaseModel): """Runtime payload for an agent step invocation.""" + model_config = {"frozen": True} + type: str = Field( ..., min_length=1, @@ -159,6 +161,13 @@ class Step(BaseModel): context: JSONObject | None = Field( None, description="Optional context (conversation history, metadata, etc.)" ) + tools: list[JSONObject] | None = Field( + None, + description="Complete structured definitions of tools available to the LLM", + ) + ground_truth: JSONValue | None = Field( + None, description="Optional expected or reference output for this step" + ) @field_validator("type") @classmethod diff --git a/models/tests/test_step_runtime_context.py b/models/tests/test_step_runtime_context.py new file mode 100644 index 00000000..2b9f80f6 --- /dev/null +++ b/models/tests/test_step_runtime_context.py @@ -0,0 +1,51 @@ +"""Tests for optional structured runtime context on Step.""" + +from agent_control_models import Step +from pydantic import ValidationError + + +def test_step_accepts_and_serializes_structured_tools_and_ground_truth() -> None: + # Given: a runtime LLM step with structured scorer context + payload = { + "type": "llm", + "name": "answer", + "input": {"question": "Capital of France?"}, + "output": "Paris", + "tools": [ + { + "name": "search", + "description": "Search documents", + "input_schema": {"type": "object"}, + } + ], + "ground_truth": {"answer": "Paris"}, + } + + # When: validating and serializing the public Step model + serialized = Step.model_validate(payload).model_dump(mode="json") + + # Then: structured values survive unchanged + assert serialized["tools"] == payload["tools"] + assert serialized["ground_truth"] == payload["ground_truth"] + + +def test_existing_step_payload_remains_valid() -> None: + # Given/When: an existing payload without structured context + step = Step(type="llm", name="answer", input="hello") + + # Then: new fields remain optional + assert step.tools is None + assert step.ground_truth is None + + +def test_step_is_immutable_runtime_context() -> None: + # Given: a validated runtime step + step = Step(type="llm", name="answer", input="hello") + + # When/Then: evaluators cannot replace request-scoped fields + try: + step.input = "changed" + except ValidationError: + pass + else: + raise AssertionError("Step must be frozen") diff --git a/sdks/python/src/agent_control/evaluation.py b/sdks/python/src/agent_control/evaluation.py index e79b736b..2a045bb5 100644 --- a/sdks/python/src/agent_control/evaluation.py +++ b/sdks/python/src/agent_control/evaluation.py @@ -15,6 +15,7 @@ EvaluationResponse, EvaluationResult, EvaluatorResult, + JSONValue, Step, ) @@ -520,6 +521,8 @@ async def evaluate_controls( input: Any | None = None, output: Any | None = None, context: dict[str, Any] | None = None, + tools: list[dict[str, JSONValue]] | None = None, + ground_truth: JSONValue | None = None, step_type: Literal["tool", "llm"] = "llm", stage: Literal["pre", "post"] = "pre", agent_name: str, @@ -552,6 +555,10 @@ async def evaluate_controls( } if context is not None: step_dict["context"] = context + if tools is not None: + step_dict["tools"] = tools + if ground_truth is not None: + step_dict["ground_truth"] = ground_truth step_obj = Step(**step_dict) # type: ignore[arg-type] resolved_controls = state.server_controls or [] diff --git a/sdks/python/src/agent_control/integrations/_core.py b/sdks/python/src/agent_control/integrations/_core.py index 27693dbf..c9587dcd 100644 --- a/sdks/python/src/agent_control/integrations/_core.py +++ b/sdks/python/src/agent_control/integrations/_core.py @@ -4,7 +4,7 @@ from typing import Any, Literal -from agent_control_models import EvaluationResult +from agent_control_models import EvaluationResult, JSONValue import agent_control from agent_control import ControlSteerError, ControlViolationError @@ -51,6 +51,8 @@ async def _evaluate_and_enforce( input: Any | None = None, output: Any | None = None, context: dict[str, Any] | None = None, + tools: list[dict[str, JSONValue]] | None = None, + ground_truth: JSONValue | None = None, step_type: Literal["tool", "llm"] = "llm", stage: Literal["pre", "post"] = "pre", ) -> EvaluationResult: @@ -61,6 +63,8 @@ async def _evaluate_and_enforce( input=input, output=output, context=context, + tools=tools, + ground_truth=ground_truth, step_type=step_type, stage=stage, agent_name=agent_name, diff --git a/sdks/python/src/agent_control/integrations/_tools.py b/sdks/python/src/agent_control/integrations/_tools.py new file mode 100644 index 00000000..84f55f90 --- /dev/null +++ b/sdks/python/src/agent_control/integrations/_tools.py @@ -0,0 +1,52 @@ +"""Normalization helpers for complete framework tool registries.""" + +from __future__ import annotations + +from typing import Any + +from agent_control_models import JSONObject, JSONValue + + +def normalized_tool_definition( + *, + name: str, + description: str | None, + input_schema: dict[str, Any] | None, +) -> JSONObject: + """Return the framework-neutral available-tool representation.""" + definition: dict[str, JSONValue] = { + "name": name, + "description": description or "", + "input_schema": input_schema or {}, + } + return definition + + +def normalize_strands_tool_specs(specs: object) -> list[JSONObject] | None: + """Normalize a complete Strands tool-spec collection, if available.""" + if not isinstance(specs, list): + return None + + normalized: list[JSONObject] = [] + for spec in specs: + if not isinstance(spec, dict): + return None + name = spec.get("name") + if not isinstance(name, str) or not name: + return None + description = spec.get("description") + raw_input_schema = spec.get("inputSchema") + if isinstance(raw_input_schema, dict) and isinstance(raw_input_schema.get("json"), dict): + input_schema = raw_input_schema["json"] + elif isinstance(raw_input_schema, dict): + input_schema = raw_input_schema + else: + input_schema = None + normalized.append( + normalized_tool_definition( + name=name, + description=description if isinstance(description, str) else None, + input_schema=input_schema, + ) + ) + return normalized diff --git a/sdks/python/src/agent_control/integrations/google_adk/plugin.py b/sdks/python/src/agent_control/integrations/google_adk/plugin.py index 28e59698..75266623 100644 --- a/sdks/python/src/agent_control/integrations/google_adk/plugin.py +++ b/sdks/python/src/agent_control/integrations/google_adk/plugin.py @@ -19,6 +19,7 @@ from agent_control._schema_derivation import derive_schemas from agent_control._state import state from agent_control.integrations._core import _evaluate_and_enforce +from agent_control.integrations._tools import normalized_tool_definition from agent_control.validation import ensure_agent_name try: @@ -109,11 +110,13 @@ def __init__( self._known_steps: dict[tuple[str, str], StepSchemaDict] = {} self._synced_step_keys: set[tuple[str, str]] = set() self._step_sync_tasks: dict[tuple[str, str], asyncio.Task[None]] = {} + self._available_tools_by_step: dict[str, list[dict[str, Any]]] = {} def bind(self, agent: Any) -> None: """Pre-register known ADK steps before the runner starts.""" steps = self._discover_steps(agent) + self._remember_available_tools(agent) self._remember_steps(steps) self._sync_steps_blocking(steps, raise_on_error=True) @@ -176,6 +179,7 @@ async def before_model_callback( step_name, input=request_text, context=context, + tools=self._available_tools_by_step.get(step_name), step_type="llm", stage="pre", ) @@ -227,6 +231,7 @@ async def after_model_callback( input=input_text, output=output_text, context=context, + tools=self._available_tools_by_step.get(step_name), step_type="llm", stage="post", ) @@ -662,6 +667,33 @@ def _iter_tools(self, agent: Any) -> Iterable[Any]: return tools return [] + def _remember_available_tools(self, root_agent: Any) -> None: + """Capture each ADK agent's complete bound tool set as structured JSON.""" + available_tools: dict[str, list[dict[str, Any]]] = {} + for agent in self._iter_agents(root_agent): + agent_name = getattr(agent, "name", None) + if not isinstance(agent_name, str) or not agent_name: + continue + step_name = self._resolve_step_name( + agent_name, + step_type="llm", + callback_context=None, + agent=agent, + ) + definitions: list[dict[str, Any]] = [] + for tool in self._iter_tools(agent): + tool_name = self._resolve_tool_step_name(tool, agent_step_name=step_name) + schema = self._build_tool_step_schema(tool, tool_name) + definitions.append( + normalized_tool_definition( + name=resolve_tool_name(tool), + description=schema.get("description"), + input_schema=schema.get("input_schema"), + ) + ) + available_tools[step_name] = definitions + self._available_tools_by_step = available_tools + def _remember_steps(self, steps: Iterable[StepSchemaDict]) -> None: for step in steps: key = (step["type"], step["name"]) diff --git a/sdks/python/src/agent_control/integrations/strands/plugin.py b/sdks/python/src/agent_control/integrations/strands/plugin.py index 1aa503cd..367a673c 100644 --- a/sdks/python/src/agent_control/integrations/strands/plugin.py +++ b/sdks/python/src/agent_control/integrations/strands/plugin.py @@ -10,6 +10,7 @@ import agent_control from agent_control import ControlSteerError, ControlViolationError +from agent_control.integrations._tools import normalize_strands_tool_specs try: from strands.hooks import ( # type: ignore[import-not-found] @@ -86,6 +87,7 @@ def __init__( self.event_control_list = event_control_list self.on_violation_callback = on_violation_callback self.enable_logging = enable_logging + self._tool_registry: Any | None = None def _invoke_callback(self, control_name: str, stage: str, result: EvaluationResult) -> None: if self.on_violation_callback: @@ -109,6 +111,7 @@ async def _evaluate_and_enforce( input: Any | None = None, output: Any | None = None, context: dict[str, Any] | None = None, + tools: list[dict[str, Any]] | None = None, step_type: Literal["tool", "llm"] = "llm", stage: Literal["pre", "post"] = "pre", use_runtime_error: bool = False, @@ -118,6 +121,7 @@ async def _evaluate_and_enforce( input=input, output=output, context=context, + tools=tools, step_type=step_type, stage=stage, agent_name=self.agent_name, @@ -181,6 +185,7 @@ async def _evaluate_and_enforce( ) def init_agent(self, agent: Any) -> None: + self._tool_registry = getattr(agent, "tool_registry", None) event_map = { BeforeInvocationEvent: self.check_before_invocation, BeforeModelCallEvent: self.check_before_model, @@ -211,6 +216,7 @@ async def check_before_invocation(self, event: BeforeInvocationEvent) -> None: await self._evaluate_and_enforce( step_name="check_before_invocation", input=input_text, + tools=self._available_tools(), step_type="llm", stage="pre", ) @@ -220,6 +226,7 @@ async def check_before_model(self, event: BeforeModelCallEvent) -> None: await self._evaluate_and_enforce( step_name="check_before_model", input=input_text, + tools=self._available_tools(), step_type="llm", stage="pre", ) @@ -232,6 +239,7 @@ async def check_after_model(self, event: AfterModelCallEvent) -> None: input=input_text, output=output_text, context=context, + tools=self._available_tools(), step_type="llm", stage="post", ) @@ -270,6 +278,7 @@ async def check_before_node(self, event: BeforeNodeCallEvent) -> None: step_name=node_id, input=input_text, context=context, + tools=self._available_tools(), step_type="llm", stage="pre", ) @@ -283,10 +292,22 @@ async def check_after_node(self, event: AfterNodeCallEvent) -> None: input=input_text, output=output_text, context=context, + tools=self._available_tools(), step_type="llm", stage="post", ) + def _available_tools(self) -> list[dict[str, Any]] | None: + """Return the complete current Strands registry in normalized form.""" + get_specs = getattr(self._tool_registry, "get_all_tool_specs", None) + if not callable(get_specs): + return None + try: + return normalize_strands_tool_specs(get_specs()) + except Exception: + logger.warning("Unable to capture complete Strands tool definitions", exc_info=True) + return None + def _extract_user_message_from_list(self, messages: list | None, reverse: bool = False) -> str: if not messages: return "" diff --git a/sdks/python/tests/test_evaluation.py b/sdks/python/tests/test_evaluation.py index 2fb92555..885a69a7 100644 --- a/sdks/python/tests/test_evaluation.py +++ b/sdks/python/tests/test_evaluation.py @@ -60,8 +60,10 @@ def json(self) -> dict[str, object]: "type": "llm", "name": "chat", "input": "hello", - "output": None, - "context": None, + "output": None, + "context": None, + "tools": None, + "ground_truth": None, }, "stage": "pre", "target_type": None, @@ -125,6 +127,34 @@ async def test_evaluate_controls_with_context(monkeypatch): assert mock_check.call_args is not None +@pytest.mark.asyncio +async def test_evaluate_controls_preserves_explicit_tools_and_ground_truth(monkeypatch): + """Explicit structured scorer context is preserved on the SDK Step.""" + # Given: a configured SDK and local evaluation boundary + mock_check = AsyncMock(return_value=EvaluationResult(is_safe=True, confidence=1.0)) + monkeypatch.setattr(evaluation, "check_evaluation_with_local", mock_check) + tools = [ + {"name": "search", "description": "Search", "input_schema": {"type": "object"}} + ] + + # When: a direct SDK caller supplies tools and ground truth + with patch("agent_control.state.server_url", "http://localhost:8000"): + await evaluation.evaluate_controls( + step_name="chat", + input="question", + output="answer", + tools=tools, + ground_truth={"answer": "expected"}, + stage="post", + agent_name="test-bot", + ) + + # Then: both fields are retained as structured JSON + step = mock_check.call_args.kwargs["step"] + assert step.tools == tools + assert step.ground_truth == {"answer": "expected"} + + @pytest.mark.asyncio async def test_evaluate_controls_uses_session_api_key_header(monkeypatch): """evaluate_controls should pass init's API-key header into the client.""" diff --git a/sdks/python/tests/test_google_adk_plugin.py b/sdks/python/tests/test_google_adk_plugin.py index f68bd341..f72057ad 100644 --- a/sdks/python/tests/test_google_adk_plugin.py +++ b/sdks/python/tests/test_google_adk_plugin.py @@ -540,6 +540,34 @@ def test_bind_discovers_root_sub_agents_and_tools(plugin_module): mock_sync.assert_called_once() +@pytest.mark.asyncio +async def test_bound_agent_passes_complete_normalized_tools_to_llm(plugin_module): + # Given: an ADK agent whose complete tool list is available during binding + plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") + root = SimpleNamespace( + name="planner", + tools=[MockTool("search_docs", "Search documentation")], + ) + with patch.object(plugin, "_sync_steps_blocking"): + plugin.bind(root) + + # When: an LLM callback is evaluated + with patch.object( + plugin_module, "_evaluate_and_enforce", AsyncMock(return_value=MagicMock()) + ) as mock_eval: + await plugin.before_model_callback( + callback_context=MockCallbackContext("planner"), + llm_request=MockLlmRequest("hello"), + ) + + # Then: available definitions are normalized, not inferred from a call + definitions = mock_eval.await_args.kwargs["tools"] + assert len(definitions) == 1 + assert definitions[0]["name"] == "search_docs" + assert definitions[0]["description"] == "Search documentation" + assert definitions[0]["input_schema"]["properties"]["city"]["type"] == "string" + + def test_bind_keeps_duplicate_tool_names_distinct_across_sub_agents(plugin_module): plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") root = SimpleNamespace( diff --git a/sdks/python/tests/test_strands_plugin.py b/sdks/python/tests/test_strands_plugin.py index 3f52848e..c14557c9 100644 --- a/sdks/python/tests/test_strands_plugin.py +++ b/sdks/python/tests/test_strands_plugin.py @@ -424,6 +424,31 @@ def test_hook_initialization(): assert hook.enable_logging is False +def test_init_agent_captures_complete_strands_tool_registry(agent_control_hook): + # Given: a Strands agent exposing its complete normalized registry + registry = MagicMock() + registry.get_all_tool_specs.return_value = [ + { + "name": "search_docs", + "description": "Search documentation", + "inputSchema": {"json": {"type": "object", "properties": {}}}, + } + ] + agent = MagicMock(tool_registry=registry) + + # When: the plugin is initialized against that agent + agent_control_hook.init_agent(agent) + + # Then: the full registry is normalized without executable objects + assert agent_control_hook._available_tools() == [ + { + "name": "search_docs", + "description": "Search documentation", + "input_schema": {"type": "object", "properties": {}}, + } + ] + + def test_hook_with_callback(): """Test AgentControlPlugin with violation callback.""" from agent_control.integrations.strands.plugin import AgentControlPlugin From 2525945ac9ce49d4ce259d4db202ae1de0d5ce8e Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 18 Aug 2026 11:02:55 -0700 Subject: [PATCH 2/6] add more coverage and fix ci --- evaluators/builtin/tests/test_base.py | 19 +++++++- .../galileo/tests/test_luna_evaluator.py | 35 ++++++++++++++ sdks/python/tests/test_google_adk_plugin.py | 12 +++++ sdks/python/tests/test_integration_tools.py | 47 +++++++++++++++++++ sdks/python/tests/test_strands_plugin.py | 16 +++++++ sdks/typescript/src/generated/models/step.ts | 24 ++++++++-- sdks/typescript/tests/generated-smoke.test.ts | 34 ++++++++++++++ 7 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 sdks/python/tests/test_integration_tools.py diff --git a/evaluators/builtin/tests/test_base.py b/evaluators/builtin/tests/test_base.py index 776a8d01..d81cc3f5 100644 --- a/evaluators/builtin/tests/test_base.py +++ b/evaluators/builtin/tests/test_base.py @@ -3,11 +3,12 @@ Architecture: Evaluators take config at __init__, evaluate() only takes data. """ -import pytest from typing import Any +import pytest + from agent_control_evaluators import Evaluator, EvaluatorConfig, EvaluatorMetadata -from agent_control_models import EvaluatorResult +from agent_control_models import EvaluatorResult, Step class MockConfig(EvaluatorConfig): @@ -106,6 +107,20 @@ async def test_mock_evaluator_evaluate_no_match(self): assert result.matched is False + @pytest.mark.asyncio + async def test_contextual_evaluation_delegates_to_existing_evaluate(self): + """Existing evaluators receive selected data through the default hook.""" + # Given: an evaluator that implements only evaluate(data) + evaluator = MockEvaluator.from_dict({"should_match": True}) + step = Step(type="llm", name="answer", input="full input") + + # When: the engine-facing contextual hook is called + result = await evaluator.evaluate_with_context("selected data", step) + + # Then: the legacy evaluate implementation handles the selected data + assert result.matched is True + assert result.metadata == {"data": "selected data"} + def test_evaluator_config_stored(self): """Test that evaluator stores config.""" evaluator = MockEvaluator.from_dict({"should_match": True}) diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 37b17c27..cb2a5f11 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -715,6 +715,41 @@ async def test_evaluator_applies_threshold_locally_to_raw_score(self) -> None: timeout=5.0, ) + @patch.dict(os.environ, LUNA_ENV) + @pytest.mark.asyncio + async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + # Given: selected scorer data and complete structured runtime context + evaluator = LunaEvaluator.from_dict( + {"scorer_id": "scorer-123", "threshold": 0.5, "operator": "gte"} + ) + step = Step( + type="llm", + name="answer", + input="full input", + output="full output", + tools=[{"name": "search", "description": "Search", "input_schema": {}}], + ground_truth="expected", + ) + + # When: evaluating through the contextual hook + with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: + mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") + result = await evaluator.evaluate_with_context("selected input", step) + + # Then: selector-selected data and the complete Step are both forwarded + assert result.matched is True + mock_invoke.assert_awaited_once_with( + scorer_id="scorer-123", + step=step, + input="selected input", + output=None, + config=None, + timeout=10.0, + ) + @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio async def test_evaluator_forwards_configured_scorer_version_id(self) -> None: diff --git a/sdks/python/tests/test_google_adk_plugin.py b/sdks/python/tests/test_google_adk_plugin.py index f72057ad..b50499a1 100644 --- a/sdks/python/tests/test_google_adk_plugin.py +++ b/sdks/python/tests/test_google_adk_plugin.py @@ -585,6 +585,18 @@ def test_bind_keeps_duplicate_tool_names_distinct_across_sub_agents(plugin_modul assert ("tool", "writer.search_docs") in plugin._known_steps +def test_available_tool_capture_skips_unnamed_agents(plugin_module): + # Given: a framework object that does not identify an ADK agent + plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") + unnamed_agent = SimpleNamespace(tools=[MockTool("search_docs")]) + + # When: available tools are captured from the bound hierarchy + plugin._remember_available_tools(unnamed_agent) + + # Then: no incomplete tool set is guessed + assert plugin._available_tools_by_step == {} + + @pytest.mark.asyncio async def test_lazy_step_sync_when_bind_skipped(plugin_module): plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") diff --git a/sdks/python/tests/test_integration_tools.py b/sdks/python/tests/test_integration_tools.py new file mode 100644 index 00000000..8f23eb4f --- /dev/null +++ b/sdks/python/tests/test_integration_tools.py @@ -0,0 +1,47 @@ +"""Tests for framework-neutral available-tool normalization.""" + +import pytest + +from agent_control.integrations._tools import normalize_strands_tool_specs + + +@pytest.mark.parametrize( + "specs", + [ + {"search": {}}, + ["not-a-tool-spec"], + [{"description": "missing name"}], + [{"name": ""}], + ], +) +def test_normalizer_rejects_incomplete_strands_registries(specs: object) -> None: + # Given/When: the purported complete registry has an invalid shape + normalized = normalize_strands_tool_specs(specs) + + # Then: the integration leaves tools absent rather than guessing + assert normalized is None + + +@pytest.mark.parametrize( + ("input_schema", "expected"), + [ + ({"type": "object", "properties": {}}, {"type": "object", "properties": {}}), + (None, {}), + ], +) +def test_normalizer_supports_plain_or_missing_strands_input_schema( + input_schema: object, + expected: dict[str, object], +) -> None: + # Given: valid Strands specs from supported schema variants + spec: dict[str, object] = {"name": "search", "description": 123} + if input_schema is not None: + spec["inputSchema"] = input_schema + + # When: normalizing the complete registry + normalized = normalize_strands_tool_specs([spec]) + + # Then: a stable JSON-only tool definition is produced + assert normalized == [ + {"name": "search", "description": "", "input_schema": expected} + ] diff --git a/sdks/python/tests/test_strands_plugin.py b/sdks/python/tests/test_strands_plugin.py index c14557c9..2210fda4 100644 --- a/sdks/python/tests/test_strands_plugin.py +++ b/sdks/python/tests/test_strands_plugin.py @@ -449,6 +449,22 @@ def test_init_agent_captures_complete_strands_tool_registry(agent_control_hook): ] +def test_available_tools_fails_closed_when_strands_registry_raises( + agent_control_hook, caplog +): + # Given: a Strands registry that cannot provide a complete tool set + registry = MagicMock() + registry.get_all_tool_specs.side_effect = RuntimeError("registry unavailable") + agent_control_hook._tool_registry = registry + + # When: tool definitions are requested + tools = agent_control_hook._available_tools() + + # Then: tools remain absent and the integration records the failure + assert tools is None + assert "Unable to capture complete Strands tool definitions" in caplog.text + + def test_hook_with_callback(): """Test AgentControlPlugin with violation callback.""" from agent_control.integrations.strands.plugin import AgentControlPlugin diff --git a/sdks/typescript/src/generated/models/step.ts b/sdks/typescript/src/generated/models/step.ts index 132cf9c9..e5dd9775 100644 --- a/sdks/typescript/src/generated/models/step.ts +++ b/sdks/typescript/src/generated/models/step.ts @@ -3,6 +3,7 @@ */ import * as z from "zod/v4-mini"; +import { remap as remap$ } from "../lib/primitives.js"; /** * Runtime payload for an agent step invocation. @@ -12,6 +13,10 @@ export type Step = { * Optional context (conversation history, metadata, etc.) */ context?: { [k: string]: any } | null | undefined; + /** + * Optional expected or reference output for this step + */ + groundTruth?: any | null | undefined; /** * Any JSON value */ @@ -24,6 +29,10 @@ export type Step = { * Output content for this step (None for pre-checks) */ output?: any | null | undefined; + /** + * Complete structured definitions of tools available to the LLM + */ + tools?: Array<{ [k: string]: any }> | null | undefined; /** * Step type (e.g., 'tool', 'llm') */ @@ -33,21 +42,30 @@ export type Step = { /** @internal */ export type Step$Outbound = { context?: { [k: string]: any } | null | undefined; + ground_truth?: any | null | undefined; input: any; name: string; output?: any | null | undefined; + tools?: Array<{ [k: string]: any }> | null | undefined; type: string; }; /** @internal */ -export const Step$outboundSchema: z.ZodMiniType = z.object( - { +export const Step$outboundSchema: z.ZodMiniType = z.pipe( + z.object({ context: z.optional(z.nullable(z.record(z.string(), z.any()))), + groundTruth: z.optional(z.nullable(z.any())), input: z.any(), name: z.string(), output: z.optional(z.nullable(z.any())), + tools: z.optional(z.nullable(z.array(z.record(z.string(), z.any())))), type: z.string(), - }, + }), + z.transform((v) => { + return remap$(v, { + groundTruth: "ground_truth", + }); + }), ); export function stepToJSON(step: Step): string { diff --git a/sdks/typescript/tests/generated-smoke.test.ts b/sdks/typescript/tests/generated-smoke.test.ts index 453267aa..dda98e3a 100644 --- a/sdks/typescript/tests/generated-smoke.test.ts +++ b/sdks/typescript/tests/generated-smoke.test.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; +import { stepToJSON } from "../src/generated/models/step"; + describe("generated client layout", () => { it("has a generated index entrypoint", () => { const generatedIndex = path.resolve(process.cwd(), "src/generated/index.ts"); @@ -24,4 +26,36 @@ describe("generated client layout", () => { } } }); + + it("serializes structured Step scorer context", () => { + const serialized = stepToJSON({ + type: "llm", + name: "answer", + input: "question", + output: "answer", + groundTruth: "expected", + tools: [ + { + name: "search", + description: "Search documents", + input_schema: { type: "object" }, + }, + ], + }); + + expect(JSON.parse(serialized)).toEqual({ + type: "llm", + name: "answer", + input: "question", + output: "answer", + ground_truth: "expected", + tools: [ + { + name: "search", + description: "Search documents", + input_schema: { type: "object" }, + }, + ], + }); + }); }); From a9a518994feb744391413d825aad672d1b61dc5f Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Mon, 24 Aug 2026 13:46:09 -0700 Subject: [PATCH 3/6] update luna client for updated scorer_invoke api contract --- evaluators/contrib/galileo/README.md | 5 + .../__init__.py | 2 + .../luna/__init__.py | 7 +- .../luna/client.py | 123 ++++++++++++++--- .../luna/config.py | 34 ++++- .../galileo/tests/test_luna_evaluator.py | 130 +++++++++++++++--- 6 files changed, 257 insertions(+), 44 deletions(-) diff --git a/evaluators/contrib/galileo/README.md b/evaluators/contrib/galileo/README.md index a951fa5e..72d8ca3a 100644 --- a/evaluators/contrib/galileo/README.md +++ b/evaluators/contrib/galileo/README.md @@ -14,6 +14,11 @@ set `threshold` and `operator` as needed. If you still need the legacy Luna2 evaluator, pin `agent-control-evaluator-galileo <8`. +The optional evaluator `config` mirrors Orbit's allowlisted scorer-invoke +configuration. Supported keys are `threshold`, `score_threshold`, and +`request_timeout_seconds`; unsupported keys are rejected locally before an HTTP +request is made. + ## Install Canonical install path: diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py index 93eb8567..749f6322 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/__init__.py @@ -25,6 +25,7 @@ LunaEvaluator, LunaEvaluatorConfig, LunaOperator, + ScorerInvokeConfig, ScorerInvokeRecord, ScorerInvokeRequest, ScorerInvokeResponse, @@ -33,6 +34,7 @@ __all__ = [ "GalileoLunaClient", "ScorerInvokeRequest", + "ScorerInvokeConfig", "ScorerInvokeRecord", "ScorerInvokeResponse", "LunaEvaluator", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py index 39f95016..3eb04102 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/__init__.py @@ -7,12 +7,17 @@ ScorerInvokeRequest, ScorerInvokeResponse, ) -from agent_control_evaluator_galileo.luna.config import LunaEvaluatorConfig, LunaOperator +from agent_control_evaluator_galileo.luna.config import ( + LunaEvaluatorConfig, + LunaOperator, + ScorerInvokeConfig, +) from agent_control_evaluator_galileo.luna.evaluator import LUNA_AVAILABLE, LunaEvaluator __all__ = [ "GalileoLunaClient", "ScorerInvokeInputs", + "ScorerInvokeConfig", "ScorerInvokeRecord", "ScorerInvokeRequest", "ScorerInvokeResponse", diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index e0b759b7..7c3f5687 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -11,12 +11,15 @@ from hmac import new as hmac_new from json import dumps from time import time +from typing import Literal, cast, get_args from urllib.parse import urlsplit import httpx from agent_control_models import JSONObject, JSONValue, Step from pydantic import BaseModel, Field, PrivateAttr, model_validator +from .config import ScorerInvokeConfig + logger = logging.getLogger(__name__) DEFAULT_TIMEOUT_SECS = 10.0 @@ -40,6 +43,20 @@ LUNA_MAX_KEEPALIVE_CONNECTIONS_ENV = "GALILEO_LUNA_MAX_KEEPALIVE_CONNECTIONS" LUNA_CLIENT_POOL_SIZE_ENV = "GALILEO_LUNA_CLIENT_POOL_SIZE" +# These values mirror Orbit's StepType discriminator. Agent Control's generic +# Step remains extensible; only the Galileo transport boundary is constrained. +ScorerInvokeRecordType = Literal[ + "llm", + "retriever", + "tool", + "workflow", + "agent", + "control", + "trace", + "session", +] +SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) + def _b64url(data: bytes) -> str: return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") @@ -148,6 +165,25 @@ def _has_value(value: JSONValue) -> bool: return True +def _serialize_dataset_output(value: JSONValue) -> str | None: + """Map Agent Control ground truth to Orbit's string dataset-output field. + + Orbit accepts the original JSON value in the legacy ``inputs.ground_truth`` + field, but its shared record hierarchy represents ``dataset_output`` as a + string. Compact, sorted JSON keeps the structured value lossless and lets + Orbit's semantic dual-write validator compare both representations. + + Args: + value: JSON-compatible ground truth from the Agent Control step. + + Returns: + The original string, compact JSON for a structured value, or ``None``. + """ + if value is None or isinstance(value, str): + return value + return dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + class ScorerInvokeInputs(BaseModel): """Input values sent to the Luna scorer invoke endpoint.""" @@ -158,14 +194,19 @@ class ScorerInvokeInputs(BaseModel): class ScorerInvokeRecord(BaseModel): - """Structured runtime record sent alongside legacy scorer inputs.""" + """Caller-controlled subset of Orbit's partial runtime-record contract. + + Identity, ownership, persistence, and execution IDs are intentionally not + represented here. Orbit hydrates those fields from trusted server context. + """ - type: str = Field(min_length=1) + type: ScorerInvokeRecordType + name: str | None = None input: JSONValue = None output: JSONValue = None context: JSONObject | None = None tools: list[JSONObject] | None = None - dataset_output: JSONValue = None + dataset_output: str | None = None class ScorerInvokeRequest(BaseModel): @@ -176,6 +217,7 @@ class ScorerInvokeRequest(BaseModel): scorer_version_id: Optional pinned scorer version identifier. scorer_label: Optional display/metadata label. inputs: Selected scorer input values. + record: Optional Orbit-compatible structured runtime record. config: Scorer-specific configuration, always emitted. """ @@ -184,7 +226,7 @@ class ScorerInvokeRequest(BaseModel): scorer_label: str | None = Field(default=None, min_length=1) inputs: ScorerInvokeInputs record: ScorerInvokeRecord | None = None - config: JSONObject = Field(default_factory=dict) + config: ScorerInvokeConfig = Field(default_factory=ScorerInvokeConfig) @model_validator(mode="after") def ensure_required_values(self) -> ScorerInvokeRequest: @@ -197,6 +239,48 @@ def to_dict(self) -> JSONObject: return self.model_dump(mode="json", exclude_none=True) +def _orbit_record_from_step( + step: Step | None, + *, + selected_input: JSONValue, + selected_output: JSONValue, +) -> ScorerInvokeRecord | None: + """Translate a generic Agent Control step into Orbit's record contract. + + Selector-selected values remain the primary scorer input. When a selector + supplies one side, that value is written to both the legacy and structured + representations so Orbit's conflict validation cannot observe two meanings. + The complete step supplies the unselected side and additional record context. + + Unknown Agent Control step types intentionally fall back to the legacy + ``inputs`` contract. This keeps the open-source Step model extensible without + sending an invalid discriminator to Orbit. + + Args: + step: Complete Agent Control step, when contextual evaluation is used. + selected_input: Selector-selected value sent as ``inputs.query``. + selected_output: Selector-selected value sent as ``inputs.response``. + + Returns: + An Orbit-compatible record, or ``None`` for absent/unsupported steps. + """ + if step is None or step.type not in SUPPORTED_SCORER_INVOKE_RECORD_TYPES: + return None + + # The membership check above narrows the runtime value to Orbit's known + # discriminator set, but static type checkers cannot infer that relationship. + record_type = cast(ScorerInvokeRecordType, step.type) + return ScorerInvokeRecord( + type=record_type, + name=step.name, + input=selected_input if selected_input is not None else step.input, + output=selected_output if selected_output is not None else step.output, + context=step.context, + tools=step.tools, + dataset_output=_serialize_dataset_output(step.ground_truth), + ) + + class ScorerInvokeResponse(BaseModel): """Response from Luna scorer invocation. @@ -370,7 +454,7 @@ async def invoke( input: JSONValue = None, output: JSONValue = None, step: Step | None = None, - config: JSONObject | None = None, + config: ScorerInvokeConfig | JSONObject | None = None, timeout: float = DEFAULT_TIMEOUT_SECS, headers: dict[str, str] | None = None, ) -> ScorerInvokeResponse: @@ -383,7 +467,7 @@ async def invoke( input: Optional user/system prompt text. output: Optional model response text. step: Optional complete runtime step used for structured dual-write. - config: Optional scorer-specific configuration. + config: Optional Orbit-supported scorer invocation configuration. timeout: Request timeout in seconds. headers: Additional request headers. @@ -391,7 +475,8 @@ async def invoke( Parsed scorer invocation response. Raises: - ValueError: If neither input nor output is provided. + ValueError: If neither input nor output is provided, or if config + contains a field Orbit does not support. RuntimeError: If the API response is not a JSON object. httpx.HTTPStatusError: If the Luna invoke endpoint returns an error status code. httpx.RequestError: If the request fails before a response is received. @@ -399,6 +484,13 @@ async def invoke( if not (_has_value(input) or _has_value(output)): raise ValueError("At least one of input or output must be provided.") + # Accept dictionaries for source compatibility with the original client, + # but validate them against Orbit's authoritative allowlist locally. + invoke_config = ( + ScorerInvokeConfig.model_validate(config) + if config is not None + else ScorerInvokeConfig() + ) request_body = ScorerInvokeRequest( scorer_id=scorer_id, scorer_version_id=scorer_version_id, @@ -409,19 +501,12 @@ async def invoke( ground_truth=step.ground_truth if step is not None else None, tools=step.tools if step is not None else None, ), - record=( - ScorerInvokeRecord( - type=step.type, - input=step.input, - output=step.output, - context=step.context, - tools=step.tools, - dataset_output=step.ground_truth, - ) - if step is not None - else None + record=_orbit_record_from_step( + step, + selected_input=input, + selected_output=output, ), - config=config if config is not None else {}, + config=invoke_config, ).to_dict() endpoint, auth_header = self._endpoint_and_auth_header() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py index 98ab94e4..b95989f4 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py @@ -5,8 +5,8 @@ from typing import Literal from agent_control_evaluators import EvaluatorConfig -from agent_control_models import JSONObject, JSONValue -from pydantic import Field, model_validator +from agent_control_models import JSONValue +from pydantic import BaseModel, ConfigDict, Field, model_validator LunaOperator = Literal["gt", "gte", "lt", "lte", "eq", "ne", "contains", "any"] LunaPayloadField = Literal["input", "output"] @@ -14,6 +14,29 @@ _NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"}) +class ScorerInvokeConfig(BaseModel): + """Orbit-supported overrides for a synchronous scorer invocation. + + Orbit owns the Galileo scorer-invoke wire contract. Keeping this model + strict makes an unsupported option fail locally instead of producing a + less actionable HTTP 422 response from Runners. + + Attributes: + threshold: Legacy threshold accepted by Orbit. Agent Control still + applies its evaluator threshold locally. + score_threshold: Legacy score threshold accepted by Orbit. + request_timeout_seconds: Optional upper bound for scorer execution in + Orbit. The Agent Control HTTP and evaluator deadlines must remain + longer than this value. + """ + + model_config = ConfigDict(extra="forbid") + + threshold: float | None = None + score_threshold: float | None = None + request_timeout_seconds: float | None = Field(default=None, gt=0) + + def coerce_number(value: JSONValue) -> float | None: """Return a numeric value for JSON scalars that can be compared numerically.""" if isinstance(value, bool) or value is None: @@ -37,7 +60,8 @@ class LunaEvaluatorConfig(EvaluatorConfig): scorer_label: Optional display/metadata label. threshold: Local threshold used by the evaluator for comparison. operator: Local comparison operator. Numeric operators use threshold as a number. - scorer_config: Optional scorer-specific config sent as ``config``. + scorer_config: Optional Orbit-supported scorer invocation config sent + as ``config``. payload_field: Explicit scorer input side for scalar selected data. timeout_ms: Request timeout in milliseconds. """ @@ -64,12 +88,12 @@ class LunaEvaluatorConfig(EvaluatorConfig): default="gte", description="Local comparison operator applied to the raw Luna score.", ) - scorer_config: JSONObject | None = Field( + scorer_config: ScorerInvokeConfig | None = Field( default=None, alias="config", serialization_alias="config", description=( - "Optional scorer-specific configuration sent to the Luna scorer invoke endpoint." + "Optional Orbit-supported configuration sent to the Luna scorer invoke endpoint." ), ) payload_field: LunaPayloadField = Field( diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index cb2a5f11..5e3a3ee9 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -10,13 +10,31 @@ import httpx import pytest -from agent_control_models import EvaluatorResult, Step -from pydantic import ValidationError +from agent_control_models import EvaluatorResult, JSONValue, Step +from pydantic import UUID4, BaseModel, ValidationError LUNA_ENV = { "GALILEO_API_SECRET_KEY": "test-secret", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", } +SCORER_ID = "3d45ef0d-5f14-4f1a-a8f1-8ab758da18b4" + + +class _LegacyOrbitInputs(BaseModel): + """Released Orbit input shape used for backward-compatibility assertions.""" + + query: JSONValue = "" + response: JSONValue = "" + + +class _LegacyOrbitRequest(BaseModel): + """Released Orbit request shape, whose default extra handling is additive.""" + + scorer_id: UUID4 + scorer_version_id: UUID4 | None = None + scorer_label: str | None = None + inputs: _LegacyOrbitInputs + config: dict[str, object] | None = None def _decode_jwt_payload(token: str) -> dict[str, object]: @@ -37,7 +55,7 @@ def test_config_accepts_scorer_id_with_all_optional_fields(self) -> None: scorer_label="toxicity", threshold=0.7, operator="gte", - config={"temperature": 0}, + config={"request_timeout_seconds": 8}, ) assert config.scorer_id == "scorer-123" @@ -45,9 +63,22 @@ def test_config_accepts_scorer_id_with_all_optional_fields(self) -> None: assert config.scorer_label == "toxicity" assert config.threshold == 0.7 assert config.operator == "gte" - assert config.scorer_config == {"temperature": 0} + assert config.scorer_config is not None + assert config.scorer_config.model_dump(exclude_none=True) == { + "request_timeout_seconds": 8.0 + } assert config.payload_field == "input" + def test_config_rejects_option_not_supported_by_orbit(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluatorConfig + + # Given: a scorer option outside Orbit's allowlisted invoke contract + unsupported_config = {"temperature": 0} + + # When/Then: Agent Control reports the contract error before making an HTTP request + with pytest.raises(ValidationError, match="temperature"): + LunaEvaluatorConfig(scorer_id="scorer-123", config=unsupported_config) + def test_config_accepts_scorer_id_without_label(self) -> None: from agent_control_evaluator_galileo.luna import LunaEvaluatorConfig @@ -94,14 +125,18 @@ def test_scorer_invoke_request_requires_scorer_id(self) -> None: ) def test_scorer_invoke_request_shape_with_all_fields(self) -> None: - from agent_control_evaluator_galileo.luna import ScorerInvokeInputs, ScorerInvokeRequest + from agent_control_evaluator_galileo.luna import ( + ScorerInvokeConfig, + ScorerInvokeInputs, + ScorerInvokeRequest, + ) request = ScorerInvokeRequest( scorer_id="scorer-123", scorer_version_id="version-123", scorer_label="toxicity", inputs=ScorerInvokeInputs(query={"messages": [{"role": "user", "content": "hello"}]}), - config={"top_k": 1}, + config=ScorerInvokeConfig(request_timeout_seconds=7), ) assert request.to_dict() == { @@ -112,9 +147,26 @@ def test_scorer_invoke_request_shape_with_all_fields(self) -> None: "query": {"messages": [{"role": "user", "content": "hello"}]}, "response": "", }, - "config": {"top_k": 1}, + "config": {"request_timeout_seconds": 7.0}, } + def test_scorer_invoke_request_rejects_unknown_config(self) -> None: + from agent_control_evaluator_galileo.luna import ( + ScorerInvokeInputs, + ScorerInvokeRequest, + ) + + # Given: an option the current Orbit scorer-invoke schema does not accept + unsupported_config = {"top_k": 1} + + # When/Then: the mirrored client contract rejects it locally + with pytest.raises(ValidationError, match="top_k"): + ScorerInvokeRequest( + scorer_id="scorer-123", + inputs=ScorerInvokeInputs(query="hello"), + config=unsupported_config, + ) + def test_scorer_invoke_request_omits_optional_fields_when_absent(self) -> None: from agent_control_evaluator_galileo.luna import ScorerInvokeInputs, ScorerInvokeRequest @@ -489,10 +541,10 @@ def handler(request: httpx.Request) -> httpx.Response: try: response = await client.invoke( - scorer_id="scorer-123", + scorer_id=SCORER_ID, input="user prompt", output="model answer", - config={"top_k": 1}, + config={"request_timeout_seconds": 7}, ) finally: await client.close() @@ -500,11 +552,12 @@ def handler(request: httpx.Request) -> httpx.Response: # Then: posts to luna invoke endpoint /api/v1/scorers/invoke with JWT, no Galileo-API-Key assert response.score == 0.82 assert captured["url"] == "http://luna-invoke:8090/api/v1/scorers/invoke" - assert captured["body"] == { - "scorer_id": "scorer-123", + expected_body = { + "scorer_id": SCORER_ID, "inputs": {"query": "user prompt", "response": "model answer"}, - "config": {"top_k": 1}, + "config": {"request_timeout_seconds": 7.0}, } + assert captured["body"] == expected_body headers = captured["headers"] assert isinstance(headers, dict) assert "galileo-api-key" not in headers @@ -545,7 +598,7 @@ def handler(request: httpx.Request) -> httpx.Response: # When: invoking the rollout-compatible Runners endpoint try: response = await client.invoke( - scorer_id="scorer-123", + scorer_id=SCORER_ID, input="selected question", output="selected answer", step=step, @@ -553,10 +606,10 @@ def handler(request: httpx.Request) -> httpx.Response: finally: await client.close() - # Then: old inputs and the expanded record are sent together + # Then: old inputs and an equivalent Orbit record are sent together assert response.score == 0.9 - assert captured["body"] == { - "scorer_id": "scorer-123", + expected_body = { + "scorer_id": SCORER_ID, "inputs": { "query": "selected question", "response": "selected answer", @@ -565,14 +618,53 @@ def handler(request: httpx.Request) -> httpx.Response: }, "record": { "type": "llm", - "input": {"messages": [{"role": "user", "content": "question"}]}, - "output": {"text": "answer"}, + "name": "answer", + "input": "selected question", + "output": "selected answer", "context": {"session": "s-1"}, "tools": [{"name": "search", "description": "Search", "input_schema": {}}], - "dataset_output": {"text": "expected"}, + "dataset_output": '{"text":"expected"}', }, "config": {}, } + assert captured["body"] == expected_body + + # The same request must remain consumable by released Orbit versions. + legacy_request = _LegacyOrbitRequest.model_validate(expected_body) + assert legacy_request.model_dump(mode="json", exclude_none=True) == { + "scorer_id": SCORER_ID, + "inputs": {"query": "selected question", "response": "selected answer"}, + "config": {}, + } + + @pytest.mark.asyncio + async def test_client_omits_record_for_step_type_orbit_does_not_support(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.4, "status": "success"}) + + # Given: a generic Agent Control step whose type is not an Orbit record discriminator + step = Step(type="custom", name="custom-step", input="full input", output="full output") + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + # When: the scorer is invoked with selector-selected input + try: + await client.invoke(scorer_id="scorer-123", input="selected input", step=step) + finally: + await client.close() + + # Then: the legacy contract remains usable by old and new Orbit versions + assert captured["body"] == { + "scorer_id": "scorer-123", + "inputs": {"query": "selected input", "response": ""}, + "config": {}, + } @pytest.mark.asyncio async def test_client_forwards_scorer_version_id_when_configured(self) -> None: From 8a2b5d52717090ed7c65b627035f73d89c46f5ff Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Wed, 26 Aug 2026 14:40:59 -0700 Subject: [PATCH 4/6] fix: align Luna invocation with Orbit rollout contract --- evaluators/contrib/galileo/README.md | 8 +- .../luna/client.py | 59 ++++-- .../luna/config.py | 8 +- .../luna/evaluator.py | 3 +- .../galileo/tests/test_luna_coverage_gaps.py | 4 +- .../galileo/tests/test_luna_evaluator.py | 177 ++++++++++++++++-- examples/galileo_luna/README.md | 5 +- examples/galileo_luna/setup_controls.py | 1 + 8 files changed, 227 insertions(+), 38 deletions(-) diff --git a/evaluators/contrib/galileo/README.md b/evaluators/contrib/galileo/README.md index 72d8ca3a..8191bb73 100644 --- a/evaluators/contrib/galileo/README.md +++ b/evaluators/contrib/galileo/README.md @@ -7,7 +7,9 @@ Integration package for Galileo Luna evaluator. The `galileo.luna2` evaluator ID has been removed. Existing controls that use `galileo.luna2` should migrate to `galileo.luna` and update their evaluator configuration to use the direct Luna scorer fields. `scorer_id` is required; -`scorer_label` and `scorer_version_id` are optional. The evaluator calls the +`scorer_label` and `scorer_version_id` are optional. `scorer_version_id` is a +deprecated optional compatibility identifier; Orbit currently invokes the +scorer's current default version. The evaluator calls the URL configured by `GALILEO_LUNA_INVOKE_URL`; the target must support the Luna scorer invoke request/response contract and internal Galileo secret auth. Also set `threshold` and `operator` as needed. If you still need the legacy Luna2 @@ -19,6 +21,10 @@ configuration. Supported keys are `threshold`, `score_threshold`, and `request_timeout_seconds`; unsupported keys are rejected locally before an HTTP request is made. +Agent Control always sends an Orbit execution timeout shorter than its HTTP +deadline. When `request_timeout_seconds` is omitted, it defaults to 80% of the +configured HTTP timeout (8 seconds for the default 10-second deadline). + ## Install Canonical install path: diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index 7c3f5687..d00d1457 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -23,6 +23,7 @@ logger = logging.getLogger(__name__) DEFAULT_TIMEOUT_SECS = 10.0 +SERVER_TIMEOUT_RATIO = 0.8 DEFAULT_INTERNAL_TOKEN_TTL_SECS = 3600 DEFAULT_LUNA_SCORER_INVOKE_PATH = "/api/v1/scorers/invoke" LUNA_INVOKE_URL_ENV = "GALILEO_LUNA_INVOKE_URL" @@ -165,23 +166,41 @@ def _has_value(value: JSONValue) -> bool: return True -def _serialize_dataset_output(value: JSONValue) -> str | None: - """Map Agent Control ground truth to Orbit's string dataset-output field. +def _effective_scorer_timeout( + config: ScorerInvokeConfig, + *, + http_timeout_seconds: float, +) -> ScorerInvokeConfig: + """Resolve an Orbit execution timeout that expires before the HTTP request. - Orbit accepts the original JSON value in the legacy ``inputs.ground_truth`` - field, but its shared record hierarchy represents ``dataset_output`` as a - string. Compact, sorted JSON keeps the structured value lossless and lets - Orbit's semantic dual-write validator compare both representations. + The server execution budget defaults to 80% of the caller's HTTP deadline, + leaving time for Orbit to serialize and return the result. Explicit caller + overrides are preserved only when they maintain the same ordering. Args: - value: JSON-compatible ground truth from the Agent Control step. + config: Caller-provided scorer-invoke configuration. + http_timeout_seconds: Agent Control's HTTP request deadline in seconds. Returns: - The original string, compact JSON for a structured value, or ``None``. + A scorer-invoke configuration with an effective execution timeout. + + Raises: + ValueError: If either deadline is invalid or the explicit server timeout + is not shorter than the HTTP deadline. """ - if value is None or isinstance(value, str): - return value - return dumps(value, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + if http_timeout_seconds <= 0: + raise ValueError("HTTP timeout must be greater than 0 seconds.") + + server_timeout = config.request_timeout_seconds + if server_timeout is None: + server_timeout = http_timeout_seconds * SERVER_TIMEOUT_RATIO + elif server_timeout >= http_timeout_seconds: + raise ValueError( + "config.request_timeout_seconds must be shorter than the HTTP " + f"timeout ({http_timeout_seconds:g} seconds)." + ) + + return config.model_copy(update={"request_timeout_seconds": server_timeout}) class ScorerInvokeInputs(BaseModel): @@ -206,7 +225,7 @@ class ScorerInvokeRecord(BaseModel): output: JSONValue = None context: JSONObject | None = None tools: list[JSONObject] | None = None - dataset_output: str | None = None + dataset_output: JSONValue = None class ScorerInvokeRequest(BaseModel): @@ -214,7 +233,8 @@ class ScorerInvokeRequest(BaseModel): Attributes: scorer_id: Required scorer identifier. - scorer_version_id: Optional pinned scorer version identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. scorer_label: Optional display/metadata label. inputs: Selected scorer input values. record: Optional Orbit-compatible structured runtime record. @@ -277,7 +297,7 @@ def _orbit_record_from_step( output=selected_output if selected_output is not None else step.output, context=step.context, tools=step.tools, - dataset_output=_serialize_dataset_output(step.ground_truth), + dataset_output=step.ground_truth, ) @@ -462,7 +482,8 @@ async def invoke( Args: scorer_id: Required scorer identifier. - scorer_version_id: Optional pinned scorer version identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. scorer_label: Optional display/metadata label. input: Optional user/system prompt text. output: Optional model response text. @@ -475,8 +496,8 @@ async def invoke( Parsed scorer invocation response. Raises: - ValueError: If neither input nor output is provided, or if config - contains a field Orbit does not support. + ValueError: If neither input nor output is provided, config contains + a field Orbit does not support, or the timeout ordering is invalid. RuntimeError: If the API response is not a JSON object. httpx.HTTPStatusError: If the Luna invoke endpoint returns an error status code. httpx.RequestError: If the request fails before a response is received. @@ -491,6 +512,10 @@ async def invoke( if config is not None else ScorerInvokeConfig() ) + invoke_config = _effective_scorer_timeout( + invoke_config, + http_timeout_seconds=timeout, + ) request_body = ScorerInvokeRequest( scorer_id=scorer_id, scorer_version_id=scorer_version_id, diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py index b95989f4..0bc0fa37 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py @@ -56,7 +56,8 @@ class LunaEvaluatorConfig(EvaluatorConfig): Attributes: scorer_id: Required scorer identifier for Luna scorer invocation. - scorer_version_id: Optional pinned scorer version identifier. + scorer_version_id: Deprecated optional compatibility identifier. Orbit + currently invokes the scorer's current default version. scorer_label: Optional display/metadata label. threshold: Local threshold used by the evaluator for comparison. operator: Local comparison operator. Numeric operators use threshold as a number. @@ -73,7 +74,10 @@ class LunaEvaluatorConfig(EvaluatorConfig): scorer_version_id: str | None = Field( default=None, min_length=1, - description="Optional pinned scorer version identifier.", + description=( + "Deprecated optional compatibility identifier. Orbit currently invokes " + "the scorer's current default version." + ), ) scorer_label: str | None = Field( default=None, diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index 7a837aaa..e4147a9a 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -258,9 +258,10 @@ async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: return self._handle_error(exc) def _base_metadata(self) -> dict[str, Any]: + """Build result metadata without implying a requested version executed.""" metadata: dict[str, Any] = {"scorer_id": self.config.scorer_id} if self.config.scorer_version_id is not None: - metadata["scorer_version_id"] = self.config.scorer_version_id + metadata["requested_scorer_version_id"] = self.config.scorer_version_id if self.config.scorer_label is not None: metadata["scorer_label"] = self.config.scorer_label return metadata diff --git a/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py b/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py index 0d249725..c5b65e7b 100644 --- a/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py +++ b/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py @@ -663,7 +663,7 @@ def handler(request: httpx.Request) -> httpx.Response: @pytest.mark.asyncio async def test_invoke_always_emits_config_field(monkeypatch): - """Regression: config must always be present in the request body, defaulting to {}.""" + """The request always carries a server timeout below its HTTP deadline.""" for key, value in LUNA_ENV.items(): monkeypatch.setenv(key, value) from agent_control_evaluator_galileo.luna.client import GalileoLunaClient @@ -683,4 +683,4 @@ def handler(request: httpx.Request) -> httpx.Response: await client.close() assert "config" in captured["body"] - assert captured["body"]["config"] == {} + assert captured["body"]["config"] == {"request_timeout_seconds": 8.0} diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 5e3a3ee9..af892368 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -6,18 +6,20 @@ import json import os from base64 import urlsafe_b64decode +from typing import Literal from unittest.mock import AsyncMock, patch import httpx import pytest from agent_control_models import EvaluatorResult, JSONValue, Step -from pydantic import UUID4, BaseModel, ValidationError +from pydantic import UUID4, BaseModel, ConfigDict, ValidationError LUNA_ENV = { "GALILEO_API_SECRET_KEY": "test-secret", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", } SCORER_ID = "3d45ef0d-5f14-4f1a-a8f1-8ab758da18b4" +SCORER_VERSION_ID = "07fb9c96-9752-4cf5-a253-1a396100e9d2" class _LegacyOrbitInputs(BaseModel): @@ -37,12 +39,61 @@ class _LegacyOrbitRequest(BaseModel): config: dict[str, object] | None = None -def _decode_jwt_payload(token: str) -> dict[str, object]: - payload_segment = token.split(".")[1] - padded = payload_segment + ("=" * (-len(payload_segment) % 4)) +class _Orbit1720Inputs(BaseModel): + """Input portion of the Orbit #1720 scorer-invoke request contract.""" + + query: JSONValue = "" + response: JSONValue = "" + ground_truth: JSONValue = None + tools: list[dict[str, JSONValue]] | None = None + + +class _Orbit1720Record(BaseModel): + """Caller-controlled record fields accepted by Orbit #1720.""" + + model_config = ConfigDict(extra="allow") + + type: Literal["llm", "retriever", "tool", "workflow", "agent", "control", "trace", "session"] + name: str | None = None + input: JSONValue = None + output: JSONValue = None + context: dict[str, JSONValue] | None = None + tools: list[dict[str, JSONValue]] | None = None + dataset_output: JSONValue = None + + +class _Orbit1720Config(BaseModel): + """Strict configuration accepted by Orbit #1720.""" + + model_config = ConfigDict(extra="forbid") + + threshold: float | None = None + score_threshold: float | None = None + request_timeout_seconds: float | None = None + + +class _Orbit1720Request(BaseModel): + """Test mirror of Orbit #1720's public scorer-invoke request schema.""" + + scorer_id: UUID4 + scorer_version_id: UUID4 | None = None + scorer_label: str | None = None + inputs: _Orbit1720Inputs + record: _Orbit1720Record | None = None + config: _Orbit1720Config | None = None + + +def _decode_jwt_segment(segment: str) -> dict[str, object]: + """Decode one base64url JSON segment from an internal JWT.""" + padded = segment + ("=" * (-len(segment) % 4)) return json.loads(urlsafe_b64decode(padded.encode()).decode()) +def _decode_jwt_payload(token: str) -> dict[str, object]: + """Decode the claims segment from an internal JWT.""" + return _decode_jwt_segment(token.split(".")[1]) + + class TestLunaEvaluatorConfig: """Tests for direct Luna evaluator configuration.""" @@ -545,6 +596,7 @@ def handler(request: httpx.Request) -> httpx.Response: input="user prompt", output="model answer", config={"request_timeout_seconds": 7}, + headers={"Galileo-API-Key": "blocked", "X-Request-ID": "safe-id"}, ) finally: await client.close() @@ -561,12 +613,77 @@ def handler(request: httpx.Request) -> httpx.Response: headers = captured["headers"] assert isinstance(headers, dict) assert "galileo-api-key" not in headers + assert headers["x-request-id"] == "safe-id" auth_header = headers["authorization"] assert isinstance(auth_header, str) assert auth_header.startswith("Bearer ") - payload = _decode_jwt_payload(auth_header.removeprefix("Bearer ")) + token = auth_header.removeprefix("Bearer ") + jwt_header = _decode_jwt_segment(token.split(".")[0]) + payload = _decode_jwt_payload(token) + assert jwt_header["alg"] == "HS256" assert payload["internal"] is True assert payload["scope"] == "scorers.invoke" + assert isinstance(payload["iat"], int) + assert isinstance(payload["exp"], int) + assert payload["exp"] > payload["iat"] + + @pytest.mark.asyncio + async def test_client_derives_server_timeout_from_custom_http_deadline(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.5, "status": "success"}) + + # Given: a custom HTTP deadline and no explicit Orbit execution timeout + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + # When: invoking the scorer with a five-second HTTP deadline + try: + await client.invoke(scorer_id=SCORER_ID, input="hello", timeout=5) + finally: + await client.close() + + # Then: Orbit receives an execution deadline at 80% of the HTTP deadline + body = captured["body"] + assert isinstance(body, dict) + assert body["config"] == {"request_timeout_seconds": 4.0} + + @pytest.mark.asyncio + @pytest.mark.parametrize("server_timeout", [10, 11]) + async def test_client_rejects_server_timeout_not_below_http_deadline( + self, server_timeout: float + ) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + request_count = 0 + + def handler(_request: httpx.Request) -> httpx.Response: + nonlocal request_count + request_count += 1 + return httpx.Response(200, json={"score": 0.5, "status": "success"}) + + # Given: an explicit Orbit timeout equal to or above the HTTP deadline + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + # When/Then: validation fails locally before the transport is used + try: + with pytest.raises(ValueError, match="must be shorter than the HTTP timeout"): + await client.invoke( + scorer_id=SCORER_ID, + input="hello", + timeout=10, + config={"request_timeout_seconds": server_timeout}, + ) + finally: + await client.close() + assert request_count == 0 @pytest.mark.asyncio async def test_client_dual_writes_legacy_inputs_and_structured_record(self) -> None: @@ -578,7 +695,14 @@ def handler(request: httpx.Request) -> httpx.Response: captured["body"] = json.loads(request.content.decode()) return httpx.Response( 200, - json={"score": 0.9, "status": "success", "additive_field": "ignored"}, + json={ + "scorer_label": "toxicity", + "score": 0.9, + "status": "success", + "execution_time": 0.12, + "error_message": None, + "additive_field": "ignored", + }, ) # Given: selected legacy values and a complete structured runtime Step @@ -587,7 +711,7 @@ def handler(request: httpx.Request) -> httpx.Response: name="answer", input={"messages": [{"role": "user", "content": "question"}]}, output={"text": "answer"}, - context={"session": "s-1"}, + context={"session": {"id": "s-1", "attributes": {"region": "west"}}}, tools=[{"name": "search", "description": "Search", "input_schema": {}}], ground_truth={"text": "expected"}, ) @@ -599,6 +723,8 @@ def handler(request: httpx.Request) -> httpx.Response: try: response = await client.invoke( scorer_id=SCORER_ID, + scorer_version_id=SCORER_VERSION_ID, + scorer_label="toxicity", input="selected question", output="selected answer", step=step, @@ -608,8 +734,14 @@ def handler(request: httpx.Request) -> httpx.Response: # Then: old inputs and an equivalent Orbit record are sent together assert response.score == 0.9 + assert response.scorer_label == "toxicity" + assert response.status == "success" + assert response.execution_time == 0.12 + assert response.error_message is None expected_body = { "scorer_id": SCORER_ID, + "scorer_version_id": SCORER_VERSION_ID, + "scorer_label": "toxicity", "inputs": { "query": "selected question", "response": "selected answer", @@ -621,20 +753,36 @@ def handler(request: httpx.Request) -> httpx.Response: "name": "answer", "input": "selected question", "output": "selected answer", - "context": {"session": "s-1"}, + "context": {"session": {"id": "s-1", "attributes": {"region": "west"}}}, "tools": [{"name": "search", "description": "Search", "input_schema": {}}], - "dataset_output": '{"text":"expected"}', + "dataset_output": {"text": "expected"}, }, - "config": {}, + "config": {"request_timeout_seconds": 8.0}, } assert captured["body"] == expected_body + # The emitted body also satisfies the exact additive Orbit #1720 shape. + orbit_request = _Orbit1720Request.model_validate(expected_body) + assert orbit_request.record is not None + assert orbit_request.inputs.query == orbit_request.record.input + assert orbit_request.inputs.response == orbit_request.record.output + assert orbit_request.inputs.tools == orbit_request.record.tools + assert orbit_request.record.dataset_output == {"text": "expected"} + assert orbit_request.inputs.ground_truth == {"text": "expected"} + assert orbit_request.record.context == { + "session": {"id": "s-1", "attributes": {"region": "west"}} + } + assert orbit_request.config is not None + assert orbit_request.config.request_timeout_seconds == 8.0 + # The same request must remain consumable by released Orbit versions. legacy_request = _LegacyOrbitRequest.model_validate(expected_body) assert legacy_request.model_dump(mode="json", exclude_none=True) == { "scorer_id": SCORER_ID, + "scorer_version_id": SCORER_VERSION_ID, + "scorer_label": "toxicity", "inputs": {"query": "selected question", "response": "selected answer"}, - "config": {}, + "config": {"request_timeout_seconds": 8.0}, } @pytest.mark.asyncio @@ -663,7 +811,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert captured["body"] == { "scorer_id": "scorer-123", "inputs": {"query": "selected input", "response": ""}, - "config": {}, + "config": {"request_timeout_seconds": 8.0}, } @pytest.mark.asyncio @@ -844,7 +992,7 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: @patch.dict(os.environ, LUNA_ENV) @pytest.mark.asyncio - async def test_evaluator_forwards_configured_scorer_version_id(self) -> None: + async def test_evaluator_labels_forwarded_scorer_version_id_as_requested(self) -> None: from agent_control_evaluator_galileo.luna import LunaEvaluator, ScorerInvokeResponse from agent_control_evaluator_galileo.luna.client import GalileoLunaClient @@ -866,7 +1014,8 @@ async def test_evaluator_forwards_configured_scorer_version_id(self) -> None: result = await evaluator.evaluate("hello") assert result.matched is True - assert result.metadata["scorer_version_id"] == "version-456" + assert result.metadata["requested_scorer_version_id"] == "version-456" + assert "scorer_version_id" not in result.metadata mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", scorer_version_id="version-456", diff --git a/examples/galileo_luna/README.md b/examples/galileo_luna/README.md index d976ff4e..5d9ada07 100644 --- a/examples/galileo_luna/README.md +++ b/examples/galileo_luna/README.md @@ -36,11 +36,14 @@ Optional scorer settings: ```bash export GALILEO_LUNA_SCORER_LABEL="toxicity" # display/metadata label only -export GALILEO_LUNA_SCORER_VERSION_ID="version-uuid" # pin a specific scorer version +export GALILEO_LUNA_SCORER_VERSION_ID="version-uuid" # deprecated compatibility ID export GALILEO_LUNA_THRESHOLD="0.5" export GALILEO_LUNA_PAYLOAD_FIELD="output" ``` +`GALILEO_LUNA_SCORER_VERSION_ID` is a deprecated optional compatibility +identifier. Orbit currently invokes the scorer's current default version. + `GALILEO_LUNA_PAYLOAD_FIELD` is explicit for scalar selected data. This example selects the agent's drafted reply with `selector.path="output"`, so it sends that scalar as the scorer `output` field. If a selector returns structured data with `input` and/or `output` keys, those keys are sent directly and override `GALILEO_LUNA_PAYLOAD_FIELD`. If the Luna invoke endpoint uses an internal certificate authority, configure one of: diff --git a/examples/galileo_luna/setup_controls.py b/examples/galileo_luna/setup_controls.py index c8d57f59..753805df 100644 --- a/examples/galileo_luna/setup_controls.py +++ b/examples/galileo_luna/setup_controls.py @@ -27,6 +27,7 @@ LUNA_SCORER_ID = os.getenv("GALILEO_LUNA_SCORER_ID") LUNA_SCORER_LABEL = os.getenv("GALILEO_LUNA_SCORER_LABEL") +# Deprecated compatibility ID; Orbit invokes the scorer's current default version. LUNA_SCORER_VERSION_ID = os.getenv("GALILEO_LUNA_SCORER_VERSION_ID") LUNA_THRESHOLD = float(os.getenv("GALILEO_LUNA_THRESHOLD", "0.5")) LUNA_PAYLOAD_FIELD = os.getenv("GALILEO_LUNA_PAYLOAD_FIELD", "output") From bdd30d5da6796c09a0b88d18e98ccb4f031a42ed Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Wed, 26 Aug 2026 15:43:15 -0700 Subject: [PATCH 5/6] fix: align evaluator dependencies and ADK tools --- engine/pyproject.toml | 4 +- evaluators/builtin/pyproject.toml | 4 +- evaluators/contrib/galileo/pyproject.toml | 4 +- sdks/python/pyproject.toml | 2 +- .../integrations/google_adk/plugin.py | 60 +++++++++++++++-- sdks/python/tests/test_google_adk_plugin.py | 67 ++++++++++++++++++- server/pyproject.toml | 2 +- 7 files changed, 130 insertions(+), 13 deletions(-) diff --git a/engine/pyproject.toml b/engine/pyproject.toml index 0bcc04f4..4e14bbd2 100644 --- a/engine/pyproject.toml +++ b/engine/pyproject.toml @@ -4,8 +4,8 @@ version = "8.6.0" description = "Control execution engine for Agent Control" requires-python = ">=3.12" dependencies = [ - "agent-control-models>=3.0.0", - "agent-control-evaluators>=3.0.0", + "agent-control-models>=8.6.0", + "agent-control-evaluators>=8.6.0", "google-re2>=1.1", ] authors = [ diff --git a/evaluators/builtin/pyproject.toml b/evaluators/builtin/pyproject.toml index 4f11bb0e..8877b86a 100644 --- a/evaluators/builtin/pyproject.toml +++ b/evaluators/builtin/pyproject.toml @@ -7,7 +7,7 @@ requires-python = ">=3.12" license = { text = "Apache-2.0" } authors = [{ name = "Agent Control Team" }] dependencies = [ - "agent-control-models>=7.5.0", + "agent-control-models>=8.6.0", "pydantic>=2.12.4", "google-re2>=1.1", "jsonschema>=4.0.0", @@ -15,7 +15,7 @@ dependencies = [ ] [project.optional-dependencies] -galileo = ["agent-control-evaluator-galileo>=7.5.0"] +galileo = ["agent-control-evaluator-galileo>=8.6.0"] budget = ["agent-control-evaluator-budget>=7.5.0"] cisco = ["agent-control-evaluator-cisco>=7.5.0"] defenseclaw = ["agent-control-evaluator-defenseclaw>=8.2.0"] diff --git a/evaluators/contrib/galileo/pyproject.toml b/evaluators/contrib/galileo/pyproject.toml index a254fa5b..152f968b 100644 --- a/evaluators/contrib/galileo/pyproject.toml +++ b/evaluators/contrib/galileo/pyproject.toml @@ -7,8 +7,8 @@ requires-python = ">=3.12" license = { text = "Apache-2.0" } authors = [{ name = "Agent Control Team" }] dependencies = [ - "agent-control-evaluators>=7.5.0", - "agent-control-models>=7.5.0", + "agent-control-evaluators>=8.6.0", + "agent-control-models>=8.6.0", "httpx>=0.24.0", "pydantic>=2.12.4", ] diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index d0df4d99..08018e8d 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -14,7 +14,7 @@ dependencies = [ "docstring-parser>=0.15", # For @tool decorator schema inference "google-re2>=1.1", # For engine (bundled) "jsonschema>=4.0.0", # For models/engine (bundled) - "agent-control-evaluators>=7.5.0", # NOT vendored - avoid conflict with galileo + "agent-control-evaluators>=8.6.0", # NOT vendored - avoid conflict with galileo ] authors = [ {name = "Agent Control Team"} diff --git a/sdks/python/src/agent_control/integrations/google_adk/plugin.py b/sdks/python/src/agent_control/integrations/google_adk/plugin.py index 75266623..4f977440 100644 --- a/sdks/python/src/agent_control/integrations/google_adk/plugin.py +++ b/sdks/python/src/agent_control/integrations/google_adk/plugin.py @@ -104,6 +104,7 @@ def __init__( self._generated_invocation_ids_by_context_id: dict[int, str] = {} self._generated_context_ids_by_invocation_id: dict[str, int] = {} self._request_text_by_call_key: dict[tuple[str, str], str] = {} + self._request_tools_by_call_key: dict[tuple[str, str], list[dict[str, Any]]] = {} self._request_object_ids_by_call_key: dict[tuple[str, str], int] = {} self._current_llm_call_ids: dict[str, list[str]] = {} self._stored_llm_call_ids: dict[int, str] = {} @@ -134,6 +135,7 @@ async def close(self) -> None: self._generated_invocation_ids_by_context_id.clear() self._generated_context_ids_by_invocation_id.clear() self._request_text_by_call_key.clear() + self._request_tools_by_call_key.clear() self._request_object_ids_by_call_key.clear() self._current_llm_call_ids.clear() self._stored_llm_call_ids.clear() @@ -157,11 +159,17 @@ async def before_model_callback( step_name = self._resolve_llm_step_name(callback_context) request_text = extract_request_text(llm_request) + available_tools = self._invocation_tools(llm_request, step_name=step_name) invocation_id: str | None = None call_id: str | None = None if "after_model" in self.enabled_hooks: invocation_id = self._resolve_invocation_id(callback_context) - call_id = self._register_llm_request(invocation_id, llm_request, request_text) + call_id = self._register_llm_request( + invocation_id, + llm_request, + request_text, + available_tools, + ) self._ensure_step_known( self._build_llm_step_schema(step_name, callback_context=callback_context), ) @@ -179,7 +187,7 @@ async def before_model_callback( step_name, input=request_text, context=context, - tools=self._available_tools_by_step.get(step_name), + tools=available_tools, step_type="llm", stage="pre", ) @@ -210,7 +218,12 @@ async def after_model_callback( step_name = self._resolve_llm_step_name(callback_context) invocation_id = self._resolve_invocation_id(callback_context) call_id = self._resolve_llm_call_id(llm_response, invocation_id) - input_text = self._request_text_by_call_key.pop((invocation_id, call_id), "") + call_key = (invocation_id, call_id) + input_text = self._request_text_by_call_key.pop(call_key, "") + available_tools = self._request_tools_by_call_key.pop( + call_key, + self._available_tools_by_step.get(step_name, []), + ) self._clear_pending_llm_state(invocation_id, call_id, llm_response=llm_response) output_text = extract_response_text(llm_response) self._ensure_step_known( @@ -231,7 +244,7 @@ async def after_model_callback( input=input_text, output=output_text, context=context, - tools=self._available_tools_by_step.get(step_name), + tools=available_tools, step_type="llm", stage="post", ) @@ -694,6 +707,42 @@ def _remember_available_tools(self, root_agent: Any) -> None: available_tools[step_name] = definitions self._available_tools_by_step = available_tools + def _invocation_tools( + self, + llm_request: LlmRequest, + *, + step_name: str, + ) -> list[dict[str, Any]]: + """Return the tool definitions resolved for this exact ADK model call. + + Modern ADK versions populate ``LlmRequest.tools_dict`` after resolving + dynamic toolsets. Older versions do not expose it, so the bind-time + registry remains a compatibility fallback rather than the primary source. + + Args: + llm_request: Prepared ADK request for the current model invocation. + step_name: Resolved Agent Control LLM step name. + + Returns: + Normalized definitions for the invocation's exact tool set. + """ + resolved_tools = getattr(llm_request, "tools_dict", None) + if not isinstance(resolved_tools, dict): + return self._available_tools_by_step.get(step_name, []) + + definitions: list[dict[str, Any]] = [] + for tool in resolved_tools.values(): + tool_name = self._resolve_tool_step_name(tool, agent_step_name=step_name) + schema = self._build_tool_step_schema(tool, tool_name) + definitions.append( + normalized_tool_definition( + name=resolve_tool_name(tool), + description=schema.get("description"), + input_schema=schema.get("input_schema"), + ) + ) + return definitions + def _remember_steps(self, steps: Iterable[StepSchemaDict]) -> None: for step in steps: key = (step["type"], step["name"]) @@ -803,11 +852,13 @@ def _register_llm_request( invocation_id: str, llm_request: LlmRequest, request_text: str, + available_tools: list[dict[str, Any]], ) -> str: call_id = self._resolve_llm_call_id(llm_request, invocation_id) call_key = (invocation_id, call_id) self._stored_llm_call_ids[id(llm_request)] = call_id self._request_text_by_call_key[call_key] = request_text + self._request_tools_by_call_key[call_key] = available_tools self._request_object_ids_by_call_key[call_key] = id(llm_request) self._current_llm_call_ids.setdefault(invocation_id, []).append(call_id) return call_id @@ -822,6 +873,7 @@ def _clear_pending_llm_state( ) -> None: call_key = (invocation_id, call_id) self._request_text_by_call_key.pop(call_key, None) + self._request_tools_by_call_key.pop(call_key, None) request_object_id = self._request_object_ids_by_call_key.pop(call_key, None) if request_object_id is not None: diff --git a/sdks/python/tests/test_google_adk_plugin.py b/sdks/python/tests/test_google_adk_plugin.py index b50499a1..5ddaf903 100644 --- a/sdks/python/tests/test_google_adk_plugin.py +++ b/sdks/python/tests/test_google_adk_plugin.py @@ -10,7 +10,6 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest - from agent_control import ControlSteerError, ControlViolationError from agent_control._state import state @@ -42,10 +41,13 @@ def __init__( text: str = "hello", config: object | None = None, request_id: str | None = None, + tools_dict: dict[str, object] | None = None, ): self.contents = [SimpleNamespace(parts=[MockPart(text)])] self.config = config if config is not None else MockConfig() self.request_id = request_id + if tools_dict is not None: + self.tools_dict = tools_dict class MockLlmResponse: @@ -568,6 +570,67 @@ async def test_bound_agent_passes_complete_normalized_tools_to_llm(plugin_module assert definitions[0]["input_schema"]["properties"]["city"]["type"] == "string" +@pytest.mark.asyncio +async def test_model_callbacks_use_invocation_resolved_tools(plugin_module): + """Dynamic ADK tools from the prepared request override the bind-time registry.""" + # Given: a bound tool that ADK replaces while preparing this model request + plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") + with patch.object(plugin, "_sync_steps_blocking"): + plugin.bind( + SimpleNamespace( + name="planner", + tools=[MockTool("bound_search", "Bound search")], + ) + ) + context = MockCallbackContext("planner", invocation_id="inv-1") + request = MockLlmRequest( + "hello", + request_id="call-1", + tools_dict={"runtime_search": MockTool("runtime_search", "Runtime search")}, + ) + response = MockLlmResponse( + MockContent(role="model", parts=[MockPart("done")]), + request_id="call-1", + ) + + # When: pre- and post-model controls evaluate the same ADK invocation + with patch.object( + plugin_module, "_evaluate_and_enforce", AsyncMock(return_value=MagicMock()) + ) as mock_eval: + await plugin.before_model_callback(callback_context=context, llm_request=request) + await plugin.after_model_callback(callback_context=context, llm_response=response) + + # Then: both stages receive the exact runtime-resolved tool set + assert mock_eval.await_count == 2 + for call in mock_eval.await_args_list: + definitions = call.kwargs["tools"] + assert len(definitions) == 1 + assert definitions[0]["name"] == "runtime_search" + assert definitions[0]["description"] == "Runtime search" + assert definitions[0]["input_schema"]["properties"]["city"]["type"] == "string" + + +@pytest.mark.asyncio +async def test_empty_invocation_tool_set_does_not_fall_back_to_bound_tools(plugin_module): + """An authoritative empty ADK tool map remains empty for scoring.""" + # Given: a bound tool that is disabled for the current model invocation + plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") + with patch.object(plugin, "_sync_steps_blocking"): + plugin.bind(SimpleNamespace(name="planner", tools=[MockTool("bound_search")])) + + # When: ADK exposes an empty resolved tool dictionary + with patch.object( + plugin_module, "_evaluate_and_enforce", AsyncMock(return_value=MagicMock()) + ) as mock_eval: + await plugin.before_model_callback( + callback_context=MockCallbackContext("planner"), + llm_request=MockLlmRequest("hello", tools_dict={}), + ) + + # Then: Agent Control scores the empty invocation set, not stale bound tools + assert mock_eval.await_args.kwargs["tools"] == [] + + def test_bind_keeps_duplicate_tool_names_distinct_across_sub_agents(plugin_module): plugin = plugin_module.AgentControlPlugin(agent_name="test-agent01") root = SimpleNamespace( @@ -700,6 +763,7 @@ async def test_close_cancels_tasks_and_clears_request_cache(plugin_module): plugin._generated_invocation_ids_by_context_id[123] = "inv-2" plugin._generated_context_ids_by_invocation_id["inv-2"] = 123 plugin._request_text_by_call_key[("inv-1", "call-1")] = "hello" + plugin._request_tools_by_call_key[("inv-1", "call-1")] = [] plugin._request_object_ids_by_call_key[("inv-1", "call-1")] = 123 plugin._current_llm_call_ids["inv-1"] = ["call-1"] plugin._stored_llm_call_ids[123] = "call-1" @@ -718,6 +782,7 @@ async def slow_task(): assert plugin._generated_invocation_ids_by_context_id == {} assert plugin._generated_context_ids_by_invocation_id == {} assert plugin._request_text_by_call_key == {} + assert plugin._request_tools_by_call_key == {} assert plugin._request_object_ids_by_call_key == {} assert plugin._current_llm_call_ids == {} assert plugin._stored_llm_call_ids == {} diff --git a/server/pyproject.toml b/server/pyproject.toml index 774d289a..d5e7e8a5 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -24,7 +24,7 @@ dependencies = [ "jsonschema-rs>=0.22.0", "PyJWT>=2.8.0", "google-re2>=1.1", # For engine (bundled) - "agent-control-evaluators>=7.5.0", # NOT vendored - avoid conflict with galileo + "agent-control-evaluators>=8.6.0", # NOT vendored - avoid conflict with galileo ] authors = [ {name = "Agent Control Team"} From 7f72a8ca46281b82bf9a422531e26a0f29cae3b5 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Wed, 26 Aug 2026 16:00:35 -0700 Subject: [PATCH 6/6] test: align packaging floor expectation --- scripts/tests/test_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tests/test_build.py b/scripts/tests/test_build.py index eeb8087c..9d7d8359 100644 --- a/scripts/tests/test_build.py +++ b/scripts/tests/test_build.py @@ -99,4 +99,4 @@ def test_builtin_evaluators_manifest_keeps_models_floor_rewritable() -> None: dependencies = manifest["project"]["dependencies"] - assert "agent-control-models>=7.5.0" in dependencies + assert "agent-control-models>=8.6.0" in dependencies