diff --git a/README.md b/README.md index 4c34a29..5cbbcc8 100644 --- a/README.md +++ b/README.md @@ -541,6 +541,7 @@ uv run python main.py [example] [flag-key] [user-input] | --- | --- | --- | | `agent` *(default)* | `uv run python main.py agent` | `config()` via the global registry — switches providers without code changes | | `graph` | `uv run python main.py graph` | `graph()` multi-agent workflow driven by a LaunchDarkly agent graph flag | +| `graph-history` | `uv run python main.py graph-history` | `graph().invoke()` with multimodal `history` forwarded to the root node | | `openai-only` | `uv run python main.py openai-only` | `config()` with a custom `Registry` restricted to OpenAI handlers | | `streaming` | `uv run python main.py streaming` | `config().stream()` — token-by-token output | diff --git a/examples/graph_history.py b/examples/graph_history.py new file mode 100644 index 0000000..f148543 --- /dev/null +++ b/examples/graph_history.py @@ -0,0 +1,100 @@ +""" +Example: graph().invoke() with multimodal conversation history. + +Passes a `history` list containing an image content block to a graph flag. Only +the root node receives the history; downstream nodes see it through the normal +node-to-node data passing. The image is a generated solid red square, so the +model naming the colour is the signal that the image actually reached the +provider. + +Usage (via main.py): + python main.py graph-history "" +""" + +from __future__ import annotations + +import json +import re +import sys +from typing import Any + +import examples.register # noqa: F401 – side-effect: populate global_registry +from examples.utils import new_context, solid_color_png_base64, write_output +from launchdarkly_ai_server import global_registry, graph + +IMAGE_BLOCK = { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": solid_color_png_base64((255, 0, 0)), + }, +} + +COLOR_QUESTION = ( + "What colour is the square in the image I shared? Answer with just the colour name." +) + +# Two supported shapes: history that carries only context (the user turn arrives +# as user_input), and history that already ends with the user turn (user_input +# is empty). +SCENARIOS: list[dict[str, Any]] = [ + { + "name": "image-in-history + question as user_input", + "history": [{"role": "user", "content": [IMAGE_BLOCK]}], + "user_input": COLOR_QUESTION, + }, + { + "name": "history ends with the user turn, empty user_input", + "history": [ + {"role": "user", "content": "I am going to share an image with you."}, + {"role": "assistant", "content": "Sure — go ahead and share it."}, + { + "role": "user", + "content": [IMAGE_BLOCK, {"type": "text", "text": COLOR_QUESTION}], + }, + ], + "user_input": "", + }, +] + + +async def run(key: str, user_input: str) -> None: + failures: list[str] = [] + + for scenario in SCENARIOS: + response = await graph(key, registry=global_registry).invoke( + user_input or scenario["user_input"], + new_context(), + {"user_id": "user-123"}, + history=scenario["history"], + ) + + text = str( + response.get("response", "") + if isinstance(response, dict) + else getattr(response, "response", "") + ) + saw_color = bool(re.search(r"\bred\b", text, re.IGNORECASE)) + + tag = "SAW" if saw_color else "DID NOT see" + print( + f"[graph-history-check] {scenario['name']}: model {tag} the image from history", + file=sys.stderr, + ) + if not saw_color: + failures.append(scenario["name"]) + print( + f"[graph-history-check] response was: {text[:300]}", + file=sys.stderr, + ) + + print(json.dumps(response, indent=2, default=str)) + write_output(response) + + if failures: + raise RuntimeError( + "graph() did not forward history to the root node for: " + + ", ".join(failures) + + ". Before the history feature lands this is the expected result." + ) diff --git a/examples/utils.py b/examples/utils.py index 6bb4f5a..23de477 100644 --- a/examples/utils.py +++ b/examples/utils.py @@ -2,10 +2,13 @@ from __future__ import annotations +import base64 import dataclasses import json import random import string +import struct +import zlib from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -35,3 +38,30 @@ def write_output(data: Any) -> None: json.dumps(data, indent=2, default=_default_encoder), encoding="utf-8" ) print(f"Output written to output/{filename}") + + +def _png_chunk(kind: bytes, data: bytes) -> bytes: + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + +def solid_color_png_base64(rgb: tuple[int, int, int], size: int = 64) -> str: + """Encodes a solid-colour PNG as base64 for multimodal examples. + + Generating the image avoids committing a binary fixture, and the colour is + the only thing the model can report back — which makes it a usable signal + for whether the image actually reached the provider. + """ + raw = b"".join(b"\x00" + bytes(rgb) * size for _ in range(size)) + ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0) + png = ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", ihdr) + + _png_chunk(b"IDAT", zlib.compress(raw)) + + _png_chunk(b"IEND", b"") + ) + return base64.b64encode(png).decode("ascii") diff --git a/main.py b/main.py index 2f957fb..a36e684 100644 --- a/main.py +++ b/main.py @@ -9,6 +9,7 @@ python main.py streaming launch-darkly-documentation-summarizer "Summarise feature flags in 3 bullets" python main.py judge launch-darkly-documentation-summarizer "What is the LaunchDarkly AI SDK?" python main.py graph my-agent-graph "What is the LaunchDarkly AI SDK?" + python main.py graph-history my-agent-graph "" python main.py openai-only my-openai-flag "Tell me about feature flags" python main.py langchain my-langchain-flag "Tell me about feature flags" python main.py claude-agents launch-darkly-documentation-summarizer "What is the LaunchDarkly AI SDK?" @@ -43,6 +44,7 @@ "agent": "examples.agent", "streaming": "examples.streaming", "graph": "examples.graph_example", + "graph-history": "examples.graph_history", "history": "examples.history", "judge": "examples.judge_example", "claude-agents": "examples.claude_agents_example", diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py index d98c831..dc22ec5 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/handler.py @@ -16,7 +16,9 @@ LDContext, NativeTool, ProviderHandler, + compose_history, config, + content_to_text, create_handler, parse_template, set_ld_span_attributes, @@ -96,17 +98,6 @@ def partition_tools( return native_tool_map, user_config_tools, list(native_tool_map.keys()) -def _format_history(history: list[dict[str, Any]] | None) -> str | None: - if not history: - return None - lines = [] - for msg in history: - role = msg.get("role", "user") - content = msg.get("content", "") - lines.append(f"{role}: {content}") - return "Conversation History:\n\n" + "\n".join(lines) - - def build_prompt( config: AiConfigRep, user_input: str | None, @@ -134,15 +125,91 @@ def build_prompt( f"{config_history}\n\n{safe_input}" if config_history else safe_input ) - history_text = _format_history(history) - if history_text: - system_prompt = ( - f"{system_prompt}\n\n{history_text}" if system_prompt else history_text - ) - return safe_input, system_prompt +def _parse_message_content(content: Any, variables: dict[str, Any]) -> Any: + """Apply templates only to string content.""" + return parse_template(content, variables) if isinstance(content, str) else content + + +def _config_conversation_turns( + config: AiConfigRep, variables: dict[str, Any] +) -> list[dict[str, Any]]: + """Return non-system config messages with string templates applied.""" + return [ + { + "role": message.get("role"), + "content": _parse_message_content(message.get("content", ""), variables), + } + for message in (config.get("messages") or []) + if message.get("role") != "system" + ] + + +def _to_anthropic_user_content(content: Any) -> Any: + """Map canonical user content to Anthropic-native content blocks.""" + if isinstance(content, str): + return content + + blocks: list[dict[str, Any]] = [] + for block in content: + if block.get("type") == "text": + blocks.append({"type": "text", "text": block.get("text", "")}) + elif block.get("type") == "image": + source = block.get("source", {}) + if source.get("type") == "url": + mapped_source = {"type": "url", "url": source.get("url", "")} + else: + mapped_source = { + "type": "base64", + "media_type": source.get("media_type", ""), + "data": source.get("data", ""), + } + blocks.append({"type": "image", "source": mapped_source}) + return blocks + + +async def _to_streamed_prompt( + turns: list[dict[str, Any]], +) -> AsyncGenerator[dict[str, Any], None]: + """Yield composed turns in the Claude Agent SDK streaming-input shape.""" + for turn in turns: + content = turn.get("content", "") + if turn.get("role") == "assistant": + content = content_to_text(content) + else: + content = _to_anthropic_user_content(content) + yield { + "type": "user", + "message": {"role": turn.get("role"), "content": content}, + "parent_tool_use_id": None, + } + + +def build_query_prompt( + config: AiConfigRep, + user_input: str | None, + variables: dict[str, Any], + history: list[dict[str, Any]] | None, + fallback_prompt: str, +) -> str | AsyncGenerator[dict[str, Any], None]: + """Build a plain prompt without history, or a structured streamed prompt.""" + if not history: + return fallback_prompt + + turns = compose_history( + history=history, + user_input=user_input, + config_messages=( + [] + if config.get("instructions") + else _config_conversation_turns(config, variables) + ), + ) + return _to_streamed_prompt(turns) + + def _is_coroutine(fn: Any) -> bool: return asyncio.iscoroutinefunction(fn) @@ -210,6 +277,7 @@ async def _call_impl( span = None prompt, system_prompt = build_prompt(config, user_input, vs, history) + query_prompt = build_query_prompt(config, user_input, vs, history, prompt) # Append outputFormat instruction to system prompt if config.get("outputFormat"): @@ -258,7 +326,7 @@ async def _call_impl( # generator — Python's asyncio finalizer later tries to aclose() it # and raises RuntimeError if the generator is suspended inside a real # await in the SDK (AIC-2950). See Appendix A.4 in TESTING.md. - gen = query_fn(prompt=prompt, options=options) + gen = query_fn(prompt=query_prompt, options=options) try: async for message in gen: if isinstance(message, ResultMessage): @@ -360,6 +428,7 @@ async def _stream_gen( span = None prompt, system_prompt = build_prompt(config, user_input, variables, history) + query_prompt = build_query_prompt(config, user_input, variables, history, prompt) if span: span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt}) prompt_msgs: list[dict[str, str]] = [] @@ -392,50 +461,58 @@ async def _stream_gen( full_output = "" - async for message in query_fn(prompt=prompt, options=options): - if isinstance(message, StreamEvent): - event = message.event - if ( - event.get("type") == "content_block_delta" - and event.get("delta", {}).get("type") == "text_delta" - ): - text = event["delta"].get("text", "") - if text: - yield {"type": "chunk", "text": text} - full_output += text - elif isinstance(message, ResultMessage): - raw_usage = message.usage or {} - input_tokens = int(raw_usage.get("input_tokens", 0)) - output_tokens = int(raw_usage.get("output_tokens", 0)) - final_output = message.result or full_output - if span: - span.set_attribute( - "gen_ai.response.model", config.get("model", {}).get("name", "") - ) - span.set_attribute("gen_ai.usage.input_tokens", input_tokens) - span.set_attribute("gen_ai.usage.output_tokens", output_tokens) - span.set_attribute( - "gen_ai.usage.total_tokens", input_tokens + output_tokens - ) - span.add_event( - "gen_ai.content.completion", - { - "gen_ai.completion": final_output + gen = query_fn(prompt=query_prompt, options=options) + try: + async for message in gen: + if isinstance(message, StreamEvent): + event = message.event + if ( + event.get("type") == "content_block_delta" + and event.get("delta", {}).get("type") == "text_delta" + ): + text = event["delta"].get("text", "") + if text: + yield {"type": "chunk", "text": text} + full_output += text + elif isinstance(message, ResultMessage): + raw_usage = message.usage or {} + input_tokens = int(raw_usage.get("input_tokens", 0)) + output_tokens = int(raw_usage.get("output_tokens", 0)) + final_output = message.result or full_output + if span: + span.set_attribute( + "gen_ai.response.model", + config.get("model", {}).get("name", ""), + ) + span.set_attribute("gen_ai.usage.input_tokens", input_tokens) + span.set_attribute("gen_ai.usage.output_tokens", output_tokens) + span.set_attribute( + "gen_ai.usage.total_tokens", input_tokens + output_tokens + ) + span.add_event( + "gen_ai.content.completion", + { + "gen_ai.completion": final_output + if isinstance(final_output, str) + else json.dumps(final_output) + }, + ) + set_openllmetry_completion( + span, + final_output if isinstance(final_output, str) - else json.dumps(final_output) - }, - ) - set_openllmetry_completion( - span, - final_output - if isinstance(final_output, str) - else json.dumps(final_output), - {"input_tokens": input_tokens, "output_tokens": output_tokens}, - ) - span.set_status(SpanStatusCode.OK) - span.end() - yield {"type": "done", "output": final_output, "usage": raw_usage} - return + else json.dumps(final_output), + { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + }, + ) + span.set_status(SpanStatusCode.OK) + span.end() + yield {"type": "done", "output": final_output, "usage": raw_usage} + return + finally: + await gen.aclose() if span: span.set_status(SpanStatusCode.OK) diff --git a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py index 063dc46..a2d9c08 100644 --- a/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py +++ b/packages/claude-agents/src/launchdarkly_ai_claude_agents/native_graph.py @@ -32,6 +32,7 @@ from launchdarkly_ai_claude_agents.handler import ( _build_hooks, build_prompt, + build_query_prompt, build_tool_mcp, partition_tools, ) @@ -103,6 +104,7 @@ async def _run_query( graph_key: str, run_id: str, child_subagent_tools: list[Any], + history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: import importlib @@ -116,6 +118,9 @@ async def _run_query( wrapped = _wrap_native_tools(tool_handlers, ld_context, track_data) prompt, system_prompt = build_prompt(node.config, input_text, variables) + query_prompt = build_query_prompt( + node.config, input_text, variables, history, prompt + ) native_tool_map, user_config_tools, native_tool_names = partition_tools( node.config.get("tools"), wrapped ) @@ -170,7 +175,7 @@ async def _run_query( # Bare `return` inside `async for` abandons the generator — Python's asyncio # finalizer later tries to aclose() it and may raise RuntimeError if the # generator is suspended inside a real await in the SDK (AIC-2950). - gen = query_fn(prompt=prompt, options=options) + gen = query_fn(prompt=query_prompt, options=options) try: async for message in gen: if isinstance(message, ResultMessage): @@ -207,6 +212,7 @@ def to_claude_agents( async def invoke( input_text: str = "", variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: import importlib @@ -335,6 +341,7 @@ async def _subagent_execute( def_obj.key, run_id, root_child_tools, + history, ) except Exception as exc: if span: diff --git a/packages/claude-agents/tests/test_handler.py b/packages/claude-agents/tests/test_handler.py index 1f7eaaf..7cbd2f8 100644 --- a/packages/claude-agents/tests/test_handler.py +++ b/packages/claude-agents/tests/test_handler.py @@ -1025,19 +1025,12 @@ class TestHistory: {"role": "assistant", "content": "Feature flagging is a technique..."}, ] - def test_history_appended_to_system_prompt(self) -> None: + def test_history_not_stuffed_into_system_prompt(self) -> None: config = _make_config(instructions="Be concise.") _, system = build_prompt(config, "hi", {}, self.SAMPLE_HISTORY) assert system is not None - assert "Conversation History:" in system assert "Be concise." in system - - def test_history_format_is_correct(self) -> None: - config = _make_config(instructions="Be helpful.") - _, system = build_prompt(config, "hi", {}, self.SAMPLE_HISTORY) - assert system is not None - assert "user: What is feature flagging?" in system - assert "assistant: Feature flagging is a technique..." in system + assert "Conversation History:" not in system def test_empty_history_treated_like_no_history(self) -> None: config = _make_config(instructions="Be concise.") @@ -1046,9 +1039,8 @@ def test_empty_history_treated_like_no_history(self) -> None: assert system_with_empty == system_without assert "Conversation History:" not in (system_with_empty or "") - def test_history_without_prior_system_prompt(self) -> None: + def test_history_without_instructions_keeps_system_none(self) -> None: + """History is structured input — it must not invent a Conversation History system prompt.""" config = _make_config() _, system = build_prompt(config, "hi", {}, self.SAMPLE_HISTORY) - assert system is not None - assert "Conversation History:" in system - assert "user: What is feature flagging?" in system + assert system is None or "Conversation History:" not in system diff --git a/packages/claude-agents/tests/test_native_graph.py b/packages/claude-agents/tests/test_native_graph.py index 9f204db..01f628c 100644 --- a/packages/claude-agents/tests/test_native_graph.py +++ b/packages/claude-agents/tests/test_native_graph.py @@ -276,6 +276,58 @@ async def test_runner_starts_at_root_and_returns_output(self) -> None: assert result["response"] == "final-output" + @pytest.mark.asyncio + async def test_multimodal_history_uses_native_root_prompt(self) -> None: + mock_sdk = _make_sdk_mock("done") + graph_def = _make_graph_def() + captured_prompts: list[Any] = [] + result_msg = _make_result_msg("done") + + async def _query(**kwargs: Any) -> AsyncIterator[Any]: + captured_prompts.append(kwargs.get("prompt")) + yield result_msg + + mock_sdk.query = _query + history = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + + with patch( + "importlib.import_module", + side_effect=lambda n: ( + mock_sdk if n == "claude_agent_sdk" else __import__(n) + ), + ): + await to_claude_agents(_make_def_promise(graph_def)).invoke( + "describe", {}, history + ) + + assert captured_prompts + prompt = captured_prompts[0] + assert not isinstance(prompt, str) + chunks = [chunk async for chunk in prompt] + image = chunks[0]["message"]["content"][0] + assert image == { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + @pytest.mark.asyncio async def test_config_tools_converted_and_passed(self) -> None: mock_sdk = _make_sdk_mock("done") diff --git a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py index 0d2f3ca..44970f9 100644 --- a/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py +++ b/packages/claude-messages/src/launchdarkly_ai_claude_messages/handler.py @@ -9,8 +9,11 @@ AiConfigRep, LDContext, ProviderHandler, + compose_history, config, + content_to_text, create_handler, + is_content_blocks, parse_template, set_ld_span_attributes, set_openllmetry_completion, @@ -44,9 +47,65 @@ def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: ] +def _anthropic_content(content: Any) -> str | list[dict[str, Any]]: + """Map LaunchDarkly-canonical content to Anthropic message content.""" + if not is_content_blocks(content): + return content if isinstance(content, str) else "" + + blocks: list[dict[str, Any]] = [] + for block in content: + if block.get("type") == "text": + blocks.append({"type": "text", "text": block.get("text", "")}) + elif block.get("type") == "image": + source = block.get("source", {}) + if source.get("type") == "url": + mapped_source = {"type": "url", "url": source.get("url", "")} + else: + mapped_source = { + "type": "base64", + "media_type": source.get("media_type", ""), + "data": source.get("data", ""), + } + blocks.append({"type": "image", "source": mapped_source}) + return blocks + + +def _template_content(content: Any, variables: dict[str, Any]) -> Any: + """Apply templates to text content without parsing structured blocks.""" + return parse_template(content, variables) if isinstance(content, str) else content + + +def _anthropic_blocks(content: Any) -> list[dict[str, Any]]: + """Normalize Anthropic message content to a list of content blocks.""" + if isinstance(content, list): + return content + return [{"type": "text", "text": content}] if content else [] + + +def _merge_adjacent_same_role( + messages: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Merge consecutive same-role turns into one multi-block message. + + Anthropic's Messages API requires strictly alternating user/assistant roles. + Composed history can place an image-only user turn immediately before the + appended ``user_input`` question, which would otherwise send two consecutive + user turns and be rejected. Merging keeps both as a single user message. + """ + merged: list[dict[str, Any]] = [] + for message in messages: + if merged and merged[-1]["role"] == message["role"]: + merged[-1]["content"] = _anthropic_blocks( + merged[-1]["content"] + ) + _anthropic_blocks(message.get("content")) + else: + merged.append({"role": message["role"], "content": message.get("content")}) + return merged + + def _build_messages( config: AiConfigRep, - user_input: str, + user_input: str | None, variables: dict[str, Any], *, include_output_format: bool = True, @@ -54,33 +113,48 @@ def _build_messages( ) -> tuple[list[dict[str, Any]], str | None]: """Returns (messages, system_prompt).""" system: str | None = None - messages: list[dict[str, Any]] = [] + config_messages: list[dict[str, Any]] = [] if config.get("messages"): system_msgs = [m for m in config["messages"] if m.get("role") == "system"] conv_msgs = [m for m in config["messages"] if m.get("role") != "system"] if system_msgs: system = parse_template( - "\n".join(m["content"] for m in system_msgs), variables + "\n".join(content_to_text(m.get("content", "")) for m in system_msgs), + variables, ) for msg in conv_msgs: - messages.append( + config_messages.append( { "role": msg["role"], - "content": parse_template(msg["content"], variables), + "content": _anthropic_content( + _template_content(msg.get("content", ""), variables) + ), } ) elif config.get("instructions"): system = parse_template(config["instructions"], variables) if history: - for msg in history: - role = msg.get("role", "user") - if role in ("user", "assistant"): - messages.append({"role": role, "content": msg.get("content", "")}) - - if not messages or messages[-1].get("role") != "user": - messages.append({"role": "user", "content": user_input or ""}) + composed = compose_history( + history=history, + user_input=user_input, + config_messages=config_messages, + ) + messages = _merge_adjacent_same_role( + [ + { + "role": msg["role"], + "content": _anthropic_content(msg.get("content", "")), + } + for msg in composed + if msg.get("role") in ("user", "assistant") + ] + ) + else: + messages = config_messages + if user_input or not messages: + messages.append({"role": "user", "content": user_input or ""}) if include_output_format and config.get("outputFormat"): schema_instruction = f"Respond with valid JSON matching this schema:\n{json.dumps(config['outputFormat'])}" @@ -199,12 +273,15 @@ async def _call_impl( messages, system = _build_messages(config, user_input, vs, history=history) if span: prompt_text = (f"system: {system}\n" if system else "") + "\n".join( - f"{m['role']}: {m['content']}" for m in messages + f"{m['role']}: {content_to_text(m['content'])}" for m in messages ) span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) prompt_msgs = ( [{"role": "system", "content": system}] if system else [] - ) + [{"role": m["role"], "content": m["content"]} for m in messages] + ) + [ + {"role": m["role"], "content": content_to_text(m["content"])} + for m in messages + ] set_openllmetry_prompt(span, prompt_msgs) try: @@ -283,11 +360,12 @@ async def _stream_gen( ) if span: prompt_text = (f"system: {system}\n" if system else "") + "\n".join( - f"{m['role']}: {m['content']}" for m in messages + f"{m['role']}: {content_to_text(m['content'])}" for m in messages ) span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) prompt_msgs = ([{"role": "system", "content": system}] if system else []) + [ - {"role": m["role"], "content": m["content"]} for m in messages + {"role": m["role"], "content": content_to_text(m["content"])} + for m in messages ] set_openllmetry_prompt(span, prompt_msgs) diff --git a/packages/claude-messages/tests/test_handler.py b/packages/claude-messages/tests/test_handler.py index ccd4d90..d67086d 100644 --- a/packages/claude-messages/tests/test_handler.py +++ b/packages/claude-messages/tests/test_handler.py @@ -1238,3 +1238,68 @@ async def test_system_role_in_history_filtered_out( msgs = mock_anthropic.messages.create.call_args.kwargs["messages"] roles = [m["role"] for m in msgs] assert "system" not in roles + + async def test_image_history_plus_user_input_merges_into_one_user_turn( + self, mock_anthropic: MagicMock + ) -> None: + # Image-only history + a separate question must not produce two + # consecutive user turns (Anthropic requires alternating roles); they + # merge into a single user message carrying both blocks. + from launchdarkly_ai_claude_messages import create_claude_messages_handler + + history = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + h = create_claude_messages_handler() + await h(CONFIG, "What colour is this?", {}, {}, history) + msgs = mock_anthropic.messages.create.call_args.kwargs["messages"] + + # No two consecutive user turns. + roles = [m["role"] for m in msgs] + assert not any( + roles[i] == "user" and roles[i + 1] == "user" for i in range(len(roles) - 1) + ) + # The trailing user turn carries both the image and the question text. + last = msgs[-1] + assert last["role"] == "user" + assert isinstance(last["content"], list) + block_types = [b["type"] for b in last["content"]] + assert "image" in block_types + assert any( + b["type"] == "text" and b["text"] == "What colour is this?" + for b in last["content"] + ) + + async def test_history_ending_in_user_text_plus_user_input_merges( + self, mock_anthropic: MagicMock + ) -> None: + from launchdarkly_ai_claude_messages import create_claude_messages_handler + + history = [{"role": "user", "content": "prior question"}] + h = create_claude_messages_handler() + await h(CONFIG, "follow-up", {}, {}, history) + msgs = mock_anthropic.messages.create.call_args.kwargs["messages"] + roles = [m["role"] for m in msgs] + assert not any( + roles[i] == "user" and roles[i + 1] == "user" for i in range(len(roles) - 1) + ) + # Both the prior question and the follow-up survive in the merged turn. + merged_text = "".join( + b["text"] + for b in msgs[-1]["content"] + if isinstance(b, dict) and b.get("type") == "text" + ) + assert "prior question" in merged_text + assert "follow-up" in merged_text diff --git a/packages/client/src/launchdarkly_ai_server/__init__.py b/packages/client/src/launchdarkly_ai_server/__init__.py index 5d1f2ee..182ebe8 100644 --- a/packages/client/src/launchdarkly_ai_server/__init__.py +++ b/packages/client/src/launchdarkly_ai_server/__init__.py @@ -4,6 +4,14 @@ from .client import ConfigInstance, config from .graph import GraphInstance, graph, resolve_graph +from .history import ( + any_multimodal, + compose_history, + content_to_text, + has_multimodal_content, + image_block_to_url, + is_content_blocks, +) from .judges import build_judge_tasks, run_judge, run_judges from .lifecycle import ( extract_variation, @@ -113,6 +121,13 @@ "set_openllmetry_completion", "set_openllmetry_prompt", "to_ld_context", + # history + "compose_history", + "content_to_text", + "image_block_to_url", + "is_content_blocks", + "has_multimodal_content", + "any_multimodal", # validation "parse_ai_config", # registry diff --git a/packages/client/src/launchdarkly_ai_server/graph.py b/packages/client/src/launchdarkly_ai_server/graph.py index 0a397cf..335ece9 100644 --- a/packages/client/src/launchdarkly_ai_server/graph.py +++ b/packages/client/src/launchdarkly_ai_server/graph.py @@ -206,6 +206,7 @@ async def run_node( tool_handlers=tool_handlers, variables=opts.get("variables"), graph_key=key, + history=opts.get("history"), ) response = ( result["response"] @@ -338,6 +339,7 @@ def _fn(*a: Any, **kw: Any) -> str: tool_handlers=merged_tool_handlers, variables=opts.get("variables"), graph_key=key, + history=opts.get("history"), ) response = ( result["response"] @@ -524,6 +526,7 @@ async def invoke( user_input: str | None, context: LDContext, variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, ) -> ProviderGraphResponse: from .judges import run_judges from .lifecycle import get_client @@ -584,6 +587,11 @@ async def invoke( opts: dict[str, Any] = {"variables": variables} if previous_node: opts["from"] = previous_node + # History seeds the entry point only. After the root hop, nodes + # stay oriented through the string threading built below, so + # history is not re-sent to downstream handlers. + elif history: + opts["history"] = history res = await graph_def.route(current, current_input, opts) path.append(current.key) diff --git a/packages/client/src/launchdarkly_ai_server/history.py b/packages/client/src/launchdarkly_ai_server/history.py new file mode 100644 index 0000000..f5f4694 --- /dev/null +++ b/packages/client/src/launchdarkly_ai_server/history.py @@ -0,0 +1,97 @@ +"""Shared conversation-history composition and multimodal content helpers. + +Mirrors the TypeScript ``history`` module so every handler composes runtime +``history`` the same way (TESTING.md §1.11) and maps LaunchDarkly-canonical +content blocks to each provider's native shape (Appendix A.7). + +History messages are plain dicts: ``{"role": ..., "content": ...}`` where +``content`` is either a string or a list of content-block dicts: + + {"type": "text", "text": str} + {"type": "image", "source": {"type": "base64", "media_type": str, "data": str}} + {"type": "image", "source": {"type": "url", "url": str}} +""" + +from __future__ import annotations + +from typing import Any + +MessageContent = str | list[dict[str, Any]] + + +def compose_history( + *, + history: list[dict[str, Any]], + user_input: str | None = None, + config_messages: list[dict[str, Any]] | None = None, +) -> list[dict[str, Any]]: + """Composes the ordered conversation turns for a handler with runtime history. + + Order: ``[config conversation messages] -> [history] -> [user_input?]``. + + - System-role history messages are dropped (system belongs on the provider + system prompt, derived separately by each handler). + - A non-empty ``user_input`` is always appended as a final user text turn, + even when history already ends with a user turn (image-only history + a + separate question). + - An empty/missing ``user_input`` appends nothing, so history that already + carries the full (possibly multimodal) user turn is sent as-is. + + Returns a list of ``{"role": "user"|"assistant", "content": ...}`` dicts. + Callers only take this structured path when ``history`` is non-empty; with no + history they keep their single-string prompt behaviour, so an empty history + stays identical to passing none. + """ + turns: list[dict[str, Any]] = list(config_messages or []) + + for message in history: + role = message.get("role") + if role == "system": + continue + turns.append({"role": role, "content": message.get("content")}) + + if user_input: + turns.append({"role": "user", "content": user_input}) + + return turns + + +def is_content_blocks(content: MessageContent) -> bool: + """True when content is the multimodal block-array shape.""" + return isinstance(content, list) + + +def has_multimodal_content(content: MessageContent) -> bool: + """True when a message carries any non-text (e.g. image) content block.""" + if not isinstance(content, list): + return False + return any(block.get("type") != "text" for block in content) + + +def any_multimodal(turns: list[dict[str, Any]]) -> bool: + """True when any turn in the list carries multimodal content.""" + return any(has_multimodal_content(turn.get("content", "")) for turn in turns) + + +def content_to_text(content: MessageContent) -> str: + """Flattens content to plain text: a string passes through; a block array + contributes only its text blocks.""" + if isinstance(content, str): + return content + return "".join( + block.get("text", "") for block in content if block.get("type") == "text" + ) + + +def image_block_to_url(block: dict[str, Any]) -> str: + """Builds a ``data:;base64,`` URL for a base64 image block, + or returns the URL directly for a URL-sourced block. + + This is the form OpenAI and LangChain expect (``image_url``); Anthropic keeps + ``media_type`` + ``data`` split, so its handlers read ``block["source"]`` + directly instead. + """ + source = block.get("source", {}) + if source.get("type") == "url": + return str(source.get("url", "")) + return f"data:{source.get('media_type', '')};base64,{source.get('data', '')}" diff --git a/packages/client/tests/test_graph.py b/packages/client/tests/test_graph.py index 7264c06..50dfd6c 100644 --- a/packages/client/tests/test_graph.py +++ b/packages/client/tests/test_graph.py @@ -423,3 +423,64 @@ async def failing_variation(key: str, ctx: dict, default: Any) -> Any: assert gd.enabled is False mock_logger.error.assert_called() + + async def test_history_forwarded_to_root_handler_only( + self, mock_ld_client: MagicMock + ) -> None: + received: list[Any] = [] + + async def capturing_handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict: + received.append(history) + return {"output": "ok", "usage": {"input_tokens": 1, "output_tokens": 1}} + + handler = ProviderHandler( + fn=capturing_handler, provides_for=("TestProvider", "messages") + ) # type: ignore[arg-type] + history = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc", + }, + } + ], + } + ] + await graph("graph-key", handlers=[handler]).invoke( + "hi", CONTEXT, history=history + ) + assert len(received) >= 2 + assert received[0] == history + assert all(h is None for h in received[1:]) + + async def test_omitted_history_leaves_root_handler_history_none( + self, mock_ld_client: MagicMock + ) -> None: + received: list[Any] = [] + + async def capturing_handler( + config: Any, + user_input: Any, + tool_handlers: Any, + variables: Any, + history: Any = None, + ) -> dict: + received.append(history) + return {"output": "ok", "usage": {"input_tokens": 1, "output_tokens": 1}} + + handler = ProviderHandler( + fn=capturing_handler, provides_for=("TestProvider", "messages") + ) # type: ignore[arg-type] + await graph("graph-key", handlers=[handler]).invoke("hi", CONTEXT) + assert received[0] is None diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py index efe8163..367fb2d 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/handler.py @@ -13,7 +13,9 @@ AiConfigRep, LDContext, ProviderHandler, + compose_history, config, + content_to_text, create_handler, parse_template, set_ld_span_attributes, @@ -21,6 +23,8 @@ set_openllmetry_prompt, ) +from .messages import to_lang_chain_messages + try: from opentelemetry import trace from opentelemetry.trace import StatusCode as SpanStatusCode @@ -60,69 +64,118 @@ async def _handler(_name: str = name, **kwargs: Any) -> str: return result -def _format_history(history: list[dict[str, Any]] | None) -> str | None: - if not history: - return None - lines = [] - for msg in history: - role = msg.get("role", "user") - content = msg.get("content", "") - lines.append(f"{role}: {content}") - return "Conversation History:\n\n" + "\n".join(lines) - - def _extract_system_prompt( config: AiConfigRep, variables: dict[str, Any], history: list[dict[str, Any]] | None = None, ) -> str | None: - system_prompt: str | None = None + """The system prompt, from ``instructions`` or the system-role config messages. + + ``history`` is accepted so callers can pass it uniformly, but it never + reaches the system prompt: history is conversation, and it goes through the + structured message path in ``_build_initial_messages`` (TESTING.md §1.11). + """ if config.get("instructions"): - system_prompt = parse_template(config["instructions"], variables) - elif config.get("messages"): + return parse_template(config["instructions"], variables) + if config.get("messages"): sys_msgs = [m for m in config["messages"] if m.get("role") == "system"] if sys_msgs: - system_prompt = parse_template( - "\n".join(m["content"] for m in sys_msgs), variables - ) + return parse_template("\n".join(m["content"] for m in sys_msgs), variables) + return None - history_text = _format_history(history) - if history_text: - system_prompt = ( - f"{system_prompt}\n\n{history_text}" if system_prompt else history_text - ) - return system_prompt +def _config_conversation_turns( + config: AiConfigRep, + variables: dict[str, Any], +) -> list[dict[str, Any]]: + """Non-system config conversation messages, template-applied, in canonical form.""" + return [ + {"role": m["role"], "content": parse_template(m["content"], variables)} + for m in (config.get("messages") or []) + if m.get("role") != "system" + ] def _build_initial_messages( config: AiConfigRep, - user_input: str, + user_input: str | None, variables: dict[str, Any], + history: list[dict[str, Any]] | None = None, ) -> list[Any]: import importlib + # With history, the whole conversation is composed as LangChain messages — + # the framework's native input path. `config.instructions` / system messages + # stay on the system prompt; history never becomes system-prompt text. + if history: + return to_lang_chain_messages( + compose_history( + history=history, + user_input=user_input, + config_messages=( + [] + if config.get("instructions") + else _config_conversation_turns(config, variables) + ), + ) + ) + msgs_mod = importlib.import_module("langchain_core.messages") HumanMessage = msgs_mod.HumanMessage AIMessage = msgs_mod.AIMessage messages: list[Any] = [] last_role: str | None = None - if config.get("messages"): - for msg in config["messages"]: - if msg.get("role") == "system": - continue - content = parse_template(msg["content"], variables) - if msg["role"] == "user": - messages.append(HumanMessage(content)) - else: - messages.append(AIMessage(content)) - last_role = msg["role"] + for msg in _config_conversation_turns(config, variables): + if msg["role"] == "user": + messages.append(HumanMessage(msg["content"])) + else: + messages.append(AIMessage(msg["content"])) + last_role = msg["role"] if last_role != "user": messages.append(HumanMessage(user_input or "")) return messages +def _message_text(message: Any) -> str: + """Span-safe text for a message. Multimodal content contributes only its text + parts, so an image never lands in a span attribute as a base64 payload.""" + content = getattr(message, "content", "") + if isinstance(content, str | list): + return content_to_text(content) + return str(content) + + +def _record_prompt( + span: Any, + system_prompt: str | None, + messages: list[Any], +) -> None: + prompt_text = "\n".join( + [ + *(["system: " + system_prompt] if system_prompt else []), + *[ + f"{getattr(m, 'type', type(m).__name__)}: {_message_text(m)}" + for m in messages + ], + ] + ) + span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) + prompt_msgs: list[dict[str, str]] = [] + if system_prompt: + prompt_msgs.append({"role": "system", "content": system_prompt}) + prompt_msgs.extend( + [ + { + "role": getattr(m, "type", type(m).__name__), + "content": _message_text(m), + } + for m in messages + ] + ) + set_openllmetry_prompt(span, prompt_msgs) + + def _make_default_chat_model(config: AiConfigRep) -> Any: """ Instantiate the appropriate LangChain chat model based on ``config.provider.name``. @@ -172,41 +225,17 @@ async def _call_impl( else: span = None - system_prompt = _extract_system_prompt(config, vs, history) + system_prompt = _extract_system_prompt(config, vs) if config.get("outputFormat"): schema_instr = f"Respond with valid JSON matching this schema:\n{json.dumps(config['outputFormat'])}" system_prompt = ( f"{system_prompt}\n\n{schema_instr}" if system_prompt else schema_instr ) - initial_messages = _build_initial_messages(config, user_input, vs) + initial_messages = _build_initial_messages(config, user_input, vs, history) if span: - prompt_text = "\n".join( - [ - *(["system: " + system_prompt] if system_prompt else []), - *[ - f"{getattr(m, 'type', type(m).__name__)}: {m.content}" - for m in initial_messages - ], - ] - ) - span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) - prompt_msgs: list[dict[str, str]] = [] - if system_prompt: - prompt_msgs.append({"role": "system", "content": system_prompt}) - prompt_msgs.extend( - [ - { - "role": getattr(m, "type", type(m).__name__), - "content": m.content - if isinstance(m.content, str) - else str(m.content), - } - for m in initial_messages - ] - ) - set_openllmetry_prompt(span, prompt_msgs) + _record_prompt(span, system_prompt, initial_messages) try: base_model = llm @@ -319,35 +348,11 @@ async def _stream_gen( else: span = None - system_prompt = _extract_system_prompt(config, variables, history) - initial_messages = _build_initial_messages(config, user_input, variables) + system_prompt = _extract_system_prompt(config, variables) + initial_messages = _build_initial_messages(config, user_input, variables, history) if span: - prompt_text = "\n".join( - [ - *(["system: " + system_prompt] if system_prompt else []), - *[ - f"{getattr(m, 'type', type(m).__name__)}: {m.content}" - for m in initial_messages - ], - ] - ) - span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) - prompt_msgs: list[dict[str, str]] = [] - if system_prompt: - prompt_msgs.append({"role": "system", "content": system_prompt}) - prompt_msgs.extend( - [ - { - "role": getattr(m, "type", type(m).__name__), - "content": m.content - if isinstance(m.content, str) - else str(m.content), - } - for m in initial_messages - ] - ) - set_openllmetry_prompt(span, prompt_msgs) + _record_prompt(span, system_prompt, initial_messages) try: base_model = llm diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/messages.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/messages.py new file mode 100644 index 0000000..f305df1 --- /dev/null +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/messages.py @@ -0,0 +1,54 @@ +"""Maps LaunchDarkly-canonical conversation turns onto LangChain messages. + +Images travel as ``image_url`` content parts with a data or remote URL — the +standard multimodal shape every LangChain chat model accepts — rather than the +LaunchDarkly-canonical ``{"type": "image", "source": ...}`` block, which no +LangChain provider understands (TESTING.md Appendix A.7). +""" + +from __future__ import annotations + +from typing import Any + +from launchdarkly_ai_server import content_to_text, image_block_to_url + + +def to_content_parts(content: str | list[dict[str, Any]]) -> list[dict[str, Any]]: + """Maps one canonical message's content into LangChain user content parts.""" + if isinstance(content, str): + return [{"type": "text", "text": content}] + parts: list[dict[str, Any]] = [] + for block in content: + if block.get("type") == "text": + parts.append({"type": "text", "text": block.get("text", "")}) + else: + parts.append( + {"type": "image_url", "image_url": {"url": image_block_to_url(block)}} + ) + return parts + + +def to_lang_chain_messages(turns: list[dict[str, Any]]) -> list[Any]: + """Turns composed canonical turns into LangChain messages. + + A string user turn stays a string-content ``HumanMessage``, so text-only + callers see exactly the message they saw before history existed. Assistant + turns are flattened to text: an ``AIMessage`` carries the model's own prior + reply, which has no image to preserve. + """ + import importlib + + msgs_mod = importlib.import_module("langchain_core.messages") + HumanMessage = msgs_mod.HumanMessage + AIMessage = msgs_mod.AIMessage + + messages: list[Any] = [] + for turn in turns: + content = turn.get("content") or "" + if turn.get("role") == "assistant": + messages.append(AIMessage(content_to_text(content))) + elif isinstance(content, str): + messages.append(HumanMessage(content)) + else: + messages.append(HumanMessage(to_content_parts(content))) + return messages diff --git a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py index d04524c..91ee6ca 100644 --- a/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py +++ b/packages/langchain-agents/src/launchdarkly_ai_langchain_agents/native_graph.py @@ -15,12 +15,15 @@ GraphDefinition, GraphNode, NativeTool, + compose_history, get_client, make_track_data, parse_template, to_ld_context, ) +from .messages import to_lang_chain_messages + try: from opentelemetry import trace from opentelemetry.trace import StatusCode as SpanStatusCode @@ -129,6 +132,7 @@ def to_lang_graph( async def invoke( input_text: str = "", variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: import importlib @@ -341,8 +345,19 @@ async def _pre_visit(node_key: str) -> None: compiled = builder.compile() + # History is a root-only concern: it seeds the initial message state the + # entry node reads. Downstream nodes are reached through handoffs and see + # the accumulated graph state, never the original `history` array. + initial_messages = ( + to_lang_chain_messages( + compose_history(history=history, user_input=input_text) + ) + if history + else [HumanMessage(input_text)] + ) + try: - result = await compiled.ainvoke({"messages": [HumanMessage(input_text)]}) + result = await compiled.ainvoke({"messages": initial_messages}) if span: span.set_status(SpanStatusCode.OK) except Exception as exc: diff --git a/packages/langchain-agents/tests/test_handler.py b/packages/langchain-agents/tests/test_handler.py index fe35713..650a174 100644 --- a/packages/langchain-agents/tests/test_handler.py +++ b/packages/langchain-agents/tests/test_handler.py @@ -1101,19 +1101,12 @@ class TestHistory: {"role": "assistant", "content": "Feature flagging is a technique..."}, ] - def test_history_appended_to_system_prompt(self) -> None: + def test_history_not_stuffed_into_system_prompt(self) -> None: config = _make_config(instructions="Be concise.") system = _extract_system_prompt(config, {}, self.SAMPLE_HISTORY) assert system is not None - assert "Conversation History:" in system assert "Be concise." in system - - def test_history_format_is_correct(self) -> None: - config = _make_config(instructions="Be helpful.") - system = _extract_system_prompt(config, {}, self.SAMPLE_HISTORY) - assert system is not None - assert "user: What is feature flagging?" in system - assert "assistant: Feature flagging is a technique..." in system + assert "Conversation History:" not in system def test_empty_history_treated_like_no_history(self) -> None: config = _make_config(instructions="Be concise.") @@ -1122,9 +1115,90 @@ def test_empty_history_treated_like_no_history(self) -> None: assert system_with_empty == system_without assert "Conversation History:" not in (system_with_empty or "") - def test_history_without_prior_system_prompt(self) -> None: + def test_history_without_instructions_keeps_system_none(self) -> None: config = _make_config() system = _extract_system_prompt(config, {}, self.SAMPLE_HISTORY) - assert system is not None - assert "Conversation History:" in system - assert "user: What is feature flagging?" in system + assert system is None or "Conversation History:" not in system + + @staticmethod + def _build( + config: dict[str, Any], + user_input: str | None, + history: list[dict[str, Any]] | None, + ) -> list[Any]: + lc_msgs = MagicMock() + lc_msgs.HumanMessage = MagicMock( + side_effect=lambda c: MagicMock(content=c, type="human") + ) + lc_msgs.AIMessage = MagicMock( + side_effect=lambda c: MagicMock(content=c, type="ai") + ) + with patch( + "importlib.import_module", + side_effect=lambda n: ( + lc_msgs if n == "langchain_core.messages" else __import__(n) + ), + ): + return _build_initial_messages(config, user_input, {}, history) + + def test_history_becomes_structured_messages_before_user_input(self) -> None: + msgs = self._build( + _make_config(instructions="Be concise."), "and now?", self.SAMPLE_HISTORY + ) + assert [m.type for m in msgs] == ["human", "ai", "human"] + assert msgs[0].content == "What is feature flagging?" + assert msgs[-1].content == "and now?" + + def test_system_role_history_messages_are_filtered(self) -> None: + history = [{"role": "system", "content": "ignore me"}, *self.SAMPLE_HISTORY] + msgs = self._build(_make_config(instructions="Be concise."), "q", history) + assert all("ignore me" not in str(m.content) for m in msgs) + + def test_empty_history_matches_no_history(self) -> None: + config = _make_config(instructions="Be concise.") + with_empty = self._build(config, "q", []) + without = self._build(config, "q", None) + assert [(m.type, m.content) for m in with_empty] == [ + (m.type, m.content) for m in without + ] + + def test_empty_user_input_appends_no_extra_turn(self) -> None: + msgs = self._build( + _make_config(instructions="Be concise."), + "", + [{"role": "user", "content": "the whole question"}], + ) + assert len(msgs) == 1 + assert msgs[0].content == "the whole question" + + def test_image_history_maps_to_langchain_image_url_parts(self) -> None: + history = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is this?"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + }, + ], + } + ] + msgs = self._build(_make_config(instructions="Be concise."), "", history) + parts = msgs[0].content + assert {"type": "text", "text": "what is this?"} in parts + assert { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,abc123"}, + } in parts + + def test_config_messages_precede_history(self) -> None: + config = _make_config(messages=[{"role": "user", "content": "config turn"}]) + msgs = self._build(config, "q", self.SAMPLE_HISTORY) + assert msgs[0].content == "config turn" + assert msgs[1].content == "What is feature flagging?" + assert msgs[-1].content == "q" diff --git a/packages/langchain-agents/tests/test_native_graph.py b/packages/langchain-agents/tests/test_native_graph.py index 39f6b47..6631d3e 100644 --- a/packages/langchain-agents/tests/test_native_graph.py +++ b/packages/langchain-agents/tests/test_native_graph.py @@ -5,6 +5,7 @@ from __future__ import annotations +import json import sys from contextlib import contextmanager from typing import Any @@ -188,6 +189,38 @@ def compile(self) -> Any: } +def _capture_compiled_state(mocks: dict[str, Any], ai_msg: Any) -> list[dict[str, Any]]: + """Swaps in a StateGraph whose compiled graph records the state it is invoked + with, so a test can assert on the root's initial messages.""" + states: list[dict[str, Any]] = [] + + class _CapturingStateGraph: + def __init__(self, *a: Any, **kw: Any) -> None: + pass + + def add_node(self, *a: Any, **kw: Any) -> None: + pass + + def add_edge(self, *a: Any, **kw: Any) -> None: + pass + + def add_conditional_edges(self, *a: Any, **kw: Any) -> None: + pass + + def compile(self) -> Any: + compiled = MagicMock() + + async def _ainvoke(state: dict[str, Any]) -> Any: + states.append(state) + return {"messages": [ai_msg]} + + compiled.ainvoke = _ainvoke + return compiled + + mocks["langgraph.graph"].StateGraph = _CapturingStateGraph + return states + + @contextmanager def _patch_imports(mocks: dict[str, Any]) -> Any: """Patch sys.modules so importlib.import_module picks up our mocks.""" @@ -692,6 +725,54 @@ async def _capture_invoke(msgs: list[Any]) -> Any: assert system_msgs, "No SystemMessage found" assert "expert" in system_msgs[0].content + @pytest.mark.asyncio + async def test_no_history_seeds_root_with_plain_human_message(self) -> None: + ai_msg = _make_ai_msg() + mocks = _make_langgraph_mocks(ai_msg) + states = _capture_compiled_state(mocks, ai_msg) + graph_def = _make_graph_def() + + with _patch_imports(mocks): + await to_lang_graph(_make_def_promise(graph_def)).invoke("hi") + + messages = states[0]["messages"] + assert len(messages) == 1 + assert messages[0].content == "hi" + + @pytest.mark.asyncio + async def test_history_seeds_root_with_native_image_content(self) -> None: + """A multimodal history reaches the root as LangChain image content.""" + ai_msg = _make_ai_msg() + mocks = _make_langgraph_mocks(ai_msg) + states = _capture_compiled_state(mocks, ai_msg) + graph_def = _make_graph_def() + + history = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + + with _patch_imports(mocks): + await to_lang_graph(_make_def_promise(graph_def)).invoke( + "describe", {}, history + ) + + serialized = json.dumps([m.content for m in states[0]["messages"]]) + assert "image_url" in serialized or '"type": "image"' in serialized + assert "abc123" in serialized + assert "describe" in serialized + @pytest.mark.asyncio async def test_config_tools_creates_tool_node(self) -> None: """When config.tools is non-empty, a ToolNode must be created and wired.""" diff --git a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py index 32fce47..c47fcff 100644 --- a/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py +++ b/packages/langchain-messages/src/launchdarkly_ai_langchain_messages/handler.py @@ -9,8 +9,11 @@ AiConfigRep, LDContext, ProviderHandler, + compose_history, config, create_handler, + image_block_to_url, + is_content_blocks, parse_template, set_ld_span_attributes, set_openllmetry_completion, @@ -40,6 +43,23 @@ def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: ] +def _to_langchain_content(content: Any) -> Any: + """Maps LD-canonical content blocks to LangChain multimodal content parts. + String content passes through so text-only callers keep plain strings.""" + if not is_content_blocks(content): + return content if content is not None else "" + + parts: list[dict[str, Any]] = [] + for block in content: + if block.get("type") == "text": + parts.append({"type": "text", "text": block.get("text", "")}) + elif block.get("type") == "image": + parts.append( + {"type": "image_url", "image_url": {"url": image_block_to_url(block)}} + ) + return parts + + def _build_messages( config: AiConfigRep, user_input: str, @@ -55,7 +75,7 @@ def _build_messages( AIMessage = msgs_mod.AIMessage messages: list[Any] = [] - last_role: str | None = None + config_messages: list[dict[str, Any]] = [] if config.get("messages"): system_msgs = [m for m in config["messages"] if m.get("role") == "system"] @@ -69,28 +89,40 @@ def _build_messages( ) ) for msg in conv_msgs: - content = parse_template(msg["content"], variables) - if msg["role"] == "user": - messages.append(HumanMessage(content)) - elif msg["role"] == "assistant": - messages.append(AIMessage(content)) - last_role = msg["role"] + content = msg.get("content", "") + if isinstance(content, str): + content = parse_template(content, variables) + if msg.get("role") in ("user", "assistant"): + config_messages.append({"role": msg["role"], "content": content}) elif config.get("instructions"): messages.append( SystemMessage(parse_template(config["instructions"], variables)) ) - if history: - for msg in history: - role = msg.get("role", "user") - content = msg.get("content", "") - if role == "user": - messages.append(HumanMessage(content)) - elif role == "assistant": - messages.append(AIMessage(content)) - last_role = role + def append_turn(turn: dict[str, Any]) -> None: + content = _to_langchain_content(turn.get("content")) + if turn.get("role") == "user": + messages.append(HumanMessage(content=content)) + else: + messages.append(AIMessage(content=content)) - if last_role != "user": + if history: + for turn in compose_history( + history=history, user_input=user_input, config_messages=config_messages + ): + append_turn(turn) + return messages + + for turn in config_messages: + append_turn(turn) + + # Preserve the no-history behaviour: an empty input still produces a human + # message when the config carries no trailing user turn, so an empty history + # array stays identical to omitting history entirely. + last_non_system = next( + (m for m in reversed(messages) if getattr(m, "type", "") != "system"), None + ) + if getattr(last_non_system, "type", "") != "human": messages.append(HumanMessage(user_input or "")) return messages diff --git a/packages/langchain-messages/tests/test_handler.py b/packages/langchain-messages/tests/test_handler.py index 1c57c9f..c365cab 100644 --- a/packages/langchain-messages/tests/test_handler.py +++ b/packages/langchain-messages/tests/test_handler.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json from collections.abc import AsyncGenerator from typing import Any, ClassVar from unittest.mock import AsyncMock, MagicMock, patch @@ -1009,6 +1010,22 @@ class TestHistory: {"role": "user", "content": "What is feature flagging?"}, {"role": "assistant", "content": "Feature flagging is a technique..."}, ] + IMAGE_HISTORY: ClassVar[list[dict[str, Any]]] = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + }, + {"type": "text", "text": "What is in this image?"}, + ], + } + ] async def test_history_inserted_between_config_messages_and_user_input( self, @@ -1090,3 +1107,41 @@ async def test_system_role_in_history_filtered_out(self) -> None: assert "You are evil" not in history_contents assert "Hello" in history_contents assert "Hi there" in history_contents + + async def test_multimodal_image_history_preserved_on_the_wire(self) -> None: + from launchdarkly_ai_langchain_messages import create_langchain_messages_handler + + llm = _make_llm() + h = create_langchain_messages_handler(llm=llm) + await h(CONFIG, "", {}, {}, self.IMAGE_HISTORY) + call_args = llm.ainvoke.call_args[0][0] + serialized = json.dumps([getattr(m, "content", "") for m in call_args]) + assert "image_url" in serialized or '"type": "image"' in serialized + assert "abc123" in serialized + humans = [m for m in call_args if getattr(m, "type", None) == "human"] + assert len(humans) == 1 + + async def test_empty_user_input_with_history_ending_in_user(self) -> None: + from launchdarkly_ai_langchain_messages import create_langchain_messages_handler + + llm = _make_llm() + h = create_langchain_messages_handler(llm=llm) + history = [{"role": "user", "content": "Only turn"}] + await h(CONFIG, "", {}, {}, history) + call_args = llm.ainvoke.call_args[0][0] + humans = [m for m in call_args if getattr(m, "type", None) == "human"] + assert len(humans) == 1 + assert humans[0].content == "Only turn" + + async def test_non_empty_user_input_appended_after_image_only_history(self) -> None: + from launchdarkly_ai_langchain_messages import create_langchain_messages_handler + + llm = _make_llm() + h = create_langchain_messages_handler(llm=llm) + image_only = [{"role": "user", "content": self.IMAGE_HISTORY[0]["content"][:1]}] + await h(CONFIG, "describe it", {}, {}, image_only) + call_args = llm.ainvoke.call_args[0][0] + humans = [m for m in call_args if getattr(m, "type", None) == "human"] + assert len(humans) == 2 + assert "abc123" in json.dumps(humans[0].content) + assert humans[1].content == "describe it" diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py index 0c6d851..ce47a39 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/handler.py @@ -13,8 +13,11 @@ AiConfigRep, LDContext, ProviderHandler, + compose_history, config, + content_to_text, create_handler, + image_block_to_url, parse_template, set_ld_span_attributes, set_openllmetry_completion, @@ -64,15 +67,40 @@ async def _execute(_ctx: Any, args_str: str, _name: str = name) -> str: return result -def _format_history(history: list[dict[str, Any]] | None) -> str | None: - if not history: - return None - lines = [] - for msg in history: - role = msg.get("role", "user") - content = msg.get("content", "") - lines.append(f"{role}: {content}") - return "Conversation History:\n\n" + "\n".join(lines) +def _parse_message_content(content: Any, variables: dict[str, Any]) -> Any: + """Apply templates to text content while preserving structured blocks.""" + return parse_template(content, variables) if isinstance(content, str) else content + + +def _to_openai_agent_items(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Map LaunchDarkly canonical turns to OpenAI Agents input items.""" + items: list[dict[str, Any]] = [] + for turn in turns: + role = turn["role"] + content = turn["content"] + if role == "assistant": + items.append({"role": "assistant", "content": content_to_text(content)}) + continue + + blocks = ( + content + if isinstance(content, list) + else [{"type": "text", "text": content}] + ) + parts: list[dict[str, Any]] = [] + for block in blocks: + if block.get("type") == "image": + parts.append( + {"type": "input_image", "image_url": image_block_to_url(block)} + ) + elif block.get("type") == "text": + parts.append({"type": "input_text", "text": block.get("text", "")}) + items.append({"role": "user", "content": parts}) + return items + + +def _prompt_to_text(prompt: str | list[dict[str, Any]]) -> str: + return prompt if isinstance(prompt, str) else json.dumps(prompt) def _build_agent_and_prompt( @@ -81,7 +109,7 @@ def _build_agent_and_prompt( tool_handlers: dict[str, Any], variables: dict[str, Any], history: list[dict[str, Any]] | None = None, -) -> tuple[Any, str, str | None]: +) -> tuple[Any, str | list[dict[str, Any]], str | None]: import importlib agents_mod = importlib.import_module("agents") @@ -89,27 +117,46 @@ def _build_agent_and_prompt( safe_input = user_input or "" instructions: str | None = None - prompt = safe_input + prompt: str | list[dict[str, Any]] = safe_input + + config_messages = config.get("messages") or [] + parsed_messages = [ + { + **message, + "content": _parse_message_content(message.get("content", ""), variables), + } + for message in config_messages + ] if config.get("instructions"): instructions = parse_template(config["instructions"], variables) - elif config.get("messages"): - system_msgs = [m for m in config["messages"] if m.get("role") == "system"] - conv_msgs = [m for m in config["messages"] if m.get("role") != "system"] + elif parsed_messages: + system_msgs = [m for m in parsed_messages if m.get("role") == "system"] + conv_msgs = [m for m in parsed_messages if m.get("role") != "system"] if system_msgs: - instructions = parse_template( - "\n".join(m["content"] for m in system_msgs), variables - ) - conv_history = "\n".join( - parse_template(m["content"], variables) for m in conv_msgs - ) + instructions = "\n".join(content_to_text(m["content"]) for m in system_msgs) + conv_history = "\n".join(content_to_text(m["content"]) for m in conv_msgs) prompt = f"{conv_history}\n\n{safe_input}" if conv_history else safe_input - history_text = _format_history(history) - if history_text: - instructions = ( - f"{instructions}\n\n{history_text}" if instructions else history_text + if history: + # When config.instructions is set, config.messages conversation turns are + # ignored (see the no-history branches above), so history composition must + # not resurrect them — mirror that priority here. + config_history_messages = ( + [] + if config.get("instructions") + else [ + message + for message in parsed_messages + if message.get("role") != "system" + ] + ) + turns = compose_history( + history=history, + user_input=user_input, + config_messages=config_history_messages, ) + prompt = _to_openai_agent_items(turns) tools = _build_agent_tools(config.get("tools") or {}, tool_handlers) @@ -176,14 +223,15 @@ async def _call_impl( ) if span: + serialized_prompt = _prompt_to_text(prompt) prompt_text = ( f"system: {instructions}\n\n" if instructions else "" - ) + prompt + ) + serialized_prompt span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) prompt_msgs: list[dict[str, str]] = [] if instructions: prompt_msgs.append({"role": "system", "content": instructions}) - prompt_msgs.append({"role": "user", "content": prompt}) + prompt_msgs.append({"role": "user", "content": serialized_prompt}) set_openllmetry_prompt(span, prompt_msgs) try: @@ -274,12 +322,15 @@ async def _stream_gen( ) if span: - prompt_text = (f"system: {instructions}\n\n" if instructions else "") + prompt + serialized_prompt = _prompt_to_text(prompt) + prompt_text = ( + f"system: {instructions}\n\n" if instructions else "" + ) + serialized_prompt span.add_event("gen_ai.content.prompt", {"gen_ai.prompt": prompt_text}) prompt_msgs: list[dict[str, str]] = [] if instructions: prompt_msgs.append({"role": "system", "content": instructions}) - prompt_msgs.append({"role": "user", "content": prompt}) + prompt_msgs.append({"role": "user", "content": serialized_prompt}) set_openllmetry_prompt(span, prompt_msgs) try: diff --git a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py index f66b1c1..88aef9c 100644 --- a/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py +++ b/packages/openai-agents/src/launchdarkly_ai_openai_agents/native_graph.py @@ -15,12 +15,15 @@ GraphDefinition, GraphNode, NativeTool, + compose_history, get_client, make_track_data, parse_template, to_ld_context, ) +from .handler import _parse_message_content, _to_openai_agent_items + try: from opentelemetry import trace from opentelemetry.trace import StatusCode as SpanStatusCode @@ -99,6 +102,7 @@ def to_openai_agents( async def invoke( input_text: str = "", variables: dict[str, Any] | None = None, + history: list[dict[str, Any]] | None = None, ) -> dict[str, Any]: import importlib @@ -216,8 +220,34 @@ async def on_agent_start(self, context: Any, agent: Any) -> None: hooks = _LDHooks() + root_prompt: str | list[dict[str, Any]] = input_text + if history: + # config.instructions takes priority over config.messages, so skip + # config conversation turns when instructions are set (parity with the + # single-node handler and TESTING.md §1.11 composition order). + config_messages = ( + [] + if root.config.get("instructions") + else [ + { + **message, + "content": _parse_message_content( + message.get("content", ""), vs + ), + } + for message in (root.config.get("messages") or []) + if message.get("role") != "system" + ] + ) + turns = compose_history( + history=history, + user_input=input_text, + config_messages=config_messages, + ) + root_prompt = _to_openai_agent_items(turns) + try: - result = await Runner.run(root_agent, input_text, hooks=hooks) + result = await Runner.run(root_agent, root_prompt, hooks=hooks) if span: span.set_status(SpanStatusCode.OK) except Exception as exc: diff --git a/packages/openai-agents/tests/test_handler.py b/packages/openai-agents/tests/test_handler.py index a629483..af14bb4 100644 --- a/packages/openai-agents/tests/test_handler.py +++ b/packages/openai-agents/tests/test_handler.py @@ -1050,6 +1050,21 @@ class TestHistory: {"role": "user", "content": "What is feature flagging?"}, {"role": "assistant", "content": "Feature flagging is a technique..."}, ] + IMAGE_HISTORY: ClassVar[list[dict[str, Any]]] = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] def _mock_agents(self) -> Any: mock = MagicMock() @@ -1057,33 +1072,39 @@ def _mock_agents(self) -> Any: mock.tool = MagicMock(side_effect=lambda **kw: lambda fn: fn) return mock - def test_history_appended_to_instructions(self) -> None: + def test_history_is_structured_input_not_system_prompt_text(self) -> None: config = _make_config(instructions="Be concise.") agents_mock = self._mock_agents() with patch( "importlib.import_module", side_effect=lambda n: agents_mock if n == "agents" else __import__(n), ): - _, _, instructions = _build_agent_and_prompt( + _, prompt, instructions = _build_agent_and_prompt( config, "hi", {}, {}, self.SAMPLE_HISTORY ) assert instructions is not None - assert "Conversation History:" in instructions assert "Be concise." in instructions + assert "Conversation History:" not in instructions + assert isinstance(prompt, list) + serialized = str(prompt) + assert "What is feature flagging?" in serialized + assert "hi" in serialized - def test_history_format_is_correct(self) -> None: + def test_history_turns_appear_before_user_input(self) -> None: config = _make_config(instructions="Be helpful.") agents_mock = self._mock_agents() with patch( "importlib.import_module", side_effect=lambda n: agents_mock if n == "agents" else __import__(n), ): - _, _, instructions = _build_agent_and_prompt( - config, "hi", {}, {}, self.SAMPLE_HISTORY + _, prompt, _ = _build_agent_and_prompt( + config, "follow up", {}, {}, self.SAMPLE_HISTORY ) - assert instructions is not None - assert "user: What is feature flagging?" in instructions - assert "assistant: Feature flagging is a technique..." in instructions + assert isinstance(prompt, list) + serialized = str(prompt) + assert serialized.index("What is feature flagging?") < serialized.rindex( + "follow up" + ) def test_empty_history_treated_like_no_history(self) -> None: config = _make_config(instructions="Be concise.") @@ -1092,21 +1113,48 @@ def test_empty_history_treated_like_no_history(self) -> None: "importlib.import_module", side_effect=lambda n: agents_mock if n == "agents" else __import__(n), ): - _, _, instr_with_empty = _build_agent_and_prompt(config, "hi", {}, {}, []) - _, _, instr_without = _build_agent_and_prompt(config, "hi", {}, {}) + _, prompt_with_empty, instr_with_empty = _build_agent_and_prompt( + config, "hi", {}, {}, [] + ) + _, prompt_without, instr_without = _build_agent_and_prompt( + config, "hi", {}, {} + ) assert instr_with_empty == instr_without assert "Conversation History:" not in (instr_with_empty or "") + assert prompt_with_empty == prompt_without - def test_history_without_prior_instructions(self) -> None: - config = _make_config() + def test_multimodal_image_history_maps_to_input_image(self) -> None: + config = _make_config(instructions="Be helpful.") agents_mock = self._mock_agents() with patch( "importlib.import_module", side_effect=lambda n: agents_mock if n == "agents" else __import__(n), ): - _, _, instructions = _build_agent_and_prompt( - config, "hi", {}, {}, self.SAMPLE_HISTORY + _, prompt, instructions = _build_agent_and_prompt( + config, "describe", {}, {}, self.IMAGE_HISTORY ) assert instructions is not None - assert "Conversation History:" in instructions - assert "user: What is feature flagging?" in instructions + assert "Conversation History:" not in instructions + assert isinstance(prompt, list) + serialized = str(prompt) + assert "input_image" in serialized + assert "data:image/png;base64,abc123" in serialized + + def test_history_with_instructions_ignores_config_messages(self) -> None: + # config.instructions takes priority over config.messages, so history + # composition must not resurrect those conversation turns. + config = _make_config( + instructions="Use instructions.", + messages=[{"role": "user", "content": "config-only turn"}], + ) + agents_mock = self._mock_agents() + with patch( + "importlib.import_module", + side_effect=lambda n: agents_mock if n == "agents" else __import__(n), + ): + _, prompt, instructions = _build_agent_and_prompt( + config, "hi", {}, {}, self.SAMPLE_HISTORY + ) + assert instructions == "Use instructions." + assert isinstance(prompt, list) + assert "config-only turn" not in str(prompt) diff --git a/packages/openai-agents/tests/test_native_graph.py b/packages/openai-agents/tests/test_native_graph.py index a30dbb3..d596a42 100644 --- a/packages/openai-agents/tests/test_native_graph.py +++ b/packages/openai-agents/tests/test_native_graph.py @@ -186,6 +186,41 @@ async def test_root_node_is_entry_point(self) -> None: call_args = agents_mock.Runner.run.call_args assert "input-text" in call_args[0] or "input-text" == call_args[0][1] + @pytest.mark.asyncio + async def test_multimodal_history_is_structured_root_input(self) -> None: + run_result = _make_run_result("out") + agents_mock = _make_agents_mock(run_result) + graph_def = _make_graph_def() + history = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + + with patch( + "importlib.import_module", + side_effect=lambda n: agents_mock if n == "agents" else __import__(n), + ): + await to_openai_agents(_make_def_promise(graph_def)).invoke( + "describe", None, history + ) + + root_input = agents_mock.Runner.run.call_args.args[1] + assert isinstance(root_input, list) + serialized = str(root_input) + assert "input_image" in serialized + assert "data:image/png;base64,abc123" in serialized + @pytest.mark.asyncio async def test_terminal_nodes_no_handoff_tools(self) -> None: run_result = _make_run_result("out") diff --git a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py index 9144e26..4b7b50f 100644 --- a/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py +++ b/packages/openai-messages/src/launchdarkly_ai_openai_messages/handler.py @@ -9,8 +9,12 @@ AiConfigRep, LDContext, ProviderHandler, + compose_history, config, + content_to_text, create_handler, + image_block_to_url, + is_content_blocks, parse_template, set_ld_span_attributes, set_openllmetry_completion, @@ -41,34 +45,88 @@ def _build_tools(config_tools: dict[str, Any]) -> list[dict[str, Any]]: def _build_input_messages( config: AiConfigRep, - user_input: str, + user_input: str | None, variables: dict[str, Any], history: list[dict[str, Any]] | None = None, ) -> list[dict[str, Any]]: + system_messages: list[dict[str, Any]] = [] + config_messages: list[dict[str, Any]] = [] + if config.get("messages"): - msgs = [ - {"role": m["role"], "content": parse_template(m["content"], variables)} - for m in config["messages"] - ] - if history: - for msg in history: - role = msg.get("role", "user") - if role in ("user", "assistant"): - msgs.append({"role": role, "content": msg.get("content", "")}) - if user_input and (not msgs or msgs[-1].get("role") != "user"): - msgs.append({"role": "user", "content": user_input}) - return msgs - instructions = parse_template(config.get("instructions") or "", variables) - result: list[dict[str, Any]] = [] - if instructions: - result.append({"role": "system", "content": instructions}) + for message in config["messages"]: + content = message.get("content", "") + if isinstance(content, str): + content = parse_template(content, variables) + mapped = {"role": message["role"], "content": content} + if message["role"] == "system": + system_messages.append(mapped) + else: + config_messages.append(mapped) + else: + instructions = parse_template(config.get("instructions") or "", variables) + if instructions: + system_messages.append({"role": "system", "content": instructions}) + if history: - for msg in history: - role = msg.get("role", "user") - if role in ("user", "assistant"): - result.append({"role": role, "content": msg.get("content", "")}) - result.append({"role": "user", "content": user_input or ""}) - return result + turns = compose_history( + history=history, + user_input=user_input, + config_messages=config_messages, + ) + else: + # No history: preserve the pre-history behaviour so an empty history is + # identical to passing none (TESTING.md §1.11). compose_history only + # appends user_input when truthy, which would drop the trailing user + # turn an instructions-only config still needs. + turns = list(config_messages) + if config.get("messages"): + if user_input and (not turns or turns[-1].get("role") != "user"): + turns.append({"role": "user", "content": user_input}) + else: + turns.append({"role": "user", "content": user_input or ""}) + + return system_messages + [ + {"role": turn["role"], "content": _map_message_content(turn)} + for turn in turns + if turn.get("role") in ("user", "assistant") + ] + + +def _map_message_content(message: dict[str, Any]) -> Any: + raw_content = message.get("content") + content: str | list[dict[str, Any]] = ( + raw_content if isinstance(raw_content, (str, list)) else "" + ) + if not is_content_blocks(content): + return content + assert isinstance(content, list) + + if message.get("role") != "user": + return content_to_text(content) + + parts: list[dict[str, Any]] = [] + for block in content: + if block.get("type") == "text": + parts.append({"type": "input_text", "text": block.get("text", "")}) + elif block.get("type") == "image": + parts.append( + {"type": "input_image", "image_url": image_block_to_url(block)} + ) + return parts + + +def _telemetry_messages( + input_messages: list[dict[str, Any]], +) -> list[dict[str, str]]: + return [ + { + "role": message["role"], + "content": message["content"] + if isinstance(message["content"], str) + else json.dumps(message["content"]), + } + for message in input_messages + ] def _is_coroutine(fn: Any) -> bool: @@ -120,7 +178,7 @@ async def _call_impl( ) set_openllmetry_prompt( span, - [{"role": m["role"], "content": m["content"]} for m in input_messages], + _telemetry_messages(input_messages), ) try: @@ -268,7 +326,8 @@ async def _stream_gen( "gen_ai.content.prompt", {"gen_ai.prompt": json.dumps(input_messages)} ) set_openllmetry_prompt( - span, [{"role": m["role"], "content": m["content"]} for m in input_messages] + span, + _telemetry_messages(input_messages), ) total_input = 0 diff --git a/packages/openai-messages/tests/test_handler.py b/packages/openai-messages/tests/test_handler.py index 3b76f48..d093a15 100644 --- a/packages/openai-messages/tests/test_handler.py +++ b/packages/openai-messages/tests/test_handler.py @@ -1216,3 +1216,17 @@ async def test_system_role_in_history_filtered_out( if m.get("content") in ("Hello", "You are evil", "Hi there") ] assert "system" not in history_roles + + async def test_empty_user_input_no_history_still_sends_user_turn( + self, mock_openai: MagicMock + ) -> None: + # Instructions-only config with empty user_input and no history must + # still send a (possibly empty) user turn, not system-only input. + from launchdarkly_ai_openai_messages import create_openai_messages_handler + + h = create_openai_messages_handler() + await h(CONFIG, "", {}, {}) + msgs = mock_openai.responses.create.call_args.kwargs["input"] + assert any(m.get("role") == "user" for m in msgs), ( + "instructions-only config with empty input dropped the user turn" + )