diff --git a/agent_core/core/impl/context/engine.py b/agent_core/core/impl/context/engine.py index 94229769..86b85609 100644 --- a/agent_core/core/impl/context/engine.py +++ b/agent_core/core/impl/context/engine.py @@ -457,8 +457,10 @@ def get_session_state(self, session_id: Optional[str] = None) -> str: f"Session ID: {session.id}", f"Session Type: {session.type}", ] - if session.title: - lines.append(f"Session Title: {session.title}") + # Session Title is intentionally omitted: it is auto-generated/ + # updated a turn or two into a session, and this block sits in the + # cacheable prefix (ahead of the event stream), so a mutating title + # would break the KV-cache prefix every time it changed. if getattr(session, "living_ui_project_id", None): lines.append(f"Living UI Project: {session.living_ui_project_id}") lines.append(f"Loaded Action Sets: {['core'] + list(session.action_sets)}") diff --git a/agent_core/core/impl/llm/interface.py b/agent_core/core/impl/llm/interface.py index 43a89489..1fb8b3a0 100644 --- a/agent_core/core/impl/llm/interface.py +++ b/agent_core/core/impl/llm/interface.py @@ -15,19 +15,15 @@ import asyncio import contextvars -import hashlib import re import time -import requests from typing import Any, Dict, List, Optional from agent_core.decorators import profile, OperationCategory from agent_core.core.impl.llm.cache import ( BytePlusCacheManager, - BytePlusContextOverflowError, GeminiCacheManager, - get_cache_config, get_cache_metrics, ) from agent_core.core.errors import ErrorCategory, FAIL_FAST_CATEGORIES @@ -35,7 +31,6 @@ LLMConsecutiveFailureError, LLMErrorInfo, classify_llm_error, - provider_display_name, ) from agent_core.core.hooks import ( GetTokenCountHook, @@ -46,6 +41,11 @@ LLMCallRecord, RecordLLMCallHook, ) +from agent_core.core.impl.llm import transports as _transports +from agent_core.core.models.registry import ( + get_registry as _get_registry, + session_cc_providers as _session_cc_providers, +) # Logging setup - use shared agent_core logger for consistency from agent_core.utils.logger import logger @@ -126,31 +126,6 @@ def _generic_empty_response_detail(provider: str, model: str) -> str: ) -def _byteplus_blocked_reason(result: Dict[str, Any]) -> Optional[str]: - """Best-effort detection of content-filter/moderation blocking in a - BytePlus Responses API result that came back with empty content but no - HTTP-level error (status 200, `choices`/`output` just empty). - - Mirrors OpenAI's Responses API `status` / `incomplete_details.reason` - shape, which BytePlus's docs describe this endpoint as following — not - independently verified against a live blocked response, so this only - fires on an unambiguous signal and otherwise returns None, leaving the - existing generic empty-response handling untouched. - """ - status = result.get("status") - if status == "incomplete": - reason = (result.get("incomplete_details") or {}).get("reason") - if reason: - return str(reason) - error = result.get("error") - if isinstance(error, dict): - code = str(error.get("code") or "").lower() - message = str(error.get("message") or "") - if any(k in code for k in ("content_filter", "moderation", "safety")): - return message or code - return None - - class LLMInterface: """LLM interface with multi-provider support and hook-based customization. @@ -187,6 +162,7 @@ def __init__( report_usage: Optional[ReportUsageHook] = None, log_to_db: Optional[LogToDbHook] = None, record_llm_call: Optional[RecordLLMCallHook] = None, + on_fallback: Optional[Any] = None, ) -> None: self.temperature = temperature self.max_tokens = max_tokens @@ -211,6 +187,28 @@ def __init__( self._consecutive_failures = 0 self._max_consecutive_failures = 5 + # Cross-provider fallback (Phase 5, FR-9). Turn-scoped: a fallback + # serves the current turn only; the next turn retries the primary. + # Fallback interfaces are lazily-built secondary LLMInterface + # instances with their OWN session buffers, so the primary's + # accumulated cache state is never disturbed (NFR-3). + self._fallback_interfaces: Dict[str, "LLMInterface"] = {} + # Set by reinitialize(): the first turn after an explicit provider + # selection runs strict (no fallback) so misconfiguration surfaces + # instead of being silently masked (OpenClaw's rule). + self._suppress_fallback_once = False + # (task_id, call_type) of the in-flight session call, stashed by the + # session dispatcher so _finalize_session_response can retry the + # same session turn on a fallback provider. + self._current_session_call: Optional[tuple] = None + self._on_fallback = on_fallback + # True on secondary interfaces built BY the fallback machinery. + # Enforces "at most one chain walk per turn" structurally: a + # fallback instance never consults the chain itself, so a + # multi-provider outage terminates instead of nesting + # primary -> fb -> fb-of-fb recursion. + self._is_fallback_instance = False + # Defer imports to avoid circular dependency from app.models.factory import ModelFactory from app.models.types import InterfaceType @@ -324,6 +322,12 @@ def reinitialize( target_provider = provider or self.provider + # Explicit selection is strict for one turn (Phase 5): the next + # generate call runs without fallback so a bad key/model surfaces. + # Also drop cached fallback interfaces — the chain may have changed. + self._suppress_fallback_once = True + self._fallback_interfaces = {} + # Read API key and base URL from settings.json if not provided if api_key is None or base_url is None: from app.config import get_api_key, get_base_url @@ -601,6 +605,15 @@ def _register_failure( path. """ category = error_info.category if error_info else ErrorCategory.UNKNOWN + # Credential-pool bookkeeping (Phase 5, FR-7): rate-limit/billing/ + # auth failures cool the credential that served this call so the + # next request rotates. No-op for single-key providers. + try: + from agent_core.core.models import credentials as _credentials + + _credentials.note_failure(self.provider, category.value) + except Exception: # pragma: no cover — pools must never break errors + pass if category in FAIL_FAST_CATEGORIES: logger.critical( f"[LLM ABORT] Non-transient category={category.value} — failing fast " @@ -623,6 +636,110 @@ def _register_failure( last_error_info=error_info, ) + # ─────────────── Cross-provider fallback (Phase 5, FR-9) ─────────────── + + def _fallback_chain(self) -> List[str]: + """Configured fallback providers, minus the active one, that have a + usable credential or need none. Empty when unconfigured (default). + + Always empty on fallback instances themselves — only the PRIMARY + interface walks the chain (one walk per turn, no nesting).""" + if self._is_fallback_instance: + return [] + try: + from app.config import get_api_key, get_fallback_providers + + chain = [] + registry = _get_registry() + for candidate in get_fallback_providers(): + if candidate == self.provider or candidate in chain: + continue + prof = registry.get(candidate) + if prof is None: + continue + if prof.requires_api_key and not get_api_key(candidate): + logger.debug( + f"[FALLBACK] skipping {candidate}: no credential configured" + ) + continue + chain.append(candidate) + return chain + except Exception: + return [] + + def _get_fallback_interface(self, provider: str) -> Optional["LLMInterface"]: + cached = self._fallback_interfaces.get(provider) + if cached is not None: + return cached + try: + from app.config import get_api_key, get_base_url + + iface = LLMInterface( + provider=provider, + api_key=get_api_key(provider) or None, + base_url=get_base_url(provider), + temperature=self.temperature, + max_tokens=self.max_tokens, + get_token_count=self._get_token_count, + set_token_count=self._set_token_count, + report_usage=self._report_usage, + log_to_db=self._log_to_db, + record_llm_call=self._record_llm_call, + ) + except Exception as e: + logger.warning(f"[FALLBACK] could not build {provider} interface: {e}") + return None + iface._is_fallback_instance = True + self._fallback_interfaces[provider] = iface + return iface + + def _notify_fallback(self, to_provider: str, reason: str) -> None: + message = ( + f"Model fallback: {self.provider} -> {to_provider} ({reason}); " + f"will retry {self.provider} next turn." + ) + logger.warning(f"[FALLBACK] {message}") + if self._on_fallback is not None: + try: + self._on_fallback(self.provider, to_provider, reason) + except Exception: # pragma: no cover — the hook must never break inference + pass + + def _try_fallback(self, response: Dict[str, Any], attempt) -> Optional[str]: + """Walk the fallback chain for this turn. ``attempt`` is a callable + (fallback_iface) -> content-or-raises. Returns served content, or + None when fallback is off / suppressed / exhausted / blocked. + + Never touches the primary's failure bookkeeping: on success the turn + is served (caller resets the counter); on None the caller proceeds + with today's exact failure path (NFR-1). + """ + if self._suppress_fallback_once: + self._suppress_fallback_once = False + return None + error_info = response.get("error_info_obj") + category = error_info.category if error_info is not None else None + if category is ErrorCategory.BLOCKED: + # The same content would be blocked on any provider. + return None + reason = category.value if category is not None else "error" + for candidate in self._fallback_chain(): + fb = self._get_fallback_interface(candidate) + if fb is None: + continue + # One attempt per candidate per turn: a broken fallback must not + # burn its own consecutive budget across turns. + fb.reset_failure_counter() + try: + content = attempt(fb) + except Exception as e: + logger.warning(f"[FALLBACK] {candidate} also failed: {e}") + continue + if content: + self._notify_fallback(candidate, reason) + return content + return None + def _generate_response_sync( self, system_prompt: Optional[str] = None, @@ -646,30 +763,20 @@ def _generate_response_sync( logger.info(f"[LLM SEND] system={system_prompt} | user={user_prompt}") try: - if self.provider in ( - "openai", - "minimax", - "deepseek", - "moonshot", - "grok", - "openrouter", - "glm", - "fugu", - ): - response = self._generate_openai(system_prompt, user_prompt) - elif self.provider == "remote": - response = self._generate_ollama(system_prompt, user_prompt) - elif self.provider == "gemini": - response = self._generate_gemini(system_prompt, user_prompt) - elif self.provider == "byteplus": - response = self._generate_byteplus(system_prompt, user_prompt) - elif self.provider == "anthropic": - response = self._generate_anthropic(system_prompt, user_prompt) - elif self.provider == "bedrock": - response = self._generate_bedrock(system_prompt, user_prompt) - else: # pragma: no cover + # Dispatch on the provider profile's wire protocol (Phase 2, + # docs/PROVIDER_LAYER_CATCHUP.md FR-2). Transports carry the + # request/response encoding; all session state stays here. The + # dynamic registry (not the static PROVIDER_CONFIG) is consulted + # so settings.json custom providers dispatch too (Phase 3). + _profile_cfg = _get_registry().get(self.provider) + _transport = ( + _transports.TRANSPORTS.get(_profile_cfg.wire) + if _profile_cfg is not None + else None + ) + if _transport is None: # pragma: no cover raise RuntimeError(f"Unknown provider {self.provider!r}") - + response = _transport(self, system_prompt, user_prompt) content = response.get("content", "").strip() # Check if response is empty and provide diagnostics @@ -688,6 +795,20 @@ def _generate_response_sync( self.provider, self.model ) logger.error(f"[LLM ERROR] {error_detail}") + # Turn-scoped cross-provider fallback (Phase 5, FR-9): try + # the configured chain BEFORE any failure bookkeeping. A + # served fallback turn is a success; an exhausted (or + # unconfigured) chain falls through to the exact historical + # failure path below. + served = self._try_fallback( + response, + lambda fb: fb._generate_response_sync( + system_prompt, user_prompt, log_response=False + ), + ) + if served is not None: + self._consecutive_failures = 0 + return served # Registers/raises based on category (fail-fast vs retry # budget) — see _register_failure. Attaches the classified # info so the agent_base error handler can show the *cause* @@ -707,6 +828,12 @@ def _generate_response_sync( # Success - reset consecutive failure counter self._consecutive_failures = 0 + try: + from agent_core.core.models import credentials as _credentials + + _credentials.note_success(self.provider) + except Exception: + pass cleaned = re.sub(self._CODE_BLOCK_RE, "", content) @@ -813,8 +940,7 @@ def create_session_cache( (self.provider == "byteplus" and self._byteplus_cache_manager) or (self.provider == "gemini" and self._gemini_cache_manager) or ( - self.provider - in ("openai", "deepseek", "grok", "openrouter", "glm", "fugu") + self.provider in _session_cc_providers() and self.client ) # OpenAI/DeepSeek/Grok/OpenRouter use automatic caching with prompt_cache_key (and cache_control for Anthropic-routed OpenRouter models) or ( @@ -934,9 +1060,20 @@ def _trim_openai_compat_history(self, history: List[dict]) -> None: MIDDLE pairs, so we never re-introduce the amnesia this fix exists to prevent. Uses a chars≈4*tokens heuristic. """ - # ~240k chars ≈ ~60k tokens: comfortably inside grok-3's 131k window - # after the system prompt, the newest turn, and the response. + # Fixed history budget (~240k chars ≈ 60k tokens), leaving room for the + # system prompt, newest turn, and response. Provider-independent by + # design: we keep no per-model context-window table (no hardcoded model + # list), so a single conservative constant governs trimming for every + # provider. A power user can raise it via model.context_window_override. max_history_chars = 240_000 + try: + from app.config import get_settings + + override = get_settings().get("model", {}).get("context_window_override") + if override: + max_history_chars = max(240_000, int(override) * 4) + except Exception: + pass def _size() -> int: return sum(len(m.get("content", "") or "") for m in history) @@ -1024,6 +1161,26 @@ def _finalize_session_response( else: error_detail = _generic_empty_response_detail(self.provider, self.model) logger.error(f"[LLM ERROR] {error_detail}") + # Session-path fallback (Phase 5): retry the SAME session turn + # on a fallback provider. The fallback interface keeps its own + # session buffers, so its history accumulates independently and + # the primary's buffers stay warm for the next-turn retry. + if self._current_session_call is not None: + task_id, call_type, fb_user_prompt = self._current_session_call + stored_system = self._session_system_prompts.get( + f"{task_id}:{call_type}" + ) + + def _session_attempt(fb, _t=task_id, _c=call_type): + fb.create_session_cache(_t, _c, stored_system or "") + return fb._generate_response_with_session_sync( + _t, _c, fb_user_prompt, log_response=False + ) + + served = self._try_fallback(response, _session_attempt) + if served is not None: + self._consecutive_failures = 0 + return served # See _generate_response_sync's equivalent call for why # raw_error is always passed, even when error_info is None. self._register_failure( @@ -1033,6 +1190,12 @@ def _finalize_session_response( # Success - reset consecutive failure counter self._consecutive_failures = 0 + try: + from agent_core.core.models import credentials as _credentials + + _credentials.note_success(self.provider) + except Exception: + pass cleaned = re.sub(self._CODE_BLOCK_RE, "", content) current_count = self._get_token_count() self._set_token_count(current_count + billable_tokens(response)) @@ -1071,6 +1234,10 @@ def _generate_response_with_session_sync( if user_prompt is None: raise ValueError("`user_prompt` cannot be None.") + # Stash the in-flight session call so _finalize_session_response can + # retry this same turn on a fallback provider (Phase 5, FR-9). + self._current_session_call = (task_id, call_type, user_prompt) + # Same consecutive-failure backstop as `_generate_response_sync`. The # session path previously had none, so a persistent provider error # (e.g. out-of-credits) retried forever instead of aborting. @@ -1135,8 +1302,10 @@ def _generate_response_with_session_sync( return self._finalize_session_response(response, log_response) - # Handle OpenAI/DeepSeek/Grok/OpenRouter with call_type-based cache routing - if self.provider in ("openai", "deepseek", "grok", "openrouter", "glm", "fugu"): + # Handle OpenAI/DeepSeek/Grok/OpenRouter with call_type-based cache routing. + # Membership is derived from the profiles (wire == chat_completions AND + # session_accumulation) — see registry.session_cc_providers(). + if self.provider in _session_cc_providers(): # Get stored system prompt or use provided one session_key = f"{task_id}:{call_type}" stored_system_prompt = self._session_system_prompts.get(session_key) @@ -1657,192 +1826,12 @@ async def generate_response_with_session_async( def _generate_byteplus_with_session( self, task_id: str, call_type: str, user_prompt: str ) -> Dict[str, Any]: - """Use Responses API with session caching for task/GUI calls. - - The context grows with each call as we chain responses via previous_response_id. - Each call type has its own session to avoid polluting different prompt structures. - - If context overflow is detected, the session is automatically reset and retried - with a fresh session containing only the system prompt and current user prompt. - """ - token_count_input = token_count_output = 0 - total_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - cached_tokens = 0 - session_key = f"{task_id}:{call_type}" - - try: - if not self._byteplus_cache_manager.has_session(task_id, call_type): - # The cache manager was rebuilt (e.g. a model-only Settings - # change recreates it since BytePlus sessions are server-side - # and model-bound), emptying its session registry — but the - # system prompt survives a model-only reinit, so reseed a - # fresh session instead of failing this turn outright. - system_prompt = self._session_system_prompts.get(session_key) - if not system_prompt: - raise ValueError(f"No session cache found for {session_key}") - - logger.info( - f"[BYTEPLUS] No session cache for {session_key} — " - f"reseeding a fresh session from the stored system prompt" - ) - result = self._byteplus_cache_manager.create_session_cache( - task_id=task_id, - call_type=call_type, - system_prompt=system_prompt, - user_prompt=user_prompt, - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - else: - result = self._byteplus_cache_manager.chat_with_session( - task_id=task_id, - call_type=call_type, - user_prompt=user_prompt, - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - - logger.info(f"BYTEPLUS SESSION RESPONSE: {result}") - - # Parse response (Responses API format) - content = self._parse_responses_api_content(result) - - # Token usage from Responses API - usage = result.get("usage") or {} - token_count_input = int(usage.get("input_tokens", 0)) - token_count_output = int(usage.get("output_tokens", 0)) - total_tokens = int(usage.get("total_tokens", 0)) or ( - token_count_input + token_count_output - ) - - # Log cache info and record metrics - # Responses API uses input_tokens_details instead of prompt_tokens_details - cached_tokens = usage.get("input_tokens_details", {}).get( - "cached_tokens", 0 - ) - metrics = get_cache_metrics() - if cached_tokens and cached_tokens > 0: - logger.info( - f"[CACHE] BytePlus session cache hit: {cached_tokens}/{token_count_input} tokens cached" - ) - metrics.record_hit( - "byteplus", - "session", - cached_tokens=cached_tokens, - total_tokens=token_count_input, - ) - else: - # First call in session or growing context - metrics.record_miss( - "byteplus", "session", total_tokens=token_count_input - ) - - status = "success" - - except BytePlusContextOverflowError: - # Context exceeded maximum length - reset session and retry with fresh context - logger.warning( - f"[BYTEPLUS] Context overflow for {session_key}, resetting session and retrying..." - ) - - # End the overflowed session - self._byteplus_cache_manager.end_session(task_id, call_type) - - # Get the stored system prompt for this session - system_prompt = self._session_system_prompts.get(session_key) - if not system_prompt: - exc_obj = ValueError( - f"Cannot reset session {session_key}: no system prompt stored" - ) - logger.error(str(exc_obj)) - else: - try: - # Create a fresh session with system prompt and current user prompt - logger.info( - f"[BYTEPLUS] Creating fresh session for {session_key} after overflow" - ) - result = self._byteplus_cache_manager.create_session_cache( - task_id=task_id, - call_type=call_type, - system_prompt=system_prompt, - user_prompt=user_prompt, - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - - logger.info(f"BYTEPLUS SESSION RESPONSE (after reset): {result}") - - # Parse response - content = self._parse_responses_api_content(result) - - # Token usage - usage = result.get("usage") or {} - token_count_input = int(usage.get("input_tokens", 0)) - token_count_output = int(usage.get("output_tokens", 0)) - total_tokens = int(usage.get("total_tokens", 0)) or ( - token_count_input + token_count_output - ) - - # Record as cache miss (fresh session) - metrics = get_cache_metrics() - metrics.record_miss( - "byteplus", "session_reset", total_tokens=token_count_input - ) - - status = "success" - logger.info( - f"[BYTEPLUS] Successfully recovered from context overflow for {session_key}" - ) - - except Exception as retry_exc: - exc_obj = retry_exc - logger.error( - f"Error retrying BytePlus Session API for {session_key} after reset: {retry_exc}" - ) - - except Exception as exc: - exc_obj = exc - logger.error(f"Error calling BytePlus Session API for {session_key}: {exc}") - - self._call_log_to_db( - f"[SESSION:{session_key}]", # Mark as session call in logs with call_type - user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - cached_tokens=cached_tokens or 0, - ) - - # Report usage - cached_tokens = 0 - if status == "success": - usage = result.get("usage") or {} if "result" in dir() else {} - cached_tokens = ( - usage.get("input_tokens_details", {}).get("cached_tokens", 0) - if usage - else 0 - ) - self._report_usage_async( - "llm_byteplus", - "byteplus", - self.model, - token_count_input, - token_count_output, - cached_tokens, + """Delegate to the byteplus_responses transport (Phase 2).""" + return _transports.byteplus_responses.generate_with_session( + self, task_id, call_type, user_prompt ) - return { - "tokens_used": total_tokens or 0, - "content": content or "", - "cached_tokens": cached_tokens or 0, - } - - # ───────────────────── Provider‑specific private helpers ───────────────────── - @profile("llm_openai_call", OperationCategory.LLM) + # ──────────── Provider-specific delegates (bodies live in transports/) ──────────── def _generate_openai( self, system_prompt: str | None, @@ -1850,308 +1839,24 @@ def _generate_openai( call_type: Optional[str] = None, messages_override: Optional[List[Dict[str, Any]]] = None, ) -> Dict[str, Any]: - """Generate response using OpenAI with automatic prompt caching. - - OpenAI's prompt caching is automatic for prompts ≥1024 tokens: - - No code changes required to enable caching - - Cached tokens are returned in usage.prompt_tokens_details.cached_tokens - - 50% discount on cached input tokens - - Cache retention: 5-10 minutes (up to 1 hour during off-peak) - - Using prompt_cache_key influences routing for better cache hit rates - - Args: - system_prompt: The system prompt. - user_prompt: The user prompt for this request. - call_type: Optional call type for cache routing (e.g., "reasoning", "action_selection"). - When provided, generates a prompt_cache_key to improve cache hit rates - when alternating between different call types. - messages_override: Optional pre-built multi-turn messages list. Used - by the OpenRouter-via-Claude session path to send a growing - conversation history so the upstream Anthropic model can cache - the accumulating prefix via OR's cache_control field. When set, - it's sent verbatim — system_prompt is still passed in for cache- - key derivation but the request body uses messages_override. - - Cache hits are logged when cached_tokens > 0 in the response. - """ - token_count_input = token_count_output = 0 - cached_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - config = get_cache_config() - cache_type = f"automatic_{call_type}" if call_type else "automatic" - - try: - if not self.client: - # No API key configured (or client construction failed) — - # shared by openai/minimax/deepseek/moonshot/grok/openrouter/ - # glm/fugu, all of which route through this method. Without - # this guard, `self.client.chat...` below raises a bare - # "'NoneType' object has no attribute 'chat'" — matches the - # explicit "client was not initialised" pattern already used - # for Anthropic/Gemini/Bedrock, so it classifies as CONFIG - # and fails fast instead of a confusing crash. - raise RuntimeError( - f"{provider_display_name(self.provider)} client was not initialised." - ) - if messages_override is not None: - messages: List[Dict[str, Any]] = messages_override - else: - messages = [] - if system_prompt: - messages.append({"role": "system", "content": system_prompt}) - messages.append({"role": "user", "content": user_prompt}) - - # Build request kwargs - request_kwargs: Dict[str, Any] = { - "model": self.model, - "messages": messages, - "temperature": self.temperature, - } - - # Newer OpenAI models (o1, o3, o4, gpt-5, etc.) require - # 'max_completion_tokens' instead of the legacy 'max_tokens' parameter. - model_lower = (self.model or "").lower() - uses_max_completion_tokens = ( - model_lower.startswith("o1") - or model_lower.startswith("o3") - or model_lower.startswith("o4") - or model_lower.startswith("gpt-5") - ) - if uses_max_completion_tokens: - request_kwargs["max_completion_tokens"] = self.max_tokens - else: - request_kwargs["max_tokens"] = self.max_tokens - - # Always enforce JSON output format - request_kwargs["response_format"] = {"type": "json_object"} - - # Build provider-specific cache hints in extra_body. - # - prompt_cache_key (OpenAI/DeepSeek/OpenRouter/Grok): improves - # prefix-cache routing stickiness across alternating call types. - # Grok DOES honor it — verified empirically: without a key a - # repeated identical prefix intermittently missed (routing bounced - # to a cold node); with prompt_cache_key the same prefix stayed a - # consistent hit. The old code skipped grok on a stale assumption. - # - cache_control (OpenRouter routing to Anthropic Claude only): Anthropic - # prompt caching is opt-in. OpenRouter accepts a top-level cache_control - # field and applies it to the last cacheable block automatically. For - # OpenAI/DeepSeek/Gemini upstreams via OpenRouter, caching is automatic - # on the upstream side, so cache_control would be ignored — we only set - # it when the slug is Anthropic-routed. - extra_body: Dict[str, Any] = {} - - long_enough = ( - system_prompt and len(system_prompt) >= config.min_cache_tokens - ) - - if call_type and long_enough: - prompt_hash = hashlib.sha256(system_prompt.encode()).hexdigest()[:16] - cache_key = f"{call_type}_{prompt_hash}" - extra_body["prompt_cache_key"] = cache_key - logger.debug(f"[OPENAI] Using prompt_cache_key: {cache_key}") - - if self.provider == "openrouter" and long_enough: - model_lower_for_cache = (self.model or "").lower() - # OpenRouter slugs are "/". Anthropic Claude routes - # are the only ones requiring opt-in cache_control. Detect by either - # the slug prefix or the "claude" substring (some aliases like - # "anthropic/claude-3.5-sonnet:beta" still match). - if ( - model_lower_for_cache.startswith("anthropic/") - or "claude" in model_lower_for_cache - ): - cache_control: Dict[str, Any] = {"type": "ephemeral"} - if call_type: - # 1-hour TTL keeps caches alive across alternating call types - # (mirrors the Anthropic-direct path). - cache_control["ttl"] = "1h" - extra_body["cache_control"] = cache_control - logger.debug( - f"[OPENROUTER] Anthropic cache_control: {cache_control} (model={self.model})" - ) - - if extra_body: - request_kwargs["extra_body"] = extra_body - - # In ChatGPT subscription mode the ``self.client`` is a - # ChatGPTSubscriptionClient that re-routes chat.completions - # calls through the Responses API (the only surface the - # chatgpt.com/backend-api/codex backend exposes). Call-site - # stays unchanged. - response = self.client.chat.completions.create(**request_kwargs) - if not response.choices: - raise ValueError(f"Provider returned no choices (model={self.model!r})") - content = (response.choices[0].message.content or "").strip() - token_count_input = response.usage.prompt_tokens - token_count_output = response.usage.completion_tokens - - # Extract cached tokens. Empirically ALL the OpenAI-compatible - # upstreams we use — including grok (xAI) — report cached tokens - # under usage.prompt_tokens_details.cached_tokens. Grok does NOT - # return the top-level prompt_cache_hit_tokens field (verified: it - # is always absent), so the old grok-specific read reported 0 even - # on real cache hits. Read the nested field first, then fall back - # to the legacy top-level field for any provider that still uses it. - prompt_tokens_details = getattr( - response.usage, "prompt_tokens_details", None - ) - if prompt_tokens_details: - cached_tokens = getattr(prompt_tokens_details, "cached_tokens", 0) or 0 - if not cached_tokens: - cached_tokens = ( - getattr(response.usage, "prompt_cache_hit_tokens", 0) or 0 - ) - - # Record cache metrics - provider_label = self.provider # "openai", "grok", "deepseek", etc. - metrics = get_cache_metrics() - if cached_tokens > 0: - logger.info( - f"[CACHE] {provider_label} {cache_type} cache hit: {cached_tokens}/{token_count_input} tokens from cache" - ) - metrics.record_hit( - provider_label, - cache_type, - cached_tokens=cached_tokens, - total_tokens=token_count_input, - ) - elif system_prompt and len(system_prompt) >= config.min_cache_tokens: - # Caching should have been attempted (prompt long enough) - # This is a miss - either first call or cache expired - metrics.record_miss( - provider_label, cache_type, total_tokens=token_count_input - ) - - status = "success" - except Exception as exc: - exc_obj = exc - logger.debug(f"Error calling OpenAI API: {exc}") - - total_tokens = token_count_input + token_count_output - - self._call_log_to_db( + """Delegate to the chat_completions transport (Phase 2).""" + return _transports.chat_completions.generate_openai( + self, system_prompt, user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - cached_tokens=cached_tokens or 0, - ) - - # Report usage. service_type stays "llm_openai" (the request shape) but - # provider attributes to the actual upstream so dashboards split out - # OpenRouter / DeepSeek / Grok separately. - self._report_usage_async( - "llm_openai", - self.provider, - self.model, - token_count_input, - token_count_output, - cached_tokens, + call_type=call_type, + messages_override=messages_override, ) - result = { - "tokens_used": total_tokens or 0, - "cached_tokens": cached_tokens, - } - - if exc_obj: - # Include error details for better diagnostics - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result["error"] = error_str - # Classify once and stash the LLMErrorInfo object so the outer - # `_generate_response_sync` can attach it to the consecutive- - # failure exception. Without this, providers that go through - # this path (OpenAI, OpenRouter, Grok, DeepSeek, MiniMax, - # Moonshot) would surface a bare "Aborted after N consecutive - # failures." with no cause when they fail. The classifier is - # wrapped in try/except so it can never break the error path. - try: - result["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result["content"] = "" - else: - result["content"] = content or "" - - return result - @profile("llm_ollama_call", OperationCategory.LLM) def _generate_ollama( self, system_prompt: str | None, user_prompt: str ) -> Dict[str, Any]: - token_count_input = token_count_output = 0 - total_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - - try: - payload = { - "model": self.model, - "prompt": user_prompt, - "stream": False, - "format": "json", - "options": { - "temperature": self.temperature, - }, - } - if system_prompt: - payload["system"] = system_prompt - url: str = f"{self.remote_url.rstrip('/')}/api/generate" - response = requests.post(url, json=payload, timeout=600) - response.raise_for_status() - result = response.json() - - content = result.get("response", "").strip() - token_count_input = result.get("prompt_eval_count", 0) - token_count_output = result.get("eval_count", 0) - total_tokens = token_count_input + token_count_output - status = "success" - except Exception as exc: - exc_obj = exc - logger.debug(f"Error calling Ollama API: {exc}") - - self._call_log_to_db( - system_prompt, - user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, + """Delegate to the chat_completions transport's Ollama path (Phase 2).""" + return _transports.chat_completions.generate_ollama( + self, system_prompt, user_prompt ) - # Report usage (no caching for Ollama) - self._report_usage_async( - "llm_ollama", "remote", self.model, token_count_input, token_count_output, 0 - ) - - result = {"tokens_used": total_tokens or 0} - if exc_obj: - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result["error"] = error_str - # Classify once and stash the LLMErrorInfo object so the - # outer `_generate_response_sync` can put `info.message` - # (the rich detailed string) into the RuntimeError it raises, - # and attach the info to LLMConsecutiveFailureError at the - # 5-failure threshold. The classifier is wrapped in try/except - # so it can never break the error path itself. - try: - result["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result["content"] = "" - else: - result["content"] = content or "" - return result - @profile("llm_gemini_call", OperationCategory.LLM) def _generate_gemini( self, @@ -2160,337 +1865,24 @@ def _generate_gemini( call_type: Optional[str] = None, contents_override: Optional[List[Dict[str, Any]]] = None, ) -> Dict[str, Any]: - """Generate response using Gemini with explicit or implicit caching. - - When call_type is provided and system_prompt is long enough, uses explicit - caching via GeminiCacheManager. This ensures different call types (reasoning, - action_selection, etc.) get separate caches for optimal cache hit rates. - - Without call_type, falls back to Gemini's implicit caching which may have - lower hit rates when alternating between different prompt structures. - - Args: - system_prompt: The system prompt (cached when using explicit caching). - user_prompt: The user prompt for this request. - call_type: Optional call type for cache keying (e.g., "reasoning", "action_selection"). - When provided, enables explicit caching per call type. - contents_override: Optional pre-built multi-turn `contents` array - from the session-cache path. When provided, skips the - explicit-cache code path and sends the full conversation - history so Gemini's implicit caching catches the growing - stable prefix automatically (caching covers more tokens with - every turn without us needing to manage a named cache object). - - Returns: - Dict with tokens_used, content, cached_tokens. - """ - from app.google_gemini_client import GeminiAPIError - - token_count_input = token_count_output = 0 - cached_tokens = 0 - total_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - config = get_cache_config() - cache_type = "implicit" # Default cache type for metrics - - try: - if not self._gemini_client: - raise RuntimeError("Gemini client was not initialised.") - - # Multi-turn implicit-cache path takes precedence when provided — - # the session-cache dispatcher accumulates history and we want - # Gemini's automatic prefix matching to do the work. - if contents_override is not None: - cache_type = f"implicit_{call_type}" if call_type else "implicit" - logger.debug( - f"[GEMINI] Using multi-turn implicit caching " - f"(call_type={call_type}, turns={len(contents_override)})" - ) - result = self._gemini_client.generate_text_multiturn( - self.model, - contents=contents_override, - system_prompt=system_prompt, - temperature=self.temperature, - max_output_tokens=self.max_tokens, - json_mode=True, - ) - else: - # Use explicit caching when: - # 1. call_type is provided - # 2. system_prompt is long enough - # 3. cache manager is available - # Note: GeminiCacheManager will automatically fall back to implicit - # caching if the system prompt is below Gemini's 1024 token minimum - use_explicit_cache = ( - call_type - and system_prompt - and len(system_prompt) >= config.min_cache_tokens - and self._gemini_cache_manager - ) - - if use_explicit_cache: - cache_type = f"explicit_{call_type}" - logger.debug( - f"[GEMINI] Using explicit caching for call_type: {call_type}" - ) - result = self._gemini_cache_manager.get_or_create_cache( - system_prompt=system_prompt, - user_prompt=user_prompt, - call_type=call_type, - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - else: - # Fall back to implicit caching (or no caching for short prompts) - result = self._gemini_client.generate_text( - self.model, - prompt=user_prompt, - system_prompt=system_prompt, - temperature=self.temperature, - max_output_tokens=self.max_tokens, - json_mode=True, - ) - - # Extract response data - content = result.get("content", "") - total_tokens = result.get("tokens_used", 0) - token_count_input = result.get("prompt_tokens", 0) - token_count_output = result.get("completion_tokens", 0) - cached_tokens = result.get("cached_tokens", 0) - - # Record cache metrics - metrics = get_cache_metrics() - if cached_tokens > 0: - logger.info( - f"[CACHE] Gemini {cache_type} cache hit: {cached_tokens}/{token_count_input} tokens from cache" - ) - metrics.record_hit( - "gemini", - cache_type, - cached_tokens=cached_tokens, - total_tokens=token_count_input, - ) - elif system_prompt and len(system_prompt) >= config.min_cache_tokens: - # Caching should have been attempted (prompt long enough) - # This is a miss - either first call or cache expired - metrics.record_miss( - "gemini", cache_type, total_tokens=token_count_input - ) - - status = "success" - except GeminiAPIError as exc: # pragma: no cover - exc_obj = exc - logger.error(f"Gemini API rejected the prompt: {exc}") - except Exception as exc: # pragma: no cover - exc_obj = exc - logger.debug(f"Error calling Gemini API: {exc}") - - self._call_log_to_db( + """Delegate to the gemini_native transport (Phase 2).""" + return _transports.gemini_native.generate( + self, system_prompt, user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - cached_tokens=cached_tokens, - ) - - # Report usage - self._report_usage_async( - "llm_gemini", - "gemini", - self.model, - token_count_input, - token_count_output, - cached_tokens, + call_type=call_type, + contents_override=contents_override, ) - result = {"tokens_used": total_tokens or 0, "cached_tokens": cached_tokens} - if exc_obj: - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result["error"] = error_str - # Classify once and stash the LLMErrorInfo object so the - # outer `_generate_response_sync` can put `info.message` - # (the rich detailed string) into the RuntimeError it raises, - # and attach the info to LLMConsecutiveFailureError at the - # 5-failure threshold. The classifier is wrapped in try/except - # so it can never break the error path itself. - try: - result["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result["content"] = "" - else: - result["content"] = content or "" - return result - @profile("llm_byteplus_call", OperationCategory.LLM) def _generate_byteplus( self, system_prompt: str | None, user_prompt: str ) -> Dict[str, Any]: - """Generate response using BytePlus with automatic prefix caching. - - Routes to prefix cache or standard API based on context. - """ - config = get_cache_config() - # Use prefix caching if: - # - System prompt is provided - # - System prompt is long enough (uses shared config) - # - Cache manager is available - if ( - system_prompt - and len(system_prompt) >= config.min_cache_tokens - and self._byteplus_cache_manager - ): - return self._generate_byteplus_with_prefix_cache(system_prompt, user_prompt) - - # Standard path (no caching) - return self._generate_byteplus_standard(system_prompt, user_prompt) - - def _generate_byteplus_with_prefix_cache( - self, system_prompt: str, user_prompt: str - ) -> Dict[str, Any]: - """Use Responses API with prefix caching. - - The system prompt is cached and reused across calls with the same content. - Only the user prompt is processed fresh each time. - Uses previous_response_id chaining for cache hits. - """ - token_count_input = token_count_output = 0 - total_tokens = 0 - cached_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - - try: - # Get response using prefix cache (creates cache on first call) - result = self._byteplus_cache_manager.get_or_create_prefix_cache( - system_prompt=system_prompt, - user_prompt=user_prompt, - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - - logger.info(f"BYTEPLUS CACHED RESPONSE: {result}") - - # Parse response (Responses API format) - content = self._parse_responses_api_content(result) - - if not content: - blocked_reason = _byteplus_blocked_reason(result) - if blocked_reason: - raise RuntimeError( - f"Response was blocked by the provider's content filter " - f"({blocked_reason})." - ) - - # Token usage from Responses API - usage = result.get("usage") or {} - token_count_input = int(usage.get("input_tokens", 0)) - token_count_output = int(usage.get("output_tokens", 0)) - total_tokens = int(usage.get("total_tokens", 0)) or ( - token_count_input + token_count_output - ) - - # Log cache hit info if available and record metrics - # Responses API uses input_tokens_details instead of prompt_tokens_details - cached_tokens = usage.get("input_tokens_details", {}).get( - "cached_tokens", 0 - ) - metrics = get_cache_metrics() - if cached_tokens and cached_tokens > 0: - logger.info( - f"[CACHE] BytePlus prefix cache hit: {cached_tokens}/{token_count_input} tokens cached" - ) - metrics.record_hit( - "byteplus", - "prefix", - cached_tokens=cached_tokens, - total_tokens=token_count_input, - ) - else: - # First call or cache miss - metrics.record_miss( - "byteplus", "prefix", total_tokens=token_count_input - ) - - status = "success" - - except requests.HTTPError as e: - # Check if this is a cache-related error (expired, not found) - if e.response is not None and e.response.status_code in (404, 410): - logger.warning(f"[CACHE] Cache expired or not found, recreating: {e}") - # Invalidate and retry once - self._byteplus_cache_manager.invalidate_prefix_cache(system_prompt) - try: - result = self._byteplus_cache_manager.get_or_create_prefix_cache( - system_prompt=system_prompt, - user_prompt=user_prompt, - temperature=self.temperature, - max_tokens=self.max_tokens, - ) - content = self._parse_responses_api_content(result) - usage = result.get("usage") or {} - token_count_input = int(usage.get("input_tokens", 0)) - token_count_output = int(usage.get("output_tokens", 0)) - total_tokens = int(usage.get("total_tokens", 0)) or ( - token_count_input + token_count_output - ) - status = "success" - except Exception as retry_exc: - exc_obj = retry_exc - logger.error(f"[CACHE] Retry failed, falling back: {retry_exc}") - return self._generate_byteplus_standard(system_prompt, user_prompt) - else: - exc_obj = e - logger.debug(f"Error calling BytePlus Responses API: {e}") - except Exception as exc: - exc_obj = exc - logger.debug(f"Error calling BytePlus Responses API: {exc}") - - self._call_log_to_db( - system_prompt, - user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - cached_tokens=cached_tokens or 0, - ) - - # Report usage - self._report_usage_async( - "llm_byteplus", - "byteplus", - self.model, - token_count_input, - token_count_output, - cached_tokens or 0, + """Delegate to the byteplus_responses transport (Phase 2).""" + return _transports.byteplus_responses.generate( + self, system_prompt, user_prompt ) - result_out: Dict[str, Any] = { - "tokens_used": total_tokens or 0, - "cached_tokens": cached_tokens or 0, - } - if exc_obj: - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result_out["error"] = error_str - try: - result_out["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result_out["content"] = "" - else: - result_out["content"] = content or "" - return result_out - def _parse_responses_api_content(self, result: Dict[str, Any]) -> str: """Parse content from BytePlus Responses API response. @@ -2514,125 +1906,6 @@ def _parse_responses_api_content(self, result: Dict[str, Any]) -> str: content += block.get("text", "") return content.strip() - def _generate_byteplus_standard( - self, system_prompt: str | None, user_prompt: str - ) -> Dict[str, Any]: - """Standard BytePlus API call without caching (uses /chat/completions).""" - token_count_input = token_count_output = 0 - total_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - - try: - # Build OpenAI-compatible messages array - messages: List[Dict[str, str]] = [] - if system_prompt: - messages.append({"role": "system", "content": system_prompt}) - messages.append({"role": "user", "content": user_prompt}) - - url = f"{self.byteplus_base_url.rstrip('/')}/chat/completions" - payload = { - "model": self.model, - "messages": messages, - # Wire through sampling + output control - "temperature": self.temperature, - "max_tokens": self.max_tokens, - # Note: response_format not supported by all BytePlus models (e.g., kimi) - # "stream": False, # default is non-streaming - } - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.api_key}", - } - - # Log the request - logger.info(f"[BYTEPLUS STANDARD REQUEST] URL: {url}") - logger.info( - f"[BYTEPLUS STANDARD REQUEST] Model: {self.model}, Temp: {self.temperature}, MaxTokens: {self.max_tokens}" - ) - logger.info(f"[BYTEPLUS STANDARD REQUEST] Messages count: {len(messages)}") - - response = requests.post(url, json=payload, headers=headers, timeout=600) - - # Log response status - logger.info(f"[BYTEPLUS STANDARD RESPONSE] Status: {response.status_code}") - - response.raise_for_status() - result = response.json() - - logger.info(f"[BYTEPLUS STANDARD RESPONSE] Body: {result}") - - # Non-streaming content location (OpenAI-compatible) - choices = result.get("choices", []) - if choices: - # choices[0].message.content is the OpenAI-compatible field - content = ( - choices[0].get("message", {}).get("content") - or choices[0].get("delta", {}).get("content", "") - or "" - ).strip() - if not content and choices[0].get("finish_reason") == "content_filter": - # OpenAI-compatible signal for moderation-blocked output — - # HTTP 200 with empty content, otherwise indistinguishable - # from a generic empty response. - raise RuntimeError( - "Response was blocked by the provider's content filter." - ) - - total_tokens = int(result.get("usage", {}).get("total_tokens", 0)) - - # Token usage (prompt/completion/total) - usage = result.get("usage") or {} - token_count_input = int(usage.get("prompt_tokens", 0)) - token_count_output = int(usage.get("completion_tokens", 0)) - status = "success" - - except Exception as exc: # pragma: no cover - exc_obj = exc - logger.debug(f"Error calling BytePlus API: {exc}") - - self._call_log_to_db( - system_prompt, - user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - ) - - # Report usage (no caching for standard path) - self._report_usage_async( - "llm_byteplus", - "byteplus", - self.model, - token_count_input, - token_count_output, - 0, - ) - - result = {"tokens_used": total_tokens or 0} - if exc_obj: - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result["error"] = error_str - # Classify once and stash the LLMErrorInfo object so the - # outer `_generate_response_sync` can put `info.message` - # (the rich detailed string) into the RuntimeError it raises, - # and attach the info to LLMConsecutiveFailureError at the - # 5-failure threshold. The classifier is wrapped in try/except - # so it can never break the error path itself. - try: - result["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result["content"] = "" - else: - result["content"] = content or "" - return result - - @profile("llm_anthropic_call", OperationCategory.LLM) def _generate_anthropic( self, system_prompt: str | None, @@ -2640,189 +1913,15 @@ def _generate_anthropic( call_type: Optional[str] = None, messages: Optional[List[dict]] = None, ) -> Dict[str, Any]: - """Generate response using Anthropic with prompt caching. - - Anthropic's prompt caching uses `cache_control` markers on content blocks. - When the system prompt is long enough (≥1024 tokens), we enable caching. - - For multi-turn sessions, pass pre-built `messages` with cache_control on the - last assistant message. This enables prefix caching of the entire conversation - history, not just the system prompt. - - TTL Options: - - Default (5 minutes): Free, uses "ephemeral" type - - Extended (1 hour): When call_type is provided, uses extended TTL for better - cache hit rates when alternating between different call types. - Note: Extended TTL cache writes cost 100% more, but reads are 90% cheaper. - - Args: - system_prompt: The system prompt (cached when long enough). - user_prompt: The user prompt for this request. - call_type: Optional call type (e.g., "reasoning", "action_selection"). - When provided, uses extended 1-hour TTL for better cache hit rates. - messages: Optional pre-built messages list for multi-turn sessions. - When provided, used instead of building a single-turn message. - - Cache hits are logged when `cache_read_input_tokens` > 0 in the response. - """ - token_count_input = token_count_output = 0 - total_tokens = 0 - cached_tokens = 0 - # Initialized here (not just inside the try) so the post-`except` - # _call_log_to_db below can reference them even when the API call - # throws before they're assigned (e.g. out-of-credits). Otherwise the - # real provider error is masked by an UnboundLocalError. - cache_creation = 0 - cache_read = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - config = get_cache_config() - cache_type = f"ephemeral_{call_type}" if call_type else "ephemeral" - - try: - if not self._anthropic_client: - raise RuntimeError("Anthropic client was not initialised.") - - # Build the message - use pre-built messages for multi-turn, or single-turn - # Anthropic requires max_tokens; use 16384 (Claude 4 default) to avoid truncation - message_kwargs: Dict[str, Any] = { - "model": self.model, - "max_tokens": 16384, - "messages": messages - if messages is not None - else [ - {"role": "user", "content": user_prompt}, - ], - } - - if system_prompt: - # Use caching if system prompt is long enough - if len(system_prompt) >= config.min_cache_tokens: - # Format system as list of content blocks with cache_control - # Use extended 1-hour TTL when call_type is provided for better - # cache hit rates when alternating between different call types - cache_control: Dict[str, str] = {"type": "ephemeral"} - if call_type: - # Extended TTL: cache writes cost 100% more, reads 90% cheaper - # Better for alternating call types where 5-minute TTL might expire - cache_control["ttl"] = "1h" - logger.debug( - f"[ANTHROPIC] Using 1-hour TTL for call_type: {call_type}" - ) - - message_kwargs["system"] = [ - { - "type": "text", - "text": system_prompt, - "cache_control": cache_control, - } - ] - else: - # Short prompt - use simple string format (no caching) - message_kwargs["system"] = system_prompt - - # Always pass temperature for Anthropic (their default is 1.0, not 0.0) - message_kwargs["temperature"] = self.temperature - - response = self._anthropic_client.messages.create(**message_kwargs) - - # Extract content from the response - content = "" - for block in response.content: - if block.type == "text": - content += block.text - content = content.strip() - - # Token usage from Anthropic response - # Anthropic reports input_tokens as non-cached input only. - # cache_creation_input_tokens: tokens written to cache (first call) - # cache_read_input_tokens: tokens read from cache (subsequent calls) - # Total input = input_tokens + cache_creation + cache_read - base_input = response.usage.input_tokens - token_count_output = response.usage.output_tokens - cache_creation = ( - getattr(response.usage, "cache_creation_input_tokens", 0) or 0 - ) - cache_read = getattr(response.usage, "cache_read_input_tokens", 0) or 0 - token_count_input = base_input + cache_creation + cache_read - total_tokens = token_count_input + token_count_output - cached_tokens = cache_read - - # Record metrics - metrics = get_cache_metrics() - if cache_read > 0: - logger.info( - f"[CACHE] Anthropic {cache_type} cache hit: {cache_read}/{token_count_input} tokens from cache" - ) - metrics.record_hit( - "anthropic", - cache_type, - cached_tokens=cache_read, - total_tokens=token_count_input, - ) - elif cache_creation > 0: - logger.info( - f"[CACHE] Anthropic {cache_type} cache created: {cache_creation} tokens cached" - ) - # Cache creation is a "miss" for the current call but sets up future hits - metrics.record_miss( - "anthropic", cache_type, total_tokens=token_count_input - ) - elif system_prompt and len(system_prompt) >= config.min_cache_tokens: - # Caching was attempted but no cache info returned - unexpected - metrics.record_miss( - "anthropic", cache_type, total_tokens=token_count_input - ) - - status = "success" - - except Exception as exc: # pragma: no cover - exc_obj = exc - logger.debug(f"Error calling Anthropic API: {exc}") - - self._call_log_to_db( + """Delegate to the anthropic_messages transport (Phase 2).""" + return _transports.anthropic_messages.generate( + self, system_prompt, user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - cached_tokens=cached_tokens, # cache_read — was MISSING (always 0) - cache_creation_tokens=cache_creation, # cache_write — to settle write-vs-expiry - ) - - # Report usage - self._report_usage_async( - "llm_anthropic", - "anthropic", - self.model, - token_count_input, - token_count_output, - cached_tokens, + call_type=call_type, + messages=messages, ) - result = {"tokens_used": total_tokens or 0, "cached_tokens": cached_tokens} - if exc_obj: - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result["error"] = error_str - # Classify once and stash the LLMErrorInfo object so the - # outer `_generate_response_sync` can put `info.message` - # (the rich detailed string) into the RuntimeError it raises, - # and attach the info to LLMConsecutiveFailureError at the - # 5-failure threshold. The classifier is wrapped in try/except - # so it can never break the error path itself. - try: - result["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result["content"] = "" - else: - result["content"] = content or "" - return result - # ─────────── Bedrock model capability detection ─────────────────── # Bedrock model ID prefixes that support cachePoint prompt caching. @@ -2840,7 +1939,6 @@ def _bedrock_model_supports_caching(self, model: Optional[str] = None) -> bool: model_id = model or self.model or "" return any(model_id.startswith(p) for p in self._BEDROCK_CACHE_PREFIXES) - @profile("llm_bedrock_call", OperationCategory.LLM) def _generate_bedrock( self, system_prompt: str | None, @@ -2848,185 +1946,15 @@ def _generate_bedrock( call_type: Optional[str] = None, messages: Optional[List[dict]] = None, ) -> Dict[str, Any]: - """Generate response via AWS Bedrock Converse API with prompt caching. - - Converse is the unified Bedrock API across Claude / Llama / Titan / - Mistral. cachePoint markers are inserted only for models that support - it (Anthropic Claude family) — other models would reject the request. - - Args: - system_prompt: The system prompt. - user_prompt: The user prompt for this request. - call_type: Optional call type for cache labelling. - messages: Optional pre-built multi-turn messages list. When provided - (from the session-cache path), the caller has already placed a - `cachePoint` block at the end of the last assistant content — - that captures the entire growing prefix. In that mode we do - NOT also put a cachePoint in the system block (only one is - needed and placing it in messages lets the cache grow with the - conversation). When messages is None, falls back to a fresh - single-turn call with cachePoint on the system block. - """ - token_count_input = token_count_output = 0 - total_tokens = 0 - cached_tokens = 0 - status = "failed" - content: Optional[str] = None - exc_obj: Optional[Exception] = None - config = get_cache_config() - cache_type = f"cachepoint_{call_type}" if call_type else "cachepoint" - - try: - if not self._bedrock_client: - raise RuntimeError("Bedrock client was not initialised.") - - # Multi-turn path: caller provided pre-built messages with cachePoint - # already placed on the last assistant message (if any). Single-turn - # path: build a fresh user-only message list. - multi_turn = messages is not None - converse_messages = ( - messages - if multi_turn - else [{"role": "user", "content": [{"text": user_prompt}]}] - ) - - converse_kwargs: Dict[str, Any] = { - "modelId": self.model, - "messages": converse_messages, - "inferenceConfig": { - "temperature": self.temperature, - "maxTokens": self.max_tokens, - }, - } - - if system_prompt: - # When messages already carry a cachePoint (multi-turn first - # call having a history assistant), don't double up by adding - # another in the system block — Bedrock would still accept it - # but a redundant checkpoint wastes a slot (max 4 per request). - msgs_have_cachepoint = multi_turn and any( - any("cachePoint" in block for block in msg.get("content", [])) - for msg in converse_messages - ) - use_system_cache = bool( - call_type - and len(system_prompt) >= config.min_cache_tokens - and self._bedrock_model_supports_caching() - and not msgs_have_cachepoint - ) - if use_system_cache: - converse_kwargs["system"] = [ - {"text": system_prompt}, - {"cachePoint": {"type": "default"}}, - ] - else: - converse_kwargs["system"] = [{"text": system_prompt}] - - response = self._bedrock_client.converse(**converse_kwargs) - - output_message = response.get("output", {}).get("message", {}) - content_blocks = output_message.get("content", []) or [] - content = "".join( - block.get("text", "") for block in content_blocks if "text" in block - ).strip() - - usage = response.get("usage", {}) or {} - token_count_input = int(usage.get("inputTokens", 0) or 0) - token_count_output = int(usage.get("outputTokens", 0) or 0) - - if self._bedrock_model_supports_caching(): - # Official Converse response uses `cacheReadInputTokens` / - # `cacheWriteInputTokens` (no "Count" suffix) per the API - # reference. The "...TokenCount" variants are tolerated as a - # defensive fallback in case older SDK builds expose them. - cache_read = int( - usage.get("cacheReadInputTokens") - or usage.get("cacheReadInputTokenCount") - or 0 - ) - cache_write = int( - usage.get("cacheWriteInputTokens") - or usage.get("cacheWriteInputTokenCount") - or 0 - ) - # Bedrock's `inputTokens` EXCLUDES cache activity, unlike the - # Anthropic API where input covers the full prompt. Normalize - # to the Anthropic shape — input = full prompt, cached = reads - # only — so downstream `input - cached` display math holds for - # every provider. - token_count_input += cache_read + cache_write - cached_tokens = cache_read - - metrics = get_cache_metrics() - if cache_read > 0: - logger.info( - f"[CACHE] Bedrock {cache_type} cache hit: " - f"{cache_read}/{token_count_input} tokens from cache" - ) - metrics.record_hit( - "bedrock", - cache_type, - cached_tokens=cache_read, - total_tokens=token_count_input, - ) - elif cache_write > 0: - logger.info( - f"[CACHE] Bedrock {cache_type} cache created: " - f"{cache_write} tokens cached" - ) - metrics.record_miss( - "bedrock", cache_type, total_tokens=token_count_input - ) - elif system_prompt and len(system_prompt) >= config.min_cache_tokens: - metrics.record_miss( - "bedrock", cache_type, total_tokens=token_count_input - ) - - total_tokens = token_count_input + token_count_output - - status = "success" - - except Exception as exc: # pragma: no cover - exc_obj = exc - logger.debug(f"Error calling Bedrock Converse API: {exc}") - - self._call_log_to_db( + """Delegate to the bedrock_converse transport (Phase 2).""" + return _transports.bedrock_converse.generate( + self, system_prompt, user_prompt, - content if content is not None else str(exc_obj), - status, - token_count_input, - token_count_output, - cached_tokens=cached_tokens or 0, + call_type=call_type, + messages=messages, ) - self._report_usage_async( - "llm_bedrock", - "bedrock", - self.model, - token_count_input, - token_count_output, - cached_tokens, - ) - - result = { - "tokens_used": total_tokens or 0, - "cached_tokens": cached_tokens, - } - if exc_obj: - error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" - result["error"] = error_str - try: - result["error_info_obj"] = classify_llm_error( - exc_obj, provider=self.provider, model=self.model - ) - except Exception: - pass - result["content"] = "" - else: - result["content"] = content or "" - return result - # ─────────────────── CLI helper for ad‑hoc testing ─────────────────── def _cli(self) -> None: # pragma: no cover """Run a quick interactive shell for manual testing.""" diff --git a/agent_core/core/impl/llm/transports/__init__.py b/agent_core/core/impl/llm/transports/__init__.py new file mode 100644 index 00000000..ed184861 --- /dev/null +++ b/agent_core/core/impl/llm/transports/__init__.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +"""Wire-protocol transports for LLMInterface (Phase 2, FR-2). + +Each transport module owns ONE wire protocol's request/response encoding, +extracted verbatim from interface.py. Transports are stateless: they receive +the live LLMInterface instance (`iface`) and read its provider context, +session buffers, cache managers, and logging/usage hooks through it — all +session state stays on LLMInterface (NFR-3 in docs/PROVIDER_LAYER_CATCHUP.md). + +TRANSPORTS maps ProviderProfile.wire -> the transport's generate callable +with signature (iface, system_prompt, user_prompt) -> response dict +({"content", "tokens_used", "cached_tokens"?, "error"?, "error_info_obj"?}). +Session-mode entry points with richer signatures are exposed as module +functions and called by the session dispatcher on LLMInterface. +""" + +from agent_core.core.impl.llm.transports import ( + anthropic_messages, + bedrock_converse, + byteplus_responses, + chat_completions, + gemini_native, +) + +TRANSPORTS = { + "chat_completions": chat_completions.generate_openai, + "ollama": chat_completions.generate_ollama, + "anthropic_messages": anthropic_messages.generate, + "bedrock_converse": bedrock_converse.generate, + "gemini_native": gemini_native.generate, + "byteplus_responses": byteplus_responses.generate, +} + +__all__ = [ + "TRANSPORTS", + "anthropic_messages", + "bedrock_converse", + "byteplus_responses", + "chat_completions", + "gemini_native", +] diff --git a/agent_core/core/impl/llm/transports/anthropic_messages.py b/agent_core/core/impl/llm/transports/anthropic_messages.py new file mode 100644 index 00000000..fcaafd7a --- /dev/null +++ b/agent_core/core/impl/llm/transports/anthropic_messages.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +"""Anthropic Messages transport (Phase 2 extraction from interface.py). + +Body moved VERBATIM from LLMInterface._generate_anthropic; ``self`` rewired +to ``iface``. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics +from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.utils.logger import logger + + +@profile("llm_anthropic_call", OperationCategory.LLM) +def generate( + iface, + system_prompt: str | None, + user_prompt: str, + call_type: Optional[str] = None, + messages: Optional[List[dict]] = None, +) -> Dict[str, Any]: + """Generate response using Anthropic with prompt caching. + + Anthropic's prompt caching uses `cache_control` markers on content blocks. + When the system prompt is long enough (≥1024 tokens), we enable caching. + + For multi-turn sessions, pass pre-built `messages` with cache_control on the + last assistant message. This enables prefix caching of the entire conversation + history, not just the system prompt. + + TTL Options: + - Default (5 minutes): Free, uses "ephemeral" type + - Extended (1 hour): When call_type is provided, uses extended TTL for better + cache hit rates when alternating between different call types. + Note: Extended TTL cache writes cost 100% more, but reads are 90% cheaper. + + Args: + system_prompt: The system prompt (cached when long enough). + user_prompt: The user prompt for this request. + call_type: Optional call type (e.g., "reasoning", "action_selection"). + When provided, uses extended 1-hour TTL for better cache hit rates. + messages: Optional pre-built messages list for multi-turn sessions. + When provided, used instead of building a single-turn message. + + Cache hits are logged when `cache_read_input_tokens` > 0 in the response. + """ + token_count_input = token_count_output = 0 + total_tokens = 0 + cached_tokens = 0 + # Initialized here (not just inside the try) so the post-`except` + # _call_log_to_db below can reference them even when the API call + # throws before they're assigned (e.g. out-of-credits). Otherwise the + # real provider error is masked by an UnboundLocalError. + cache_creation = 0 + cache_read = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + config = get_cache_config() + cache_type = f"ephemeral_{call_type}" if call_type else "ephemeral" + + try: + if not iface._anthropic_client: + raise RuntimeError("Anthropic client was not initialised.") + + # Build the message - use pre-built messages for multi-turn, or single-turn + # Anthropic requires max_tokens; use 16384 (Claude 4 default) to avoid truncation + message_kwargs: Dict[str, Any] = { + "model": iface.model, + "max_tokens": 16384, + "messages": messages + if messages is not None + else [ + {"role": "user", "content": user_prompt}, + ], + } + + if system_prompt: + # Use caching if system prompt is long enough + if len(system_prompt) >= config.min_cache_tokens: + # Format system as list of content blocks with cache_control + # Use extended 1-hour TTL when call_type is provided for better + # cache hit rates when alternating between different call types + cache_control: Dict[str, str] = {"type": "ephemeral"} + if call_type: + # Extended TTL: cache writes cost 100% more, reads 90% cheaper + # Better for alternating call types where 5-minute TTL might expire + cache_control["ttl"] = "1h" + logger.debug( + f"[ANTHROPIC] Using 1-hour TTL for call_type: {call_type}" + ) + + message_kwargs["system"] = [ + { + "type": "text", + "text": system_prompt, + "cache_control": cache_control, + } + ] + else: + # Short prompt - use simple string format (no caching) + message_kwargs["system"] = system_prompt + + # Always pass temperature for Anthropic (their default is 1.0, not 0.0) + message_kwargs["temperature"] = iface.temperature + + response = iface._anthropic_client.messages.create(**message_kwargs) + + # Extract content from the response + content = "" + for block in response.content: + if block.type == "text": + content += block.text + content = content.strip() + + # Token usage from Anthropic response + # Anthropic reports input_tokens as non-cached input only. + # cache_creation_input_tokens: tokens written to cache (first call) + # cache_read_input_tokens: tokens read from cache (subsequent calls) + # Total input = input_tokens + cache_creation + cache_read + base_input = response.usage.input_tokens + token_count_output = response.usage.output_tokens + cache_creation = ( + getattr(response.usage, "cache_creation_input_tokens", 0) or 0 + ) + cache_read = getattr(response.usage, "cache_read_input_tokens", 0) or 0 + token_count_input = base_input + cache_creation + cache_read + total_tokens = token_count_input + token_count_output + cached_tokens = cache_read + + # Record metrics + metrics = get_cache_metrics() + if cache_read > 0: + logger.info( + f"[CACHE] Anthropic {cache_type} cache hit: {cache_read}/{token_count_input} tokens from cache" + ) + metrics.record_hit( + "anthropic", + cache_type, + cached_tokens=cache_read, + total_tokens=token_count_input, + ) + elif cache_creation > 0: + logger.info( + f"[CACHE] Anthropic {cache_type} cache created: {cache_creation} tokens cached" + ) + # Cache creation is a "miss" for the current call but sets up future hits + metrics.record_miss( + "anthropic", cache_type, total_tokens=token_count_input + ) + elif system_prompt and len(system_prompt) >= config.min_cache_tokens: + # Caching was attempted but no cache info returned - unexpected + metrics.record_miss( + "anthropic", cache_type, total_tokens=token_count_input + ) + + status = "success" + + except Exception as exc: # pragma: no cover + exc_obj = exc + logger.debug(f"Error calling Anthropic API: {exc}") + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + cached_tokens=cached_tokens, # cache_read — was MISSING (always 0) + cache_creation_tokens=cache_creation, # cache_write — to settle write-vs-expiry + ) + + # Report usage + iface._report_usage_async( + "llm_anthropic", + "anthropic", + iface.model, + token_count_input, + token_count_output, + cached_tokens, + ) + + result = {"tokens_used": total_tokens or 0, "cached_tokens": cached_tokens} + if exc_obj: + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result["error"] = error_str + # Classify once and stash the LLMErrorInfo object so the + # outer `_generate_response_sync` can put `info.message` + # (the rich detailed string) into the RuntimeError it raises, + # and attach the info to LLMConsecutiveFailureError at the + # 5-failure threshold. The classifier is wrapped in try/except + # so it can never break the error path itself. + try: + result["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result["content"] = "" + else: + result["content"] = content or "" + return result diff --git a/agent_core/core/impl/llm/transports/bedrock_converse.py b/agent_core/core/impl/llm/transports/bedrock_converse.py new file mode 100644 index 00000000..478e794c --- /dev/null +++ b/agent_core/core/impl/llm/transports/bedrock_converse.py @@ -0,0 +1,204 @@ +# -*- coding: utf-8 -*- +"""Bedrock Converse transport (Phase 2 extraction from interface.py). + +Body moved VERBATIM from LLMInterface._generate_bedrock; ``self`` rewired to +``iface``. Cache capability detection (`_bedrock_model_supports_caching`) +stays on LLMInterface — the session dispatcher and this transport share it. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics +from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.utils.logger import logger + + +@profile("llm_bedrock_call", OperationCategory.LLM) +def generate( + iface, + system_prompt: str | None, + user_prompt: str, + call_type: Optional[str] = None, + messages: Optional[List[dict]] = None, +) -> Dict[str, Any]: + """Generate response via AWS Bedrock Converse API with prompt caching. + + Converse is the unified Bedrock API across Claude / Llama / Titan / + Mistral. cachePoint markers are inserted only for models that support + it (Anthropic Claude family) — other models would reject the request. + + Args: + system_prompt: The system prompt. + user_prompt: The user prompt for this request. + call_type: Optional call type for cache labelling. + messages: Optional pre-built multi-turn messages list. When provided + (from the session-cache path), the caller has already placed a + `cachePoint` block at the end of the last assistant content — + that captures the entire growing prefix. In that mode we do + NOT also put a cachePoint in the system block (only one is + needed and placing it in messages lets the cache grow with the + conversation). When messages is None, falls back to a fresh + single-turn call with cachePoint on the system block. + """ + token_count_input = token_count_output = 0 + total_tokens = 0 + cached_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + config = get_cache_config() + cache_type = f"cachepoint_{call_type}" if call_type else "cachepoint" + + try: + if not iface._bedrock_client: + raise RuntimeError("Bedrock client was not initialised.") + + # Multi-turn path: caller provided pre-built messages with cachePoint + # already placed on the last assistant message (if any). Single-turn + # path: build a fresh user-only message list. + multi_turn = messages is not None + converse_messages = ( + messages + if multi_turn + else [{"role": "user", "content": [{"text": user_prompt}]}] + ) + + converse_kwargs: Dict[str, Any] = { + "modelId": iface.model, + "messages": converse_messages, + "inferenceConfig": { + "temperature": iface.temperature, + "maxTokens": iface.max_tokens, + }, + } + + if system_prompt: + # When messages already carry a cachePoint (multi-turn first + # call having a history assistant), don't double up by adding + # another in the system block — Bedrock would still accept it + # but a redundant checkpoint wastes a slot (max 4 per request). + msgs_have_cachepoint = multi_turn and any( + any("cachePoint" in block for block in msg.get("content", [])) + for msg in converse_messages + ) + use_system_cache = bool( + call_type + and len(system_prompt) >= config.min_cache_tokens + and iface._bedrock_model_supports_caching() + and not msgs_have_cachepoint + ) + if use_system_cache: + converse_kwargs["system"] = [ + {"text": system_prompt}, + {"cachePoint": {"type": "default"}}, + ] + else: + converse_kwargs["system"] = [{"text": system_prompt}] + + response = iface._bedrock_client.converse(**converse_kwargs) + + output_message = response.get("output", {}).get("message", {}) + content_blocks = output_message.get("content", []) or [] + content = "".join( + block.get("text", "") for block in content_blocks if "text" in block + ).strip() + + usage = response.get("usage", {}) or {} + token_count_input = int(usage.get("inputTokens", 0) or 0) + token_count_output = int(usage.get("outputTokens", 0) or 0) + + if iface._bedrock_model_supports_caching(): + # Official Converse response uses `cacheReadInputTokens` / + # `cacheWriteInputTokens` (no "Count" suffix) per the API + # reference. The "...TokenCount" variants are tolerated as a + # defensive fallback in case older SDK builds expose them. + cache_read = int( + usage.get("cacheReadInputTokens") + or usage.get("cacheReadInputTokenCount") + or 0 + ) + cache_write = int( + usage.get("cacheWriteInputTokens") + or usage.get("cacheWriteInputTokenCount") + or 0 + ) + # Bedrock's `inputTokens` EXCLUDES cache activity, unlike the + # Anthropic API where input covers the full prompt. Normalize + # to the Anthropic shape — input = full prompt, cached = reads + # only — so downstream `input - cached` display math holds for + # every provider. + token_count_input += cache_read + cache_write + cached_tokens = cache_read + + metrics = get_cache_metrics() + if cache_read > 0: + logger.info( + f"[CACHE] Bedrock {cache_type} cache hit: " + f"{cache_read}/{token_count_input} tokens from cache" + ) + metrics.record_hit( + "bedrock", + cache_type, + cached_tokens=cache_read, + total_tokens=token_count_input, + ) + elif cache_write > 0: + logger.info( + f"[CACHE] Bedrock {cache_type} cache created: " + f"{cache_write} tokens cached" + ) + metrics.record_miss( + "bedrock", cache_type, total_tokens=token_count_input + ) + elif system_prompt and len(system_prompt) >= config.min_cache_tokens: + metrics.record_miss( + "bedrock", cache_type, total_tokens=token_count_input + ) + + total_tokens = token_count_input + token_count_output + + status = "success" + + except Exception as exc: # pragma: no cover + exc_obj = exc + logger.debug(f"Error calling Bedrock Converse API: {exc}") + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + cached_tokens=cached_tokens or 0, + ) + + iface._report_usage_async( + "llm_bedrock", + "bedrock", + iface.model, + token_count_input, + token_count_output, + cached_tokens, + ) + + result = { + "tokens_used": total_tokens or 0, + "cached_tokens": cached_tokens, + } + if exc_obj: + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result["error"] = error_str + try: + result["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result["content"] = "" + else: + result["content"] = content or "" + return result diff --git a/agent_core/core/impl/llm/transports/byteplus_responses.py b/agent_core/core/impl/llm/transports/byteplus_responses.py new file mode 100644 index 00000000..6e2f5373 --- /dev/null +++ b/agent_core/core/impl/llm/transports/byteplus_responses.py @@ -0,0 +1,522 @@ +# -*- coding: utf-8 -*- +"""BytePlus Responses transport (Phase 2 extraction from interface.py). + +Bodies moved VERBATIM from LLMInterface._generate_byteplus, +._generate_byteplus_with_prefix_cache, ._generate_byteplus_standard and +._generate_byteplus_with_session; ``self`` rewired to ``iface``. The +BytePlusCacheManager and the Responses-API content parser +(`_parse_responses_api_content`) stay owned by LLMInterface — the session +dispatcher's _process_* helpers share them. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +import requests + +from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm.cache import ( + BytePlusContextOverflowError, + get_cache_config, + get_cache_metrics, +) +from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.utils.logger import logger + + +@profile("llm_byteplus_call", OperationCategory.LLM) +def generate( + iface, system_prompt: str | None, user_prompt: str +) -> Dict[str, Any]: + """Generate response using BytePlus with automatic prefix caching. + + Routes to prefix cache or standard API based on context. + """ + config = get_cache_config() + # Use prefix caching if: + # - System prompt is provided + # - System prompt is long enough (uses shared config) + # - Cache manager is available + if ( + system_prompt + and len(system_prompt) >= config.min_cache_tokens + and iface._byteplus_cache_manager + ): + return generate_with_prefix_cache(iface, system_prompt, user_prompt) + + # Standard path (no caching) + return generate_standard(iface, system_prompt, user_prompt) + + +def generate_with_prefix_cache( + iface, system_prompt: str, user_prompt: str +) -> Dict[str, Any]: + """Use Responses API with prefix caching. + + The system prompt is cached and reused across calls with the same content. + Only the user prompt is processed fresh each time. + Uses previous_response_id chaining for cache hits. + """ + token_count_input = token_count_output = 0 + total_tokens = 0 + cached_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + + try: + # Get response using prefix cache (creates cache on first call) + result = iface._byteplus_cache_manager.get_or_create_prefix_cache( + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=iface.temperature, + max_tokens=iface.max_tokens, + ) + + logger.info(f"BYTEPLUS CACHED RESPONSE: {result}") + + # Parse response (Responses API format) + content = iface._parse_responses_api_content(result) + + if not content: + blocked_reason = _byteplus_blocked_reason(result) + if blocked_reason: + raise RuntimeError( + f"Response was blocked by the provider's content filter " + f"({blocked_reason})." + ) + + # Token usage from Responses API + usage = result.get("usage") or {} + token_count_input = int(usage.get("input_tokens", 0)) + token_count_output = int(usage.get("output_tokens", 0)) + total_tokens = int(usage.get("total_tokens", 0)) or ( + token_count_input + token_count_output + ) + + # Log cache hit info if available and record metrics + # Responses API uses input_tokens_details instead of prompt_tokens_details + cached_tokens = usage.get("input_tokens_details", {}).get( + "cached_tokens", 0 + ) + metrics = get_cache_metrics() + if cached_tokens and cached_tokens > 0: + logger.info( + f"[CACHE] BytePlus prefix cache hit: {cached_tokens}/{token_count_input} tokens cached" + ) + metrics.record_hit( + "byteplus", + "prefix", + cached_tokens=cached_tokens, + total_tokens=token_count_input, + ) + else: + # First call or cache miss + metrics.record_miss( + "byteplus", "prefix", total_tokens=token_count_input + ) + + status = "success" + + except requests.HTTPError as e: + # Check if this is a cache-related error (expired, not found) + if e.response is not None and e.response.status_code in (404, 410): + logger.warning(f"[CACHE] Cache expired or not found, recreating: {e}") + # Invalidate and retry once + iface._byteplus_cache_manager.invalidate_prefix_cache(system_prompt) + try: + result = iface._byteplus_cache_manager.get_or_create_prefix_cache( + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=iface.temperature, + max_tokens=iface.max_tokens, + ) + content = iface._parse_responses_api_content(result) + usage = result.get("usage") or {} + token_count_input = int(usage.get("input_tokens", 0)) + token_count_output = int(usage.get("output_tokens", 0)) + total_tokens = int(usage.get("total_tokens", 0)) or ( + token_count_input + token_count_output + ) + status = "success" + except Exception as retry_exc: + exc_obj = retry_exc + logger.error(f"[CACHE] Retry failed, falling back: {retry_exc}") + return generate_standard(iface, system_prompt, user_prompt) + else: + exc_obj = e + logger.debug(f"Error calling BytePlus Responses API: {e}") + except Exception as exc: + exc_obj = exc + logger.debug(f"Error calling BytePlus Responses API: {exc}") + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + cached_tokens=cached_tokens or 0, + ) + + # Report usage + iface._report_usage_async( + "llm_byteplus", + "byteplus", + iface.model, + token_count_input, + token_count_output, + cached_tokens or 0, + ) + + result_out: Dict[str, Any] = { + "tokens_used": total_tokens or 0, + "cached_tokens": cached_tokens or 0, + } + if exc_obj: + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result_out["error"] = error_str + try: + result_out["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result_out["content"] = "" + else: + result_out["content"] = content or "" + return result_out + + +def generate_standard( + iface, system_prompt: str | None, user_prompt: str +) -> Dict[str, Any]: + """Standard BytePlus API call without caching (uses /chat/completions).""" + token_count_input = token_count_output = 0 + total_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + + try: + # Build OpenAI-compatible messages array + messages: List[Dict[str, str]] = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + + url = f"{iface.byteplus_base_url.rstrip('/')}/chat/completions" + payload = { + "model": iface.model, + "messages": messages, + # Wire through sampling + output control + "temperature": iface.temperature, + "max_tokens": iface.max_tokens, + # Note: response_format not supported by all BytePlus models (e.g., kimi) + # "stream": False, # default is non-streaming + } + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {iface.api_key}", + } + + # Log the request + logger.info(f"[BYTEPLUS STANDARD REQUEST] URL: {url}") + logger.info( + f"[BYTEPLUS STANDARD REQUEST] Model: {iface.model}, Temp: {iface.temperature}, MaxTokens: {iface.max_tokens}" + ) + logger.info(f"[BYTEPLUS STANDARD REQUEST] Messages count: {len(messages)}") + + response = requests.post(url, json=payload, headers=headers, timeout=600) + + # Log response status + logger.info(f"[BYTEPLUS STANDARD RESPONSE] Status: {response.status_code}") + + response.raise_for_status() + result = response.json() + + logger.info(f"[BYTEPLUS STANDARD RESPONSE] Body: {result}") + + # Non-streaming content location (OpenAI-compatible) + choices = result.get("choices", []) + if choices: + # choices[0].message.content is the OpenAI-compatible field + content = ( + choices[0].get("message", {}).get("content") + or choices[0].get("delta", {}).get("content", "") + or "" + ).strip() + if not content and choices[0].get("finish_reason") == "content_filter": + # OpenAI-compatible signal for moderation-blocked output — + # HTTP 200 with empty content, otherwise indistinguishable + # from a generic empty response. + raise RuntimeError( + "Response was blocked by the provider's content filter." + ) + + total_tokens = int(result.get("usage", {}).get("total_tokens", 0)) + + # Token usage (prompt/completion/total) + usage = result.get("usage") or {} + token_count_input = int(usage.get("prompt_tokens", 0)) + token_count_output = int(usage.get("completion_tokens", 0)) + status = "success" + + except Exception as exc: # pragma: no cover + exc_obj = exc + logger.debug(f"Error calling BytePlus API: {exc}") + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + ) + + # Report usage (no caching for standard path) + iface._report_usage_async( + "llm_byteplus", + "byteplus", + iface.model, + token_count_input, + token_count_output, + 0, + ) + + result = {"tokens_used": total_tokens or 0} + if exc_obj: + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result["error"] = error_str + # Classify once and stash the LLMErrorInfo object so the + # outer `_generate_response_sync` can put `info.message` + # (the rich detailed string) into the RuntimeError it raises, + # and attach the info to LLMConsecutiveFailureError at the + # 5-failure threshold. The classifier is wrapped in try/except + # so it can never break the error path itself. + try: + result["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result["content"] = "" + else: + result["content"] = content or "" + return result + + +def generate_with_session( + iface, task_id: str, call_type: str, user_prompt: str +) -> Dict[str, Any]: + """Use Responses API with session caching for task/GUI calls. + + The context grows with each call as we chain responses via previous_response_id. + Each call type has its own session to avoid polluting different prompt structures. + + If context overflow is detected, the session is automatically reset and retried + with a fresh session containing only the system prompt and current user prompt. + """ + token_count_input = token_count_output = 0 + total_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + cached_tokens = 0 + session_key = f"{task_id}:{call_type}" + + try: + if not iface._byteplus_cache_manager.has_session(task_id, call_type): + # The cache manager was rebuilt (e.g. a model-only Settings + # change recreates it since BytePlus sessions are server-side + # and model-bound), emptying its session registry — but the + # system prompt survives a model-only reinit, so reseed a + # fresh session instead of failing this turn outright. + system_prompt = iface._session_system_prompts.get(session_key) + if not system_prompt: + raise ValueError(f"No session cache found for {session_key}") + + logger.info( + f"[BYTEPLUS] No session cache for {session_key} — " + f"reseeding a fresh session from the stored system prompt" + ) + result = iface._byteplus_cache_manager.create_session_cache( + task_id=task_id, + call_type=call_type, + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=iface.temperature, + max_tokens=iface.max_tokens, + ) + else: + result = iface._byteplus_cache_manager.chat_with_session( + task_id=task_id, + call_type=call_type, + user_prompt=user_prompt, + temperature=iface.temperature, + max_tokens=iface.max_tokens, + ) + + logger.info(f"BYTEPLUS SESSION RESPONSE: {result}") + + # Parse response (Responses API format) + content = iface._parse_responses_api_content(result) + + # Token usage from Responses API + usage = result.get("usage") or {} + token_count_input = int(usage.get("input_tokens", 0)) + token_count_output = int(usage.get("output_tokens", 0)) + total_tokens = int(usage.get("total_tokens", 0)) or ( + token_count_input + token_count_output + ) + + # Log cache info and record metrics + # Responses API uses input_tokens_details instead of prompt_tokens_details + cached_tokens = usage.get("input_tokens_details", {}).get( + "cached_tokens", 0 + ) + metrics = get_cache_metrics() + if cached_tokens and cached_tokens > 0: + logger.info( + f"[CACHE] BytePlus session cache hit: {cached_tokens}/{token_count_input} tokens cached" + ) + metrics.record_hit( + "byteplus", + "session", + cached_tokens=cached_tokens, + total_tokens=token_count_input, + ) + else: + # First call in session or growing context + metrics.record_miss( + "byteplus", "session", total_tokens=token_count_input + ) + + status = "success" + + except BytePlusContextOverflowError: + # Context exceeded maximum length - reset session and retry with fresh context + logger.warning( + f"[BYTEPLUS] Context overflow for {session_key}, resetting session and retrying..." + ) + + # End the overflowed session + iface._byteplus_cache_manager.end_session(task_id, call_type) + + # Get the stored system prompt for this session + system_prompt = iface._session_system_prompts.get(session_key) + if not system_prompt: + exc_obj = ValueError( + f"Cannot reset session {session_key}: no system prompt stored" + ) + logger.error(str(exc_obj)) + else: + try: + # Create a fresh session with system prompt and current user prompt + logger.info( + f"[BYTEPLUS] Creating fresh session for {session_key} after overflow" + ) + result = iface._byteplus_cache_manager.create_session_cache( + task_id=task_id, + call_type=call_type, + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=iface.temperature, + max_tokens=iface.max_tokens, + ) + + logger.info(f"BYTEPLUS SESSION RESPONSE (after reset): {result}") + + # Parse response + content = iface._parse_responses_api_content(result) + + # Token usage + usage = result.get("usage") or {} + token_count_input = int(usage.get("input_tokens", 0)) + token_count_output = int(usage.get("output_tokens", 0)) + total_tokens = int(usage.get("total_tokens", 0)) or ( + token_count_input + token_count_output + ) + + # Record as cache miss (fresh session) + metrics = get_cache_metrics() + metrics.record_miss( + "byteplus", "session_reset", total_tokens=token_count_input + ) + + status = "success" + logger.info( + f"[BYTEPLUS] Successfully recovered from context overflow for {session_key}" + ) + + except Exception as retry_exc: + exc_obj = retry_exc + logger.error( + f"Error retrying BytePlus Session API for {session_key} after reset: {retry_exc}" + ) + + except Exception as exc: + exc_obj = exc + logger.error(f"Error calling BytePlus Session API for {session_key}: {exc}") + + iface._call_log_to_db( + f"[SESSION:{session_key}]", # Mark as session call in logs with call_type + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + cached_tokens=cached_tokens or 0, + ) + + # Report usage + cached_tokens = 0 + if status == "success": + usage = result.get("usage") or {} if "result" in dir() else {} + cached_tokens = ( + usage.get("input_tokens_details", {}).get("cached_tokens", 0) + if usage + else 0 + ) + iface._report_usage_async( + "llm_byteplus", + "byteplus", + iface.model, + token_count_input, + token_count_output, + cached_tokens, + ) + + return { + "tokens_used": total_tokens or 0, + "content": content or "", + "cached_tokens": cached_tokens or 0, + } + + +def _byteplus_blocked_reason(result: Dict[str, Any]) -> Optional[str]: + """Best-effort detection of content-filter/moderation blocking in a + BytePlus Responses API result that came back with empty content but no + HTTP-level error (status 200, `choices`/`output` just empty). + + Mirrors OpenAI's Responses API `status` / `incomplete_details.reason` + shape, which BytePlus's docs describe this endpoint as following — not + independently verified against a live blocked response, so this only + fires on an unambiguous signal and otherwise returns None, leaving the + existing generic empty-response handling untouched. + """ + status = result.get("status") + if status == "incomplete": + reason = (result.get("incomplete_details") or {}).get("reason") + if reason: + return str(reason) + error = result.get("error") + if isinstance(error, dict): + code = str(error.get("code") or "").lower() + message = str(error.get("message") or "") + if any(k in code for k in ("content_filter", "moderation", "safety")): + return message or code + return None diff --git a/agent_core/core/impl/llm/transports/chat_completions.py b/agent_core/core/impl/llm/transports/chat_completions.py new file mode 100644 index 00000000..94351a9f --- /dev/null +++ b/agent_core/core/impl/llm/transports/chat_completions.py @@ -0,0 +1,394 @@ +# -*- coding: utf-8 -*- +"""Chat Completions transport (Phase 2 extraction from interface.py). + +Covers every OpenAI-compatible provider (openai, minimax, deepseek, +moonshot, grok, openrouter, glm, fugu — including the ChatGPT-subscription +translator client, which keeps the same call surface) plus the Ollama +native ``/api/generate`` path (wire "ollama"). + +Bodies moved VERBATIM from LLMInterface._generate_openai and +LLMInterface._generate_ollama; ``self`` rewired to ``iface``. +""" + +from __future__ import annotations + +import hashlib +import re +from typing import Any, Dict, List, Optional + +import requests + +from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics +from agent_core.core.impl.llm.errors import classify_llm_error, provider_display_name +from agent_core.core.models.registry import ( + get_registry as _get_registry, + supports_prompt_cache_key as _supports_pck, +) +from agent_core.core.models.provider_config import ( + OMIT_TEMPERATURE as _OMIT_TEMPERATURE, + resolve_temperature as _resolve_temperature, +) +from agent_core.utils.logger import logger + +# Some reasoning models (e.g. MiniMax M2.x by default) inline their +# chain-of-thought in the message content wrapped in ... +# instead of a separate reasoning_content field. Strip it so the downstream +# JSON-action parser sees only the answer. Non-greedy + DOTALL. +_THINK_RE = re.compile(r".*?\s*", re.DOTALL | re.IGNORECASE) + + +def _strip_reasoning_tags(text: Optional[str]) -> str: + return _THINK_RE.sub("", text or "").strip() + + +@profile("llm_openai_call", OperationCategory.LLM) +def generate_openai( + iface, + system_prompt: str | None, + user_prompt: str, + call_type: Optional[str] = None, + messages_override: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """Generate response using OpenAI with automatic prompt caching. + + OpenAI's prompt caching is automatic for prompts ≥1024 tokens: + - No code changes required to enable caching + - Cached tokens are returned in usage.prompt_tokens_details.cached_tokens + - 50% discount on cached input tokens + - Cache retention: 5-10 minutes (up to 1 hour during off-peak) + - Using prompt_cache_key influences routing for better cache hit rates + + Args: + system_prompt: The system prompt. + user_prompt: The user prompt for this request. + call_type: Optional call type for cache routing (e.g., "reasoning", "action_selection"). + When provided, generates a prompt_cache_key to improve cache hit rates + when alternating between different call types. + messages_override: Optional pre-built multi-turn messages list. Used + by the OpenRouter-via-Claude session path to send a growing + conversation history so the upstream Anthropic model can cache + the accumulating prefix via OR's cache_control field. When set, + it's sent verbatim — system_prompt is still passed in for cache- + key derivation but the request body uses messages_override. + + Cache hits are logged when cached_tokens > 0 in the response. + """ + token_count_input = token_count_output = 0 + cached_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + config = get_cache_config() + cache_type = f"automatic_{call_type}" if call_type else "automatic" + + try: + if not iface.client: + # No API key configured (or client construction failed) — + # shared by openai/minimax/deepseek/moonshot/grok/openrouter/ + # glm/fugu, all of which route through this method. Without + # this guard, `iface.client.chat...` below raises a bare + # "'NoneType' object has no attribute 'chat'" — matches the + # explicit "client was not initialised" pattern already used + # for Anthropic/Gemini/Bedrock, so it classifies as CONFIG + # and fails fast instead of a confusing crash. + raise RuntimeError( + f"{provider_display_name(iface.provider)} client was not initialised." + ) + if messages_override is not None: + messages: List[Dict[str, Any]] = messages_override + else: + messages = [] + if system_prompt: + messages.append({"role": "system", "content": system_prompt}) + messages.append({"role": "user", "content": user_prompt}) + + # Build request kwargs. Temperature follows the provider's policy + # (resolve_temperature): most providers send the caller's value, but + # Kimi/Moonshot thinking models reject an explicit temperature, so it + # is omitted entirely for them (docs/PROVIDER_SETTINGS_UX_FIX.md; the + # Kimi API rejects "invalid temperature: only 1 is allowed"). + request_kwargs: Dict[str, Any] = { + "model": iface.model, + "messages": messages, + } + _profile = _get_registry().get(iface.provider) + _temp = _resolve_temperature(_profile, iface.temperature) + if _temp is not _OMIT_TEMPERATURE: + request_kwargs["temperature"] = _temp + + # Output tokens: cap the VALUE to the provider's output limit (several + # providers — NVIDIA, Cerebras, Together, Groq — 400 rather than clamp + # when it's exceeded), and pick the FIELD NAME per provider policy. + # Newer OpenAI models (o1/o3/o4/gpt-5) and Cerebras/MiniMax require + # 'max_completion_tokens'; everyone else uses legacy 'max_tokens'. + _max_tokens_value = iface.max_tokens + if _profile is not None and _profile.max_output_tokens: + _max_tokens_value = min(_max_tokens_value, _profile.max_output_tokens) + model_lower = (iface.model or "").lower() + uses_max_completion_tokens = ( + (_profile is not None and _profile.uses_max_completion_tokens) + or model_lower.startswith("o1") + or model_lower.startswith("o3") + or model_lower.startswith("o4") + or model_lower.startswith("gpt-5") + ) + if uses_max_completion_tokens: + request_kwargs["max_completion_tokens"] = _max_tokens_value + else: + request_kwargs["max_tokens"] = _max_tokens_value + + # Enforce JSON output where the provider accepts json_object. + # Perplexity (only text/json_schema) and LM Studio reject/ignore it, + # so their profiles opt out and rely on prompt-instructed JSON (the + # request messages already instruct JSON). See _profile above. + if _profile is None or _profile.supports_json_object: + request_kwargs["response_format"] = {"type": "json_object"} + + # Build provider-specific cache hints in extra_body. + # - prompt_cache_key (OpenAI/DeepSeek/OpenRouter/Grok): improves + # prefix-cache routing stickiness across alternating call types. + # Grok DOES honor it — verified empirically: without a key a + # repeated identical prefix intermittently missed (routing bounced + # to a cold node); with prompt_cache_key the same prefix stayed a + # consistent hit. The old code skipped grok on a stale assumption. + # - cache_control (OpenRouter routing to Anthropic Claude only): Anthropic + # prompt caching is opt-in. OpenRouter accepts a top-level cache_control + # field and applies it to the last cacheable block automatically. For + # OpenAI/DeepSeek/Gemini upstreams via OpenRouter, caching is automatic + # on the upstream side, so cache_control would be ignored — we only set + # it when the slug is Anthropic-routed. + extra_body: Dict[str, Any] = {} + + long_enough = ( + system_prompt and len(system_prompt) >= config.min_cache_tokens + ) + + # prompt_cache_key pins requests with the same key to the same + # cache node (sticky routing), so a repeated stable prefix stays a + # HIT instead of bouncing to cold nodes. It is sent for ANY + # long-enough prompt — including the agent's main sessionless + # reasoning loop (call_type=None), whose 32k-char system prompt is + # byte-identical every turn yet was getting 0% cache because we only + # sent the key on call_type-tagged (session) calls. The key is + # hash(system_prompt), which is stable across turns, so identical + # system prompts route together. Opt-in per profile: some + # OpenAI-compatible endpoints reject unknown top-level fields. + if long_enough and _supports_pck(iface.provider): + prompt_hash = hashlib.sha256(system_prompt.encode()).hexdigest()[:16] + cache_key = f"{call_type}_{prompt_hash}" if call_type else prompt_hash + extra_body["prompt_cache_key"] = cache_key + logger.debug(f"[OPENAI] Using prompt_cache_key: {cache_key}") + + if iface.provider == "openrouter" and long_enough: + model_lower_for_cache = (iface.model or "").lower() + # OpenRouter slugs are "/". Anthropic Claude routes + # are the only ones requiring opt-in cache_control. Detect by either + # the slug prefix or the "claude" substring (some aliases like + # "anthropic/claude-3.5-sonnet:beta" still match). + if ( + model_lower_for_cache.startswith("anthropic/") + or "claude" in model_lower_for_cache + ): + cache_control: Dict[str, Any] = {"type": "ephemeral"} + if call_type: + # 1-hour TTL keeps caches alive across alternating call types + # (mirrors the Anthropic-direct path). + cache_control["ttl"] = "1h" + extra_body["cache_control"] = cache_control + logger.debug( + f"[OPENROUTER] Anthropic cache_control: {cache_control} (model={iface.model})" + ) + + if extra_body: + request_kwargs["extra_body"] = extra_body + + # In ChatGPT subscription mode the ``iface.client`` is a + # ChatGPTSubscriptionClient that re-routes chat.completions + # calls through the Responses API (the only surface the + # chatgpt.com/backend-api/codex backend exposes). Call-site + # stays unchanged. + response = iface.client.chat.completions.create(**request_kwargs) + if not response.choices: + raise ValueError(f"Provider returned no choices (model={iface.model!r})") + content = _strip_reasoning_tags(response.choices[0].message.content) + token_count_input = response.usage.prompt_tokens + token_count_output = response.usage.completion_tokens + + # Extract cached tokens. Empirically ALL the OpenAI-compatible + # upstreams we use — including grok (xAI) — report cached tokens + # under usage.prompt_tokens_details.cached_tokens. Grok does NOT + # return the top-level prompt_cache_hit_tokens field (verified: it + # is always absent), so the old grok-specific read reported 0 even + # on real cache hits. Read the nested field first, then fall back + # to the legacy top-level field for any provider that still uses it. + # Cached-token field varies by provider (verified against docs). Read + # in priority order so automatic prompt caching is COUNTED everywhere: + # 1. usage.prompt_tokens_details.cached_tokens — OpenAI/OpenRouter/ + # Grok/GLM/Cerebras/Qwen/Perplexity/MiniMax/Mistral (OpenAI-style) + # 2. usage.cached_tokens (flat) — Together (non-reasoning + # models), legacy Qwen + # 3. usage.prompt_cache_hit_tokens (top-level) — DeepSeek + # (Fireworks reports cached tokens only via a response HEADER, and + # HF-router / hosted NVIDIA NIM don't report them at all — those stay + # 0% in metrics even though the provider may still cache server-side.) + prompt_tokens_details = getattr( + response.usage, "prompt_tokens_details", None + ) + if prompt_tokens_details: + cached_tokens = getattr(prompt_tokens_details, "cached_tokens", 0) or 0 + if not cached_tokens: + cached_tokens = getattr(response.usage, "cached_tokens", 0) or 0 + if not cached_tokens: + cached_tokens = ( + getattr(response.usage, "prompt_cache_hit_tokens", 0) or 0 + ) + + # Record cache metrics + provider_label = iface.provider # "openai", "grok", "deepseek", etc. + metrics = get_cache_metrics() + if cached_tokens > 0: + logger.info( + f"[CACHE] {provider_label} {cache_type} cache hit: {cached_tokens}/{token_count_input} tokens from cache" + ) + metrics.record_hit( + provider_label, + cache_type, + cached_tokens=cached_tokens, + total_tokens=token_count_input, + ) + elif system_prompt and len(system_prompt) >= config.min_cache_tokens: + # Caching should have been attempted (prompt long enough) + # This is a miss - either first call or cache expired + metrics.record_miss( + provider_label, cache_type, total_tokens=token_count_input + ) + + status = "success" + except Exception as exc: + exc_obj = exc + logger.debug(f"Error calling OpenAI API: {exc}") + + total_tokens = token_count_input + token_count_output + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + cached_tokens=cached_tokens or 0, + ) + + # Report usage. service_type stays "llm_openai" (the request shape) but + # provider attributes to the actual upstream so dashboards split out + # OpenRouter / DeepSeek / Grok separately. + iface._report_usage_async( + "llm_openai", + iface.provider, + iface.model, + token_count_input, + token_count_output, + cached_tokens, + ) + + result = { + "tokens_used": total_tokens or 0, + "cached_tokens": cached_tokens, + } + + if exc_obj: + # Include error details for better diagnostics + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result["error"] = error_str + # Classify once and stash the LLMErrorInfo object so the outer + # `_generate_response_sync` can attach it to the consecutive- + # failure exception. Without this, providers that go through + # this path (OpenAI, OpenRouter, Grok, DeepSeek, MiniMax, + # Moonshot) would surface a bare "Aborted after N consecutive + # failures." with no cause when they fail. The classifier is + # wrapped in try/except so it can never break the error path. + try: + result["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result["content"] = "" + else: + result["content"] = content or "" + + return result + + +@profile("llm_ollama_call", OperationCategory.LLM) +def generate_ollama( + iface, system_prompt: str | None, user_prompt: str +) -> Dict[str, Any]: + token_count_input = token_count_output = 0 + total_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + + try: + payload = { + "model": iface.model, + "prompt": user_prompt, + "stream": False, + "format": "json", + "options": { + "temperature": iface.temperature, + }, + } + if system_prompt: + payload["system"] = system_prompt + url: str = f"{iface.remote_url.rstrip('/')}/api/generate" + response = requests.post(url, json=payload, timeout=600) + response.raise_for_status() + result = response.json() + + content = result.get("response", "").strip() + token_count_input = result.get("prompt_eval_count", 0) + token_count_output = result.get("eval_count", 0) + total_tokens = token_count_input + token_count_output + status = "success" + except Exception as exc: + exc_obj = exc + logger.debug(f"Error calling Ollama API: {exc}") + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + ) + + # Report usage (no caching for Ollama) + iface._report_usage_async( + "llm_ollama", "remote", iface.model, token_count_input, token_count_output, 0 + ) + + result = {"tokens_used": total_tokens or 0} + if exc_obj: + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result["error"] = error_str + # Classify once and stash the LLMErrorInfo object so the + # outer `_generate_response_sync` can put `info.message` + # (the rich detailed string) into the RuntimeError it raises, + # and attach the info to LLMConsecutiveFailureError at the + # 5-failure threshold. The classifier is wrapped in try/except + # so it can never break the error path itself. + try: + result["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result["content"] = "" + else: + result["content"] = content or "" + return result diff --git a/agent_core/core/impl/llm/transports/gemini_native.py b/agent_core/core/impl/llm/transports/gemini_native.py new file mode 100644 index 00000000..8813354a --- /dev/null +++ b/agent_core/core/impl/llm/transports/gemini_native.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +"""Gemini native transport (Phase 2 extraction from interface.py). + +Body moved VERBATIM from LLMInterface._generate_gemini; ``self`` rewired to +``iface``. The GeminiCacheManager stays owned by LLMInterface. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional + +from agent_core.decorators import profile, OperationCategory +from agent_core.core.impl.llm.cache import get_cache_config, get_cache_metrics +from agent_core.core.impl.llm.errors import classify_llm_error +from agent_core.utils.logger import logger + + +@profile("llm_gemini_call", OperationCategory.LLM) +def generate( + iface, + system_prompt: str | None, + user_prompt: str, + call_type: Optional[str] = None, + contents_override: Optional[List[Dict[str, Any]]] = None, +) -> Dict[str, Any]: + """Generate response using Gemini with explicit or implicit caching. + + When call_type is provided and system_prompt is long enough, uses explicit + caching via GeminiCacheManager. This ensures different call types (reasoning, + action_selection, etc.) get separate caches for optimal cache hit rates. + + Without call_type, falls back to Gemini's implicit caching which may have + lower hit rates when alternating between different prompt structures. + + Args: + system_prompt: The system prompt (cached when using explicit caching). + user_prompt: The user prompt for this request. + call_type: Optional call type for cache keying (e.g., "reasoning", "action_selection"). + When provided, enables explicit caching per call type. + contents_override: Optional pre-built multi-turn `contents` array + from the session-cache path. When provided, skips the + explicit-cache code path and sends the full conversation + history so Gemini's implicit caching catches the growing + stable prefix automatically (caching covers more tokens with + every turn without us needing to manage a named cache object). + + Returns: + Dict with tokens_used, content, cached_tokens. + """ + from app.google_gemini_client import GeminiAPIError + + token_count_input = token_count_output = 0 + cached_tokens = 0 + total_tokens = 0 + status = "failed" + content: Optional[str] = None + exc_obj: Optional[Exception] = None + config = get_cache_config() + cache_type = "implicit" # Default cache type for metrics + + try: + if not iface._gemini_client: + raise RuntimeError("Gemini client was not initialised.") + + # Multi-turn implicit-cache path takes precedence when provided — + # the session-cache dispatcher accumulates history and we want + # Gemini's automatic prefix matching to do the work. + if contents_override is not None: + cache_type = f"implicit_{call_type}" if call_type else "implicit" + logger.debug( + f"[GEMINI] Using multi-turn implicit caching " + f"(call_type={call_type}, turns={len(contents_override)})" + ) + result = iface._gemini_client.generate_text_multiturn( + iface.model, + contents=contents_override, + system_prompt=system_prompt, + temperature=iface.temperature, + max_output_tokens=iface.max_tokens, + json_mode=True, + ) + else: + # Use explicit caching when: + # 1. call_type is provided + # 2. system_prompt is long enough + # 3. cache manager is available + # Note: GeminiCacheManager will automatically fall back to implicit + # caching if the system prompt is below Gemini's 1024 token minimum + use_explicit_cache = ( + call_type + and system_prompt + and len(system_prompt) >= config.min_cache_tokens + and iface._gemini_cache_manager + ) + + if use_explicit_cache: + cache_type = f"explicit_{call_type}" + logger.debug( + f"[GEMINI] Using explicit caching for call_type: {call_type}" + ) + result = iface._gemini_cache_manager.get_or_create_cache( + system_prompt=system_prompt, + user_prompt=user_prompt, + call_type=call_type, + temperature=iface.temperature, + max_tokens=iface.max_tokens, + ) + else: + # Fall back to implicit caching (or no caching for short prompts) + result = iface._gemini_client.generate_text( + iface.model, + prompt=user_prompt, + system_prompt=system_prompt, + temperature=iface.temperature, + max_output_tokens=iface.max_tokens, + json_mode=True, + ) + + # Extract response data + content = result.get("content", "") + total_tokens = result.get("tokens_used", 0) + token_count_input = result.get("prompt_tokens", 0) + token_count_output = result.get("completion_tokens", 0) + cached_tokens = result.get("cached_tokens", 0) + + # Record cache metrics + metrics = get_cache_metrics() + if cached_tokens > 0: + logger.info( + f"[CACHE] Gemini {cache_type} cache hit: {cached_tokens}/{token_count_input} tokens from cache" + ) + metrics.record_hit( + "gemini", + cache_type, + cached_tokens=cached_tokens, + total_tokens=token_count_input, + ) + elif system_prompt and len(system_prompt) >= config.min_cache_tokens: + # Caching should have been attempted (prompt long enough) + # This is a miss - either first call or cache expired + metrics.record_miss( + "gemini", cache_type, total_tokens=token_count_input + ) + + status = "success" + except GeminiAPIError as exc: # pragma: no cover + exc_obj = exc + logger.error(f"Gemini API rejected the prompt: {exc}") + except Exception as exc: # pragma: no cover + exc_obj = exc + logger.debug(f"Error calling Gemini API: {exc}") + + iface._call_log_to_db( + system_prompt, + user_prompt, + content if content is not None else str(exc_obj), + status, + token_count_input, + token_count_output, + cached_tokens=cached_tokens, + ) + + # Report usage + iface._report_usage_async( + "llm_gemini", + "gemini", + iface.model, + token_count_input, + token_count_output, + cached_tokens, + ) + + result = {"tokens_used": total_tokens or 0, "cached_tokens": cached_tokens} + if exc_obj: + error_str = f"{type(exc_obj).__name__}: {str(exc_obj)}" + result["error"] = error_str + # Classify once and stash the LLMErrorInfo object so the + # outer `_generate_response_sync` can put `info.message` + # (the rich detailed string) into the RuntimeError it raises, + # and attach the info to LLMConsecutiveFailureError at the + # 5-failure threshold. The classifier is wrapped in try/except + # so it can never break the error path itself. + try: + result["error_info_obj"] = classify_llm_error( + exc_obj, provider=iface.provider, model=iface.model + ) + except Exception: + pass + result["content"] = "" + else: + result["content"] = content or "" + return result diff --git a/agent_core/core/impl/vlm/interface.py b/agent_core/core/impl/vlm/interface.py index a9d14432..827c9991 100644 --- a/agent_core/core/impl/vlm/interface.py +++ b/agent_core/core/impl/vlm/interface.py @@ -268,14 +268,23 @@ def describe_image_bytes( if log_response: logger.info(f"[LLM SEND] system={system_prompt} | user={user_prompt}") + # Native-wire providers are matched by name first; every other + # OpenAI-compatible provider (all Phase-3 additions: groq, + # mistral, together, fireworks, qwen, huggingface, nvidia, + # lmstudio, vllm, ... and openrouter) falls through to the + # OpenAI image_url path via a WIRE check — replacing the old + # hardcoded ("openai","minimax","moonshot","grok","glm") tuple + # that raised "Unknown provider" for any new VLM-capable + # provider (docs/PROVIDER_SETTINGS_UX_FIX.md A1). + from agent_core.core.models.registry import get_registry + + _profile = get_registry().get(self.provider) + _wire = _profile.wire if _profile is not None else "chat_completions" + if self.provider == "deepseek": raise RuntimeError( "DeepSeek does not support vision/VLM. Use a different provider for image description." ) - elif self.provider in ("openai", "minimax", "moonshot", "grok", "glm"): - response = self._openai_describe_bytes( - image_bytes, system_prompt, user_prompt, json_mode=json_mode - ) elif self.provider == "remote": response = self._ollama_describe_bytes( image_bytes, system_prompt, user_prompt @@ -296,6 +305,10 @@ def describe_image_bytes( response = self._bedrock_describe_bytes( image_bytes, system_prompt, user_prompt ) + elif _wire == "chat_completions": + response = self._openai_describe_bytes( + image_bytes, system_prompt, user_prompt, json_mode=json_mode + ) else: raise RuntimeError(f"Unknown provider {self.provider!r}") @@ -555,9 +568,21 @@ def _openai_describe_bytes( request_kwargs: Dict[str, Any] = { "model": self.model, "messages": messages, - "temperature": self.temperature, } - if json_mode: + # Same temperature + json policy as the LLM chat_completions + # transport: omit temperature for Kimi/Moonshot; omit response_format + # json_object for providers that reject it (Perplexity/LM Studio). + from agent_core.core.models.registry import get_registry + from agent_core.core.models.provider_config import ( + OMIT_TEMPERATURE, + resolve_temperature, + ) + + _profile = get_registry().get(self.provider) + _temp = resolve_temperature(_profile, self.temperature) + if _temp is not OMIT_TEMPERATURE: + request_kwargs["temperature"] = _temp + if json_mode and (_profile is None or _profile.supports_json_object): request_kwargs["response_format"] = {"type": "json_object"} model_lower = (self.model or "").lower() uses_max_completion_tokens = ( diff --git a/agent_core/core/models/connection_tester.py b/agent_core/core/models/connection_tester.py index 619e7aaf..efc2281c 100644 --- a/agent_core/core/models/connection_tester.py +++ b/agent_core/core/models/connection_tester.py @@ -14,6 +14,7 @@ import httpx from agent_core.core.models.provider_config import PROVIDER_CONFIG +from agent_core.core.models.registry import get_registry def test_provider_connection( @@ -40,15 +41,34 @@ def test_provider_connection( Returns: Dictionary with success/message/provider/error. """ - if provider not in PROVIDER_CONFIG: + return _test_provider_connection_inner( + provider, + api_key=api_key, + base_url=base_url, + timeout=timeout, + model=model, + aws_credentials=aws_credentials, + ) + + +def _test_provider_connection_inner( + provider: str, + api_key: Optional[str] = None, + base_url: Optional[str] = None, + timeout: float = 15.0, + model: Optional[str] = None, + aws_credentials: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + registry = get_registry() + if provider not in registry: return { "success": False, "message": f"Unknown provider: {provider}", "provider": provider, - "error": f"Supported providers: {', '.join(PROVIDER_CONFIG.keys())}", + "error": f"Supported providers: {', '.join(registry.keys())}", } - cfg = PROVIDER_CONFIG[provider] + cfg = registry[provider] try: if provider == "openai": @@ -94,6 +114,18 @@ def test_provider_connection( timeout=timeout, aws_credentials=aws_credentials, ) + elif cfg.wire == "chat_completions": + # Generic OpenAI-compatible branch (Phase 3): covers every new + # profile (groq, mistral, together, fireworks, cerebras, qwen, + # huggingface, nvidia, perplexity), the -cn region variants, + # local servers, and settings.json custom providers. + url = base_url or cfg.default_base_url + effective_key = api_key + if not effective_key and not cfg.requires_api_key: + # Local servers ignore auth but the request shape needs a + # bearer string. + effective_key = "local" + return _test_openai_compat(provider, effective_key, url, timeout, model) else: return { "success": False, @@ -111,23 +143,20 @@ def test_provider_connection( # ─── OpenRouter proxy helpers (Moonshot / MiniMax) ──────────────────── +# Derived from the provider profiles (Phase 3) — this used to be a second +# hand-maintained copy of the factory's slug map. _OR_MODEL_MAP: dict = { - "moonshot": { - "kimi-k2.5": "moonshotai/kimi-k2.5", - "moonshot-v1-8k": "moonshotai/moonshot-v1-8k", - "moonshot-v1-32k": "moonshotai/moonshot-v1-32k", - "moonshot-v1-128k": "moonshotai/moonshot-v1-128k", - "moonshot-v1-8k-vision-preview": "moonshotai/moonshot-v1-8k-vision-preview", - }, - "minimax": { - "MiniMax-Text-01": "minimax/minimax-01", - "MiniMax-VL-01": "minimax/minimax-01", - "abab6.5s-chat": "minimax/abab6.5s-chat", - }, + key: dict(p.openrouter_slug_map) + for key, p in PROVIDER_CONFIG.items() + if p.openrouter_slug_map } -_OR_NAMESPACE = {"moonshot": "moonshotai", "minimax": "minimax"} +_OR_NAMESPACE = { + key: p.openrouter_namespace + for key, p in PROVIDER_CONFIG.items() + if p.openrouter_namespace +} _OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" @@ -218,8 +247,10 @@ def _classified_error_result( def _resolve_test_model(provider: str, model: Optional[str], fallback: str) -> str: - """Use the user's model when provided; otherwise pull the default test - model from connection_test_models.json (auth-only validation).""" + """Use the user's model when provided; otherwise the profile's + connection_test_model (Phase 1: absorbed from + connection_test_models.json); app/config override kept for users who + still carry that file locally.""" if model: return model try: @@ -230,6 +261,9 @@ def _resolve_test_model(provider: str, model: Optional[str], fallback: str) -> s return configured except Exception: pass + profile = PROVIDER_CONFIG.get(provider) + if profile is not None and profile.connection_test_model: + return profile.connection_test_model return fallback @@ -242,21 +276,9 @@ def _success(provider: str, model: Optional[str]) -> Dict[str, Any]: } -_DISPLAY = { - "openai": "OpenAI", - "anthropic": "Anthropic", - "gemini": "Google Gemini", - "byteplus": "BytePlus", - "deepseek": "DeepSeek", - "moonshot": "Moonshot", - "minimax": "MiniMax", - "grok": "Grok (xAI)", - "glm": "Z.ai (GLM)", - "fugu": "Sakana (Fugu)", - "openrouter": "OpenRouter", - "remote": "Ollama", - "bedrock": "AWS Bedrock", -} +# Derived from the provider profiles (Phase 1). Note: "remote" now reads +# "Local (Ollama)" (the settings-UI name) instead of the old "Ollama". +_DISPLAY = {key: p.display_name for key, p in PROVIDER_CONFIG.items()} # ─── OpenAI / OpenAI-compat ─────────────────────────────────────────── diff --git a/agent_core/core/models/credentials.py b/agent_core/core/models/credentials.py new file mode 100644 index 00000000..58ff09d4 --- /dev/null +++ b/agent_core/core/models/credentials.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +"""Credential pools with per-error-class cooldowns (Phase 5, FR-7). + +Pool per provider = [primary api_keys key] + extra_api_keys extras from +settings.json. Strategy is fill-first: always serve the FIRST key that is +not cooling down, which keeps traffic pinned to the primary (provider-side +prompt caches stay warm) and only rotates while a key is cooling. + +Cooldown table (adapted from Hermes' error classes to our ErrorCategory, +docs/PROVIDER_LAYER_CATCHUP.md section 11.1): + + RATE_LIMIT keep once; from the 2nd consecutive hit cool 60s, + doubling per repeat up to 15 min + CREDIT/QUOTA cool 1h immediately (billing exhaustion) + AUTH cool 5 min (bad/revoked key) + others no credential action (provider-level, handled by fallback) + +State is in-memory with best-effort persistence to +/.credentials/pool_state.json; keys are stored as SHA-256 +fingerprints, never raw. Everything fails open: with no extras configured, +resolve() returns the primary key every time — bit-identical to the +pre-pool behavior (NFR-3/NFR-1). +""" + +from __future__ import annotations + +import hashlib +import json +import threading +import time +from pathlib import Path +from typing import Any, Dict, List, Optional + +from agent_core.utils.logger import logger + +_RATE_LIMIT_BASE_COOLDOWN = 60.0 +_RATE_LIMIT_MAX_COOLDOWN = 900.0 +_BILLING_COOLDOWN = 3600.0 +_AUTH_COOLDOWN = 300.0 + +_lock = threading.Lock() +# fingerprint -> {"cooling_until": float, "reason": str, "consecutive_rl": int} +_state: Optional[Dict[str, Dict[str, Any]]] = None +# provider -> fingerprint of the credential most recently served +_last_served: Dict[str, str] = {} + + +def _fingerprint(key: str) -> str: + return hashlib.sha256(key.encode("utf-8")).hexdigest()[:16] + + +def _state_path() -> Optional[Path]: + # Same .credentials/ directory the OAuth backends use. + try: + from craftos_integrations.credentials_store import _credentials_dir # type: ignore + + return _credentials_dir() / "pool_state.json" + except Exception: + try: + from app.config import SETTINGS_CONFIG_PATH # type: ignore + + return ( + Path(SETTINGS_CONFIG_PATH).parents[2] + / ".credentials" + / "pool_state.json" + ) + except Exception: + return None + + +def _load_state() -> Dict[str, Dict[str, Any]]: + global _state + if _state is not None: + return _state + path = _state_path() + try: + _state = json.loads(path.read_text(encoding="utf-8")) if path and path.exists() else {} + except Exception: + _state = {} + if not isinstance(_state, dict): + _state = {} + return _state + + +def _save_state() -> None: + path = _state_path() + if path is None: + return + try: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(_load_state(), indent=1, sort_keys=True), encoding="utf-8" + ) + except Exception as e: # pragma: no cover — persistence is best-effort + logger.debug(f"[POOL] state persist failed: {e}") + + +def reset_state_for_tests() -> None: + global _state, _last_served + with _lock: + _state = {} + _last_served = {} + + +def _pool_for(provider: str) -> List[str]: + """[primary] + extras from settings; empty when the app layer is absent.""" + try: + from app.config import get_api_key, get_extra_api_keys # type: ignore + + primary = get_api_key(provider) or "" + extras = get_extra_api_keys(provider) + pool = ([primary] if primary else []) + [k for k in extras if k] + # De-dup preserving order. + seen: set = set() + return [k for k in pool if not (k in seen or seen.add(k))] + except Exception: + return [] + + +def has_pool(provider: str) -> bool: + """True when more than one credential is configured for the provider.""" + return len(_pool_for(provider)) > 1 + + +def resolve(provider: str, default: Optional[str] = None) -> Optional[str]: + """Fill-first: the first non-cooling credential; all cooling -> primary.""" + pool = _pool_for(provider) + if not pool: + return default + now = time.time() + with _lock: + state = _load_state() + chosen = pool[0] + for key in pool: + entry = state.get(_fingerprint(key)) + if not entry or float(entry.get("cooling_until", 0)) <= now: + chosen = key + break + _last_served[provider] = _fingerprint(chosen) + return chosen + + +def make_resolver(provider: str, fallback_key: str): + """Callable for per-request auth-header resolution in the SDK clients.""" + + def _resolve() -> str: + return resolve(provider, default=fallback_key) or fallback_key + + return _resolve + + +def note_failure(provider: str, category: Optional[str]) -> None: + """Apply the cooldown table to the credential last served for provider. + + ``category`` is an ErrorCategory.value string (decoupled from the enum so + this module has no import edge into the error layer). + """ + if not category: + return + fp = _last_served.get(provider) + if fp is None: + return + if not has_pool(provider): + return # single key: nothing to rotate to; leave state untouched + now = time.time() + with _lock: + state = _load_state() + entry = state.setdefault(fp, {"consecutive_rl": 0}) + if category == "rate_limit": + entry["consecutive_rl"] = int(entry.get("consecutive_rl", 0)) + 1 + n = entry["consecutive_rl"] + if n >= 2: + cooldown = min( + _RATE_LIMIT_BASE_COOLDOWN * (2 ** (n - 2)), + _RATE_LIMIT_MAX_COOLDOWN, + ) + entry["cooling_until"] = now + cooldown + entry["reason"] = "rate_limit" + logger.warning( + f"[POOL] {provider}: credential {fp} cooling {cooldown:.0f}s (rate limit)" + ) + elif category in ("credit", "quota"): + entry["cooling_until"] = now + _BILLING_COOLDOWN + entry["reason"] = "billing" + logger.warning( + f"[POOL] {provider}: credential {fp} cooling 1h (billing)" + ) + elif category == "auth": + entry["cooling_until"] = now + _AUTH_COOLDOWN + entry["reason"] = "auth" + logger.warning( + f"[POOL] {provider}: credential {fp} cooling 5m (auth)" + ) + else: + return + _save_state() + + +def note_success(provider: str) -> None: + """Clear failure bookkeeping for the credential that just served.""" + fp = _last_served.get(provider) + if fp is None: + return + with _lock: + state = _load_state() + if fp in state: + state.pop(fp, None) + _save_state() diff --git a/agent_core/core/models/factory.py b/agent_core/core/models/factory.py index efa07bb6..c62e1365 100644 --- a/agent_core/core/models/factory.py +++ b/agent_core/core/models/factory.py @@ -15,49 +15,53 @@ boto3 = None # type: ignore[assignment] from agent_core.core.models.types import InterfaceType -from agent_core.core.models.model_registry import MODEL_REGISTRY from agent_core.core.models.provider_config import PROVIDER_CONFIG +from agent_core.core.models.registry import ( + error_display_map as _error_display_map, + get_registry, +) from agent_core.core.llm.google_gemini_client import GeminiClient logger = logging.getLogger(__name__) -# Providers that should route through OpenRouter when OR is configured, -# because their direct APIs are geo-restricted for most international users. -_OPENROUTER_PROXIED = {"moonshot", "minimax"} +# Derived from provider profiles (Phase 1, docs/PROVIDER_LAYER_CATCHUP.md). +# OpenRouter proxy routing exists because some direct APIs are geo-restricted +# for most international users; the per-provider data lives on the profiles. +_OPENROUTER_PROXIED = { + key for key, p in PROVIDER_CONFIG.items() if p.openrouter_proxy +} # OpenRouter namespace per provider (for auto-slugging unknown model IDs). _OR_NAMESPACE = { - "moonshot": "moonshotai", - "minimax": "minimax", + key: p.openrouter_namespace + for key, p in PROVIDER_CONFIG.items() + if p.openrouter_namespace } # Explicit model-ID → OpenRouter slug overrides. _OR_MODEL_MAP: dict = { - "moonshot": { - "kimi-k2.5": "moonshotai/kimi-k2.5", - "moonshot-v1-8k": "moonshotai/moonshot-v1-8k", - "moonshot-v1-32k": "moonshotai/moonshot-v1-32k", - "moonshot-v1-128k": "moonshotai/moonshot-v1-128k", - "moonshot-v1-8k-vision-preview": "moonshotai/moonshot-v1-8k-vision-preview", - }, - "minimax": { - "MiniMax-Text-01": "minimax/minimax-01", - "MiniMax-VL-01": "minimax/minimax-01", - "abab6.5s-chat": "minimax/abab6.5s-chat", - }, + key: dict(p.openrouter_slug_map) + for key, p in PROVIDER_CONFIG.items() + if p.openrouter_slug_map } _OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" +_PROVIDER_DISPLAY = _error_display_map() -_PROVIDER_DISPLAY = { - "openai": "OpenAI", - "deepseek": "DeepSeek", - "grok": "Grok", - "moonshot": "Moonshot", - "minimax": "MiniMax", - "openrouter": "OpenRouter", -} + +def _pool_resolver(provider: str, api_key: str): + """Per-request key resolver when a credential pool is configured + (Phase 5, FR-7). Returns None when the provider has a single key, so + client construction stays bit-identical to the pre-pool behavior.""" + try: + from agent_core.core.models import credentials as _credentials + + if _credentials.has_pool(provider): + return _credentials.make_resolver(provider, api_key) + except Exception: + pass + return None def _create_openai_client( @@ -67,6 +71,7 @@ def _create_openai_client( base_url: Optional[str] = None, default_headers: Optional[dict] = None, oauth_provider: Optional[str] = None, + resolve_key=None, ): """Create an OpenAI SDK client for OpenAI-compatible providers. @@ -95,6 +100,22 @@ def _create_openai_client( kwargs["base_url"] = base_url if default_headers: kwargs["default_headers"] = default_headers + + if oauth_provider is None and resolve_key is not None: + # Credential pool: re-resolve the key per request (same SDK property + # mechanism as subscription OAuth below), so a cooldown-driven + # rotation needs no client rebuild. + class _PooledOpenAI(OpenAI): + @property + def auth_headers(self) -> dict: + try: + self.api_key = resolve_key() or self.api_key + except Exception: + pass + return {"Authorization": f"Bearer {self.api_key}"} + + return _PooledOpenAI(**kwargs) + if oauth_provider is None: return OpenAI(**kwargs) @@ -136,7 +157,7 @@ def auth_headers(self) -> dict: return _SubscriptionOpenAI(**kwargs) -def _create_anthropic_client(*, api_key: str): +def _create_anthropic_client(*, api_key: str, resolve_key=None): try: from anthropic import Anthropic except ImportError as exc: @@ -145,7 +166,19 @@ def _create_anthropic_client(*, api_key: str): "Install it with the Python that launches CraftBot: " "`python -m pip install 'anthropic>=0.97.0'`." ) from exc - return Anthropic(api_key=api_key) + if resolve_key is None: + return Anthropic(api_key=api_key) + + class _PooledAnthropic(Anthropic): + @property + def auth_headers(self) -> dict: + try: + self.api_key = resolve_key() or self.api_key + except Exception: + pass + return {"X-Api-Key": self.api_key} + + return _PooledAnthropic(api_key=api_key) def _to_openrouter_slug(provider: str, model: str) -> str: @@ -235,22 +268,22 @@ def create( Returns: Dictionary with provider context including client instances """ - # OpenAI-compatible providers that use OpenAI client with a custom base_url + # Registry lookup covers built-ins AND settings.json custom + # providers (Phase 3). OpenAI-compatible providers are every + # chat_completions-wire profile except openai itself, which keeps + # its own arm for ChatGPT-subscription handling. + registry = get_registry() _OPENAI_COMPAT = { - "minimax", - "deepseek", - "moonshot", - "grok", - "openrouter", - "glm", - "fugu", + key + for key, p in registry.items() + if p.wire == "chat_completions" and key != "openai" } - if provider not in PROVIDER_CONFIG: + if provider not in registry: raise ValueError(f"Unsupported provider: {provider}") - cfg = PROVIDER_CONFIG[provider] - model = model_override or MODEL_REGISTRY[provider].get(interface) + cfg = registry[provider] + model = model_override or cfg.default_models.get(interface) if model is None: if deferred: return { @@ -264,8 +297,19 @@ def create( "bedrock_client": None, "initialized": False, } + # Local/custom chat_completions servers legitimately have no + # default model (they serve whatever the user loaded) — the + # provider supports the interface, we just need a model name. + if cfg.wire == "chat_completions": + raise ValueError( + f"No model configured for '{provider}'. Pick or type the " + f"model in Settings (this server does not advertise a " + f"default model)." + ) supported = ", ".join( - p for p, caps in MODEL_REGISTRY.items() if caps.get(interface) + p + for p, prof in registry.items() + if prof.default_models.get(interface) ) raise ValueError( f"Provider '{provider}' does not support {interface.value}. " @@ -360,6 +404,7 @@ def create( "client": _create_openai_client( provider=provider, api_key=api_key, + resolve_key=_pool_resolver(provider, api_key), ), "gemini_client": None, "remote_url": None, @@ -406,7 +451,10 @@ def create( "gemini_client": None, "remote_url": None, "byteplus": None, - "anthropic_client": _create_anthropic_client(api_key=api_key), + "anthropic_client": _create_anthropic_client( + api_key=api_key, + resolve_key=_pool_resolver(provider, api_key), + ), "bedrock_client": None, "initialized": True, } @@ -506,16 +554,22 @@ def create( } if not api_key: - if deferred: + if not cfg.requires_api_key: + # Local OpenAI-compatible servers (LM Studio, vLLM, + # llama.cpp) need no key, but the OpenAI SDK requires a + # non-empty string. + api_key = "local" + elif deferred: return empty_context - from app.errors import CatalogError, make_error + else: + from app.errors import CatalogError, make_error - raise CatalogError( - make_error( - "CONFIG_NO_API_KEY", - provider=_PROVIDER_DISPLAY.get(provider, provider), + raise CatalogError( + make_error( + "CONFIG_NO_API_KEY", + provider=_PROVIDER_DISPLAY.get(provider, provider), + ) ) - ) return { "provider": provider, @@ -524,6 +578,8 @@ def create( provider=provider, api_key=api_key, base_url=resolved_base_url, + default_headers=dict(cfg.default_headers) or None, + resolve_key=_pool_resolver(provider, api_key), ), "gemini_client": None, "remote_url": None, diff --git a/agent_core/core/models/model_registry.py b/agent_core/core/models/model_registry.py index 7cf2e175..85891e3a 100644 --- a/agent_core/core/models/model_registry.py +++ b/agent_core/core/models/model_registry.py @@ -1,117 +1,12 @@ # -*- coding: utf-8 -*- -"""Model registry mapping providers to default models.""" +"""Model registry mapping providers to default models. -from agent_core.core.models.types import InterfaceType +Since Phase 1 (docs/PROVIDER_LAYER_CATCHUP.md) this is DERIVED from the +provider profiles in provider_config.py — the per-provider default models +live on ``ProviderProfile.default_models``. The dict shape and import path +are unchanged for all existing consumers. +""" -MODEL_REGISTRY = { - "openai": { - InterfaceType.LLM: "gpt-5.2-2025-12-11", - InterfaceType.VLM: "gpt-5.2-2025-12-11", - InterfaceType.EMBEDDING: "text-embedding-3-small", - InterfaceType.IMAGE_GEN: "gpt-image-2", - InterfaceType.VIDEO_GEN: "sora-2", - }, - "gemini": { - InterfaceType.LLM: "gemini-2.5-pro", - InterfaceType.VLM: "gemini-2.5-pro", - InterfaceType.EMBEDDING: "text-embedding-004", - InterfaceType.IMAGE_GEN: "gemini-3-pro-image", - InterfaceType.VIDEO_GEN: "veo-3.1-generate-preview", - }, - "anthropic": { - InterfaceType.LLM: "claude-sonnet-4-6", - InterfaceType.VLM: "claude-sonnet-4-6", - InterfaceType.EMBEDDING: None, # Anthropic does not provide native embedding models - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "byteplus": { - InterfaceType.LLM: "seed-2-0-pro-260328", - InterfaceType.VLM: "seed-2-0-pro-260328", - InterfaceType.EMBEDDING: "skylark-embedding-vision-250615", - InterfaceType.IMAGE_GEN: None, - # BytePlus international (ap-southeast.bytepluses.com) model IDs use - # dated build suffixes, no dots, no `doubao-` prefix (`doubao-*` is - # the Volcengine China naming). Verified from BytePlus ModelArk docs. - InterfaceType.VIDEO_GEN: "seedance-1-0-pro-fast-251015", - }, - "remote": { - InterfaceType.LLM: "llama3.2:3b", - InterfaceType.VLM: "llava:7b", - InterfaceType.EMBEDDING: "nomic-embed-text", - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "minimax": { - InterfaceType.LLM: "MiniMax-Text-01", - InterfaceType.VLM: "MiniMax-VL-01", - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "deepseek": { - InterfaceType.LLM: "deepseek-chat", - InterfaceType.VLM: None, - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "moonshot": { - InterfaceType.LLM: "kimi-k2.5", - InterfaceType.VLM: "moonshot-v1-8k-vision-preview", - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "grok": { - InterfaceType.LLM: "grok-3", - InterfaceType.VLM: "grok-4-0709", - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "glm": { - # Z.ai (Zhipu AI) GLM-5.2 -- 1M-context, OpenAI-compatible, multimodal. - InterfaceType.LLM: "glm-5.2", - InterfaceType.VLM: "glm-5.2", - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "fugu": { - # Sakana AI Fugu -- OpenAI-compatible orchestration model. Text/LLM - # only here; no native vision/embedding/image/video models exposed. - InterfaceType.LLM: "fugu", - InterfaceType.VLM: None, - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "openrouter": { - # OpenRouter slugs follow `/` format. Default to a Claude - # model so KV caching exercises the cache_control path on first use. - InterfaceType.LLM: "anthropic/claude-sonnet-4.5", - InterfaceType.VLM: "anthropic/claude-sonnet-4.5", - InterfaceType.EMBEDDING: None, - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, - "bedrock": { - # Default to Claude Haiku 4.5 — best price/performance on Bedrock with - # cachePoint support (5-min + 1-hour TTL). The `us.` prefix is the - # cross-region inference profile, which is required because Claude 4.x - # models reject on-demand invocations against the bare `anthropic.*` - # ID ("Invocation of model ID ... with on-demand throughput isn't - # supported. Retry your request with the ID or ARN of an inference - # profile that contains this model."). The `us.anthropic.` prefix - # still matches `_BEDROCK_CACHE_PREFIXES`, so cachePoint is exercised. - # Users in EU / APAC regions should change `us.` to `eu.` / `ap.`. - # Haiku 4.5 also accepts image content blocks via Converse, so it - # doubles as the VLM default. Embedding stays on Titan. - InterfaceType.LLM: "us.anthropic.claude-haiku-4-5-20251001-v1:0", - InterfaceType.VLM: "us.anthropic.claude-haiku-4-5-20251001-v1:0", - InterfaceType.EMBEDDING: "amazon.titan-embed-text-v2:0", - InterfaceType.IMAGE_GEN: None, - InterfaceType.VIDEO_GEN: None, - }, -} +from agent_core.core.models.registry import default_models_registry + +MODEL_REGISTRY = default_models_registry() diff --git a/agent_core/core/models/provider_config.py b/agent_core/core/models/provider_config.py index da79d237..bdd9dc2a 100644 --- a/agent_core/core/models/provider_config.py +++ b/agent_core/core/models/provider_config.py @@ -1,67 +1,815 @@ # -*- coding: utf-8 -*- -"""Provider configuration for model factories.""" +"""Provider profiles: the single source of truth for provider identity. -from dataclasses import dataclass -from typing import Optional +Phase 1 of docs/PROVIDER_LAYER_CATCHUP.md (FR-1). A ProviderProfile declares +everything about a provider in one place: auth env vars, endpoints, display +names, settings.json mapping, subscription OAuth, default models, and the +OpenRouter proxy fallback. Structures that used to be hand-synced across six +files (PROVIDER_INFO, PROVIDER_TO_SETTINGS_KEY, the /provider CLI list, +MODEL_REGISTRY, the factory's OpenRouter maps, connection-test models) are +now DERIVED from these profiles — see agent_core/core/models/registry.py. + +Profiles are declarative data. They do not own client construction, session +state, caching, or error handling; those stay on ModelFactory/LLMInterface. + +``ProviderConfig`` is kept as an alias of ``ProviderProfile`` so every +existing import keeps working. +""" + +from dataclasses import dataclass, field +from typing import Any, Dict, Mapping, Optional, Tuple + +from agent_core.core.models.types import InterfaceType + + +# Sentinel for ProviderProfile.fixed_temperature meaning "do NOT send the +# temperature field at all" — the model manages it server-side. Adopted from +# Hermes Agent (providers/base.py); required by Moonshot/Kimi thinking models +# (kimi-k2.5+), whose API rejects an explicit temperature +# ("invalid temperature: only 1 is allowed for this model"). Verified against +# the Kimi API docs ("temperature is not modifiable and should not be passed") +# and Hermes' kimi-coding profile (fixed_temperature=OMIT_TEMPERATURE). +OMIT_TEMPERATURE = object() @dataclass(frozen=True) -class ProviderConfig: +class ProviderProfile: + # ── auth & endpoints (original ProviderConfig fields) ───────────── api_key_env: Optional[str] = None base_url_env: Optional[str] = None default_base_url: Optional[str] = None + # ── identity / display ──────────────────────────────────────────── + key: str = "" + display_name: str = "" # settings-UI name (was PROVIDER_INFO["name"]) + # Short name used in factory error messages (was factory._PROVIDER_DISPLAY). + # None -> callers fall back to the raw provider key, preserving the old + # ``_PROVIDER_DISPLAY.get(provider, provider)`` behavior exactly. + error_display_name: Optional[str] = None + # Wire protocol / transport key (consumed by Phase 2 transports): + # chat_completions | anthropic_messages | bedrock_converse | + # gemini_native | byteplus_responses | ollama + wire: str = "chat_completions" + + # ── settings.json / self-config mapping ─────────────────────────── + # api_keys key in settings.json (was PROVIDER_TO_SETTINGS_KEY). + # None -> provider has no single API key (bedrock, remote). + settings_key: Optional[str] = None + requires_api_key: bool = True + # True -> the settings UI renders the AWS credentials block + # (was PROVIDER_INFO["is_bedrock"]). + aws_credential_block: bool = False + + # ── subscription OAuth (was scattered across PROVIDER_INFO) ─────── + # Backend name in craftos_integrations/integrations/llm_oauth + # ("chatgpt" flows live under provider key "openai"; the backend is + # addressed by the provider key, so this field just marks support). + oauth_backend: Optional[str] = None + subscription_label: Optional[str] = None + subscription_models: Tuple[str, ...] = () + subscription_default_model: Optional[str] = None + + # ── UI capabilities ─────────────────────────────────────────────── + # Frontend opts into the catalog-aware model picker (OpenRouter). + supports_catalog_picker: bool = False + + # ── models ──────────────────────────────────────────────────────── + # {InterfaceType: default model id} (was MODEL_REGISTRY row). + default_models: Mapping[InterfaceType, Optional[str]] = field( + default_factory=dict + ) + # Tiny known-good model for auth checks (was connection_test_models.json). + connection_test_model: Optional[str] = None + connection_test_max_tokens: Optional[int] = None + + # ── model discovery (Phase U2, docs/PROVIDER_SETTINGS_UX_FIX.md) ─── + # True when the provider exposes an OpenAI-standard GET {base_url}/models + # that lists usable models, so the settings UI offers a live dropdown + + # Refresh instead of a blind text box. + supports_model_discovery: bool = False + # Local-server flavor for richer native handling: "lmstudio" unlocks the + # native list-all (/api/v1/models) + load (/api/v1/models/load) UI. + local_kind: Optional[str] = None -PROVIDER_CONFIG = { - "openai": ProviderConfig(api_key_env="OPENAI_API_KEY"), - "gemini": ProviderConfig(api_key_env="GOOGLE_API_KEY"), - "anthropic": ProviderConfig(api_key_env="ANTHROPIC_API_KEY"), - "byteplus": ProviderConfig( + # ── caching / request quirks (consumed by Phase 2/3) ────────────── + # Whether the endpoint documents ``prompt_cache_key``. True only for + # providers that already receive it today (NFR-3: golden payloads + # must not change). + supports_prompt_cache_key: bool = False + # Temperature policy for the chat_completions wire (Hermes-style): + # None -> send the caller's temperature (default behavior) + # OMIT_TEMPERATURE -> do NOT send temperature (Kimi/Moonshot thinking + # models reject it; the server manages it) + # a float -> always send this fixed value + fixed_temperature: Any = None + # Whether the provider accepts response_format={"type":"json_object"} on + # its chat endpoint. True by default (the major OpenAI-compatible + # providers do). Set False for endpoints that only accept json_schema + # (Perplexity hard-400s on json_object; LM Studio ignores/rejects it) — + # those fall back to prompt-instructed JSON, exactly like Hermes (which + # never sends response_format at all). Verified against provider docs. + supports_json_object: bool = True + # Output-token cap: many providers 400 (not clamp) when the requested + # max_tokens exceeds the model's output limit. When set, the transport + # sends min(caller_max_tokens, max_output_tokens). None = no cap. + # Values are battle-tested (Hermes default_max_tokens) or from docs. + max_output_tokens: Optional[int] = None + # True -> always send the OpenAI `max_completion_tokens` field instead of + # the legacy `max_tokens` (Cerebras/MiniMax require it; Groq deprecates + # max_tokens). None/False -> the model-name heuristic (o1/o3/o4/gpt-5) + # decides, preserving existing OpenAI behavior. + uses_max_completion_tokens: bool = False + # Whether the chat_completions session path accumulates a growing + # [user, assistant, ...] history for this provider (the + # _openai_compat_session_messages buffer). False preserves the + # historical behavior for minimax/moonshot, whose session turns fall + # through to stateless generation. Only meaningful on the + # chat_completions wire. + session_accumulation: bool = False + default_headers: Mapping[str, str] = field(default_factory=dict) + + # ── OpenRouter proxy fallback (was factory module maps) ─────────── + # True -> route through OpenRouter when no direct key is configured + # (geo-restricted direct APIs; was _OPENROUTER_PROXIED membership). + openrouter_proxy: bool = False + openrouter_namespace: Optional[str] = None # was _OR_NAMESPACE + openrouter_slug_map: Mapping[str, str] = field(default_factory=dict) + + +# Legacy alias — every existing `from ... import ProviderConfig` keeps working. +ProviderConfig = ProviderProfile + + +def resolve_temperature(profile: Optional[ProviderProfile], caller_temperature): + """Temperature to send on a chat_completions request, per the provider's + policy. Returns OMIT_TEMPERATURE when the field must be dropped entirely. + + - profile.fixed_temperature is OMIT_TEMPERATURE -> OMIT_TEMPERATURE + - profile.fixed_temperature is a value -> that value + - otherwise -> caller's temperature + """ + fixed = profile.fixed_temperature if profile is not None else None + if fixed is OMIT_TEMPERATURE: + return OMIT_TEMPERATURE + if fixed is not None: + return fixed + return caller_temperature + + +PROVIDER_CONFIG: Dict[str, ProviderProfile] = { + "openai": ProviderProfile( + key="openai", + session_accumulation=True, + api_key_env="OPENAI_API_KEY", + display_name="OpenAI", + error_display_name="OpenAI", + settings_key="openai", + oauth_backend="chatgpt", + subscription_label="Sign in with ChatGPT", + # Codex-accepted models for ChatGPT subscription auth. + subscription_models=( + "gpt-5.4", + "gpt-5.5", + "gpt-5.4-mini", + "gpt-5.3-codex-spark", + ), + subscription_default_model="gpt-5.4", + supports_prompt_cache_key=True, + connection_test_model="gpt-4o-mini", + default_models={ + InterfaceType.LLM: "gpt-5.2-2025-12-11", + InterfaceType.VLM: "gpt-5.2-2025-12-11", + InterfaceType.EMBEDDING: "text-embedding-3-small", + InterfaceType.IMAGE_GEN: "gpt-image-2", + InterfaceType.VIDEO_GEN: "sora-2", + }, + ), + "gemini": ProviderProfile( + key="gemini", + api_key_env="GOOGLE_API_KEY", + display_name="Google Gemini", + wire="gemini_native", + settings_key="google", + connection_test_model="gemini-2.0-flash", + default_models={ + InterfaceType.LLM: "gemini-2.5-pro", + InterfaceType.VLM: "gemini-2.5-pro", + InterfaceType.EMBEDDING: "text-embedding-004", + InterfaceType.IMAGE_GEN: "gemini-3-pro-image", + InterfaceType.VIDEO_GEN: "veo-3.1-generate-preview", + }, + ), + "anthropic": ProviderProfile( + key="anthropic", + api_key_env="ANTHROPIC_API_KEY", + display_name="Anthropic", + wire="anthropic_messages", + settings_key="anthropic", + connection_test_model="claude-haiku-4-5-20251001", + connection_test_max_tokens=1, + default_models={ + InterfaceType.LLM: "claude-sonnet-4-6", + InterfaceType.VLM: "claude-sonnet-4-6", + # Anthropic does not provide native embedding models. + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "byteplus": ProviderProfile( + key="byteplus", api_key_env="BYTEPLUS_API_KEY", base_url_env="BYTEPLUS_BASE_URL", default_base_url="https://ark.ap-southeast.bytepluses.com/api/v3", + display_name="BytePlus", + wire="byteplus_responses", + settings_key="byteplus", + connection_test_model="kimi-k2-250905", + default_models={ + InterfaceType.LLM: "seed-2-0-pro-260328", + InterfaceType.VLM: "seed-2-0-pro-260328", + InterfaceType.EMBEDDING: "skylark-embedding-vision-250615", + InterfaceType.IMAGE_GEN: None, + # BytePlus international (ap-southeast.bytepluses.com) model IDs + # use dated build suffixes, no dots, no `doubao-` prefix + # (`doubao-*` is the Volcengine China naming). Verified from + # BytePlus ModelArk docs. + InterfaceType.VIDEO_GEN: "seedance-1-0-pro-fast-251015", + }, ), - "remote": ProviderConfig( + "remote": ProviderProfile( + key="remote", base_url_env="REMOTE_MODEL_URL", default_base_url="http://localhost:11434", + display_name="Local (Ollama)", + wire="ollama", + requires_api_key=False, + connection_test_model="llama3", + default_models={ + InterfaceType.LLM: "llama3.2:3b", + InterfaceType.VLM: "llava:7b", + InterfaceType.EMBEDDING: "nomic-embed-text", + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "minimax": ProviderConfig( + "minimax": ProviderProfile( + key="minimax", api_key_env="MINIMAX_API_KEY", - default_base_url="https://api.minimax.chat/v1", + # International OpenAI-compatible endpoint (verified 2026-08-17: + # platform.minimax.io). The old api.minimax.chat domain is RETIRED. + base_url_env="MINIMAX_BASE_URL", + default_base_url="https://api.minimax.io/v1", + display_name="MiniMax", + error_display_name="MiniMax", + settings_key="minimax", + # MiniMax caches passively (no request key) and reports hits in + # usage.prompt_tokens_details.cached_tokens (which the reader counts). + # Sending prompt_cache_key is undocumented for MiniMax -> don't. + supports_prompt_cache_key=False, + # MiniMax's OpenAI-compat /v1 endpoint uses max_completion_tokens + # (not max_tokens); M2.x also inlines ... reasoning in + # content, which the transport strips. + uses_max_completion_tokens=True, + # MiniMax has no /v1/models endpoint — the connection tester must do + # a real (tiny) chat call against this model. + connection_test_model="MiniMax-M2.1", + openrouter_proxy=True, + openrouter_namespace="minimax", + openrouter_slug_map={ + # Slugs follow OpenRouter's lowercase convention; verify against + # openrouter.ai/models when bumping the MiniMax model family. + "MiniMax-M3": "minimax/minimax-m3", + "MiniMax-M2.1": "minimax/minimax-m2.1", + "MiniMax-M2": "minimax/minimax-m2", + }, + default_models={ + # MiniMax-Text-01 was retired upstream; M-series is current + # (M3 flagship, M2.x cheap tier). + InterfaceType.LLM: "MiniMax-M2.1", + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "deepseek": ProviderConfig( + "deepseek": ProviderProfile( + key="deepseek", + session_accumulation=True, api_key_env="DEEPSEEK_API_KEY", default_base_url="https://api.deepseek.com", + display_name="DeepSeek", + error_display_name="DeepSeek", + settings_key="deepseek", + supports_prompt_cache_key=True, + connection_test_model="deepseek-chat", + default_models={ + InterfaceType.LLM: "deepseek-chat", + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "moonshot": ProviderConfig( + "moonshot": ProviderProfile( + key="moonshot", api_key_env="MOONSHOT_API_KEY", - default_base_url="https://api.moonshot.cn/v1", + # International endpoint (verified 2026-08-17: platform.kimi.ai). + base_url_env="MOONSHOT_BASE_URL", + default_base_url="https://api.moonshot.ai/v1", + display_name="Moonshot", + error_display_name="Moonshot", + settings_key="moonshot", + # Kimi/Moonshot is a strict provider (rejects unknown request fields); + # prompt_cache_key acceptance is undocumented, so don't send it. Kimi + # caches context automatically; the reader counts cached_tokens. + supports_prompt_cache_key=False, + # Kimi thinking models (k2.5+) reject an explicit temperature — omit + # it (verified: Kimi API docs + Hermes kimi-coding profile). + fixed_temperature=OMIT_TEMPERATURE, + connection_test_model="kimi-k2.5", + openrouter_proxy=True, + openrouter_namespace="moonshotai", + openrouter_slug_map={ + "kimi-k2.5": "moonshotai/kimi-k2.5", + "moonshot-v1-8k": "moonshotai/moonshot-v1-8k", + "moonshot-v1-32k": "moonshotai/moonshot-v1-32k", + "moonshot-v1-128k": "moonshotai/moonshot-v1-128k", + "moonshot-v1-8k-vision-preview": ( + "moonshotai/moonshot-v1-8k-vision-preview" + ), + }, + default_models={ + InterfaceType.LLM: "kimi-k2.5", + InterfaceType.VLM: "moonshot-v1-8k-vision-preview", + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "grok": ProviderConfig( + "grok": ProviderProfile( + key="grok", + session_accumulation=True, api_key_env="XAI_API_KEY", default_base_url="https://api.x.ai/v1", + display_name="Grok (xAI)", + error_display_name="Grok", + settings_key="grok", + # Subscription OAuth (SuperGrok / X Premium+). xAI publicly endorsed + # this path in May 2026. + oauth_backend="grok", + subscription_label="Sign in with Grok", + subscription_models=("grok-4-0709", "grok-3"), + supports_prompt_cache_key=True, + default_models={ + InterfaceType.LLM: "grok-3", + InterfaceType.VLM: "grok-4-0709", + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "glm": ProviderConfig( + "glm": ProviderProfile( + key="glm", + session_accumulation=True, # Z.ai (Zhipu AI) GLM models -- OpenAI-compatible API. api_key_env="ZAI_API_KEY", default_base_url="https://api.z.ai/api/paas/v4", + display_name="Z.ai (GLM)", + settings_key="glm", + supports_prompt_cache_key=True, + connection_test_model="glm-5.2", + default_models={ + # Z.ai (Zhipu AI) GLM-5.2 -- 1M-context, OpenAI-compatible, + # multimodal. + InterfaceType.LLM: "glm-5.2", + InterfaceType.VLM: "glm-5.2", + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "fugu": ProviderConfig( + "fugu": ProviderProfile( + key="fugu", + session_accumulation=True, # Sakana AI Fugu -- OpenAI-compatible API. api_key_env="SAKANA_API_KEY", default_base_url="https://api.sakana.ai/v1", + display_name="Sakana (Fugu)", + settings_key="fugu", + supports_prompt_cache_key=True, + connection_test_model="fugu", + default_models={ + # Sakana AI Fugu -- OpenAI-compatible orchestration model. + # Text/LLM only; no native vision/embedding/image/video models. + InterfaceType.LLM: "fugu", + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "openrouter": ProviderConfig( + "openrouter": ProviderProfile( + key="openrouter", + session_accumulation=True, api_key_env="OPENROUTER_API_KEY", base_url_env="OPENROUTER_BASE_URL", default_base_url="https://openrouter.ai/api/v1", + display_name="OpenRouter", + error_display_name="OpenRouter", + settings_key="openrouter", + supports_prompt_cache_key=True, + supports_catalog_picker=True, + default_models={ + # OpenRouter slugs follow `/` format. Default to + # a Claude model so KV caching exercises the cache_control path on + # first use. + InterfaceType.LLM: "anthropic/claude-sonnet-4.5", + InterfaceType.VLM: "anthropic/claude-sonnet-4.5", + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), - "bedrock": ProviderConfig( + "bedrock": ProviderProfile( + key="bedrock", # Bedrock uses the boto3 credential chain (access_key / secret_key / # session_token) read from settings.json by the factory. There is no - # single API key, so api_key_env is left None. base_url_env carries - # the AWS region (e.g. "us-east-1") through the factory plumbing. + # single API key, so api_key_env stays None. base_url_env carries the + # AWS region (e.g. "us-east-1") through the factory plumbing. base_url_env="AWS_REGION", default_base_url="us-east-1", + display_name="AWS Bedrock", + wire="bedrock_converse", + requires_api_key=False, + aws_credential_block=True, + connection_test_model="us.anthropic.claude-haiku-4-5-20251001-v1:0", + default_models={ + # Default to Claude Haiku 4.5 — best price/performance on Bedrock + # with cachePoint support (5-min + 1-hour TTL). The `us.` prefix + # is the cross-region inference profile, required because Claude + # 4.x models reject on-demand invocations against the bare + # `anthropic.*` ID. The `us.anthropic.` prefix still matches + # `_BEDROCK_CACHE_PREFIXES`, so cachePoint is exercised. Users in + # EU / APAC regions should change `us.` to `eu.` / `ap.`. + # Haiku 4.5 also accepts image content blocks via Converse, so it + # doubles as the VLM default. Embedding stays on Titan. + InterfaceType.LLM: "us.anthropic.claude-haiku-4-5-20251001-v1:0", + InterfaceType.VLM: "us.anthropic.claude-haiku-4-5-20251001-v1:0", + InterfaceType.EMBEDDING: "amazon.titan-embed-text-v2:0", + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + # ── Phase 3 additions (endpoints verified 2026-08-17, see + # docs/PROVIDER_LAYER_CATCHUP.md section 8.3/8.4) ────────────────── + "groq": ProviderProfile( + key="groq", + session_accumulation=True, + api_key_env="GROQ_API_KEY", + base_url_env="GROQ_BASE_URL", + default_base_url="https://api.groq.com/openai/v1", + display_name="Groq", + settings_key="groq", + # The llama-3.x / llama-4-scout ids were decommissioned (Groq + # deprecations page, Aug 2026); gpt-oss are Groq's current + # general models. Groq deprecates `max_tokens` -> use + # max_completion_tokens; cap output to the model's limit. + connection_test_model="openai/gpt-oss-20b", + supports_model_discovery=True, + uses_max_completion_tokens=True, + max_output_tokens=32768, + default_models={ + InterfaceType.LLM: "openai/gpt-oss-120b", + # Groq's vision lineup churned (llama-4-scout gone); its current + # VLM id could not be confirmed against docs, so leave VLM off and + # let the model dropdown (discovery) surface it. Better no default + # than a dead/guessed id. + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "mistral": ProviderProfile( + key="mistral", + session_accumulation=True, + api_key_env="MISTRAL_API_KEY", + base_url_env="MISTRAL_BASE_URL", + default_base_url="https://api.mistral.ai/v1", + display_name="Mistral", + settings_key="mistral", + # "-latest" aliases insulate against Mistral's dated concrete ids. + connection_test_model="mistral-small-latest", + supports_model_discovery=True, + # Mistral La Plateforme has prompt caching and honors the + # `prompt_cache_key` field (cached tokens billed at 10%, reported in + # usage.prompt_tokens_details.cached_tokens — the field we read). + # Without this we send no cache key and get 0% hits on repeated + # prefixes (observed in a live session at slow_mode's TPM ceiling). + supports_prompt_cache_key=True, + default_models={ + InterfaceType.LLM: "mistral-large-latest", + # pixtral-large-latest was retired (2026-05-31). mistral-small is + # multimodal and current, so it doubles as the VLM default. + InterfaceType.VLM: "mistral-small-latest", + InterfaceType.EMBEDDING: "mistral-embed", + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "together": ProviderProfile( + key="together", + session_accumulation=True, + api_key_env="TOGETHER_API_KEY", + base_url_env="TOGETHER_BASE_URL", + # api.together.ai is the current docs domain; the legacy + # api.together.xyz still resolves. + default_base_url="https://api.together.ai/v1", + display_name="Together AI", + settings_key="together", + connection_test_model="meta-llama/Llama-3.1-8B-Instruct-Turbo", + supports_model_discovery=True, + # Together's serverless models cap output well below our default; + # Llama-3.3-70B ~16k. Exceeding it 4xxes. + max_output_tokens=16384, + default_models={ + InterfaceType.LLM: "meta-llama/Llama-3.3-70B-Instruct-Turbo", + InterfaceType.VLM: "meta-llama/Llama-4-Scout-17B-16E-Instruct", + InterfaceType.EMBEDDING: "BAAI/bge-large-en-v1.5", + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "fireworks": ProviderProfile( + key="fireworks", + session_accumulation=True, + api_key_env="FIREWORKS_API_KEY", + base_url_env="FIREWORKS_BASE_URL", + default_base_url="https://api.fireworks.ai/inference/v1", + display_name="Fireworks", + settings_key="fireworks", + connection_test_model="accounts/fireworks/models/llama-v3p1-8b-instruct", + supports_model_discovery=True, + default_models={ + # `p`-for-dot version naming: v3p3 = 3.3, qwen2p5-vl = Qwen2.5-VL. + # llama-v3p3-70b's serverless availability is ambiguous; glm-5p2 is + # a Hermes-verified served Fireworks model (their aux default). + InterfaceType.LLM: "accounts/fireworks/models/glm-5p2", + InterfaceType.VLM: "accounts/fireworks/models/qwen2p5-vl-32b-instruct", + InterfaceType.EMBEDDING: ( + "accounts/fireworks/models/nomic-embed-text-v1.5" + ), + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "cerebras": ProviderProfile( + key="cerebras", + session_accumulation=True, + api_key_env="CEREBRAS_API_KEY", + base_url_env="CEREBRAS_BASE_URL", + default_base_url="https://api.cerebras.ai/v1", + display_name="Cerebras", + settings_key="cerebras", + # Cerebras' catalog is small and churns; gpt-oss-120b is their + # long-lived production model (stable pick for the auth test too). + connection_test_model="gpt-oss-120b", + supports_model_discovery=True, + # Cerebras' OpenAI-compat endpoint documents max_completion_tokens; + # gpt-oss-120b output cap is 32k (free) — exceeding it errors. + uses_max_completion_tokens=True, + max_output_tokens=32000, + # Cerebras documents prompt_cache_key as a routing hint (max 1024 + # chars) and reports hits in prompt_tokens_details.cached_tokens. + supports_prompt_cache_key=True, + default_models={ + InterfaceType.LLM: "gpt-oss-120b", + # Cerebras is a speed-focused, text-only inference stack — no + # vision model on the chat endpoint (VLM field stays hidden). + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "qwen": ProviderProfile( + key="qwen", + session_accumulation=True, + api_key_env="DASHSCOPE_API_KEY", + base_url_env="DASHSCOPE_BASE_URL", + # International (Singapore) endpoint; keys are region-scoped. + # Alibaba is migrating to workspace-scoped maas.aliyuncs.com + # domains, but this legacy intl domain needs no WorkspaceId. + default_base_url="https://dashscope-intl.aliyuncs.com/compatible-mode/v1", + display_name="Qwen (Alibaba)", + settings_key="qwen", + connection_test_model="qwen-flash", + supports_model_discovery=True, + # DashScope documents temperature range [0, 2) and "do not set to 0" + # — send a small positive value instead of the caller's 0.0. + # (json_object works because our prompts contain the word "json".) + fixed_temperature=0.01, + # Qwen output caps are well under our default; 8k is a safe ceiling. + max_output_tokens=8192, + default_models={ + InterfaceType.LLM: "qwen-max", + InterfaceType.VLM: "qwen-vl-max", + InterfaceType.EMBEDDING: "text-embedding-v4", + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "huggingface": ProviderProfile( + key="huggingface", + session_accumulation=True, + api_key_env="HF_TOKEN", + base_url_env="HF_ROUTER_BASE_URL", + default_base_url="https://router.huggingface.co/v1", + display_name="Hugging Face", + settings_key="huggingface", + # Hub ids, optionally suffixed with a provider (":groq") or policy + # (":fastest" default, ":cheapest"). + connection_test_model="meta-llama/Llama-3.1-8B-Instruct", + supports_model_discovery=True, + default_models={ + InterfaceType.LLM: "deepseek-ai/DeepSeek-V3-0324", + InterfaceType.VLM: "Qwen/Qwen2.5-VL-7B-Instruct", + # The router's OpenAI-compatible /v1 surface is chat-only; no + # /v1/embeddings, so no embedding default here. + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "nvidia": ProviderProfile( + key="nvidia", + session_accumulation=True, + api_key_env="NVIDIA_API_KEY", + base_url_env="NVIDIA_BASE_URL", + default_base_url="https://integrate.api.nvidia.com/v1", + display_name="NVIDIA NIM", + settings_key="nvidia", + connection_test_model="meta/llama-3.1-8b-instruct", + supports_model_discovery=True, + # NIM caps output low; Hermes ships 16384 as its battle-tested value. + max_output_tokens=16384, + default_models={ + InterfaceType.LLM: "meta/llama-3.3-70b-instruct", + InterfaceType.VLM: "meta/llama-3.2-90b-vision-instruct", + InterfaceType.EMBEDDING: "nvidia/nv-embedqa-e5-v5", + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "perplexity": ProviderProfile( + key="perplexity", + session_accumulation=True, + api_key_env="PERPLEXITY_API_KEY", + base_url_env="PERPLEXITY_BASE_URL", + # Legacy Sonar chat-completions surface. NOTE: Perplexity retires the + # Sonar tiers on 2026-09-27; the successor is the Agent API + # (https://api.perplexity.ai/docs/agent-api). Migrate before then. + default_base_url="https://api.perplexity.ai", + display_name="Perplexity", + settings_key="perplexity", + connection_test_model="sonar", + # Perplexity's chat API hard-400s on response_format json_object (it + # only accepts text/json_schema); fall back to prompt-instructed JSON. + supports_json_object=False, + # Perplexity does NOT expose GET /v1/models and we don't ship a + # hardcoded model list (it would go stale), so the settings UI renders + # a free-text model box. sonar-pro is the pre-filled default below. + default_models={ + InterfaceType.LLM: "sonar-pro", + # Sonar accepts image input but is search-grounded, not a general + # describe-image VLM — intentionally no VLM default. + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + # GitHub Copilot is temporarily disabled (commented out, not removed). + # The OAuth backend (craftos_integrations/.../llm_oauth/copilot.py) and its + # tests remain intact; re-enable by uncommenting this profile. + # "copilot": ProviderProfile( + # key="copilot", + # session_accumulation=True, + # # Subscription-only provider: no API-key mode exists. Without a + # # connected GitHub Copilot seat, calls fail with a classified auth + # # error pointing at Settings. + # default_base_url="https://api.githubcopilot.com", + # display_name="GitHub Copilot", + # requires_api_key=False, + # oauth_backend="copilot", + # subscription_label="Sign in with GitHub", + # subscription_models=("gpt-4o", "gpt-5.2"), + # subscription_default_model="gpt-4o", + # connection_test_model="gpt-4o", + # default_models={ + # InterfaceType.LLM: "gpt-4o", + # InterfaceType.VLM: None, + # InterfaceType.EMBEDDING: None, + # InterfaceType.IMAGE_GEN: None, + # InterfaceType.VIDEO_GEN: None, + # }, + # ), + "lmstudio": ProviderProfile( + key="lmstudio", + session_accumulation=True, + base_url_env="LMSTUDIO_BASE_URL", + default_base_url="http://localhost:1234/v1", + display_name="LM Studio (Local)", + requires_api_key=False, + # LM Studio's OpenAI-compat endpoint supports json_schema, not + # json_object — omit response_format and rely on prompt-instructed + # JSON (vLLM and llama.cpp DO accept json_object, so they keep it). + supports_json_object=False, + # Discovery fills the real loaded model from the running server; the + # default below is a common LM Studio download so the field is never + # blank pre-discovery (mirrors Ollama's llama3.2:3b default). + supports_model_discovery=True, + local_kind="lmstudio", + connection_test_model="openai/gpt-oss-20b", + default_models={ + InterfaceType.LLM: "openai/gpt-oss-20b", + # Vision runs if the user has a VLM loaded; Qwen2.5-VL is a common + # LM Studio vision download. + InterfaceType.VLM: "qwen2.5-vl-7b-instruct", + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "vllm": ProviderProfile( + key="vllm", + session_accumulation=True, + base_url_env="VLLM_BASE_URL", + default_base_url="http://localhost:8000/v1", + display_name="vLLM (Local)", + requires_api_key=False, + # vLLM serves exactly one model — discovery reads /v1/models and + # fills it; the default is the canonical vLLM example model. + supports_model_discovery=True, + connection_test_model="meta-llama/Llama-3.1-8B-Instruct", + default_models={ + InterfaceType.LLM: "meta-llama/Llama-3.1-8B-Instruct", + InterfaceType.VLM: "Qwen/Qwen2.5-VL-7B-Instruct", + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ), + "llamacpp": ProviderProfile( + key="llamacpp", + session_accumulation=True, + base_url_env="LLAMACPP_BASE_URL", + default_base_url="http://localhost:8080/v1", + display_name="llama.cpp (Local)", + requires_api_key=False, + # llama-server serves one loaded model — discovery reads /v1/models + # (always a single element) and fills it. + supports_model_discovery=True, + connection_test_model="llama-3.1-8b-instruct", + default_models={ + InterfaceType.LLM: "llama-3.1-8b-instruct", + InterfaceType.VLM: None, + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, ), } + + +def get_profile(provider: str) -> ProviderProfile: + """Return the profile for ``provider``. + + Raises the same ``ValueError`` the factory has always raised for unknown + providers, so the error contract is unchanged. + """ + try: + return PROVIDER_CONFIG[provider] + except KeyError: + raise ValueError(f"Unsupported provider: {provider}") from None + + +def _sanity_check_profiles() -> None: + """Registry invariants (Phase 0/1 gate; cheap, import-time).""" + seen_settings_keys: Dict[str, str] = {} + for key, profile in PROVIDER_CONFIG.items(): + assert profile.key == key, f"profile.key mismatch for {key!r}" + assert profile.display_name, f"missing display_name for {key!r}" + if profile.requires_api_key: + assert profile.api_key_env, f"missing api_key_env for {key!r}" + assert profile.settings_key, f"missing settings_key for {key!r}" + if profile.settings_key: + prior = seen_settings_keys.setdefault(profile.settings_key, key) + assert prior == key, ( + f"settings_key {profile.settings_key!r} shared by " + f"{prior!r} and {key!r}" + ) + + +_sanity_check_profiles() diff --git a/agent_core/core/models/registry.py b/agent_core/core/models/registry.py new file mode 100644 index 00000000..05b5a732 --- /dev/null +++ b/agent_core/core/models/registry.py @@ -0,0 +1,254 @@ +# -*- coding: utf-8 -*- +"""Registry derivations over ProviderProfile (Phase 1, FR-1). + +Every structure that used to be a hand-maintained literal is derived here +from PROVIDER_CONFIG, so a provider added to provider_config.py appears +everywhere (settings UI, CLI, connection tester, factory) with zero extra +code. Derived outputs are shape-identical to the old literals; the contract +tests in tests/settings/test_self_config_contract.py pin this. + +Phase 3 extends ``get_registry`` with user-defined custom providers loaded +from settings.json (custom_providers block). +""" + +from __future__ import annotations + +from typing import Any, Dict, Optional, Tuple + +from agent_core.core.models.provider_config import ( + PROVIDER_CONFIG, + ProviderProfile, + get_profile, +) + +__all__ = [ + "PROVIDER_CONFIG", + "ProviderProfile", + "get_profile", + "get_registry", + "provider_info", + "provider_settings_keys", + "cli_providers", + "display_name", + "error_display_map", + "default_models_registry", +] + + +# Custom providers are restricted to the OpenAI-compatible wire for now: +# the factory's other wires bind provider-specific clients (Anthropic SDK +# without base_url override, boto3, Gemini) that a user endpoint can't use. +_ALLOWED_CUSTOM_WIRES = ("chat_completions",) + + +def _load_custom_specs() -> Dict[str, Any]: + """settings.json custom_providers block, via the app layer when present. + + agent_core stays importable without the app package (same deferred-import + pattern the factory uses for app.config); no app -> no custom providers. + """ + try: + from app.config import get_custom_providers + + specs = get_custom_providers() + return specs if isinstance(specs, dict) else {} + except Exception: + return {} + + +def _build_custom_profile(name: str, spec: Any) -> Optional[ProviderProfile]: + """Validate one custom_providers entry into a ProviderProfile. + + Invalid entries are skipped with a warning — a misconfigured custom + provider must never brick startup (the agent edits this block itself). + """ + import logging + from urllib.parse import urlparse + + log = logging.getLogger(__name__) + + if not isinstance(spec, dict): + log.warning(f"[REGISTRY] custom provider {name!r}: spec must be an object") + return None + if name in PROVIDER_CONFIG: + log.warning( + f"[REGISTRY] custom provider {name!r} collides with a built-in; ignored" + ) + return None + base_url = spec.get("base_url") + parsed = urlparse(base_url) if isinstance(base_url, str) else None + if parsed is None or parsed.scheme not in ("http", "https") or not parsed.netloc: + log.warning( + f"[REGISTRY] custom provider {name!r}: base_url must be an http(s) URL" + ) + return None + wire = spec.get("wire", "chat_completions") + if wire not in _ALLOWED_CUSTOM_WIRES: + log.warning( + f"[REGISTRY] custom provider {name!r}: wire {wire!r} not supported " + f"(allowed: {', '.join(_ALLOWED_CUSTOM_WIRES)})" + ) + return None + + models = spec.get("models") or [] + default_model = spec.get("default_model") or (models[0] if models else None) + + from agent_core.core.models.types import InterfaceType + + return ProviderProfile( + key=name, + display_name=str(spec.get("display_name") or name), + wire=wire, + api_key_env=spec.get("api_key_env"), + default_base_url=base_url, + settings_key=name, + requires_api_key=bool(spec.get("requires_api_key", True)), + supports_prompt_cache_key=bool( + spec.get("supports_prompt_cache_key", False) + ), + # New chat_completions providers get session accumulation — it is + # the correct behavior; only legacy minimax/moonshot opt out. + session_accumulation=True, + default_headers=dict(spec.get("headers") or {}), + connection_test_model=spec.get("connection_test_model") or default_model, + default_models={ + InterfaceType.LLM: default_model, + InterfaceType.VLM: spec.get("vlm_model"), + InterfaceType.EMBEDDING: None, + InterfaceType.IMAGE_GEN: None, + InterfaceType.VIDEO_GEN: None, + }, + ) + + +def get_registry() -> Dict[str, ProviderProfile]: + """Built-in profiles merged with settings.json custom providers. + + Called per lookup (factory create, connection test, session-list + derivations) so a reload_settings() after editing custom_providers takes + effect without restart. The import-time constants (PROVIDER_INFO, + MODEL_REGISTRY, PROVIDER_TO_SETTINGS_KEY) cover built-ins only. + """ + registry = dict(PROVIDER_CONFIG) + for name, spec in _load_custom_specs().items(): + profile = _build_custom_profile(name, spec) + if profile is not None: + registry[name] = profile + return registry + + +def provider_info() -> Dict[str, Dict[str, Any]]: + """Derive the PROVIDER_INFO dict consumed by the settings frontend. + + Key-presence rules mirror the retired literal exactly (NFR-4): + - api_key_env / settings_key only when set; + - base_url_env only for providers WITHOUT an API key (remote, bedrock); + byteplus/openrouter have overridable base URLs but never exposed them + here (OpenRouter's is hidden intentionally — power users set + endpoints.openrouter_base_url in settings.json by hand); + - subscription fields only for OAuth-capable providers; + - supports_catalog / is_bedrock only when true. + """ + info: Dict[str, Dict[str, Any]] = {} + for key, p in get_registry().items(): + entry: Dict[str, Any] = {"name": p.display_name} + if p.api_key_env: + entry["api_key_env"] = p.api_key_env + if p.settings_key: + entry["settings_key"] = p.settings_key + entry["requires_api_key"] = p.requires_api_key + if p.oauth_backend: + entry["supports_subscription_oauth"] = True + entry["subscription_label"] = p.subscription_label + entry["subscription_models"] = list(p.subscription_models) + if p.subscription_default_model: + entry["subscription_default_model"] = p.subscription_default_model + if p.supports_catalog_picker: + entry["supports_catalog"] = True + if p.aws_credential_block: + entry["is_bedrock"] = True + if p.base_url_env and not p.api_key_env: + entry["base_url_env"] = p.base_url_env + info[key] = entry + return info + + +def provider_settings_keys() -> Dict[str, str]: + """Derive PROVIDER_TO_SETTINGS_KEY (settings.json api_keys mapping). + + Includes the legacy "google" alias: callers historically passed either + the provider key ("gemini") or the settings key ("google"); both resolve + to api_keys.google. Bedrock is deliberately absent (no single API key — + credentials live under aws_credentials), so ``.get("bedrock")`` returns + None and the save path routes accordingly. + """ + mapping = { + key: p.settings_key + for key, p in get_registry().items() + if p.settings_key + } + mapping["google"] = "google" + return mapping + + +def cli_providers() -> Dict[str, Tuple[Optional[str], str]]: + """Derive the /provider command's provider table. + + Shape: {provider: (api_key_env or None, display_name)}. Unlike the + retired literal this covers EVERY registry provider (the old dict was + missing minimax/moonshot/bedrock — a hand-sync failure this derivation + makes impossible). + """ + return { + key: (p.api_key_env, p.display_name) + for key, p in get_registry().items() + } + + +def display_name(provider: str) -> str: + """Settings-UI display name, falling back to the raw key.""" + p = get_registry().get(provider) + return p.display_name if p else provider + + +def error_display_map() -> Dict[str, str]: + """Derive the factory's short error-message display map. + + Only providers with an explicit error_display_name appear; callers keep + the historical ``.get(provider, provider)`` fallback so the output is + byte-identical to the retired factory._PROVIDER_DISPLAY literal. + """ + return { + key: p.error_display_name + for key, p in get_registry().items() + if p.error_display_name + } + + +def default_models_registry() -> Dict[str, Dict[Any, Optional[str]]]: + """Derive MODEL_REGISTRY ({provider: {InterfaceType: default model}}).""" + return {key: dict(p.default_models) for key, p in get_registry().items()} + + +def session_cc_providers() -> frozenset: + """chat_completions providers whose session path accumulates history + (the _openai_compat_session_messages / openrouter-anthropic buffers). + + Replaces the hand-maintained tuple in interface.py's session dispatcher + and create_session_cache. minimax/moonshot stay excluded + (session_accumulation=False) to preserve their historical stateless + session behavior. + """ + return frozenset( + key + for key, p in get_registry().items() + if p.wire == "chat_completions" and p.session_accumulation + ) + + +def supports_prompt_cache_key(provider: str) -> bool: + """Whether the chat_completions transport may emit ``prompt_cache_key`` + for this provider. Opt-in per profile: some OpenAI-compatible endpoints + reject unknown top-level fields rather than ignoring them.""" + p = get_registry().get(provider) + return bool(p and p.supports_prompt_cache_key) diff --git a/agent_core/core/prompts/action.py b/agent_core/core/prompts/action.py index d35958a9..27d48ca1 100644 --- a/agent_core/core/prompts/action.py +++ b/agent_core/core/prompts/action.py @@ -9,7 +9,11 @@ # The one action-selection prompt for session turns. # core.impl.action.router.ActionRouter.select_action_in_session -# KV CACHING OPTIMIZED: Static content FIRST, session-static in MIDDLE, dynamic (event_stream) LAST +# KV CACHING OPTIMIZED: the cacheable prefix runs static content FIRST, then +# session-static, then the append-only {event_stream}. The only truly per-turn +# volatile block, {current_turn}/{query}, goes LAST so it never sits in front +# of the growing stream and cap the cache. (The trigger is already written into +# the event stream at claim time, so {query} here is a redundant restatement.) SELECT_ACTION_PROMPT = """ You are running one turn of a persistent session. A "run" starts when input @@ -235,21 +239,21 @@ {session_state} +--- + +{event_stream} + This run woke up because of the following trigger: {query} -The trigger is the reason for this turn — not the whole picture. Your +The trigger is the reason for this turn, not the whole picture. Your objective lives in the session itself: the conversation and events in the stream, your todos, and any requirements you have set. Reason about the session's current state, then select the next action(s) and provide the input parameters so they can be executed immediately. ---- - -{event_stream} - {integration_essentials} """ diff --git a/agent_file_system/AGENT.md b/agent_file_system/AGENT.md index cdb3b9d5..a030107d 100644 --- a/agent_file_system/AGENT.md +++ b/agent_file_system/AGENT.md @@ -924,7 +924,7 @@ workspace/living_ui/_/ - `logs/pocketbase.log` (server-side) and `logs/frontend_console.log` (browser console): first place to grep when a project misbehaves. - Imported non-V2 apps register as **external** apps: they carry `craftbot.json` (install/build/start/health verbs, `{{PORT}}`) instead of `manifest.json` and log to `logs/app.log`. -The fresh-project scaffold lives at [living-ui-v2/blueprint/](living-ui-v2/blueprint/). For lifecycle, see `## Living UI`. +The fresh-project scaffold lives at [living-ui/blueprint/](living-ui/blueprint/). For lifecycle, see `## Living UI`. ### Files outside agent_file_system/ @@ -1146,7 +1146,7 @@ DO NOT silently change FORMAT.md. The user owns their style guide. ## Living UI -"Living UI" = generated web apps served from CraftBot. Every project is a React frontend (vendored kit, shadcn-conventional components) plus one PocketBase backend process. Lifecycle is driven through the `living_ui` action set ([app/data/action/living_ui_actions.py](app/data/action/living_ui_actions.py)). The fresh-project scaffold lives at [living-ui-v2/blueprint/](living-ui-v2/blueprint/). File layout: see `## File System` "Living UI projects". +"Living UI" = generated web apps served from CraftBot. Every project is a React frontend (vendored kit, shadcn-conventional components) plus one PocketBase backend process. Lifecycle is driven through the `living_ui` action set ([app/data/action/living_ui_actions.py](app/data/action/living_ui_actions.py)). The fresh-project scaffold lives at [living-ui/blueprint/](living-ui/blueprint/). File layout: see `## File System` "Living UI projects". ### Action surface (`living_ui` set) @@ -1182,9 +1182,9 @@ living_ui_marketplace_list() / living_ui_marketplace_install(app_id, ...) Install pre-built marketplace apps. As-is installs skip walk_verify. living_ui_import_zip(zip_path) / -living_ui_import(source) Import a V2 project from ZIP / local folder / git URL. - Non-V2 sources register as external apps (craftbot.json). -living_ui_convert(source, ...) Rebuild a foreign app as V2: fresh scaffold, original kept +living_ui_import(source) Import a Living UI project from ZIP / local folder / git URL. + Non-Living-UI sources register as external apps (craftbot.json). +living_ui_convert(source, ...) Rebuild a foreign app as a Living UI: fresh scaffold, original kept in reference/source/, requirements synthesized, supervised build dispatched. ``` @@ -1194,10 +1194,10 @@ living_ui_convert(source, ...) Rebuild a foreign app as V2: fresh s Read/write a project's live data with the lui CLI via `run_shell` (absolute paths required): ``` -node /living-ui-v2/tools/src/cli.ts data schema -node /living-ui-v2/tools/src/cli.ts data list|create|update|delete ... -node /living-ui-v2/tools/src/cli.ts run --param value -node /living-ui-v2/tools/src/cli.ts ops +node /living-ui/tools/src/cli.ts data schema +node /living-ui/tools/src/cli.ts data list|create|update|delete ... +node /living-ui/tools/src/cli.ts run --param value +node /living-ui/tools/src/cli.ts ops ``` `living_ui_usage(project_id)` returns the exact commands for a given project. Use `living_ui_http` only when the CLI cannot do it. Writes to a delivered app's real data outside a staging arc are refused. @@ -1510,8 +1510,8 @@ Run `/help` for the live list. If you need to verify a specific command, read it /exit quit the application /update (alias /upgrade) check for updates and update CraftBot [--check] /tokens show this session's token usage (input / cached / output / total) -/provider [name] [key] view or switch LLM provider (openai, gemini, anthropic, byteplus, - deepseek, grok, glm, fugu, openrouter, remote) and set its key +/provider [name] [key] view or switch LLM provider (any registered provider, + see ## Models) and set its key ``` ### Credential and integration overview @@ -1759,9 +1759,8 @@ memory: item_word_limit: int (default 150; words per stored memory item) model: - llm_provider: "openai" | "anthropic" | "gemini" | "byteplus" | "deepseek" | - "minimax" | "moonshot" | "grok" | "glm" | "fugu" | "openrouter" | - "bedrock" | "remote" + llm_provider: any provider key registered in the code (see ## Models); + this doc does not enumerate them; the registry is authoritative vlm_provider: same options (VLM-capable providers only) image_gen_provider / video_gen_provider: string llm_model: string | null (null = provider default; e.g. "claude-sonnet-4-6") @@ -2832,29 +2831,26 @@ Each interface picks its provider and model independently: `model.llm_provider`, ### Providers and what they support -From [MODEL_REGISTRY](agent_core/core/models/model_registry.py) — 13 providers: - -``` -provider LLM default model VLM default model notes -───────── ───────────────────────────────────── ────────────────────────── ───────────────────────────── -openai gpt-5.2-2025-12-11 gpt-5.2-2025-12-11 embedding text-embedding-3-small; image gpt-image-2; video sora-2 -anthropic claude-sonnet-4-6 claude-sonnet-4-6 no embedding -gemini gemini-2.5-pro gemini-2.5-pro embedding text-embedding-004; image gemini-3-pro-image; video veo-3.1-generate-preview -byteplus seed-2-0-pro-260328 seed-2-0-pro-260328 embedding skylark; video seedance-1-0-pro-fast-251015 -remote llama3.2:3b llava:7b Ollama or OpenAI-compat; embedding nomic-embed-text -deepseek deepseek-chat (none) text only -moonshot kimi-k2.5 moonshot-v1-8k-vision-preview -grok grok-3 grok-4-0709 xAI -minimax MiniMax-Text-01 MiniMax-VL-01 -glm glm-5.2 glm-5.2 Z.ai (GLM), OpenAI-compat -fugu fugu (none) Sakana (Fugu), text only -openrouter anthropic/claude-sonnet-4.5 anthropic/claude-sonnet-4.5 proxy to many models -bedrock us.anthropic.claude-haiku-4-5-20251001-v1:0 same AWS; embedding amazon.titan-embed-text-v2:0; model IDs need the us. cross-region prefix -``` - -If you set `model.llm_model: null` in settings.json, the default from MODEL_REGISTRY is used. Set an explicit string to override. - -A provider with `(none)` for VLM cannot be used as `vlm_provider`. If the user asks for vision but only has a text-only provider configured, tell them to set a separate `vlm_provider`. +The set of providers is defined in code (PROVIDER_CONFIG in +[provider_config.py](agent_core/core/models/provider_config.py), surfaced as +[MODEL_REGISTRY](agent_core/core/models/model_registry.py)) and GROWS over +time. This document deliberately does NOT enumerate the providers: any list +here goes stale the moment a provider is added. The code registry is the +single source of truth. + +What this means for you: +- NEVER refuse a provider switch because a name is not in a list you + remember. If the user names a provider, attempt the switch (procedure + below). If the name is not registered, the switch code returns a clear, + classified error, which you surface. Do not pre-judge from this doc. +- To see the providers that exist right now, read PROVIDER_CONFIG in the + file above, or point the user at the Settings UI (it renders the live + list). +- Each provider ships a default LLM model id (and a default VLM id where the + provider supports vision). Setting `model.llm_model: null` uses that + registry default; an explicit string overrides it. + +A provider with no VLM default model cannot be used as `vlm_provider`; the code raises a clear error if you try. If the user asks for vision but only a text-only provider is configured, tell them to set a separate `vlm_provider`. Image generation falls back through providers in priority order `gemini, openai`; video generation `gemini, openai, byteplus`. Reinit paths: `reinitialize_image_gen` / `reinitialize_video_gen` (driven by the Settings UI save). @@ -2883,7 +2879,7 @@ bedrock (none — uses aws_credentials NO — Settings UI only block + endpoints.aws_region) ``` -When setting an API key for Gemini, edit `api_keys.google`, NOT `api_keys.gemini`. Same translation in the `api_keys_configured` block. Bedrock uses `aws_credentials.{access_key_id, secret_access_key, session_token}`, not `api_keys.*`. +When setting an API key for Gemini, edit `api_keys.google`, NOT `api_keys.gemini`. Same translation in the `api_keys_configured` block. Bedrock uses `aws_credentials.{access_key_id, secret_access_key, session_token}`, not `api_keys.*`. This table covers the original providers only; providers added later follow the default rule (their key lives in `api_keys.`). ### Model section schema (in settings.json) @@ -2915,22 +2911,23 @@ At construction (and on `reinitialize_llm`), `ModelFactory.create(provider, inte 4. Returns ctx with provider, model, client/handles, base URL, etc. ``` -The LLMInterface is constructed ONCE at startup (and reconstructed by `reinitialize_llm`). It is NOT recreated when settings.json is hot-reloaded. This is the most important gotcha in this section — see "Switching provider or model" below. +The LLMInterface is constructed ONCE at startup and reconstructed by `reinitialize_llm`; it does not re-read settings per call. BUT a config-watcher reload callback calls `reinitialize_llm` AUTOMATICALLY whenever the `model` section of settings.json changes, so editing settings.json switches the live provider/model on its own. See "Switching provider or model" below. ### Switching provider or model — through chat The user asks: "switch to GPT-5" or "use Gemini" or "I'd like to try Claude". -The one rule: **every model change requires a reinitialize.** The LLMInterface holds its provider client AND model name from construction; editing `settings.json` alone changes NOTHING on the live interface — nothing re-reads settings per call. This applies to same-provider model swaps too. +The one rule: **every model change requires a reinitialize, and the config watcher does that for you.** The LLMInterface holds its provider client and model name from construction and does not re-read settings per call, BUT a config-watcher reload callback calls `agent.reinitialize_llm` automatically when the `model` section of settings.json changes (llm_provider, llm_model, vlm_provider, or vlm_model). So editing settings.json yourself IS enough to switch, for both provider changes and same-provider model swaps. Reinitialize paths: ``` -Provider switch → user runs /provider [] - (saves settings + calls agent.reinitialize_llm) -Model-only swap → Settings UI save (persists + reinitializes; - /provider takes no model argument) -minimax / moonshot / → Settings UI only (/provider does not accept them) -bedrock +Provider or model switch → stream_edit the model section of settings.json; + the config watcher calls reinitialize_llm for + you (PRIMARY, self-service, every provider) +Also (user-driven) → /provider [] slash command, or a + Settings UI save; both persist + reinitialize. + /provider does not accept minimax/moonshot/ + bedrock; stream_edit does. Image / video gen change → Settings UI save (reinitialize_image_gen / _video_gen) ``` @@ -2939,17 +2936,17 @@ Procedure for a provider switch: 1. Ensure api_keys. for the new provider is set. Remember the gemini → "google" name translation. If empty: ask the user for a key, then stream_edit api_keys + api_keys_configured. -2. Tell the user to run: /provider [] - Examples: /provider openai sk-... - /provider anthropic - /provider gemini AIza... -3. Verify by waiting for the next LLM-driven response; mention the new provider - is in effect. +2. stream_edit model.llm_provider in settings.json (app/config/settings.json) + to the new provider name. If the user named a specific model, also set + model.llm_model; leave it null to use the provider's registry default. +3. The config watcher reinitializes the live LLM/VLM automatically, no + /provider needed. Verify by waiting for the next LLM-driven response and + confirm the new provider is in effect. ``` `reinitialize()` is a no-op if provider+model+key+base_url are all unchanged. A provider-unchanged reinit preserves session histories; a true provider change wipes them. -Symptoms of editing settings without reinit: replies still come from the old model, or `LLMConsecutiveFailureError` if the old client now lacks credentials. If the user cannot run the slash command or open Settings, the fallback is restarting CraftBot. State that explicitly. +If the config watcher is disabled or a reinit fails, replies still come from the old model, or `LLMConsecutiveFailureError` if the old client now lacks credentials. Fallbacks then are the /provider command, a Settings UI save, or restarting CraftBot. State that explicitly. ### Setting a missing API key (no provider switch) @@ -4514,7 +4511,7 @@ LIVING_UI.md per-project doc inside a Living UI project Living UI generated React + PocketBase apps served from CraftBot ## Living UI LLM large language model used for text generation ## Models LLMConsecutiveFailureError circuit-breaker on repeated LLM failures ## Errors / ## Models -lui CLI node CLI for Living UI data/ops (living-ui-v2/tools) ## Living UI +lui CLI node CLI for Living UI data/ops (living-ui/tools) ## Living UI MCP Model Context Protocol; external tool servers ## MCP mcp_ action set name registered when an MCP server connects ## MCP / ## Action Sets memory_search hybrid vector+BM25 action over indexed agent_file_system files ## Memory diff --git a/app/agent_base.py b/app/agent_base.py index 8e8068b4..f731ca64 100644 --- a/app/agent_base.py +++ b/app/agent_base.py @@ -3247,6 +3247,40 @@ async def _initialize_config_watcher(self) -> None: lambda new_settings, old_settings: invalidate_settings_cache() ) + # Reinitialize the live LLM/VLM when the model section changes, so + # editing settings.json alone (e.g. the agent's own stream_edit, or + # a hand edit) switches provider/model WITHOUT /provider, the + # Settings UI, or a restart. The interface holds its client from + # construction; only reinitialize_llm() rebuilds it. Registered + # AFTER the cache-invalidation callback above so the getters that + # reinitialize_llm() reads (api key, base URL, vlm/model) already + # return fresh values. + def _reinit_llm_on_model_change(new_settings, old_settings): + try: + old_model = (old_settings or {}).get("model", {}) or {} + new_model = (new_settings or {}).get("model", {}) or {} + watched = ( + "llm_provider", + "llm_model", + "vlm_provider", + "vlm_model", + ) + if any(old_model.get(k) != new_model.get(k) for k in watched): + new_provider = new_model.get("llm_provider") + logger.info( + "[CONFIG_WATCHER] model config changed " + f"(llm_provider={new_provider}); reinitializing " + "live LLM/VLM" + ) + self.reinitialize_llm(new_provider) + except Exception as exc: + logger.warning( + "[CONFIG_WATCHER] LLM reinit on settings change " + f"failed: {exc}" + ) + + settings_manager.register_reload_callback(_reinit_llm_on_model_change) + # Get event loop for async callbacks event_loop = asyncio.get_event_loop() diff --git a/app/config.py b/app/config.py index ac92ea20..396f1bcf 100644 --- a/app/config.py +++ b/app/config.py @@ -280,6 +280,52 @@ def get_api_key(provider: str) -> str: return api_keys.get(settings_key, "") +def get_extra_api_keys(provider: str) -> list: + """Extra pool credentials for a provider (Phase 5, FR-7). + + settings.json: {"extra_api_keys": {"": ["key2", "key3"]}}. + The primary key stays in api_keys (untouched agent self-config path); + extras only ever matter when the primary is cooling down. + """ + settings = get_settings() + block = settings.get("extra_api_keys", {}) + if not isinstance(block, dict): + return [] + # Accept both the provider key and its settings_key alias (gemini/google). + key_map = {"gemini": "google"} + entries = block.get(provider) or block.get(key_map.get(provider, provider)) or [] + return [k for k in entries if isinstance(k, str) and k] if isinstance(entries, list) else [] + + +def get_fallback_providers() -> list: + """Ordered cross-provider fallback chain (Phase 5, FR-9). + + settings.json: {"model": {"fallback_providers": ["openrouter", ...]}}. + Empty by default — fallback is strictly opt-in. + """ + settings = get_settings() + chain = settings.get("model", {}).get("fallback_providers", []) + return [p for p in chain if isinstance(p, str) and p] if isinstance(chain, list) else [] + + +def get_custom_providers() -> Dict[str, Any]: + """Return the user-defined custom_providers block from settings.json. + + Shape (Phase 3, docs/PROVIDER_LAYER_CATCHUP.md section 7.2): + {"": {"base_url": ..., "wire": ..., "api_key_env": ..., + "display_name": ..., "models": [...], "headers": {...}, + "supports_prompt_cache_key": bool}} + + Inline API keys are NOT stored here — save_custom_provider() routes them + into the regular api_keys block under the provider's name, so the whole + existing key plumbing (get_api_key, settings UI, agent self-config) + works unchanged for custom providers. + """ + settings = get_settings() + block = settings.get("custom_providers", {}) + return block if isinstance(block, dict) else {} + + def get_base_url(provider: str) -> Optional[str]: """Get base URL for a provider. diff --git a/app/data/agent_file_system_template/AGENT.md b/app/data/agent_file_system_template/AGENT.md index 28675368..a030107d 100644 --- a/app/data/agent_file_system_template/AGENT.md +++ b/app/data/agent_file_system_template/AGENT.md @@ -1510,8 +1510,8 @@ Run `/help` for the live list. If you need to verify a specific command, read it /exit quit the application /update (alias /upgrade) check for updates and update CraftBot [--check] /tokens show this session's token usage (input / cached / output / total) -/provider [name] [key] view or switch LLM provider (openai, gemini, anthropic, byteplus, - deepseek, grok, glm, fugu, openrouter, remote) and set its key +/provider [name] [key] view or switch LLM provider (any registered provider, + see ## Models) and set its key ``` ### Credential and integration overview @@ -1759,9 +1759,8 @@ memory: item_word_limit: int (default 150; words per stored memory item) model: - llm_provider: "openai" | "anthropic" | "gemini" | "byteplus" | "deepseek" | - "minimax" | "moonshot" | "grok" | "glm" | "fugu" | "openrouter" | - "bedrock" | "remote" + llm_provider: any provider key registered in the code (see ## Models); + this doc does not enumerate them; the registry is authoritative vlm_provider: same options (VLM-capable providers only) image_gen_provider / video_gen_provider: string llm_model: string | null (null = provider default; e.g. "claude-sonnet-4-6") @@ -2832,29 +2831,26 @@ Each interface picks its provider and model independently: `model.llm_provider`, ### Providers and what they support -From [MODEL_REGISTRY](agent_core/core/models/model_registry.py) — 13 providers: - -``` -provider LLM default model VLM default model notes -───────── ───────────────────────────────────── ────────────────────────── ───────────────────────────── -openai gpt-5.2-2025-12-11 gpt-5.2-2025-12-11 embedding text-embedding-3-small; image gpt-image-2; video sora-2 -anthropic claude-sonnet-4-6 claude-sonnet-4-6 no embedding -gemini gemini-2.5-pro gemini-2.5-pro embedding text-embedding-004; image gemini-3-pro-image; video veo-3.1-generate-preview -byteplus seed-2-0-pro-260328 seed-2-0-pro-260328 embedding skylark; video seedance-1-0-pro-fast-251015 -remote llama3.2:3b llava:7b Ollama or OpenAI-compat; embedding nomic-embed-text -deepseek deepseek-chat (none) text only -moonshot kimi-k2.5 moonshot-v1-8k-vision-preview -grok grok-3 grok-4-0709 xAI -minimax MiniMax-Text-01 MiniMax-VL-01 -glm glm-5.2 glm-5.2 Z.ai (GLM), OpenAI-compat -fugu fugu (none) Sakana (Fugu), text only -openrouter anthropic/claude-sonnet-4.5 anthropic/claude-sonnet-4.5 proxy to many models -bedrock us.anthropic.claude-haiku-4-5-20251001-v1:0 same AWS; embedding amazon.titan-embed-text-v2:0; model IDs need the us. cross-region prefix -``` - -If you set `model.llm_model: null` in settings.json, the default from MODEL_REGISTRY is used. Set an explicit string to override. - -A provider with `(none)` for VLM cannot be used as `vlm_provider`. If the user asks for vision but only has a text-only provider configured, tell them to set a separate `vlm_provider`. +The set of providers is defined in code (PROVIDER_CONFIG in +[provider_config.py](agent_core/core/models/provider_config.py), surfaced as +[MODEL_REGISTRY](agent_core/core/models/model_registry.py)) and GROWS over +time. This document deliberately does NOT enumerate the providers: any list +here goes stale the moment a provider is added. The code registry is the +single source of truth. + +What this means for you: +- NEVER refuse a provider switch because a name is not in a list you + remember. If the user names a provider, attempt the switch (procedure + below). If the name is not registered, the switch code returns a clear, + classified error, which you surface. Do not pre-judge from this doc. +- To see the providers that exist right now, read PROVIDER_CONFIG in the + file above, or point the user at the Settings UI (it renders the live + list). +- Each provider ships a default LLM model id (and a default VLM id where the + provider supports vision). Setting `model.llm_model: null` uses that + registry default; an explicit string overrides it. + +A provider with no VLM default model cannot be used as `vlm_provider`; the code raises a clear error if you try. If the user asks for vision but only a text-only provider is configured, tell them to set a separate `vlm_provider`. Image generation falls back through providers in priority order `gemini, openai`; video generation `gemini, openai, byteplus`. Reinit paths: `reinitialize_image_gen` / `reinitialize_video_gen` (driven by the Settings UI save). @@ -2883,7 +2879,7 @@ bedrock (none — uses aws_credentials NO — Settings UI only block + endpoints.aws_region) ``` -When setting an API key for Gemini, edit `api_keys.google`, NOT `api_keys.gemini`. Same translation in the `api_keys_configured` block. Bedrock uses `aws_credentials.{access_key_id, secret_access_key, session_token}`, not `api_keys.*`. +When setting an API key for Gemini, edit `api_keys.google`, NOT `api_keys.gemini`. Same translation in the `api_keys_configured` block. Bedrock uses `aws_credentials.{access_key_id, secret_access_key, session_token}`, not `api_keys.*`. This table covers the original providers only; providers added later follow the default rule (their key lives in `api_keys.`). ### Model section schema (in settings.json) @@ -2915,22 +2911,23 @@ At construction (and on `reinitialize_llm`), `ModelFactory.create(provider, inte 4. Returns ctx with provider, model, client/handles, base URL, etc. ``` -The LLMInterface is constructed ONCE at startup (and reconstructed by `reinitialize_llm`). It is NOT recreated when settings.json is hot-reloaded. This is the most important gotcha in this section — see "Switching provider or model" below. +The LLMInterface is constructed ONCE at startup and reconstructed by `reinitialize_llm`; it does not re-read settings per call. BUT a config-watcher reload callback calls `reinitialize_llm` AUTOMATICALLY whenever the `model` section of settings.json changes, so editing settings.json switches the live provider/model on its own. See "Switching provider or model" below. ### Switching provider or model — through chat The user asks: "switch to GPT-5" or "use Gemini" or "I'd like to try Claude". -The one rule: **every model change requires a reinitialize.** The LLMInterface holds its provider client AND model name from construction; editing `settings.json` alone changes NOTHING on the live interface — nothing re-reads settings per call. This applies to same-provider model swaps too. +The one rule: **every model change requires a reinitialize, and the config watcher does that for you.** The LLMInterface holds its provider client and model name from construction and does not re-read settings per call, BUT a config-watcher reload callback calls `agent.reinitialize_llm` automatically when the `model` section of settings.json changes (llm_provider, llm_model, vlm_provider, or vlm_model). So editing settings.json yourself IS enough to switch, for both provider changes and same-provider model swaps. Reinitialize paths: ``` -Provider switch → user runs /provider [] - (saves settings + calls agent.reinitialize_llm) -Model-only swap → Settings UI save (persists + reinitializes; - /provider takes no model argument) -minimax / moonshot / → Settings UI only (/provider does not accept them) -bedrock +Provider or model switch → stream_edit the model section of settings.json; + the config watcher calls reinitialize_llm for + you (PRIMARY, self-service, every provider) +Also (user-driven) → /provider [] slash command, or a + Settings UI save; both persist + reinitialize. + /provider does not accept minimax/moonshot/ + bedrock; stream_edit does. Image / video gen change → Settings UI save (reinitialize_image_gen / _video_gen) ``` @@ -2939,17 +2936,17 @@ Procedure for a provider switch: 1. Ensure api_keys. for the new provider is set. Remember the gemini → "google" name translation. If empty: ask the user for a key, then stream_edit api_keys + api_keys_configured. -2. Tell the user to run: /provider [] - Examples: /provider openai sk-... - /provider anthropic - /provider gemini AIza... -3. Verify by waiting for the next LLM-driven response; mention the new provider - is in effect. +2. stream_edit model.llm_provider in settings.json (app/config/settings.json) + to the new provider name. If the user named a specific model, also set + model.llm_model; leave it null to use the provider's registry default. +3. The config watcher reinitializes the live LLM/VLM automatically, no + /provider needed. Verify by waiting for the next LLM-driven response and + confirm the new provider is in effect. ``` `reinitialize()` is a no-op if provider+model+key+base_url are all unchanged. A provider-unchanged reinit preserves session histories; a true provider change wipes them. -Symptoms of editing settings without reinit: replies still come from the old model, or `LLMConsecutiveFailureError` if the old client now lacks credentials. If the user cannot run the slash command or open Settings, the fallback is restarting CraftBot. State that explicitly. +If the config watcher is disabled or a reinit fails, replies still come from the old model, or `LLMConsecutiveFailureError` if the old client now lacks credentials. Fallbacks then are the /provider command, a Settings UI save, or restarting CraftBot. State that explicitly. ### Setting a missing API key (no provider switch) diff --git a/app/models/provider_config.py b/app/models/provider_config.py index d20f824c..d3c1b151 100644 --- a/app/models/provider_config.py +++ b/app/models/provider_config.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- -"""Re-export PROVIDER_CONFIG from agent_core.""" +"""Re-export provider profiles from agent_core.""" from agent_core import PROVIDER_CONFIG +from agent_core.core.models.provider_config import ( + ProviderConfig, + ProviderProfile, + get_profile, +) -__all__ = ["PROVIDER_CONFIG"] +__all__ = ["PROVIDER_CONFIG", "ProviderConfig", "ProviderProfile", "get_profile"] diff --git a/app/ui_layer/adapters/browser_adapter.py b/app/ui_layer/adapters/browser_adapter.py index 7a7c8834..aee0ce26 100644 --- a/app/ui_layer/adapters/browser_adapter.py +++ b/app/ui_layer/adapters/browser_adapter.py @@ -58,6 +58,7 @@ test_connection, validate_can_save, get_ollama_models, + get_provider_models, # Subscription OAuth (ChatGPT Plus/Pro, SuperGrok) complete_subscription, connect_subscription_async, @@ -1486,6 +1487,13 @@ async def _handle_ws_message(self, data: Dict[str, Any], ws=None) -> None: base_url = data.get("baseUrl") await self._handle_ollama_models_get(base_url) + elif msg_type == "provider_models_get": + await self._handle_provider_models_get( + provider=data.get("provider", ""), + base_url=data.get("baseUrl"), + api_key=data.get("apiKey"), + ) + elif msg_type == "openrouter_models_get": await self._handle_openrouter_models_get( base_url=data.get("baseUrl"), @@ -5465,6 +5473,31 @@ async def _handle_ollama_models_get(self, base_url: Optional[str] = None) -> Non } ) + async def _handle_provider_models_get( + self, + provider: str, + base_url: Optional[str] = None, + api_key: Optional[str] = None, + ) -> None: + """Fetch a provider's models via GET /v1/models and broadcast them. + + Wire-generic sibling of _handle_ollama_models_get; drives the model + dropdown for the new cloud + local providers. Runs the blocking HTTP + call off the event loop. + """ + try: + result = await asyncio.to_thread( + get_provider_models, provider, base_url, api_key + ) + await self._broadcast({"type": "provider_models_get", "data": result}) + except Exception as e: + await self._broadcast( + { + "type": "provider_models_get", + "data": {"success": False, "models": [], "error": str(e)}, + } + ) + async def _handle_openrouter_models_get( self, base_url: Optional[str] = None, diff --git a/app/ui_layer/browser/frontend/src/pages/Settings/ModelSettings.tsx b/app/ui_layer/browser/frontend/src/pages/Settings/ModelSettings.tsx index 849e8aa6..fbf2f0cd 100644 --- a/app/ui_layer/browser/frontend/src/pages/Settings/ModelSettings.tsx +++ b/app/ui_layer/browser/frontend/src/pages/Settings/ModelSettings.tsx @@ -138,6 +138,12 @@ export function ModelSettings() { // Ollama list loading flag (transient). Models + availability are slice-backed. const [ollamaModelsLoading, setOllamaModelsLoading] = useState(false) + // Generic /v1/models discovery (any provider with has_model_discovery: + // the new cloud providers + LM Studio/vLLM/llama.cpp). Mirrors the Ollama + // dropdown but wire-generic. docs/PROVIDER_SETTINGS_UX_FIX.md A2. + const [discoveredModels, setDiscoveredModels] = useState([]) + const [discoveredLoading, setDiscoveredLoading] = useState(false) + // Ollama auto-install state const [ollamaInstallPhase, setOllamaInstallPhase] = useState<'idle' | 'installing' | 'error'>('idle') const [ollamaInstallLog, setOllamaInstallLog] = useState([]) @@ -269,6 +275,22 @@ export function ModelSettings() { const rec = d.models?.find(m => m.recommended) if (rec) setSelectedPullModel(rec.name) }), + onMessage('provider_models_get', (data: unknown) => { + const d = data as { success: boolean; models?: string[] } + setDiscoveredLoading(false) + const models = d.success && d.models ? d.models : [] + setDiscoveredModels(models) + // If exactly one model is served (vLLM / llama.cpp) or the current + // selection isn't offered, auto-select the first discovered id so + // the field reflects what the server actually has. + if (models.length > 0) { + setNewLlmModel(prev => { + const eff = prev || currentLlmModel + if (!eff || !models.includes(eff)) { setHasChanges(true); return models[0] } + return prev + }) + } + }), onMessage('local_llm_pull_progress', (data: unknown) => { const d = data as { message: string; total: number; completed: number; percent: number } setPullStatus(d.message || '') @@ -354,6 +376,31 @@ export function ModelSettings() { const currentProvider = providers.find(p => p.id === provider) + // Generic /v1/models discovery: fetch whenever the active provider + // advertises has_model_discovery (new cloud providers + local servers). + // Ollama keeps its own native path above. + useEffect(() => { + setDiscoveredModels([]) + if (!isConnected || !currentProvider?.has_model_discovery || provider === 'remote') return + setDiscoveredLoading(true) + send('provider_models_get', { + provider, + baseUrl: newBaseUrl || baseUrls[provider] || undefined, + apiKey: newApiKey || undefined, + }) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [provider, isConnected, currentProvider?.has_model_discovery, baseUrls, send]) + + // Options for the LLM/VLM dropdowns: Ollama native list, else the + // live-discovered /v1/models list. We never ship a hardcoded model list — + // providers we can't enumerate online fall back to a free-text input so the + // suggestions can't go stale. + const modelOptions: string[] = + provider === 'remote' + ? ollamaModels + : discoveredModels + const modelsLoading = provider === 'remote' ? ollamaModelsLoading : discoveredLoading + // Update models when provider changes — only before settings have loaded (fallback to // registry defaults for the initial render). After hasInitialized is true, provider // changes are handled explicitly in handleProviderChange so we don't race against @@ -520,7 +567,7 @@ export function ModelSettings() { {/* Model Configuration */} {currentProvider && ( <> - {provider === 'openrouter' && currentProvider.supports_catalog ? ( + {currentProvider.supports_catalog ? ( - {provider === 'remote' && ollamaModels.length > 0 ? ( + {modelOptions.length > 0 ? ( ) : ( { setNewLlmModel(e.target.value); setHasChanges(true) }} placeholder={ - provider === 'remote' && ollamaModelsLoading + modelsLoading ? 'Loading models...' : currentLlmModel || 'Enter LLM model name...' } @@ -717,7 +771,7 @@ export function ModelSettings() { )} {currentProvider.has_vlm && ( - provider === 'openrouter' && currentProvider.supports_catalog ? ( + currentProvider.supports_catalog ? ( {(() => { - const visionKeywords = ['llava', 'vision', 'moondream', 'bakllava'] - const visionModels = ollamaModels.filter(m => - visionKeywords.some(kw => m.toLowerCase().includes(kw)) - ) - const vlmOptions = provider === 'remote' && ollamaModels.length > 0 - ? (visionModels.length > 0 ? visionModels : ollamaModels) - : [] + // Ollama filters its list to vision-tagged names; other + // providers expose opaque ids, so offer the full + // discovered/curated list (the profile's vlm_model + // default is pre-selected). + const visionKeywords = ['llava', 'vision', 'moondream', 'bakllava', 'vl'] + const vlmOptions = provider === 'remote' + ? (() => { + const vm = ollamaModels.filter(m => + visionKeywords.some(kw => m.toLowerCase().includes(kw))) + return vm.length > 0 ? vm : ollamaModels + })() + : modelOptions return vlmOptions.length > 0 ? ( ) : ( @@ -752,7 +817,7 @@ export function ModelSettings() { value={newVlmModel || currentVlmModel || ''} onChange={(e) => { setNewVlmModel(e.target.value); setHasChanges(true) }} placeholder={ - provider === 'remote' && ollamaModelsLoading + modelsLoading ? 'Loading models...' : currentVlmModel || 'Enter VLM model name...' } @@ -937,7 +1002,7 @@ export function ModelSettings() { onChange={(e) => { setNewApiKey(e.target.value); setHasChanges(true) }} placeholder={hasStoredKey ? 'Enter new key to replace...' : 'Enter API key...'} /> - {(['moonshot', 'minimax'] as string[]).includes(provider) && ( + {currentProvider?.openrouter_proxy && (

{apiKeys['openrouter']?.has_key ? 'OpenRouter is configured and will be used automatically if the direct API is unavailable in your region.' @@ -963,8 +1028,8 @@ export function ModelSettings() { ) })()} - {/* OpenRouter credits */} - {provider === 'openrouter' && currentProvider?.supports_catalog && ( + {/* OpenRouter credits (catalog providers only — OR today) */} + {currentProvider?.supports_catalog && (

)} @@ -1074,13 +1139,11 @@ export function ModelSettings() { onClick={handleTestConnection} disabled={ isTesting || - (provider !== 'remote' && - provider !== 'bedrock' && + (!!currentProvider?.requires_api_key && !apiKeys[provider]?.has_key) } title={ - provider !== 'remote' && - provider !== 'bedrock' && + currentProvider?.requires_api_key && !apiKeys[provider]?.has_key ? 'API key required for testing' : '' @@ -1353,7 +1416,7 @@ export function ModelSettings() { ) : ( {testResult.error || testResult.message} - {(['moonshot', 'minimax'] as string[]).includes(provider) && ( + {currentProvider?.openrouter_proxy && ( This provider may be geo-restricted in your region. {apiKeys['openrouter']?.has_key diff --git a/app/ui_layer/browser/frontend/src/store/slices/modelSettingsSlice.ts b/app/ui_layer/browser/frontend/src/store/slices/modelSettingsSlice.ts index 8916646b..d2334598 100644 --- a/app/ui_layer/browser/frontend/src/store/slices/modelSettingsSlice.ts +++ b/app/ui_layer/browser/frontend/src/store/slices/modelSettingsSlice.ts @@ -5,6 +5,7 @@ export interface ProviderInfo { id: string name: string requires_api_key: boolean + default_base_url?: string | null api_key_env?: string base_url_env?: string llm_model: string | null @@ -22,6 +23,10 @@ export interface ProviderInfo { supports_subscription_oauth?: boolean subscription_label?: string | null subscription_models?: string[] + // UX generalization (docs/PROVIDER_SETTINGS_UX_FIX.md) + has_model_discovery?: boolean // live GET /v1/models dropdown available + local_kind?: string | null // "lmstudio" → native list-all + load UI + openrouter_proxy?: boolean // geo-restricted → show OpenRouter hint } // One entry per provider that supports subscription OAuth. The backend diff --git a/app/ui_layer/commands/builtin/provider.py b/app/ui_layer/commands/builtin/provider.py index 24e83f8e..4846847d 100644 --- a/app/ui_layer/commands/builtin/provider.py +++ b/app/ui_layer/commands/builtin/provider.py @@ -4,6 +4,7 @@ from typing import List +from agent_core.core.models.registry import cli_providers from app.ui_layer.commands.base import Command, CommandResult from app.ui_layer.settings.provider_settings import ( save_settings_to_json, @@ -15,18 +16,11 @@ class ProviderCommand(Command): """Manage LLM provider settings.""" - PROVIDERS = { - "openai": ("OPENAI_API_KEY", "OpenAI"), - "gemini": ("GOOGLE_API_KEY", "Google Gemini"), - "anthropic": ("ANTHROPIC_API_KEY", "Anthropic"), - "byteplus": ("BYTEPLUS_API_KEY", "BytePlus"), - "deepseek": ("DEEPSEEK_API_KEY", "DeepSeek"), - "grok": ("XAI_API_KEY", "Grok (xAI)"), - "glm": ("ZAI_API_KEY", "Z.ai (GLM)"), - "fugu": ("SAKANA_API_KEY", "Sakana (Fugu)"), - "openrouter": ("OPENROUTER_API_KEY", "OpenRouter"), - "remote": (None, "Ollama (Local)"), - } + # Derived from the provider profiles (Phase 1, + # docs/PROVIDER_LAYER_CATCHUP.md): {provider: (api_key_env, display)}. + # The old hand-maintained dict had drifted (missing minimax/moonshot/ + # bedrock); derivation makes that class of bug impossible. + PROVIDERS = cli_providers() @property def name(self) -> str: @@ -42,7 +36,12 @@ def usage(self) -> str: @property def help_text(self) -> str: - return """Manage LLM provider settings. + width = max(len(key) for key in self.PROVIDERS) + listing = "\n".join( + f" {key:<{width}} - {display}" + for key, (_env, display) in self.PROVIDERS.items() + ) + return f"""Manage LLM provider settings. Usage: /provider - Show current provider @@ -50,16 +49,7 @@ def help_text(self) -> str: /provider - Set provider and API key Providers: - openai - OpenAI GPT models - gemini - Google Gemini models - anthropic - Anthropic Claude models - byteplus - BytePlus Kimi models - deepseek - DeepSeek models - grok - Grok (xAI) models - glm - Z.ai (GLM) models - fugu - Sakana (Fugu) models - openrouter - OpenRouter (300+ models, one key) - remote - Ollama (local models) +{listing} Examples: /provider diff --git a/app/ui_layer/metrics/collector.py b/app/ui_layer/metrics/collector.py index 8d200857..b716da3d 100644 --- a/app/ui_layer/metrics/collector.py +++ b/app/ui_layer/metrics/collector.py @@ -33,13 +33,6 @@ class TimePeriod(Enum): # ───────────────────────────────────────────────────────────────────── -# Pricing Data (USD per 1M tokens) -# ───────────────────────────────────────────────────────────────────── -# Single source of truth lives in app.usage.pricing (cached-aware, current -# models, longest-match resolution). Re-exported here for existing callers. -from app.usage.pricing import MODEL_PRICING, get_model_pricing # noqa: E402,F401 - - # ───────────────────────────────────────────────────────────────────── # Data Classes # ───────────────────────────────────────────────────────────────────── @@ -579,15 +572,8 @@ def record_llm_call( cached_tokens: int = 0, task_id: Optional[str] = None, ) -> None: - """Record an LLM call for cost tracking.""" - pricing = get_model_pricing(model) - - # Calculate cost (per million tokens) - input_cost = (input_tokens / 1_000_000) * pricing["input"] - output_cost = (output_tokens / 1_000_000) * pricing["output"] - # Cached tokens are typically free or heavily discounted - total_cost = input_cost + output_cost - + """Record an LLM call. Tokens only — we keep no per-model price table, + so no USD cost is derived (cost_usd stays 0).""" record = LLMCallRecord( timestamp=time.time(), provider=provider, @@ -595,7 +581,7 @@ def record_llm_call( input_tokens=input_tokens, output_tokens=output_tokens, cached_tokens=cached_tokens, - cost_usd=total_cost, + cost_usd=0.0, task_id=task_id, ) diff --git a/app/ui_layer/settings/__init__.py b/app/ui_layer/settings/__init__.py index 1e76cb0e..db03eb15 100644 --- a/app/ui_layer/settings/__init__.py +++ b/app/ui_layer/settings/__init__.py @@ -108,6 +108,7 @@ test_connection, validate_can_save, get_ollama_models, + get_provider_models, ) # Subscription OAuth (ChatGPT Plus/Pro, SuperGrok). Anthropic is excluded @@ -204,6 +205,7 @@ "test_connection", "validate_can_save", "get_ollama_models", + "get_provider_models", # Subscription OAuth "connect_subscription", "connect_subscription_async", diff --git a/app/ui_layer/settings/model_settings.py b/app/ui_layer/settings/model_settings.py index 6abc6287..1e2190d2 100644 --- a/app/ui_layer/settings/model_settings.py +++ b/app/ui_layer/settings/model_settings.py @@ -10,7 +10,7 @@ """ import json -from typing import Dict, Any, Optional +from typing import Any, Dict, List, Optional, Tuple import httpx @@ -22,114 +22,15 @@ ) -# Provider display names and settings.json key mapping -PROVIDER_INFO = { - "openai": { - "name": "OpenAI", - "api_key_env": "OPENAI_API_KEY", - "settings_key": "openai", - "requires_api_key": True, - "supports_subscription_oauth": True, - "subscription_label": "Sign in with ChatGPT", - # Codex-accepted models for ChatGPT subscription auth. - "subscription_models": [ - "gpt-5.4", - "gpt-5.5", - "gpt-5.4-mini", - "gpt-5.3-codex-spark", - ], - "subscription_default_model": "gpt-5.4", - }, - "anthropic": { - "name": "Anthropic", - "api_key_env": "ANTHROPIC_API_KEY", - "settings_key": "anthropic", - "requires_api_key": True, - }, - "gemini": { - "name": "Google Gemini", - "api_key_env": "GOOGLE_API_KEY", - "settings_key": "google", - "requires_api_key": True, - }, - "byteplus": { - "name": "BytePlus", - "api_key_env": "BYTEPLUS_API_KEY", - "settings_key": "byteplus", - "requires_api_key": True, - }, - "minimax": { - "name": "MiniMax", - "api_key_env": "MINIMAX_API_KEY", - "settings_key": "minimax", - "requires_api_key": True, - }, - "deepseek": { - "name": "DeepSeek", - "api_key_env": "DEEPSEEK_API_KEY", - "settings_key": "deepseek", - "requires_api_key": True, - }, - "moonshot": { - "name": "Moonshot", - "api_key_env": "MOONSHOT_API_KEY", - "settings_key": "moonshot", - "requires_api_key": True, - }, - "grok": { - "name": "Grok (xAI)", - "api_key_env": "XAI_API_KEY", - "settings_key": "grok", - "requires_api_key": True, - # Subscription OAuth (SuperGrok / X Premium+). xAI publicly endorsed - # this path in May 2026. - "supports_subscription_oauth": True, - "subscription_label": "Sign in with Grok", - "subscription_models": ["grok-4-0709", "grok-3"], - }, - "glm": { - "name": "Z.ai (GLM)", - "api_key_env": "ZAI_API_KEY", - "settings_key": "glm", - "requires_api_key": True, - }, - "fugu": { - "name": "Sakana (Fugu)", - "api_key_env": "SAKANA_API_KEY", - "settings_key": "fugu", - "requires_api_key": True, - }, - "openrouter": { - "name": "OpenRouter", - "api_key_env": "OPENROUTER_API_KEY", - # Intentionally no base_url_env — the OpenRouter endpoint is fixed for - # almost everyone, and exposing the field confused users into thinking - # they had to fill it in. Power users who need a custom gateway can - # still set endpoints.openrouter_base_url in settings.json by hand; - # the backend still reads it (see app/config.py get_base_url). - "settings_key": "openrouter", - "requires_api_key": True, - # Frontend opts in to a catalog-aware picker for this provider. - "supports_catalog": True, - }, - "remote": { - "name": "Local (Ollama)", - "base_url_env": "REMOTE_MODEL_URL", - "requires_api_key": False, - }, - "bedrock": { - "name": "AWS Bedrock", - # Bedrock uses the boto3 credential chain — there is no single key, - # so `requires_api_key` is False and the frontend renders an AWS - # credentials block instead (access key / secret key / region). - "requires_api_key": False, - "is_bedrock": True, - # Region is exposed via the base_url slot so the existing plumbing - # threads it through. The frontend uses `is_bedrock` to swap the - # generic "Server URL" field for an AWS-specific form. - "base_url_env": "AWS_REGION", - }, -} +# Provider display names and settings.json key mapping — DERIVED from the +# provider profiles (Phase 1, docs/PROVIDER_LAYER_CATCHUP.md). The JSON shape +# is a frozen frontend contract pinned by +# tests/settings/snapshots/provider_info.json; per-provider data (names, env +# vars, subscription OAuth, is_bedrock, base_url_env visibility rules) lives +# on ProviderProfile in agent_core/core/models/provider_config.py. +from agent_core.core.models.registry import provider_info as _derive_provider_info + +PROVIDER_INFO = _derive_provider_info() def _load_settings() -> Dict[str, Any]: @@ -194,11 +95,15 @@ def get_available_providers() -> Dict[str, Any]: Dict with provider info including name and models """ try: + from agent_core.core.models.registry import get_registry + + registry = get_registry() providers = [] for provider_id, info in PROVIDER_INFO.items(): # Get models for this provider provider_models = MODEL_REGISTRY.get(provider_id, {}) + profile = registry.get(provider_id) llm_model = provider_models.get(InterfaceType.LLM) vlm_model = provider_models.get(InterfaceType.VLM) @@ -212,6 +117,12 @@ def get_available_providers() -> Dict[str, Any]: "requires_api_key": info.get("requires_api_key", True), "api_key_env": info.get("api_key_env"), "base_url_env": info.get("base_url_env"), + # Default endpoint, so the UI can show a helpful + # placeholder (e.g. http://localhost:1234/v1 for LM + # Studio) instead of a generic "Enter base URL...". + "default_base_url": ( + profile.default_base_url if profile else None + ), "llm_model": llm_model, "vlm_model": vlm_model, "has_vlm": vlm_model is not None, @@ -226,6 +137,19 @@ def get_available_providers() -> Dict[str, Any]: ), "subscription_label": info.get("subscription_label"), "subscription_models": info.get("subscription_models", []), + # ── UX generalization (docs/PROVIDER_SETTINGS_UX_FIX.md) ── + # Live GET /v1/models dropdown (all new cloud + local + # servers except Perplexity). + "has_model_discovery": bool( + profile.supports_model_discovery if profile else False + ), + # "lmstudio" unlocks the native list-all + load UI. + "local_kind": profile.local_kind if profile else None, + # Drives the OpenRouter geo-fallback hint by flag instead + # of a hardcoded ['moonshot','minimax'] list. + "openrouter_proxy": bool( + profile.openrouter_proxy if profile else False + ), } ) @@ -649,6 +573,28 @@ def test_connection( } +def _sort_models_by_recency(items: List[Tuple[str, Any]]) -> List[str]: + """Order model ids newest-first, falling back to alphabetical. + + ``items`` is a list of ``(model_id, recency)`` pairs, where ``recency`` is + a comparable value the provider reports for how new a model is — a unix + timestamp for /v1/models' ``created``, an ISO-8601 string for Ollama's + ``modified_at`` — or ``None`` when the provider doesn't report it (all + recency values in one call are the same type). + + Ids that carry a recency sort newest-first; ids without one sort after + them, case-insensitively A→Z. When no id reports a recency at all, the + whole list is alphabetical. + """ + if not any(r is not None for _, r in items): + return sorted((mid for mid, _ in items), key=str.lower) + dated = [(mid, r) for mid, r in items if r is not None] + undated = sorted((mid for mid, r in items if r is None), key=str.lower) + dated.sort(key=lambda t: t[0].lower()) # A→Z tiebreak (stable) + dated.sort(key=lambda t: t[1], reverse=True) # then newest first + return [mid for mid, _ in dated] + undated + + def get_ollama_models(base_url: Optional[str] = None) -> Dict[str, Any]: """Fetch available models from a running Ollama instance. @@ -663,7 +609,14 @@ def get_ollama_models(base_url: Optional[str] = None) -> Dict[str, Any]: with httpx.Client(timeout=5.0) as client: response = client.get(f"{url.rstrip('/')}/api/tags") if response.status_code == 200: - models = [m["name"] for m in response.json().get("models", [])] + raw = response.json().get("models", []) + models = _sort_models_by_recency( + [ + (m["name"], m.get("modified_at") or None) + for m in raw + if isinstance(m, dict) and m.get("name") + ] + ) return {"success": True, "models": models} else: return { @@ -675,6 +628,74 @@ def get_ollama_models(base_url: Optional[str] = None) -> Dict[str, Any]: return {"success": False, "models": [], "error": str(e)} +def get_provider_models( + provider: str, + base_url: Optional[str] = None, + api_key: Optional[str] = None, +) -> Dict[str, Any]: + """List a provider's models via the OpenAI-standard GET {base_url}/models. + + The wire-generic analogue of get_ollama_models (which uses Ollama's + native /api/tags): powers the settings model dropdown for every provider + with supports_model_discovery — the 9 new cloud providers and the local + servers (LM Studio / vLLM / llama.cpp). See + docs/PROVIDER_SETTINGS_UX_FIX.md A2. + + Returns {success, models: [id,...], error?}. Never raises. + """ + from agent_core.core.models.registry import get_registry + + profile = get_registry().get(provider) + if profile is None: + return {"success": False, "models": [], "error": f"Unknown provider: {provider}"} + + url = base_url or profile.default_base_url + if not url: + return {"success": False, "models": [], "error": "No base URL configured."} + + # Resolve a bearer: explicit key, else the stored key, else a placeholder + # for keyless local servers (which ignore auth). + key = api_key + if not key: + try: + from app.config import get_api_key + + key = get_api_key(provider) or None + except Exception: + key = None + if not key and not profile.requires_api_key: + key = "local" + + headers = {"Authorization": f"Bearer {key}"} if key else {} + try: + with httpx.Client(timeout=8.0) as client: + response = client.get(f"{url.rstrip('/')}/models", headers=headers) + if response.status_code == 200: + data = response.json().get("data", []) or [] + + def _created(m: Dict[str, Any]) -> Optional[int]: + c = m.get("created") + # Several OpenAI-compatible providers stub `created` as 0/absent + # — treat those as "no recency" so they fall back to alpha. + return int(c) if isinstance(c, (int, float)) and c > 0 else None + + models = _sort_models_by_recency( + [ + (m["id"], _created(m)) + for m in data + if isinstance(m, dict) and m.get("id") + ] + ) + return {"success": True, "models": models} + return { + "success": False, + "models": [], + "error": f"Provider returned status {response.status_code}", + } + except Exception as e: + return {"success": False, "models": [], "error": str(e)} + + def validate_can_save( llm_provider: str, vlm_provider: Optional[str] = None, diff --git a/app/ui_layer/settings/provider_settings.py b/app/ui_layer/settings/provider_settings.py index 93f77712..2bfbe077 100644 --- a/app/ui_layer/settings/provider_settings.py +++ b/app/ui_layer/settings/provider_settings.py @@ -11,25 +11,16 @@ from app.config import SETTINGS_CONFIG_PATH -# Provider to settings.json api_keys key mapping -PROVIDER_TO_SETTINGS_KEY = { - "openai": "openai", - "gemini": "google", - "google": "google", - "byteplus": "byteplus", - "anthropic": "anthropic", - "deepseek": "deepseek", - "minimax": "minimax", - "moonshot": "moonshot", - "grok": "grok", - "glm": "glm", - "fugu": "fugu", - "openrouter": "openrouter", - # Bedrock has no single API key — credentials live under "aws_credentials" - # in settings.json (handled separately from the api_keys map). The entry - # here is left so PROVIDER_TO_SETTINGS_KEY.get("bedrock") returns None, - # which the save path can detect and route accordingly. -} +# Provider to settings.json api_keys key mapping — DERIVED from the provider +# profiles (Phase 1, docs/PROVIDER_LAYER_CATCHUP.md). Bedrock has no single +# API key (credentials live under "aws_credentials"), so it is absent and +# ``.get("bedrock")`` returns None, which the save path detects and routes +# accordingly. The legacy "google" alias entry is preserved. +from agent_core.core.models.registry import ( + provider_settings_keys as _derive_settings_keys, +) + +PROVIDER_TO_SETTINGS_KEY = _derive_settings_keys() def _load_settings() -> Dict[str, Any]: @@ -148,6 +139,77 @@ def save_remote_endpoint(url: str) -> bool: return False +def save_custom_provider(name: str, spec: Dict[str, Any]) -> Tuple[bool, str]: + """Create or update a user-defined provider (Phase 3, FR-5). + + ``spec`` follows docs/PROVIDER_LAYER_CATCHUP.md section 7.2: + base_url (required), wire, api_key or api_key_env, display_name, + models, default_model, headers, supports_prompt_cache_key, + requires_api_key. An inline "api_key" is routed into the regular + api_keys block under ``name`` (never stored in custom_providers), so + the whole existing key plumbing works unchanged. + + The entry is validated through the registry builder before saving — + the agent can call this and get an actionable error instead of + persisting a broken provider. + """ + from agent_core.core.models.provider_config import PROVIDER_CONFIG + from agent_core.core.models.registry import _build_custom_profile + + name = (name or "").strip().lower() + if not name or not name.replace("-", "").replace("_", "").isalnum(): + return False, "Provider name must be a slug (letters/digits/-/_)." + if name in PROVIDER_CONFIG: + return False, f"'{name}' collides with a built-in provider." + if not isinstance(spec, dict): + return False, "Provider spec must be an object." + + spec = dict(spec) + inline_key = spec.pop("api_key", None) + + if _build_custom_profile(name, spec) is None: + return False, ( + "Invalid provider spec: base_url must be an http(s) URL and " + "wire (if set) must be 'chat_completions'." + ) + + try: + settings = _load_settings() + settings.setdefault("custom_providers", {})[name] = spec + if inline_key: + settings.setdefault("api_keys", {})[name] = inline_key + if not _save_settings(settings): + return False, "Failed to write settings.json." + from app.config import reload_settings + + reload_settings() + logger.info(f"[SETTINGS] Saved custom provider {name!r}") + return True, f"Custom provider '{name}' saved." + except Exception as e: + logger.error(f"[SETTINGS] Failed to save custom provider {name!r}: {e}") + return False, f"Failed to save custom provider: {e}" + + +def delete_custom_provider(name: str) -> Tuple[bool, str]: + """Remove a custom provider and its stored API key.""" + try: + settings = _load_settings() + removed = settings.get("custom_providers", {}).pop(name, None) + settings.get("api_keys", {}).pop(name, None) + if removed is None: + return False, f"No custom provider named '{name}'." + if not _save_settings(settings): + return False, "Failed to write settings.json." + from app.config import reload_settings + + reload_settings() + logger.info(f"[SETTINGS] Deleted custom provider {name!r}") + return True, f"Custom provider '{name}' deleted." + except Exception as e: + logger.error(f"[SETTINGS] Failed to delete custom provider {name!r}: {e}") + return False, f"Failed to delete custom provider: {e}" + + def get_api_key_env_name(provider: str) -> Optional[str]: """Get the environment variable name for a provider's API key.""" if provider not in PROVIDER_CONFIG: diff --git a/app/usage/pricing.py b/app/usage/pricing.py deleted file mode 100644 index 647a66e8..00000000 --- a/app/usage/pricing.py +++ /dev/null @@ -1,101 +0,0 @@ -# -*- coding: utf-8 -*- -""" -app.usage.pricing - -Single source of per-model token pricing (USD per 1M tokens) for cost + -cache-savings math, used by the prompt profiler and the dashboard metrics -collector. - -Each entry has three rates: - input - standard (uncached) input tokens - cached - input tokens served from cache (provider discounts vary: - Gemini / Anthropic cache-read ≈ 10% of input, OpenAI ≈ 50%) - output - output tokens - -Values are approximate and drift over time — update against provider pricing -pages. Sources (2026-06): Gemini https://ai.google.dev/gemini-api/docs/pricing, -Anthropic & OpenAI public pricing. -""" - -from __future__ import annotations - -from typing import Dict - -# Per 1M tokens, USD. Keys are matched as substrings of the model id; matching -# prefers the LONGEST (most specific) key, so e.g. "gpt-4o-mini" wins over -# "gpt-4o". -MODEL_PRICING: Dict[str, Dict[str, float]] = { - # ─ OpenAI (cached ≈ 50% of input) ─ - "gpt-4o-mini": {"input": 0.15, "cached": 0.075, "output": 0.60}, - "gpt-4o": {"input": 2.50, "cached": 1.25, "output": 10.00}, - "gpt-4-turbo": {"input": 10.00, "cached": 10.00, "output": 30.00}, - "gpt-4": {"input": 30.00, "cached": 30.00, "output": 60.00}, - "gpt-3.5-turbo": {"input": 0.50, "cached": 0.50, "output": 1.50}, - "o1-mini": {"input": 3.00, "cached": 1.50, "output": 12.00}, - "o1-preview": {"input": 15.00, "cached": 7.50, "output": 60.00}, - "o1": {"input": 15.00, "cached": 7.50, "output": 60.00}, - "o3-mini": {"input": 1.10, "cached": 0.55, "output": 4.40}, - # ─ Anthropic (cache-read ≈ 10% of input) ─ - "claude-opus-4": {"input": 15.00, "cached": 1.50, "output": 75.00}, - "claude-sonnet-4": {"input": 3.00, "cached": 0.30, "output": 15.00}, - "claude-haiku-4": {"input": 1.00, "cached": 0.10, "output": 5.00}, - "claude-3-5-sonnet": {"input": 3.00, "cached": 0.30, "output": 15.00}, - "claude-3-5-haiku": {"input": 0.80, "cached": 0.08, "output": 4.00}, - "claude-3-opus": {"input": 15.00, "cached": 1.50, "output": 75.00}, - "claude-3-sonnet": {"input": 3.00, "cached": 0.30, "output": 15.00}, - "claude-3-haiku": {"input": 0.25, "cached": 0.03, "output": 1.25}, - # ─ Google Gemini (cached ≈ 10% of input) ─ - "gemini-2.5-pro": {"input": 1.25, "cached": 0.125, "output": 10.00}, - "gemini-2.5-flash": {"input": 0.30, "cached": 0.075, "output": 2.50}, - "gemini-2.0-flash": {"input": 0.10, "cached": 0.025, "output": 0.40}, - "gemini-1.5-pro": {"input": 1.25, "cached": 0.3125, "output": 5.00}, - "gemini-1.5-flash": {"input": 0.075, "cached": 0.01875, "output": 0.30}, - # ─ Fallback ─ - "default": {"input": 1.00, "cached": 0.25, "output": 3.00}, -} - - -def get_model_pricing(model: str) -> Dict[str, float]: - """Return the pricing dict for a model via longest-substring match. - - Longest-match avoids the classic bug where "gpt-4o" shadows "gpt-4o-mini". - Falls back to the "default" entry when nothing matches. - """ - model_lower = (model or "").lower() - best_key = None - for key in MODEL_PRICING: - if key == "default": - continue - if key in model_lower and (best_key is None or len(key) > len(best_key)): - best_key = key - return MODEL_PRICING[best_key] if best_key else MODEL_PRICING["default"] - - -def estimate_cost( - model: str, - input_tokens: int, - output_tokens: int, - cached_tokens: int = 0, -) -> Dict[str, float]: - """Estimate the USD cost of a call and the savings from cache reuse. - - `cached_tokens` is the subset of `input_tokens` served from cache (billed at - the cached rate); the remainder is billed at the standard input rate. - - Returns a dict with input_cost, output_cost, total_cost, and saved (vs. - paying the full input rate for the cached tokens). - """ - p = get_model_pricing(model) - cached = max(0, min(cached_tokens, input_tokens)) - uncached = input_tokens - cached - - input_cost = (uncached * p["input"] + cached * p["cached"]) / 1_000_000 - output_cost = (output_tokens * p["output"]) / 1_000_000 - saved = (cached * (p["input"] - p["cached"])) / 1_000_000 - - return { - "input_cost": input_cost, - "output_cost": output_cost, - "total_cost": input_cost + output_cost, - "saved": saved, - } diff --git a/craftos_integrations/integrations/llm_oauth/copilot.py b/craftos_integrations/integrations/llm_oauth/copilot.py new file mode 100644 index 00000000..f124c47c --- /dev/null +++ b/craftos_integrations/integrations/llm_oauth/copilot.py @@ -0,0 +1,262 @@ +# -*- coding: utf-8 -*- +"""GitHub Copilot subscription OAuth backend (Phase 6 subset, +docs/PROVIDER_LAYER_CATCHUP.md). + +Flow (both competitors ship the same shape): + 1. GitHub DEVICE FLOW: POST github.com/login/device/code -> user_code + + verification_uri; open the browser; poll login/oauth/access_token until + the user authorizes. Yields a long-lived ``gho_``/``ghu_`` OAuth token + (no refresh token — it does not expire). + 2. COPILOT BEARER: exchange the GitHub token at + api.github.com/copilot_internal/v2/token for a short-lived (~30 min) + bearer valid against api.githubcopilot.com. ``load_and_refresh`` + re-exchanges when <5 min from expiry — same refresh contract as the + other backends, driven per-request by ``tokens.get_bearer``. + +The inference surface is OpenAI-compatible chat completions at +https://api.githubcopilot.com with Copilot's editor headers, so the +existing chat_completions transport works unchanged. + +NOTE: written against GitHub's documented device flow + the de-facto +Copilot token exchange used by the ecosystem (OpenClaw, Hermes); not yet +exercised against a live Copilot seat from this environment — the first +real connect is the acceptance test. +""" + +from __future__ import annotations + +import asyncio +import os +import time +import webbrowser +from dataclasses import dataclass, field +from typing import Dict, Optional, Tuple + +import httpx + +from ...credentials_store import ( + has_credential as _store_has, + load_credential as _store_load, + remove_credential as _store_remove, + save_credential as _store_save, +) +from ...logger import get_logger + +logger = get_logger(__name__) + +DEVICE_CODE_URL = "https://github.com/login/device/code" +ACCESS_TOKEN_URL = "https://github.com/login/oauth/access_token" +COPILOT_TOKEN_URL = "https://api.github.com/copilot_internal/v2/token" +USER_URL = "https://api.github.com/user" +API_BASE_URL = "https://api.githubcopilot.com" + +# VS Code's public GitHub OAuth client id — the id the Copilot ecosystem's +# device-flow tools authenticate as. Override with COPILOT_CLIENT_ID. +DEFAULT_CLIENT_ID = "01ab8ac9400c4e429b23" +SCOPES = "read:user" + +CRED_FILE = "copilot_oauth.json" +REFRESH_THRESHOLD_SECONDS = 5 * 60 +DEVICE_FLOW_TIMEOUT_SECONDS = 15 * 60 + +# Copilot's API rejects requests without editor identification headers. +EDITOR_HEADERS = { + "Editor-Version": "vscode/1.99.0", + "Editor-Plugin-Version": "copilot-chat/0.26.0", + "Copilot-Integration-Id": "vscode-chat", +} + + +@dataclass +class CopilotOAuthCredential: + github_token: str = "" # long-lived gho_/ghu_ OAuth token + access_token: str = "" # short-lived Copilot bearer + expires_at: float = 0.0 # Copilot bearer expiry (epoch seconds) + email: str = "" # GitHub login, for the settings UI + plan: str = field(default="copilot") + + +def _client_id() -> str: + return os.environ.get("COPILOT_CLIENT_ID") or DEFAULT_CLIENT_ID + + +# ─────────────────────────── store plumbing ─────────────────────────── + + +def has_credential() -> bool: + return _store_has(CRED_FILE) + + +def load() -> Optional[CopilotOAuthCredential]: + return _store_load(CRED_FILE, CopilotOAuthCredential) + + +def remove() -> Tuple[bool, str]: + removed = _store_remove(CRED_FILE) + return (True, "GitHub Copilot disconnected.") if removed else ( + False, + "No Copilot credential to remove.", + ) + + +# ─────────────────────────── bearer contract ─────────────────────────── + + +def _exchange_copilot_bearer(github_token: str) -> Tuple[str, float]: + """GitHub OAuth token -> short-lived Copilot bearer + expiry.""" + with httpx.Client(timeout=30) as client: + resp = client.get( + COPILOT_TOKEN_URL, + headers={ + "Authorization": f"token {github_token}", + "Accept": "application/json", + **EDITOR_HEADERS, + }, + ) + if resp.status_code in (401, 403): + raise RuntimeError( + "GitHub rejected the stored token (revoked, or the account has " + "no active Copilot subscription). Reconnect from Settings." + ) + resp.raise_for_status() + data = resp.json() + token = data.get("token") + if not token: + raise RuntimeError(f"Copilot token exchange returned no token: {data}") + expires_at = float(data.get("expires_at") or (time.time() + 25 * 60)) + return token, expires_at + + +def load_and_refresh() -> CopilotOAuthCredential: + cred = load() + if cred is None or not cred.github_token: + raise RuntimeError("No Copilot credential on disk.") + if cred.access_token and cred.expires_at - time.time() > REFRESH_THRESHOLD_SECONDS: + return cred + token, expires_at = _exchange_copilot_bearer(cred.github_token) + cred.access_token = token + cred.expires_at = expires_at + _store_save(CRED_FILE, cred) + return cred + + +def api_base_url(_cred: CopilotOAuthCredential) -> Optional[str]: + return API_BASE_URL + + +def extra_headers(_cred: CopilotOAuthCredential) -> Dict[str, str]: + return dict(EDITOR_HEADERS) + + +# ─────────────────────────── device-flow login ─────────────────────────── + + +async def run_login() -> Tuple[bool, str]: + """GitHub device flow: open the verification page, poll until authorized.""" + + def _start() -> dict: + with httpx.Client(timeout=30) as client: + resp = client.post( + DEVICE_CODE_URL, + data={"client_id": _client_id(), "scope": SCOPES}, + headers={"Accept": "application/json"}, + ) + resp.raise_for_status() + return resp.json() + + try: + start = await asyncio.to_thread(_start) + except Exception as e: + return False, f"Could not start the GitHub device flow: {e}" + + device_code = start.get("device_code") + user_code = start.get("user_code") + verification_uri = start.get("verification_uri") or "https://github.com/login/device" + interval = int(start.get("interval") or 5) + if not device_code or not user_code: + return False, f"GitHub device flow returned an unexpected payload: {start}" + + logger.info(f"[COPILOT] Enter code {user_code} at {verification_uri}") + try: + webbrowser.open(verification_uri) + except Exception: + pass + + def _poll_once() -> dict: + with httpx.Client(timeout=30) as client: + resp = client.post( + ACCESS_TOKEN_URL, + data={ + "client_id": _client_id(), + "device_code": device_code, + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + }, + headers={"Accept": "application/json"}, + ) + resp.raise_for_status() + return resp.json() + + deadline = time.time() + DEVICE_FLOW_TIMEOUT_SECONDS + github_token: Optional[str] = None + while time.time() < deadline: + await asyncio.sleep(interval) + try: + result = await asyncio.to_thread(_poll_once) + except Exception as e: + return False, f"Device-flow polling failed: {e}" + error = result.get("error") + if error == "authorization_pending": + continue + if error == "slow_down": + interval += 5 + continue + if error in ("expired_token", "access_denied"): + return False, f"GitHub device flow ended: {error}. Try again." + if error: + return False, f"GitHub device flow error: {error}" + github_token = result.get("access_token") + if github_token: + break + if not github_token: + return False, ( + f"Timed out waiting for authorization. Enter code {user_code} at " + f"{verification_uri} and try again." + ) + + # Verify the seat by exchanging for a Copilot bearer right away. + try: + access_token, expires_at = await asyncio.to_thread( + _exchange_copilot_bearer, github_token + ) + except Exception as e: + return False, str(e) + + login = "" + try: + + def _whoami() -> str: + with httpx.Client(timeout=15) as client: + resp = client.get( + USER_URL, + headers={ + "Authorization": f"token {github_token}", + "Accept": "application/json", + }, + ) + return resp.json().get("login", "") if resp.status_code == 200 else "" + + login = await asyncio.to_thread(_whoami) + except Exception: + pass + + _store_save( + CRED_FILE, + CopilotOAuthCredential( + github_token=github_token, + access_token=access_token, + expires_at=expires_at, + email=login, + ), + ) + who = f" as {login}" if login else "" + return True, f"GitHub Copilot connected{who}." diff --git a/craftos_integrations/integrations/llm_oauth/tokens.py b/craftos_integrations/integrations/llm_oauth/tokens.py index ad9c4a05..8e581ac5 100644 --- a/craftos_integrations/integrations/llm_oauth/tokens.py +++ b/craftos_integrations/integrations/llm_oauth/tokens.py @@ -52,6 +52,10 @@ def _backend_for(provider: str): from . import grok return grok + if provider == "copilot": + from . import copilot + + return copilot return None diff --git a/scripts/check_no_secrets.py b/scripts/check_no_secrets.py new file mode 100644 index 00000000..1ab2fb70 --- /dev/null +++ b/scripts/check_no_secrets.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +"""Fail if provider secrets appear in tracked config files (Phase 0, NFR-7). + +Usage: + python scripts/check_no_secrets.py # scan tracked config files + python scripts/check_no_secrets.py --staged # scan the staged diff only + +Intended wiring: CI step and/or pre-commit hook. Exit code 1 on any finding. + +Scope is deliberately narrow (config files, not the whole tree) so the scan +is fast and has no false positives from test fixtures or docs that discuss +key FORMATS without containing real keys. +""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] + +# Key-material patterns per provider family. Each must match REAL key shapes +# but not placeholders like "sk-ant-..." or empty strings. +SECRET_PATTERNS = { + "anthropic": re.compile(r"sk-ant-[A-Za-z0-9_\-]{20,}"), + "openai": re.compile(r"sk-proj-[A-Za-z0-9_\-]{20,}"), + "openai-legacy": re.compile(r"sk-[A-Za-z0-9]{40,}"), + "openrouter": re.compile(r"sk-or-v1-[A-Za-z0-9]{20,}"), + "aws-access-key": re.compile(r"AKIA[0-9A-Z]{16}"), + "xai": re.compile(r"xai-[A-Za-z0-9]{20,}"), + "groq": re.compile(r"gsk_[A-Za-z0-9]{20,}"), + "google": re.compile(r"AIza[0-9A-Za-z_\-]{30,}"), + "hf": re.compile(r"hf_[A-Za-z0-9]{30,}"), +} + +# Tracked files worth scanning in full-scan mode. +CONFIG_GLOBS = [ + "app/config/*.json", + "app/config/**/*.json", + "*.json", + "*.env", +] + + +def _tracked_files() -> list[Path]: + out = subprocess.run( + ["git", "ls-files", *CONFIG_GLOBS], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + return [REPO_ROOT / line for line in out.stdout.splitlines() if line.strip()] + + +def _staged_diff_text() -> str: + out = subprocess.run( + ["git", "diff", "--cached", "--unified=0"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + check=True, + ) + # Only ADDED lines can introduce a new secret. + return "\n".join( + line[1:] + for line in out.stdout.splitlines() + if line.startswith("+") and not line.startswith("+++") + ) + + +def _scan_text(label: str, text: str) -> list[str]: + findings = [] + for family, pattern in SECRET_PATTERNS.items(): + for match in pattern.finditer(text): + secret = match.group(0) + masked = secret[:12] + "..." + secret[-4:] + findings.append(f"{label}: {family} key material ({masked})") + return findings + + +def main() -> int: + staged_only = "--staged" in sys.argv + findings: list[str] = [] + + if staged_only: + findings.extend(_scan_text("staged diff", _staged_diff_text())) + else: + for path in _tracked_files(): + try: + text = path.read_text(encoding="utf-8", errors="ignore") + except OSError: + continue + findings.extend(_scan_text(str(path.relative_to(REPO_ROOT)), text)) + + if findings: + print("SECRETS DETECTED (rotate the key, then remove it from git):") + for f in findings: + print(f" - {f}") + return 1 + + print("No committed secrets detected.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/prompt_profile.py b/scripts/prompt_profile.py index 8aa9f40c..338d6ebd 100644 --- a/scripts/prompt_profile.py +++ b/scripts/prompt_profile.py @@ -3,8 +3,9 @@ Prompt profiler (issue #322, P2). Aggregates the captured `llm_calls` table per (prompt_name, provider, model) and -reports the cost/efficiency picture for each named prompt on real traffic: -latency (p50/p95), token volume, cache hit-ratio, $ cost, and $ saved by caching. +reports the efficiency picture for each named prompt on real traffic: latency +(p50/p95), token volume, and cache hit-ratio. No pricing (we keep no per-model +price table); rank prompts by token volume as the cost proxy. The data comes from the capture substrate (P1) — see docs/design/prompt-optimization.md. This is a read-only view; it never writes to @@ -34,8 +35,6 @@ if _REPO_ROOT not in sys.path: sys.path.insert(0, _REPO_ROOT) -from app.usage.pricing import estimate_cost # noqa: E402 - def _default_db_path() -> str: from app.config import APP_DATA_PATH @@ -110,7 +109,6 @@ def aggregate(rows: List[sqlite3.Row]) -> List[Dict[str, Any]]: out: List[Dict[str, Any]] = [] for (prompt_name, provider, model), g in groups.items(): lat = sorted(g["latencies"]) - cost = estimate_cost(model, g["input"], g["output"], g["cached"]) calls = g["calls"] out.append( { @@ -124,12 +122,12 @@ def aggregate(rows: List[sqlite3.Row]) -> List[Dict[str, Any]]: "avg_input_tokens": round(g["input"] / calls), "avg_output_tokens": round(g["output"] / calls), "cache_hit_ratio": (g["cached"] / g["input"]) if g["input"] else 0.0, - "total_cost_usd": round(cost["total_cost"], 4), - "cost_per_call_usd": round(cost["total_cost"] / calls, 6), - "saved_usd": round(cost["saved"], 4), + "total_input_tokens": g["input"], + "total_output_tokens": g["output"], } ) - out.sort(key=lambda d: d["total_cost_usd"], reverse=True) + # No price table, so rank by total input tokens (the cost proxy). + out.sort(key=lambda d: d["total_input_tokens"], reverse=True) return out @@ -143,16 +141,14 @@ def _fmt_table(agg: List[Dict[str, Any]]) -> str: ("avg_input_tokens", "AVG_IN", "r"), ("avg_output_tokens", "AVG_OUT", "r"), ("cache_hit_ratio", "CACHE%", "r"), - ("total_cost_usd", "$ TOTAL", "r"), - ("saved_usd", "$ SAVED", "r"), + ("total_input_tokens", "TOT_IN", "r"), + ("total_output_tokens", "TOT_OUT", "r"), ] def cell(row: Dict[str, Any], key: str) -> str: v = row[key] if key == "cache_hit_ratio": return f"{v * 100:.0f}%" - if key in ("total_cost_usd", "saved_usd"): - return f"{v:.4f}" return str(v) widths = { @@ -182,8 +178,8 @@ def _totals(agg: List[Dict[str, Any]]) -> Dict[str, Any]: return { "groups": len(agg), "calls": sum(r["calls"] for r in agg), - "total_cost_usd": round(sum(r["total_cost_usd"] for r in agg), 4), - "saved_usd": round(sum(r["saved_usd"] for r in agg), 4), + "total_input_tokens": sum(r["total_input_tokens"] for r in agg), + "total_output_tokens": sum(r["total_output_tokens"] for r in agg), } @@ -197,8 +193,8 @@ def _markdown(agg: List[Dict[str, Any]], totals: Dict[str, Any]) -> str: "avg_input_tokens", "avg_output_tokens", "cache_hit_ratio", - "total_cost_usd", - "saved_usd", + "total_input_tokens", + "total_output_tokens", ] head = "| " + " | ".join(cols) + " |" sep = "| " + " | ".join("---" for _ in cols) + " |" @@ -214,8 +210,8 @@ def _markdown(agg: List[Dict[str, Any]], totals: Dict[str, Any]) -> str: body.append("| " + " | ".join(cells) + " |") summary = ( f"\n**Totals:** {totals['calls']} calls across {totals['groups']} " - f"prompt/model groups — ${totals['total_cost_usd']:.4f} spent, " - f"${totals['saved_usd']:.4f} saved by caching.\n" + f"prompt/model groups, {totals['total_input_tokens']} input / " + f"{totals['total_output_tokens']} output tokens.\n" ) return "# Prompt profile\n\n" + "\n".join([head, sep, *body]) + "\n" + summary @@ -226,7 +222,7 @@ def main() -> int: except (AttributeError, ValueError): pass - ap = argparse.ArgumentParser(description="Profile prompt cost/cache/latency.") + ap = argparse.ArgumentParser(description="Profile prompt cache/latency/tokens.") ap.add_argument("--db", help="Path to llm_calls.db (default: app data dir).") ap.add_argument("--since", help="Only calls newer than e.g. 24h, 7d, 90m.") ap.add_argument("--json", metavar="PATH", help="Write the report as JSON.") @@ -252,8 +248,8 @@ def main() -> int: print("-" * 40) print( f"{totals['calls']} calls / {totals['groups']} groups " - f"${totals['total_cost_usd']:.4f} spent " - f"${totals['saved_usd']:.4f} saved by caching" + f"{totals['total_input_tokens']} in / " + f"{totals['total_output_tokens']} out tokens" ) if args.json: