From f8f87604dba6530c9cd494c227a493315395d944 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 10:35:04 +0200 Subject: [PATCH 1/7] LCORE-2343: add unified-mode e2e configuration fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create tests/e2e/configuration/unified-mode/ with library-mode/server-mode variants (same two-subdir layout configure_service resolves), covering the five unified-mode feature files: - unified-providers: minimal unified config driven only by top-level inference.providers over the default baseline (R1/S5); openai-specific. - unified-config-only / unified-relative-profile: profile: run.yaml — the CI-materialized repo-root run.yaml as baseline, provider-agnostic (R1/R8; two files because the features pin the intents separately). - unified-absolute-profile: container-absolute profile paths, differing per mode subdir (/app-root vs /opt/app-root mounts). - unified-native-override-{scalar,list}: R5 replacement semantics fixtures, synthesis-only, never booted. - invalid-{providers,config}-and-legacy: mutual-exclusion validation fixtures (R3); invalid-version-legacy-unified-body: R11 marker mismatch (needs LCORE-2872's cross-validation to fail for the right reason). - legacy-for-migration: legacy half of the migration fixture pair, kept free of enrichment sections so migrate-then-synthesize round-trips losslessly (LCORE-3370). Every fixture is validated against the real Configuration model: bootable ones load, invalid ones fail with the intended error. The test-generated lightspeed-stack-unified-migrated.yaml is gitignored, and the directory README documents each fixture's purpose. --- .gitignore | 3 ++ .../e2e/configuration/unified-mode/README.md | 24 ++++++++++++++ ...speed-stack-invalid-config-and-legacy.yaml | 25 ++++++++++++++ ...ed-stack-invalid-providers-and-legacy.yaml | 31 +++++++++++++++++ ...k-invalid-version-legacy-unified-body.yaml | 31 +++++++++++++++++ ...lightspeed-stack-legacy-for-migration.yaml | 23 +++++++++++++ ...tspeed-stack-unified-absolute-profile.yaml | 24 ++++++++++++++ .../lightspeed-stack-unified-config-only.yaml | 24 ++++++++++++++ ...ed-stack-unified-native-override-list.yaml | 29 ++++++++++++++++ ...-stack-unified-native-override-scalar.yaml | 28 ++++++++++++++++ .../lightspeed-stack-unified-providers.yaml | 29 ++++++++++++++++ ...tspeed-stack-unified-relative-profile.yaml | 24 ++++++++++++++ ...speed-stack-invalid-config-and-legacy.yaml | 27 +++++++++++++++ ...ed-stack-invalid-providers-and-legacy.yaml | 33 +++++++++++++++++++ ...k-invalid-version-legacy-unified-body.yaml | 33 +++++++++++++++++++ ...lightspeed-stack-legacy-for-migration.yaml | 23 +++++++++++++ ...tspeed-stack-unified-absolute-profile.yaml | 26 +++++++++++++++ .../lightspeed-stack-unified-config-only.yaml | 26 +++++++++++++++ ...ed-stack-unified-native-override-list.yaml | 31 +++++++++++++++++ ...-stack-unified-native-override-scalar.yaml | 30 +++++++++++++++++ .../lightspeed-stack-unified-providers.yaml | 31 +++++++++++++++++ ...tspeed-stack-unified-relative-profile.yaml | 26 +++++++++++++++ 22 files changed, 581 insertions(+) create mode 100644 tests/e2e/configuration/unified-mode/README.md create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml create mode 100644 tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml create mode 100644 tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml diff --git a/.gitignore b/.gitignore index 15f06684a..35c352250 100644 --- a/.gitignore +++ b/.gitignore @@ -202,3 +202,6 @@ local-run.yaml .sisyphus/ # Per-developer feature design overrides (see docs/contributing/feature-design.config) .feature-design.config.local + +# Generated at e2e test time by the unified-mode --migrate-config step +tests/e2e/configuration/unified-mode/**/lightspeed-stack-unified-migrated.yaml diff --git a/tests/e2e/configuration/unified-mode/README.md b/tests/e2e/configuration/unified-mode/README.md new file mode 100644 index 000000000..79eaaabe0 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/README.md @@ -0,0 +1,24 @@ +# Unified-mode e2e configuration fixtures + +Fixtures for the five `unified-mode-*.feature` files (LCORE-2341/LCORE-2343). +Same layout as the parent directory: `library-mode/` and `server-mode/` +variants differing only in the `llama_stack` block; the harness resolves +`//` via the standard `configure_service` logic. + +All profile-based fixtures reference `run.yaml` — the repo-root copy the CI +harness materializes from `tests/e2e/configs/run-.yaml` — so they stay +provider-agnostic across the providers matrix. + +| Fixture | Purpose | +|---|---| +| `lightspeed-stack-unified-providers.yaml` | Minimal unified config driven only by top-level `inference.providers` (default baseline, R1/S5). openai-specific — used by `@openai-only` scenarios. | +| `lightspeed-stack-unified-config-only.yaml` | Unified config driven only by `llama_stack.config` (`profile: run.yaml`, R1). | +| `lightspeed-stack-unified-relative-profile.yaml` | Same shape as config-only; exists to pin R8 (relative `profile:` resolves against the config file's directory) as a distinct intent. | +| `lightspeed-stack-unified-absolute-profile.yaml` | `profile:` as a container-absolute path (differs per mode subdir). | +| `lightspeed-stack-unified-native-override-scalar.yaml` | `native_override` replaces an overlapping scalar key (R5). Synthesis-only; never booted. | +| `lightspeed-stack-unified-native-override-list.yaml` | `native_override` replaces an overlapping list wholesale (R5). Synthesis-only; never booted. | +| `lightspeed-stack-invalid-providers-and-legacy.yaml` | INVALID: `inference.providers` + `library_client_config_path` (mutual exclusion, R3). Validation-only. | +| `lightspeed-stack-invalid-config-and-legacy.yaml` | INVALID: `llama_stack.config` + `library_client_config_path` (R3). Validation-only. | +| `lightspeed-stack-invalid-version-legacy-unified-body.yaml` | INVALID: `config_format_version: legacy` on a unified-shaped body (R11, LCORE-2872). Validation-only. | +| `lightspeed-stack-legacy-for-migration.yaml` | Legacy half of "the legacy migration fixture pair"; paired with the repo-root `run.yaml`. Deliberately free of enrichment sections so migrate→synthesize round-trips losslessly (see LCORE-3370). | +| `lightspeed-stack-unified-migrated.yaml` | Generated at test time by the `--migrate-config` step; gitignored and cleaned up after each scenario. | diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml new file mode 100644 index 000000000..bdb8af911 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-config-and-legacy.yaml @@ -0,0 +1,25 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + profile: run.yaml + # INVALID: config block plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml new file mode 100644 index 000000000..124fd453c --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-providers-and-legacy.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + # INVALID: synthesis input plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml new file mode 100644 index 000000000..9ae7b389d --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} +# INVALID: explicit legacy marker on a unified-shaped body (R11, LCORE-2872) +config_format_version: legacy diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml new file mode 100644 index 000000000..6393142b5 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-legacy-for-migration.yaml @@ -0,0 +1,23 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + # Legacy two-file shape: external run.yaml, no synthesis input + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml new file mode 100644 index 000000000..089fb0afb --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-absolute-profile.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + # Absolute path as mounted in the library-mode container + profile: /app-root/run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml new file mode 100644 index 000000000..b3df26828 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-config-only.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + # Synthesis baseline: the CI-materialized run.yaml (provider-agnostic) + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml new file mode 100644 index 000000000..403cc7a00 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-list.yaml @@ -0,0 +1,29 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + profile: run.yaml + # R5: lists replace wholesale - the synthesized apis must equal exactly + # this list, not a merge with the baseline's (never booted - synthesis only) + native_override: + apis: + - inference + - tool_runtime +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml new file mode 100644 index 000000000..02f67f028 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-native-override-scalar.yaml @@ -0,0 +1,28 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + profile: run.yaml + # R5: the raw escape hatch wins; this scalar replaces the baseline's + # safety.excluded_categories value wholesale (never booted - synthesis only) + native_override: + safety: + excluded_categories: unified-override-marker +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml new file mode 100644 index 000000000..731c39b5d --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-providers.yaml @@ -0,0 +1,29 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml new file mode 100644 index 000000000..228b2d40b --- /dev/null +++ b/tests/e2e/configuration/unified-mode/library-mode/lightspeed-stack-unified-relative-profile.yaml @@ -0,0 +1,24 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Library mode - embeds the stack in-process + use_as_library_client: true + config: + # R8: relative profile resolves against this file's loaded location + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml new file mode 100644 index 000000000..34dbfb06f --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-config-and-legacy.yaml @@ -0,0 +1,27 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + profile: run.yaml + # INVALID: config block plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml new file mode 100644 index 000000000..6cbd5f50a --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-providers-and-legacy.yaml @@ -0,0 +1,33 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + # INVALID: synthesis input plus the legacy path (mutual exclusion, R3) + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml new file mode 100644 index 000000000..f3f9b9cdc --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-invalid-version-legacy-unified-body.yaml @@ -0,0 +1,33 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} +# INVALID: explicit legacy marker on a unified-shaped body (R11, LCORE-2872) +config_format_version: legacy diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml new file mode 100644 index 000000000..76b2ac36d --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-legacy-for-migration.yaml @@ -0,0 +1,23 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml new file mode 100644 index 000000000..1a1c4d8a1 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-absolute-profile.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + # Absolute path as mounted in the llama-stack container + profile: /opt/app-root/run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml new file mode 100644 index 000000000..3881fdf4e --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-config-only.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + # Synthesis baseline: the CI-materialized run.yaml (provider-agnostic) + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml new file mode 100644 index 000000000..7454bab43 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-list.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + profile: run.yaml + # R5: lists replace wholesale - the synthesized apis must equal exactly + # this list, not a merge with the baseline's (never booted - synthesis only) + native_override: + apis: + - inference + - tool_runtime +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml new file mode 100644 index 000000000..3451fb526 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-native-override-scalar.yaml @@ -0,0 +1,30 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + profile: run.yaml + # R5: the raw escape hatch wins; this scalar replaces the baseline's + # safety.excluded_categories value wholesale (never booted - synthesis only) + native_override: + safety: + excluded_categories: unified-override-marker +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml new file mode 100644 index 000000000..4ca585947 --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-providers.yaml @@ -0,0 +1,31 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini + # Unified synthesis input (Decision S5): the high-level provider entry + # replaces the default baseline's openai provider by id at synthesis time. + providers: + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + allowed_models: + - ${env.E2E_OPENAI_MODEL:=gpt-4o-mini} diff --git a/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml new file mode 100644 index 000000000..3c6c8512a --- /dev/null +++ b/tests/e2e/configuration/unified-mode/server-mode/lightspeed-stack-unified-relative-profile.yaml @@ -0,0 +1,26 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Server mode - connects to the separate llama-stack service + use_as_library_client: false + url: http://${env.E2E_LLAMA_HOSTNAME}:8321 + api_key: xyzzy + config: + # R8: relative profile resolves against this file's loaded location + profile: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini From 5f0c589b67b619e61c73dc89199cdaeaa145b9e6 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 10:35:04 +0200 Subject: [PATCH 2/7] LCORE-2343: restore legacy library-mode boot coverage with a dedicated fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LCORE-2342 migrated the standard library-mode baseline to unified mode (config.profile: run.yaml), which silently changed what unified-mode-legacy.feature's library scenario exercises: it now boots the unified baseline, not the deprecated two-file path, so R2's library-mode legacy coverage was gone. Add lightspeed-stack-legacy.yaml — identical to the baseline except its llama_stack block uses the true legacy shape (use_as_library_client + library_client_config_path: run.yaml, no synthesis input) — and point the library scenario's Given at it. This is the one deliberate Gherkin edit in LCORE-2343, agreed with Maxim in planning; the server-mode scenario is untouched since container-side enrichment there is genuinely legacy. --- .../library-mode/lightspeed-stack-legacy.yaml | 46 +++++++++++++++++++ .../e2e/features/unified-mode-legacy.feature | 5 +- 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml diff --git a/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml new file mode 100644 index 000000000..993f0d812 --- /dev/null +++ b/tests/e2e/configuration/library-mode/lightspeed-stack-legacy.yaml @@ -0,0 +1,46 @@ +name: Lightspeed Core Service (LCS) +service: + host: 0.0.0.0 + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + # Legacy two-file shape (R2 deprecation window): external run.yaml consumed + # via library_client_config_path; no unified synthesis input. Kept as a + # dedicated fixture because the standard library-mode baseline migrated to + # unified mode in LCORE-2342, which silently removed legacy boot coverage. + use_as_library_client: true + library_client_config_path: run.yaml +user_data_collection: + feedback_enabled: true + feedback_storage: "/tmp/data/feedback" + transcripts_enabled: true + transcripts_storage: "/tmp/data/transcripts" +authentication: + module: "noop" +inference: + default_provider: openai + default_model: gpt-4o-mini +byok_rag: + - rag_id: e2e-test-docs + rag_type: inline::faiss + embedding_model: sentence-transformers/all-mpnet-base-v2 + embedding_dimension: 768 + vector_db_id: ${env.FAISS_VECTOR_STORE_ID} + db_path: ${env.KV_RAG_PATH:=~/.llama/storage/rag/kv_store.db} + score_multiplier: 1.0 + +rag: + tool: + - e2e-test-docs + +shields: + - name: pii-redaction + provider_id: redaction + config: + rules: + - pattern: '\d+' + replacement: '[NUM]' + diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index 407c43e6a..d4b9c1402 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -12,7 +12,10 @@ Feature: Legacy two-file configuration during deprecation window @skip-in-server-mode Scenario: Legacy two-file configuration still boots and serves requests in library mode - Given The service uses the lightspeed-stack.yaml configuration + # lightspeed-stack-legacy.yaml (not the standard baseline): LCORE-2342 + # migrated the library-mode baseline to unified mode, so only a dedicated + # legacy-shaped fixture still exercises the deprecated two-file path (R2). + Given The service uses the lightspeed-stack-legacy.yaml configuration And The service is restarted When I access endpoint "readiness" using HTTP GET method Then The status code of the response is 200 From 06a2a7e40fde8f2b181d808f68762a144613fba2 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:13:02 +0200 Subject: [PATCH 3/7] LCORE-2343: implement unified-mode step definitions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tests/e2e/features/steps/unified_mode.py — the 16 step patterns the validation, migration, and synthesis features need (boot and legacy resolve entirely through existing generic steps). Per the planning decisions: - All artifact steps operate on the ON-DISK configuration (the repo-root lightspeed-stack.yaml copy configure_service applied), never the live service. - Validation runs the service CLI (--dump-configuration) as a black-box subprocess from the repo root and asserts a non-zero exit, so the error-contains assertions can never pass against a healthy load. - Migration runs the real --migrate-config CLI; the output lands in the active mode subdir under the gitignored name later Gherkin references, and is cleaned up per scenario. - Synthesis runs the config CLI exactly as the server entrypoint does (unified auto-detection -> synthesize_to_file, giving the 0600 mode the permissions scenario asserts). Round-trip and override assertions parse YAML and compare data, never bytes; override assertions are self-referential against the fixture's native_override and additionally assert the baseline differed, so replacements can't pass vacuously. - The --synthesized-config-output scenario launches a short-lived local service from the library-mode fixture variant on a rewritten port and polls for the custom output file (the flag is library-mode-only by design; running containers cannot be restarted with new CLI args). - The startup-log step is mode-aware: in server mode the synthesis evidence is emitted by the llama-stack container (entrypoint + CLI), not the lightspeed-stack container the Gherkin names — asserted against the synthesizing container with the rationale documented in the step. behave --dry-run over the five features: 24 scenarios, 200 steps, zero undefined. --- tests/e2e/features/steps/unified_mode.py | 477 +++++++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 tests/e2e/features/steps/unified_mode.py diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py new file mode 100644 index 000000000..880db4171 --- /dev/null +++ b/tests/e2e/features/steps/unified_mode.py @@ -0,0 +1,477 @@ +"""Step definitions for the unified-mode e2e features (LCORE-2343). + +Covers configuration validation, legacy-to-unified migration, and run.yaml +synthesis for the five ``unified-mode-*.feature`` files. + +Design rules (from the LCORE-2343 planning notes): + +- Validation, migration, and synthesis steps operate on the **on-disk** + configuration artifacts — never the live service. "The active + configuration" is the repo-root ``lightspeed-stack.yaml`` copy that + ``configure_service`` applied. +- Migration and synthesis run the real CLIs as subprocesses — exactly the + surface the server entrypoint and operators use — and assertions parse + the produced YAML (data equality, never byte comparison). +- The synthesis-log step is mode-aware: in server mode the synthesis + evidence is emitted by the llama-stack container (entrypoint + CLI), not + the lightspeed-stack container the Gherkin names; the scenario's intent + (R10: the synthesized path is logged at startup) is asserted against the + container that actually synthesizes. +""" + +import difflib +import os +import re +import shutil +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Optional + +import yaml +from behave import given, step, then, when # pyright: ignore +from behave.runner import Context + +# Generated by the --migrate-config step; matches the .gitignore entry. +MIGRATED_CONFIG_BASENAME = "lightspeed-stack-unified-migrated.yaml" +# Legacy half of "the legacy migration fixture pair"; its run.yaml half is +# the repo-root run.yaml the CI harness materializes. +MIGRATION_PAIR_LCS_BASENAME = "lightspeed-stack-legacy-for-migration.yaml" + +CLI_TIMEOUT_SECONDS = 120 +CUSTOM_OUTPUT_POLL_SECONDS = 60 + + +def _mode_subdir(context: Context) -> str: + """Return the mode fixture subdirectory name for the current harness mode.""" + return "library-mode" if context.is_library_mode else "server-mode" + + +def _config_dir(context: Context) -> Path: + """Resolve the active fixture directory, mode subdir included when present. + + Mirrors ``configure_service``'s resolution so files referenced by name in + Gherkin (fixture pairs, migrated output) land where that step finds them. + """ + base = Path( + getattr(context, "lightspeed_stack_config_directory", "") + or "tests/e2e/configuration" + ) + mode_base = base / _mode_subdir(context) + return mode_base if mode_base.is_dir() else base + + +def _active_config_path() -> Path: + """Return the on-disk active configuration (the applied repo-root copy).""" + return Path("lightspeed-stack.yaml") + + +def _run_cli( + args: list[str], cwd: Optional[Path] = None +) -> subprocess.CompletedProcess: + """Run a repo CLI as a subprocess, capturing output, never raising.""" + return subprocess.run( + [sys.executable, *args], + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + timeout=CLI_TIMEOUT_SECONDS, + check=False, + ) + + +def _load_yaml(path: Path) -> Any: + """Parse a YAML file.""" + with open(path, "r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def _native_override(config_path: Path) -> dict[str, Any]: + """Extract llama_stack.config.native_override from a config file.""" + config = _load_yaml(config_path) + override = ((config.get("llama_stack") or {}).get("config") or {}).get( + "native_override" + ) + assert override, f"{config_path} carries no llama_stack.config.native_override" + return override + + +def _synthesized(context: Context) -> Path: + """Return the synthesized run.yaml path recorded by an earlier step.""" + path = getattr(context, "synthesized_run_yaml_path", None) + assert path, "no synthesis step ran before this assertion" + return Path(path) + + +# --------------------------------------------------------------------------- +# Validation (unified-mode-validation.feature) +# --------------------------------------------------------------------------- + + +@when("configuration validation is attempted for the active configuration") +def attempt_configuration_validation(context: Context) -> None: + """Validate the on-disk active configuration via the service CLI. + + Runs ``lightspeed_stack.py --dump-configuration -c lightspeed-stack.yaml`` + as a black-box subprocess: ``main()`` loads (and thereby validates) the + configuration before any dump handling, so a Pydantic validation failure + surfaces on stderr with a non-zero exit code. cwd is the repo root so the + invalid fixtures' ``library_client_config_path: run.yaml`` resolves to the + harness-materialized run.yaml and the captured failure is the intended + cross-field error, not a file-not-found. + """ + result = _run_cli( + [ + "src/lightspeed_stack.py", + "--dump-configuration", + "-c", + str(_active_config_path()), + ] + ) + context.validation_returncode = result.returncode + context.validation_output = result.stdout + result.stderr + assert result.returncode != 0, ( + "expected the active configuration to fail validation, but the load " + f"succeeded (rc=0). Output:\n{context.validation_output}" + ) + + +@then("the validation error contains {text}") +def validation_error_contains(context: Context, text: str) -> None: + """Assert the captured validation failure mentions the given text.""" + output = getattr(context, "validation_output", None) + assert output is not None, "no validation attempt ran before this assertion" + assert ( + text.strip() in output + ), f"validation error does not contain {text!r}. Full output:\n{output}" + + +# --------------------------------------------------------------------------- +# Migration (unified-mode-migration.feature) +# --------------------------------------------------------------------------- + + +@step("lightspeed-stack --migrate-config is run for the legacy migration fixture pair") +def run_migrate_config(context: Context) -> None: + """Migrate the legacy fixture pair into the active fixture directory. + + The pair is ``lightspeed-stack-legacy-for-migration.yaml`` (mode subdir) + plus the repo-root ``run.yaml`` the harness materializes. The output lands + in the same mode subdir under the name later Gherkin steps reference, so + ``configure_service`` can boot it; it is gitignored and cleaned up after + the scenario. + """ + pair_lcs = _config_dir(context) / MIGRATION_PAIR_LCS_BASENAME + pair_run = Path("run.yaml") + output = _config_dir(context) / MIGRATED_CONFIG_BASENAME + assert pair_lcs.is_file(), f"missing migration fixture {pair_lcs}" + assert pair_run.is_file(), "repo-root run.yaml (harness-materialized) missing" + + result = _run_cli( + [ + "src/lightspeed_stack.py", + "--migrate-config", + "--run-yaml", + str(pair_run), + "-c", + str(pair_lcs), + "--migrate-output", + str(output), + ] + ) + assert result.returncode == 0 and output.is_file(), ( + f"--migrate-config failed (rc={result.returncode}).\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + context.migrated_config_path = output + context.migration_pair_run_yaml = pair_run + context.add_cleanup(lambda: output.unlink(missing_ok=True)) + + +@then("the file {filename} contains {text}") +def file_contains(context: Context, filename: str, text: str) -> None: + """Assert a file in the active fixture directory contains a substring.""" + path = _config_dir(context) / filename.strip() + content = path.read_text(encoding="utf-8") + assert text.strip() in content, f"{path} does not contain {text!r}" + + +@then("the file {filename} does not contain {text}") +def file_does_not_contain(context: Context, filename: str, text: str) -> None: + """Assert a file in the active fixture directory lacks a substring.""" + path = _config_dir(context) / filename.strip() + content = path.read_text(encoding="utf-8") + assert text.strip() not in content, f"{path} unexpectedly contains {text!r}" + + +# --------------------------------------------------------------------------- +# Synthesis (unified-mode-synthesis.feature + migration round-trip) +# --------------------------------------------------------------------------- + + +@step("the active unified configuration is synthesized to run.yaml") +def synthesize_active_configuration(context: Context) -> None: + """Synthesize a run.yaml from the on-disk unified configuration. + + Runs the config CLI exactly as the server entrypoint does (unified + auto-detection dispatches to ``synthesize_to_file``, which also gives the + 0600 output mode). Source precedence: the migrated config when the + migration step ran in this scenario, else the active on-disk config. + When the custom-output service step ran instead, this step is a + pass-through — the service subprocess performs the synthesis. + """ + if getattr(context, "custom_output_path", None): + return + + source = getattr(context, "migrated_config_path", None) or _active_config_path() + scratch = Path(tempfile.mkdtemp(prefix="lcore-e2e-synthesis-")) + context.add_cleanup(lambda: shutil.rmtree(scratch, ignore_errors=True)) + output = scratch / "run.yaml" + + result = _run_cli( + ["src/llama_stack_configuration.py", "-c", str(source), "-o", str(output)] + ) + assert result.returncode == 0 and output.is_file(), ( + f"synthesis CLI failed (rc={result.returncode}) for {source}.\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + context.synthesized_run_yaml_path = output + + +@then( + "the synthesized run.yaml parses to the same data as the legacy migration fixture run.yaml" +) +def synthesized_round_trips(context: Context) -> None: + """Assert migrate-then-synthesize reproduces the pair's run.yaml (data equality).""" + synthesized = _load_yaml(_synthesized(context)) + original = _load_yaml( + getattr(context, "migration_pair_run_yaml", None) or Path("run.yaml") + ) + if synthesized != original: + diff = "\n".join( + difflib.unified_diff( + yaml.dump(original, sort_keys=True).splitlines(), + yaml.dump(synthesized, sort_keys=True).splitlines(), + fromfile="pair run.yaml", + tofile="synthesized", + lineterm="", + ) + ) + raise AssertionError(f"round-trip data mismatch:\n{diff}") + + +@then( + "the synthesized run.yaml contains the native_override scalar value for safety.excluded_categories" +) +def synthesized_scalar_override(context: Context) -> None: + """Assert the override's scalar replaced the baseline value at that key (R5).""" + override_value = _native_override(_active_config_path())["safety"][ + "excluded_categories" + ] + synthesized = _load_yaml(_synthesized(context)) + actual = (synthesized.get("safety") or {}).get("excluded_categories") + assert actual == override_value, ( + f"safety.excluded_categories is {actual!r}, expected the " + f"native_override value {override_value!r}" + ) + baseline = _load_yaml(Path("run.yaml")) + baseline_value = (baseline.get("safety") or {}).get("excluded_categories") + assert baseline_value != override_value, ( + "fixture and baseline agree on safety.excluded_categories — the " + "replacement assertion would be vacuous" + ) + + +@then("the synthesized run.yaml contains exactly the native_override list for apis") +def synthesized_list_override(context: Context) -> None: + """Assert the override list replaced the baseline's apis wholesale (R5).""" + override_list = _native_override(_active_config_path())["apis"] + synthesized = _load_yaml(_synthesized(context)) + assert ( + synthesized.get("apis") == override_list + ), f"apis is {synthesized.get('apis')!r}, expected exactly {override_list!r}" + baseline = _load_yaml(Path("run.yaml")) + assert ( + baseline.get("apis") != override_list + ), "fixture and baseline agree on apis — wholesale replacement would be vacuous" + + +@then("the synthesized run.yaml contains ${{env.OPENAI_API_KEY}}") +def synthesized_keeps_env_reference(context: Context) -> None: + """Assert the emitted secret stays an environment reference on disk (R6).""" + content = _synthesized(context).read_text(encoding="utf-8") + assert ( + "${env.OPENAI_API_KEY}" in content + ), "synthesized run.yaml does not carry the ${env.OPENAI_API_KEY} reference" + + +@then("the synthesized run.yaml does not contain the resolved OPENAI_API_KEY value") +def synthesized_no_literal_secret(context: Context) -> None: + """Assert the literal secret value never lands on disk (R6).""" + secret = os.environ.get("OPENAI_API_KEY", "") + assert secret, ( + "OPENAI_API_KEY is not set in the harness environment — the " + "no-literal-secret assertion would be vacuous" + ) + content = _synthesized(context).read_text(encoding="utf-8") + assert ( + secret not in content + ), "synthesized run.yaml contains the resolved OPENAI_API_KEY value" + + +@then("the synthesized run.yaml file permissions are 0600") +def synthesized_permissions(context: Context) -> None: + """Assert the synthesized file is owner-read/write only (R10).""" + mode = stat.S_IMODE(os.stat(_synthesized(context)).st_mode) + assert mode == 0o600, f"synthesized run.yaml mode is {oct(mode)}, expected 0o600" + + +# --------------------------------------------------------------------------- +# --synthesized-config-output (unified-mode-synthesis.feature) +# --------------------------------------------------------------------------- + + +@given( + "lightspeed-stack is started with --synthesized-config-output set to a custom path" +) +def start_with_custom_synthesis_output(context: Context) -> None: + """Launch a short-lived local service with a custom synthesis output path. + + The flag only affects library-mode in-process synthesis, and the running + containers cannot be restarted with different CLI args — so this step + always uses the library-mode variant of the active fixture, copied to a + scratch directory with ``service.port`` rewritten to avoid clashing with + the running stack. The subprocess synthesizes during app startup; the + following Then steps poll for the file, and the process is killed on + scenario cleanup. + """ + active_basename = _active_config_path().name + fixture_basename = Path( + getattr(context, "feature_config", "") + or "lightspeed-stack-unified-providers.yaml" + ).name + base = Path( + getattr(context, "lightspeed_stack_config_directory", "") + or "tests/e2e/configuration" + ) + library_fixture = base / "library-mode" / fixture_basename + if not library_fixture.is_file(): + library_fixture = _config_dir(context) / fixture_basename + assert library_fixture.is_file(), f"no library-mode fixture for {active_basename}" + + scratch = Path(tempfile.mkdtemp(prefix="lcore-e2e-synthout-")) + context.add_cleanup(lambda: shutil.rmtree(scratch, ignore_errors=True)) + + config = _load_yaml(library_fixture) + config.setdefault("service", {})["port"] = 8099 + scratch_config = scratch / "lightspeed-stack.yaml" + with open(scratch_config, "w", encoding="utf-8") as f: + yaml.safe_dump(config, f) + + custom_output = scratch / "custom-run.yaml" + process = subprocess.Popen( # pylint: disable=consider-using-with + [ + sys.executable, + str(Path("src/lightspeed_stack.py").resolve()), + "-c", + str(scratch_config), + "--synthesized-config-output", + str(custom_output), + ], + cwd=str(scratch), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + context.custom_output_path = custom_output + context.custom_output_scratch = scratch + context.custom_output_process = process + + def _kill() -> None: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + context.add_cleanup(_kill) + + +@then("the synthesized run.yaml is written to the custom output path") +def custom_output_written(context: Context) -> None: + """Poll for the custom-path synthesis output and validate it parses.""" + custom_output = Path(context.custom_output_path) + process = context.custom_output_process + deadline = time.monotonic() + CUSTOM_OUTPUT_POLL_SECONDS + while time.monotonic() < deadline: + if custom_output.is_file() and custom_output.stat().st_size > 0: + break + if process.poll() is not None and not custom_output.is_file(): + out = process.stdout.read() if process.stdout else "" + raise AssertionError( + f"service exited (rc={process.returncode}) before writing the " + f"custom synthesis output.\n{out[-2000:]}" + ) + time.sleep(0.5) + assert custom_output.is_file(), ( + f"custom synthesis output {custom_output} did not appear within " + f"{CUSTOM_OUTPUT_POLL_SECONDS}s" + ) + assert isinstance(_load_yaml(custom_output), dict) + context.synthesized_run_yaml_path = custom_output + + +@then("the default synthesized run.yaml path does not exist") +def default_output_absent(context: Context) -> None: + """Assert the default synthesis location was not used (override took effect).""" + scratch = Path(context.custom_output_scratch) + default_path = scratch / ".generated" / "run.yaml" + assert ( + not default_path.exists() + ), f"default synthesis path {default_path} exists despite the override" + + +# --------------------------------------------------------------------------- +# Startup logging (unified-mode-synthesis.feature) — mode-aware, see module +# docstring and LCORE-2343 planning decision Q2. +# --------------------------------------------------------------------------- + + +@then("the lightspeed-stack container logs contain synthesized run.yaml") +def container_logs_show_synthesis(context: Context) -> None: + """Assert the container that synthesizes logged the synthesized-config path. + + Library mode: the lightspeed-stack container itself synthesizes in-process + and logs "Using synthesized Llama Stack config at ". Server mode: + synthesis happens in the llama-stack container (entrypoint + config CLI), + which logs the generated-config path — the Gherkin names lightspeed-stack, + but the scenario's intent (R10: the path is logged at startup) can only be + observed on the synthesizing container. Deviation agreed in planning (Q2). + """ + if context.is_library_mode: + container = "lightspeed-stack" + pattern = r"synthesized.*run\.yaml|Using synthesized Llama Stack config" + else: + container = "llama-stack" + pattern = ( + r"Wrote synthesized Llama Stack configuration" + r"|Using generated config:.*run\.yaml" + r"|mode auto-detected" + ) + + result = subprocess.run( + ["docker", "logs", container], + capture_output=True, + text=True, + timeout=60, + check=False, + ) + assert ( + result.returncode == 0 + ), f"docker logs {container} failed: {result.stderr[-500:]}" + logs = result.stdout + result.stderr + assert re.search(pattern, logs), ( + f"{container} logs carry no synthesis-path evidence " + f"(pattern {pattern!r} not found)" + ) From 74b497d37c7d2a021ff6fb7df7f494aea792a2e5 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:13:02 +0200 Subject: [PATCH 4/7] LCORE-2343: gate unified-mode features for Prow and the providers matrix Tag all five unified-mode features @skip-in-prow: the new steps rely on Docker containers and local subprocesses, neither of which exists in the Prow environment (existing convention, handled in before_scenario). Add an @openai-only tag on the two inference.providers boot scenarios and a matching before_scenario skip keyed on E2E_DEFAULT_PROVIDER_OVERRIDE: the providers workflow runs the full unsharded test list against azure/watsonx/bedrock matrices, and the unified-providers fixture hardcodes an openai provider that cannot serve those models' queries. Profile-based fixtures stay provider-agnostic (they consume the CI-materialized run.yaml) and need no gating. --- tests/e2e/features/environment.py | 15 +++++++++++++++ tests/e2e/features/unified-mode-boot.feature | 6 +++--- tests/e2e/features/unified-mode-legacy.feature | 2 +- tests/e2e/features/unified-mode-migration.feature | 2 +- tests/e2e/features/unified-mode-synthesis.feature | 2 +- .../e2e/features/unified-mode-validation.feature | 2 +- 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/tests/e2e/features/environment.py b/tests/e2e/features/environment.py index cd05d3bdc..78314f830 100644 --- a/tests/e2e/features/environment.py +++ b/tests/e2e/features/environment.py @@ -242,6 +242,21 @@ def before_scenario(context: Context, scenario: Scenario) -> None: scenario.skip("Skipped in Prow (requires Docker Compose services)") return + # Skip openai-specific scenarios on non-openai provider matrices: the + # providers workflow runs the full test list with E2E_DEFAULT_PROVIDER_OVERRIDE + # set (azure/watsonx/...), and fixtures that hardcode an openai provider + # (e.g. the unified-mode inference.providers fixture) cannot serve queries + # for those models. + provider_override = os.getenv("E2E_DEFAULT_PROVIDER_OVERRIDE", "") + if "openai-only" in scenario.effective_tags and provider_override not in ( + "", + "openai", + ): + scenario.skip( + f"Skipped on provider matrix '{provider_override}' (openai-only fixture)" + ) + return + # In Prow, verify the lightspeed port-forward is alive before each scenario. # Port-forwards can silently die between scenarios (e.g. pod restart, TCP reset). if is_prow_environment(): diff --git a/tests/e2e/features/unified-mode-boot.feature b/tests/e2e/features/unified-mode-boot.feature index 7c1413818..fc91713de 100644 --- a/tests/e2e/features/unified-mode-boot.feature +++ b/tests/e2e/features/unified-mode-boot.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Unified mode configuration boot Background: @@ -10,7 +10,7 @@ Feature: Unified mode configuration boot # --- library mode (@skip-in-server-mode) --- - @skip-in-server-mode + @skip-in-server-mode @openai-only Scenario: Unified config with inference.providers boots and serves requests in library mode Given The service uses the lightspeed-stack-unified-providers.yaml configuration And The service is restarted @@ -62,7 +62,7 @@ Feature: Unified mode configuration boot # --- server mode (@skip-in-library-mode) --- - @skip-in-library-mode + @skip-in-library-mode @openai-only Scenario: Unified config with inference.providers boots and serves requests in server mode Given The service uses the lightspeed-stack-unified-providers.yaml configuration And Llama Stack is restarted diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index d4b9c1402..9404e09a8 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Legacy two-file configuration during deprecation window Background: diff --git a/tests/e2e/features/unified-mode-migration.feature b/tests/e2e/features/unified-mode-migration.feature index b5a2d06eb..f6773327e 100644 --- a/tests/e2e/features/unified-mode-migration.feature +++ b/tests/e2e/features/unified-mode-migration.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Legacy to unified configuration migration Background: diff --git a/tests/e2e/features/unified-mode-synthesis.feature b/tests/e2e/features/unified-mode-synthesis.feature index 52b4254fa..83a4f9366 100644 --- a/tests/e2e/features/unified-mode-synthesis.feature +++ b/tests/e2e/features/unified-mode-synthesis.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Unified mode configuration synthesis Background: diff --git a/tests/e2e/features/unified-mode-validation.feature b/tests/e2e/features/unified-mode-validation.feature index 5a1688335..41563a4c5 100644 --- a/tests/e2e/features/unified-mode-validation.feature +++ b/tests/e2e/features/unified-mode-validation.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip +@cfg_unified @skip @skip-in-prow Feature: Unified mode configuration validation Background: From 68a236e10d814877bc68040b61e39131ffca3888 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:20:09 +0200 Subject: [PATCH 5/7] LCORE-2343: unskip the unified-mode feature files Remove the @skip placeholder tag from the five unified-mode features: the step definitions and fixtures they need now exist. The features keep @skip-in-prow (Docker/subprocess dependencies) and stay in test_list.txt under @e2e_group_2, so CI shards pick them up via 'not @skip and @e2e_group_2' and local runs via --tags=-skip. --- tests/e2e/features/unified-mode-boot.feature | 2 +- tests/e2e/features/unified-mode-legacy.feature | 2 +- tests/e2e/features/unified-mode-migration.feature | 2 +- tests/e2e/features/unified-mode-synthesis.feature | 2 +- tests/e2e/features/unified-mode-validation.feature | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/e2e/features/unified-mode-boot.feature b/tests/e2e/features/unified-mode-boot.feature index fc91713de..a78928200 100644 --- a/tests/e2e/features/unified-mode-boot.feature +++ b/tests/e2e/features/unified-mode-boot.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Unified mode configuration boot Background: diff --git a/tests/e2e/features/unified-mode-legacy.feature b/tests/e2e/features/unified-mode-legacy.feature index 9404e09a8..e54a64950 100644 --- a/tests/e2e/features/unified-mode-legacy.feature +++ b/tests/e2e/features/unified-mode-legacy.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Legacy two-file configuration during deprecation window Background: diff --git a/tests/e2e/features/unified-mode-migration.feature b/tests/e2e/features/unified-mode-migration.feature index f6773327e..68cd2aaeb 100644 --- a/tests/e2e/features/unified-mode-migration.feature +++ b/tests/e2e/features/unified-mode-migration.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Legacy to unified configuration migration Background: diff --git a/tests/e2e/features/unified-mode-synthesis.feature b/tests/e2e/features/unified-mode-synthesis.feature index 83a4f9366..3bfba3a65 100644 --- a/tests/e2e/features/unified-mode-synthesis.feature +++ b/tests/e2e/features/unified-mode-synthesis.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Unified mode configuration synthesis Background: diff --git a/tests/e2e/features/unified-mode-validation.feature b/tests/e2e/features/unified-mode-validation.feature index 41563a4c5..0a03bb4f3 100644 --- a/tests/e2e/features/unified-mode-validation.feature +++ b/tests/e2e/features/unified-mode-validation.feature @@ -1,4 +1,4 @@ -@cfg_unified @skip @skip-in-prow +@cfg_unified @skip-in-prow Feature: Unified mode configuration validation Background: From dcd4316fb8b52546625d6c5ef9832950dc07513e Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 11:34:05 +0200 Subject: [PATCH 6/7] LCORE-2343: close the health-vs-listen race in lightspeed restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restart_container waits on docker health, but docker can report healthy before uvicorn binds the published port — the exact race wait_for_lightspeed_stack_http_ready documents and was, until now, only closed in the proxy steps. The unified-mode boot scenarios are the slowest restarts in the suite (first unified/default-baseline boots) and hit that window reliably: the restart step passed while the following readiness GET got connection-refused. Call the existing HTTP-ready wait from restart_container for the lightspeed-stack container, closing the race for every restarting scenario; when the port is already accepting, the first poll returns immediately. --- tests/e2e/utils/utils.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/e2e/utils/utils.py b/tests/e2e/utils/utils.py index 548d3e6a4..3365b0d6b 100644 --- a/tests/e2e/utils/utils.py +++ b/tests/e2e/utils/utils.py @@ -486,6 +486,14 @@ def restart_container(container_name: str) -> None: # that restart the container don't time out. wait_for_container_health(container_name) + # Docker health can report healthy before uvicorn binds the published + # port (the documented race wait_for_lightspeed_stack_http_ready exists + # for). Unified-mode first boots are the slowest restarts in the suite + # and hit that window reliably, so close it here for every restart + # rather than only in the proxy steps. + if container_name == "lightspeed-stack": + wait_for_lightspeed_stack_http_ready() + if container_name == "llama-stack": from tests.e2e.features.steps.health import ( reset_llama_stack_disrupt_once_tracking, From ba9f5ebe2fb66ae42927c01b7d6b609a879cc9e9 Mon Sep 17 00:00:00 2001 From: Maxim Svistunov Date: Tue, 4 Aug 2026 12:09:05 +0200 Subject: [PATCH 7/7] LCORE-2343: make the migrated e2e config readable by the container user MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --migrate-config writes its output 0600 (R10: migrated files may carry lifted secrets), but the boot scenarios copy that file to the repo root for the container to consume, and the container user cannot read a host-owned 0600 file — the migrated-config boot scenario died on config read. Relax the harness copy to 0644 after a successful migration; the fixture pair is env-reference-only by design, so no secret can leak. --- tests/e2e/features/steps/unified_mode.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/e2e/features/steps/unified_mode.py b/tests/e2e/features/steps/unified_mode.py index 880db4171..f865f913a 100644 --- a/tests/e2e/features/steps/unified_mode.py +++ b/tests/e2e/features/steps/unified_mode.py @@ -186,6 +186,11 @@ def run_migrate_config(context: Context) -> None: f"--migrate-config failed (rc={result.returncode}).\n" f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" ) + # The CLI writes 0600 (R10: migrated files may carry lifted secrets), but + # the container user must be able to read the copy configure_service puts + # at the repo root to boot it. The fixture pair is env-reference-only by + # design, so relaxing the harness copy is safe. + os.chmod(output, 0o644) context.migrated_config_path = output context.migration_pair_run_yaml = pair_run context.add_cleanup(lambda: output.unlink(missing_ok=True))