From aec75cb5fd729a0d7ed7a6042fbf5a8a28eb9b68 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 15 Jul 2026 15:04:10 -0700 Subject: [PATCH 1/5] test(e2e): run VM suite in CI Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 65 ++++++++++++++++++++++++++++++++++ CI.md | 9 ++++- TESTING.md | 13 +++++++ architecture/build.md | 3 +- 4 files changed, 88 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index ea870d98e1..4069522ccc 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -205,3 +205,68 @@ jobs: - name: Print AppArmor denials if: always() run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true + + e2e-vm: + name: E2E (rust-vm) + # libkrun needs KVM, so this job must run directly on a GitHub-hosted + # Linux VM. GitHub-hosted macOS runners do not support nested + # virtualization, and a job container would hide the host KVM device. + runs-on: ubuntu-24.04 + timeout-minutes: 90 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs['checkout-ref'] || github.sha }} + persist-credentials: false + + - name: Enable KVM access + run: | + set -euo pipefail + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + test -c /dev/kvm + test -r /dev/kvm + test -w /dev/kvm + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + cmake \ + libclang-dev \ + libssl-dev \ + libz3-dev \ + openssh-client \ + pkg-config \ + socat \ + zstd + + - name: Install mise + run: | + curl https://mise.run | MISE_VERSION=v2026.4.25 sh + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH" + + - name: Install tools + run: mise install --locked + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2 + with: + shared-key: e2e-vm-linux-amd64 + cache-on-failure: "true" + + - name: Run VM smoke and host gateway E2E + run: mise run --no-deps --skip-deps e2e:vm + + - name: Run VM gateway resume E2E + env: + OPENSHELL_E2E_VM_TEST: vm_gateway_resume + run: mise run --no-deps --skip-deps e2e:vm diff --git a/CI.md b/CI.md index e14d5bdb9d..93d58b925e 100644 --- a/CI.md +++ b/CI.md @@ -14,7 +14,8 @@ Merge queue validation is a second integration gate for `main`. After a PR has p Three opt-in labels enable the long-running E2E suites: -- `test:e2e` runs the standard E2E suite in `Branch E2E Checks` +- `test:e2e` runs the standard Docker, rootless Podman, Kubernetes, and VM E2E + suites in `Branch E2E Checks` - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks` - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay (`replicaCount: 2` and bundled PostgreSQL) in `Branch E2E Checks` @@ -22,6 +23,12 @@ Three opt-in labels enable the long-running E2E suites: When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once and fans out all enabled suites in parallel. The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while HA behavior is under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. +The VM suite runs directly on a standard GitHub-hosted Ubuntu runner. It grants +the runner user access to KVM, then exercises VM sandbox creation, host gateway +networking, and sandbox resume after a gateway restart. GitHub-hosted macOS +runners cannot run this suite because they do not support nested +virtualization. + The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. ## Commit signing diff --git a/TESTING.md b/TESTING.md index d32dc385c2..238f2fbfbd 100644 --- a/TESTING.md +++ b/TESTING.md @@ -150,6 +150,7 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway resume. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. +- VM suite (`--features e2e-vm`) - VM sandbox smoke coverage and host gateway networking, with managed VM gateway resume available as a targeted test. The host must expose KVM on Linux or Hypervisor.framework on macOS. GPU device-selection tests compare OpenShell sandboxes against a plain Docker or Podman container that requests `--device nvidia.com/gpu=all`. The probe image @@ -173,6 +174,18 @@ Run the Podman-backed Rust CLI e2e suite: mise run e2e:podman ``` +Run the VM-backed Rust CLI e2e suite: + +```shell +mise run e2e:vm +``` + +Run the VM gateway-resume test by itself: + +```shell +OPENSHELL_E2E_VM_TEST=vm_gateway_resume mise run e2e:vm +``` + Run a single test directly with cargo: ```shell diff --git a/architecture/build.md b/architecture/build.md index e3ac5012e3..63d4e56849 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -148,7 +148,8 @@ The high-level CI model: 1. PR-context gate jobs publish required statuses for the PR head commit. 2. Standard branch checks run from trusted mirror branches. -3. Label-gated E2E, GPU, and Kubernetes checks run from trusted mirror branches. +3. Label-gated Docker, Podman, VM, GPU, and Kubernetes E2E checks run from + trusted mirror branches. 4. Merge-group checks run against GitHub's temporary queue branch for the final integration state. 5. Gate jobs verify that the mirror branch matches the PR head, or that the merge-group workflow ran for the queued SHA, and that the expected non-gate workflow actually ran. 6. Release workflows rebuild and publish binaries, wheels, images, and docs. From 5043372f7ec7985f08ae76caf6365a73ac9fb369 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 15 Jul 2026 15:24:08 -0700 Subject: [PATCH 2/5] fix(ci): configure KVM permissions directly Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 4069522ccc..ba9e1c91c6 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -225,11 +225,15 @@ jobs: - name: Enable KVM access run: | set -euo pipefail - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ - | sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - test -c /dev/kvm + if [[ ! -c /dev/kvm ]]; then + echo "::error::The GitHub-hosted runner did not expose /dev/kvm" + lscpu + grep -m1 -E '^(flags|Features)' /proc/cpuinfo || true + ls -la /dev + exit 1 + fi + sudo chmod 0666 /dev/kvm + ls -l /dev/kvm test -r /dev/kvm test -w /dev/kvm From 0b41a5277c70c8b7fc67d4b30b1b91683ed5f74a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 15 Jul 2026 16:02:49 -0700 Subject: [PATCH 3/5] fix(e2e): flush VM overlay before restart Signed-off-by: Drew Newberry --- e2e/rust/tests/vm_gateway_resume.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/rust/tests/vm_gateway_resume.rs b/e2e/rust/tests/vm_gateway_resume.rs index b2ec6c8b57..4c502bb9c9 100644 --- a/e2e/rust/tests/vm_gateway_resume.rs +++ b/e2e/rust/tests/vm_gateway_resume.rs @@ -36,8 +36,11 @@ async fn vm_gateway_restart_resumes_running_sandbox() { .await .expect("gateway should start healthy"); + // The gateway restart terminates the VM process before re-adopting its + // overlay. Flush the marker before reporting readiness so the assertion + // verifies durable overlay state rather than guest page-cache timing. let script = format!( - "echo before-restart > {RESUME_FILE}; echo {READY_MARKER}; while true; do sleep 1; done" + "echo before-restart > {RESUME_FILE}; sync; echo {READY_MARKER}; while true; do sleep 1; done" ); let mut sandbox = SandboxGuard::create_keep(&["sh", "-lc", &script], READY_MARKER) .await From 4bb250bdf743d79e499291019ef4e5526aaf9a7a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 15 Jul 2026 16:35:15 -0700 Subject: [PATCH 4/5] docs: simplify VM test documentation Signed-off-by: Drew Newberry --- CI.md | 6 ------ TESTING.md | 8 +------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/CI.md b/CI.md index 93d58b925e..19fad7bd35 100644 --- a/CI.md +++ b/CI.md @@ -23,12 +23,6 @@ Three opt-in labels enable the long-running E2E suites: When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once and fans out all enabled suites in parallel. The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while HA behavior is under active iteration: failures are visible in the workflow run but do not publish a required CI gate status. -The VM suite runs directly on a standard GitHub-hosted Ubuntu runner. It grants -the runner user access to KVM, then exercises VM sandbox creation, host gateway -networking, and sandbox resume after a gateway restart. GitHub-hosted macOS -runners cannot run this suite because they do not support nested -virtualization. - The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly. ## Commit signing diff --git a/TESTING.md b/TESTING.md index 238f2fbfbd..a032baa5ea 100644 --- a/TESTING.md +++ b/TESTING.md @@ -150,7 +150,7 @@ Suites: - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests. - Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway resume. - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage. -- VM suite (`--features e2e-vm`) - VM sandbox smoke coverage and host gateway networking, with managed VM gateway resume available as a targeted test. The host must expose KVM on Linux or Hypervisor.framework on macOS. +- VM suite (`--features e2e-vm`) - runs e2e tests on a VM. GPU device-selection tests compare OpenShell sandboxes against a plain Docker or Podman container that requests `--device nvidia.com/gpu=all`. The probe image @@ -180,12 +180,6 @@ Run the VM-backed Rust CLI e2e suite: mise run e2e:vm ``` -Run the VM gateway-resume test by itself: - -```shell -OPENSHELL_E2E_VM_TEST=vm_gateway_resume mise run e2e:vm -``` - Run a single test directly with cargo: ```shell From 37667e2e52709aed6cf9d18bbe0ea86cb792de92 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 15 Jul 2026 16:40:53 -0700 Subject: [PATCH 5/5] test(e2e): include gateway resume in VM run Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 7 +------ e2e/rust/e2e-vm.sh | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index ba9e1c91c6..cb7bc67d19 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -267,10 +267,5 @@ jobs: shared-key: e2e-vm-linux-amd64 cache-on-failure: "true" - - name: Run VM smoke and host gateway E2E - run: mise run --no-deps --skip-deps e2e:vm - - - name: Run VM gateway resume E2E - env: - OPENSHELL_E2E_VM_TEST: vm_gateway_resume + - name: Run VM E2E run: mise run --no-deps --skip-deps e2e:vm diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index ef798a8ea9..8005d09bd0 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -315,4 +315,5 @@ if [ -n "${E2E_TEST_OVERRIDE}" ]; then else run_e2e_test smoke run_e2e_test host_gateway_alias + run_e2e_test vm_gateway_resume fi