From f5ed92a1b7aa73dcb8d457987d7d43e614d1aa6e Mon Sep 17 00:00:00 2001 From: mintaka Date: Sun, 30 Aug 2026 16:42:52 -0400 Subject: [PATCH 1/4] feat(runtime): deliver egress policy to the guest; skip host arm on self-arming backends (V3 W2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread the egress policy from the agent spec into the microVM guest and teach the host to skip its own arm when the backend arms in-guest, per the frozen microVM Runner V3 record (§(a)/(c)/(e)). - `ContainerSpec.Egress EgressPolicy` — the microVM backend delivers it to guestd; the podman backend IGNORES it (its argv stays byte-identical and it keeps arming via `AgentRuntime.armEgress`). `createAndStart` copies `spec.Egress` into the ContainerSpec. - `microvmSession.nftScript` recorded at Create from `spec.Egress.NftScript()` (never empty — the zero-value policy still emits the default-deny base ruleset, §(e)); `Start`'s Provision RPC carries it as `nft_script`. - Hermetic Start seams (mirroring the supervisor's `newCredential` seam): `launchFunc guestLaunchFunc` over `microvm.Launch` and `newGuestClient guestClientFunc` over `microvm.GuestClient`, wired by `installSeamDefaults`. `guestVM` is a new unexported interface retyping the shared `microvmSession.vm` field (`*microvm.VM` → `guestVM`); its method set is the exhaustive set invoked on that field across ALL build tags — `Health`, `Shutdown`, `WaitVMMExit`, `PSS` — so the `-tags microvm` contract test's `session.vm.PSS()` still compiles. Only Start's Provision client is seamed; Stop's dial stays direct. - Probe-and-skip: `MicroVMRuntime.EgressArmedInGuest() bool` is an off-interface marker (the frozen `ContainerRuntime` grows no verb); `AgentRuntime.provision` type-asserts the unexported `inGuestEgressArmer` and skips `armEgress` on a self-arming backend — which on the microVM backend would run capability-less and fail. A pointer comment beside the ContainerRuntime freeze note flags the decorator hazard (a decorator that swallowed the marker would silently re-enable the host arm). - Tests (hermetic, no VM boot): script delivered verbatim + correct uid; a Provision error fails Start and tears the booted VM down (Shutdown), storing no handle; the marker skips the host nft exec while credentials + checkout still run; the zero-value policy records a non-empty default-deny script; `createArgs` is byte-identical with vs without `Egress`. Refs RIG-3019 Co-authored-by: Matt Wilkinson --- go/internal/runtime/agent.go | 25 ++- go/internal/runtime/agent_test.go | 58 ++++++ go/internal/runtime/microvm.go | 11 +- go/internal/runtime/microvm_lifecycle.go | 84 +++++++-- go/internal/runtime/microvm_start_test.go | 206 ++++++++++++++++++++++ go/internal/runtime/podman.go | 17 ++ 6 files changed, 386 insertions(+), 15 deletions(-) create mode 100644 go/internal/runtime/microvm_start_test.go diff --git a/go/internal/runtime/agent.go b/go/internal/runtime/agent.go index 520f6206..e8aaf85f 100644 --- a/go/internal/runtime/agent.go +++ b/go/internal/runtime/agent.go @@ -269,6 +269,9 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont // Keep the container alive so the Runner can exec into it; the agent is // driven via exec, not as the container's main process. Command: []string{"sleep", "infinity"}, + // The microVM backend delivers this to guestd for an in-guest arm; the + // podman backend ignores it and arms via provision's armEgress exec. + Egress: spec.Egress, } id, err := r.runtime.Create(ctx, container) @@ -285,11 +288,27 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont return id, nil } +// inGuestEgressArmer is a backend that arms the egress firewall itself, inside +// its isolation boundary (as guest root, before the exec gate opens), so the +// host-side armEgress exec must be skipped. It is a marker, deliberately NOT a +// verb on the frozen ContainerRuntime interface (podman.go): AgentRuntime probes +// for it and skips arming when a backend self-arms (design §(c)). Only +// MicroVMRuntime implements it; PodmanCLI and the test fakes do not, so the +// host-side arm runs byte-identically for them. +type inGuestEgressArmer interface { + EgressArmedInGuest() bool +} + // provision runs the post-start steps, all inside the running container: -// firewall (root), credentials (agent user), checkout dir (agent user). +// firewall (root), credentials (agent user), checkout dir (agent user). A +// backend that self-arms egress in-guest (inGuestEgressArmer, the microVM +// backend) has already armed by Start, so the host-side armEgress exec — which +// on that backend would run capability-less and fail — is skipped. func (r *AgentRuntime) provision(ctx context.Context, id ContainerID, spec AgentSpec) error { - if err := r.armEgress(ctx, id, spec.Egress); err != nil { - return err + if armer, ok := r.runtime.(inGuestEgressArmer); !ok || !armer.EgressArmedInGuest() { + if err := r.armEgress(ctx, id, spec.Egress); err != nil { + return err + } } if err := r.installCredentials(ctx, id, spec.Workspace); err != nil { return err diff --git a/go/internal/runtime/agent_test.go b/go/internal/runtime/agent_test.go index 052a8ca2..18b27702 100644 --- a/go/internal/runtime/agent_test.go +++ b/go/internal/runtime/agent_test.go @@ -294,3 +294,61 @@ func TestTeardownStopsThenRemoves(t *testing.T) { t.Errorf("stop (%d) must precede remove (%d)", stop, remove) } } + +// inGuestArmingFakeRuntime is a fakeRuntime that self-arms egress in-guest: it +// implements the inGuestEgressArmer marker (EgressArmedInGuest), so +// AgentRuntime.provision must skip the host-side armEgress exec — mirroring the +// microVM backend without booting a VM. +type inGuestArmingFakeRuntime struct { + *fakeRuntime +} + +func (f *inGuestArmingFakeRuntime) EgressArmedInGuest() bool { return true } + +// TestInGuestArmerSkipsHostArmEgress: a backend that self-arms in-guest receives +// NO nft exec, and provision still proceeds to credentials + checkout dir. This +// is the (c) probe-and-skip contract — the microVM path arms inside Start, so +// the host-side capability-less arm exec must not run. +func TestInGuestArmerSkipsHostArmEgress(t *testing.T) { + fake := &inGuestArmingFakeRuntime{fakeRuntime: newFakeRuntime(t)} + rt := NewAgentRuntime(fake) + + if _, err := rt.Launch(t.Context(), specWithCreds(true)); err != nil { + t.Fatalf("Launch error = %v", err) + } + + // No exec may carry the egress ruleset: the backend armed in-guest. + for _, e := range fake.execsSnapshot() { + if slices.ContainsFunc(e.Command, func(tok string) bool { return strings.Contains(tok, "compass_egress") }) { + t.Errorf("a self-arming backend must not run the host armEgress exec; command = %v", e.Command) + } + } + // Provision still proceeds: credentials install + checkout dir are made. + calls := fake.callsSnapshot() + mkdir := slices.IndexFunc(calls, func(c string) bool { return strings.Contains(c, "mkdir") }) + if mkdir < 0 { + t.Errorf("provision must still create the checkout dir after skipping the arm; calls = %v", calls) + } + creds := slices.ContainsFunc(fake.execsSnapshot(), func(e ExecSpec) bool { + return e.Stdin != nil && strings.Contains(*e.Stdin, "git-credentials") + }) + if !creds { + t.Errorf("provision must still install credentials after skipping the arm; execs = %v", fake.execsSnapshot()) + } +} + +// TestCreateArgsIgnoresEgress pins the podman byte-identical constraint: setting +// ContainerSpec.Egress must not change the `podman create` argv at all. The +// podman backend arms via AgentRuntime.armEgress, never from the spec field, so +// createArgs output for a spec with Egress set equals its output without. +func TestCreateArgsIgnoresEgress(t *testing.T) { + base := ContainerSpec{Name: "c", Image: "img", UID: 1000, CapAdd: []string{capNetAdmin}} + withEgress := base + withEgress.Egress = MustAllowEgress("github.com", "example.com") + + got := createArgs(withEgress) + want := createArgs(base) + if !slices.Equal(got, want) { + t.Errorf("createArgs changed when Egress was set:\n with = %q\n without = %q", got, want) + } +} diff --git a/go/internal/runtime/microvm.go b/go/internal/runtime/microvm.go index 4cb56390..9ffec7bd 100644 --- a/go/internal/runtime/microvm.go +++ b/go/internal/runtime/microvm.go @@ -74,15 +74,24 @@ type MicroVMRuntime struct { // this map for a matching spec.Name — a scan is cheap at one-VM-per-session // scale and keeps a single source of truth. sessions map[ContainerID]*microvmSession + // launchFunc boots a session guest behind the guestVM seam; newGuestClient + // dials the guest control plane. Both default to the real microvm + // implementations (installSeamDefaults, //go:build unix) and are overridden + // in hermetic Start tests so no real VMM boots and no real vsock is dialed + // (design §W2 seams, mirroring the supervisor's newCredential seam). + launchFunc guestLaunchFunc + newGuestClient guestClientFunc } // NewMicroVMRuntime builds a MicroVMRuntime from the supplied config, mirroring // NewPodmanCLI's shape, with an empty session table ready for Create. func NewMicroVMRuntime(cfg MicroVMConfig) *MicroVMRuntime { - return &MicroVMRuntime{ + m := &MicroVMRuntime{ config: cfg, sessions: make(map[ContainerID]*microvmSession), } + m.installSeamDefaults() + return m } // SelectBackend chooses the container runtime backend from cfg. An empty or diff --git a/go/internal/runtime/microvm_lifecycle.go b/go/internal/runtime/microvm_lifecycle.go index 7e4c60cc..7d0f642a 100644 --- a/go/internal/runtime/microvm_lifecycle.go +++ b/go/internal/runtime/microvm_lifecycle.go @@ -81,6 +81,47 @@ const healthPollInterval = 200 * time.Millisecond // timeout error, never block the calling task forever. const execDefaultTimeout = 120 * time.Second +// guestVM is the running-guest handle MicroVMRuntime drives, an interface over +// *microvm.VM so Start is hermetically testable behind a fake handle. Its method +// set is NOT merely what Start calls — it retypes the shared microvmSession.vm +// field, so it must cover EVERY method invoked on that field anywhere in the +// package across ALL build tags: Health/Shutdown (Start/awaitHealthy and Stop, +// launch.go), WaitVMMExit (Stop, microvm_lifecycle.go), and PSS (the Q-budget +// contract test's session.vm.PSS(), contract_microvm_test.go, //go:build microvm +// && unix). *microvm.VM satisfies all four as-is (design §W2 seams). +type guestVM interface { + Health(ctx context.Context) (*compassv1.HealthResponse, error) + Shutdown(ctx context.Context) error + WaitVMMExit(timeout time.Duration) bool + PSS() (map[string]int64, error) +} + +// guestLaunchFunc boots a session guest, returning it behind the guestVM seam. +// It defaults to a thin adapter over microvm.Launch (installSeamDefaults) and is +// overridden in hermetic Start tests so no real VMM boots. +type guestLaunchFunc func(context.Context, microvm.BootConfig) (guestVM, error) + +// guestClientFunc dials the guest control plane, returning a GuestControlClient. +// It defaults to microvm.GuestClient (installSeamDefaults) and is overridden in +// hermetic Start tests so a fake client answers Provision with no real vsock +// dial. It seams Start's Provision client ONLY; Stop's own dial stays direct. +type guestClientFunc func(socket string, port uint32) compassv1internalconnect.GuestControlClient + +// installSeamDefaults wires the production launch + client implementations onto a +// freshly constructed MicroVMRuntime. It is //go:build unix (like the microvm +// package it names) and is called by NewMicroVMRuntime, so hermetic tests can +// override the seams after construction. +func (m *MicroVMRuntime) installSeamDefaults() { + m.launchFunc = func(ctx context.Context, cfg microvm.BootConfig) (guestVM, error) { + vm, err := microvm.Launch(ctx, cfg) + if err != nil { + return nil, err + } + return vm, nil + } + m.newGuestClient = microvm.GuestClient +} + // microvmSession is one allocated microVM session's state. Created by Create // (not yet booted), populated with the running VM handle and exec client by // Start, and dropped by Remove. All fields are read/written under @@ -100,11 +141,20 @@ type microvmSession struct { // nonce is the per-session boot nonce (raw bytes); its hex encoding rides // the cmdline, and Start verifies guestd echoes it before opening the gate. nonce []byte + // nftScript is the egress ruleset delivered to guestd on Start's Provision + // RPC (as ProvisionRequest.nft_script). Recorded at Create from + // spec.Egress.NftScript(); NEVER empty for a ContainerSpec-created session, + // since the zero-value EgressPolicy still emits the full default-deny base + // ruleset (design §(e), egress.go). guestd arms it as guest root before the + // exec gate opens. + nftScript string // runtimeDir is /microvm//, holding the session's sockets. runtimeDir string // vm and guestExec are nil until Start boots the guest; Start sets both - // under the lock once the boot + Provision succeed. - vm *microvm.VM + // under the lock once the boot + Provision succeed. vm is typed as the + // unexported guestVM interface (not *microvm.VM directly) so Start is + // hermetically testable behind a fake handle (design §W2 seams). + vm guestVM guestExec *microvm.GuestExec } @@ -162,12 +212,15 @@ func (m *MicroVMRuntime) Create(_ context.Context, spec ContainerSpec) (Containe } session := µvmSession{ - id: id, - name: spec.Name, - cfg: m.bootConfig(runtimeDir, nonce, shared), - uid: spec.UID, - env: spec.Env, - nonce: nonce, + id: id, + name: spec.Name, + cfg: m.bootConfig(runtimeDir, nonce, shared), + uid: spec.UID, + env: spec.Env, + nonce: nonce, + // Never empty: the zero-value EgressPolicy still emits the default-deny + // base ruleset, so every ContainerSpec-created session boots armed (§(e)). + nftScript: spec.Egress.NftScript(), runtimeDir: runtimeDir, } @@ -279,7 +332,7 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error { return err } - vm, err := microvm.Launch(ctx, session.cfg) + vm, err := m.launchFunc(ctx, session.cfg) if err != nil { return fmt.Errorf("microvm: launching session %s: %w", id, err) } @@ -301,8 +354,9 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error { if session.uid == 0 { return fmt.Errorf("microvm: session %s has a zero exec uid; Provision requires a non-zero default_exec_uid", id) } - client := microvm.GuestClient(session.cfg.VsockSocket, session.cfg.VsockPort) + client := m.newGuestClient(session.cfg.VsockSocket, session.cfg.VsockPort) if _, err := client.Provision(ctx, connect.NewRequest(&compassv1.ProvisionRequest{ + NftScript: session.nftScript, DefaultExecUid: session.uid, BaseEnv: session.env, })); err != nil { @@ -326,12 +380,20 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error { return nil } +// EgressArmedInGuest marks this backend as self-arming egress in-guest: the +// Provision RPC Start issues carries nft_script, so guestd arms as guest root +// before the exec gate opens (§(b)/(c)). AgentRuntime.provision probes for this +// marker (the unexported inGuestEgressArmer, agent.go) and skips its host-side +// armEgress exec — which on this backend would run capability-less and fail. +// Deliberately NOT a verb on the frozen ContainerRuntime interface (podman.go). +func (m *MicroVMRuntime) EgressArmedInGuest() bool { return true } + // awaitHealthy polls the guest's Health until it reports net_provisioned && // workspace_mounted (the V2a fail-closed readiness proof) within the boot // deadline, then verifies the echoed boot_nonce equals the minted nonce before // returning — a mismatch is an error (§(e) identity binding). The deadline // derives from ctx when it carries one, else bootDeadline. -func (m *MicroVMRuntime) awaitHealthy(ctx context.Context, vm *microvm.VM, nonce []byte) error { +func (m *MicroVMRuntime) awaitHealthy(ctx context.Context, vm guestVM, nonce []byte) error { pollCtx, cancel := bootPollContext(ctx) defer cancel() diff --git a/go/internal/runtime/microvm_start_test.go b/go/internal/runtime/microvm_start_test.go new file mode 100644 index 00000000..5fc8066e --- /dev/null +++ b/go/internal/runtime/microvm_start_test.go @@ -0,0 +1,206 @@ +//go:build unix + +package runtime + +// The hermetic MicroVMRuntime.Start suite: it drives Start behind the launchFunc +// + newGuestClient seams so no real cloud-hypervisor boots and no real vsock is +// dialed. A fake guestVM answers Health (with the minted boot nonce) and records +// Shutdown; a fake GuestControlClient records the ProvisionRequest. Together they +// prove the script-delivery contract (§(a)/(e)) and the fail-closed teardown +// (§(b)/(d)) with no KVM. It is //go:build unix because the seams and the +// guestVM interface it fakes are unix-only. +// +// This is deliberately NOT modelled on serveFakeGuest, which returns a +// *GuestExec over a plain unix listener that cannot speak the vsock CONNECT +// preamble and cannot answer Health. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "connectrpc.com/connect" + + compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1" + "github.com/RigelBuild/compass/go/internal/gen/compass/v1/compassv1internalconnect" + "github.com/RigelBuild/compass/go/internal/runtime/microvm" +) + +// fakeGuestVM is a guestVM handle that answers Health ready (echoing the minted +// nonce so awaitHealthy's identity binding passes) and records whether Shutdown +// was called (the fail-closed teardown assertion). WaitVMMExit/PSS satisfy the +// interface's remaining method set but are unexercised here. +type fakeGuestVM struct { + nonce []byte + mu sync.Mutex + shutdown bool + healthCalls int +} + +func (f *fakeGuestVM) Health(context.Context) (*compassv1.HealthResponse, error) { + f.mu.Lock() + f.healthCalls++ + f.mu.Unlock() + return &compassv1.HealthResponse{ + NetProvisioned: true, + WorkspaceMounted: true, + BootNonce: f.nonce, + }, nil +} + +func (f *fakeGuestVM) Shutdown(context.Context) error { + f.mu.Lock() + f.shutdown = true + f.mu.Unlock() + return nil +} + +func (f *fakeGuestVM) WaitVMMExit(_ time.Duration) bool { return true } + +// PSS returns an empty map: it satisfies the guestVM interface's method set (the +// //go:build microvm contract test calls session.vm.PSS()) but is never invoked +// on the hermetic Start path this fake serves. +func (f *fakeGuestVM) PSS() (map[string]int64, error) { return map[string]int64{}, nil } + +func (f *fakeGuestVM) wasShutdown() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.shutdown +} + +// fakeGuestClient is a GuestControlClient that records the ProvisionRequest and +// optionally fails it. Only Provision is exercised by Start; the other verbs +// satisfy the interface but are never called on this hermetic path. +type fakeGuestClient struct { + mu sync.Mutex + provisioned *compassv1.ProvisionRequest + provErr error +} + +func (c *fakeGuestClient) Provision(_ context.Context, req *connect.Request[compassv1.ProvisionRequest]) (*connect.Response[compassv1.ProvisionResponse], error) { + c.mu.Lock() + c.provisioned = req.Msg + c.mu.Unlock() + if c.provErr != nil { + return nil, c.provErr + } + return connect.NewResponse(&compassv1.ProvisionResponse{}), nil +} + +func (c *fakeGuestClient) Health(context.Context, *connect.Request[compassv1.HealthRequest]) (*connect.Response[compassv1.HealthResponse], error) { + return nil, errors.New("fakeGuestClient: Health not used on the hermetic Start path") +} + +func (c *fakeGuestClient) Exec(context.Context, *connect.Request[compassv1.ExecRequest]) (*connect.Response[compassv1.ExecResponse], error) { + return nil, errors.New("fakeGuestClient: Exec not used on the hermetic Start path") +} + +func (c *fakeGuestClient) ExecStream(context.Context) *connect.BidiStreamForClient[compassv1.ExecStreamRequest, compassv1.ExecStreamResponse] { + return nil +} + +func (c *fakeGuestClient) Signal(context.Context, *connect.Request[compassv1.SignalRequest]) (*connect.Response[compassv1.SignalResponse], error) { + return nil, errors.New("fakeGuestClient: Signal not used on the hermetic Start path") +} + +func (c *fakeGuestClient) recorded() *compassv1.ProvisionRequest { + c.mu.Lock() + defer c.mu.Unlock() + return c.provisioned +} + +var _ guestVM = (*fakeGuestVM)(nil) +var _ compassv1internalconnect.GuestControlClient = (*fakeGuestClient)(nil) + +// seamStart wires a MicroVMRuntime's launch + client seams to the supplied fakes +// and creates one session, returning the runtime, the created id, and the fakes. +// The Create records the zero-value default-deny script (unless spec overrides), +// which Start must then deliver verbatim. +func seamStart(t *testing.T, spec ContainerSpec, provErr error) (*MicroVMRuntime, ContainerID, *fakeGuestVM, *fakeGuestClient) { + t.Helper() + m := NewMicroVMRuntime(MicroVMConfig{RunRoot: t.TempDir()}) + id, err := m.Create(t.Context(), spec) + if err != nil { + t.Fatalf("Create: %v", err) + } + session, err := m.session(id) + if err != nil { + t.Fatalf("session after Create: %v", err) + } + vm := &fakeGuestVM{nonce: session.nonce} + client := &fakeGuestClient{provErr: provErr} + m.launchFunc = func(context.Context, microvm.BootConfig) (guestVM, error) { return vm, nil } + m.newGuestClient = func(string, uint32) compassv1internalconnect.GuestControlClient { return client } + return m, id, vm, client +} + +// TestCreateRecordsDefaultDenyScript pins §(e): Create records the zero-value +// EgressPolicy's full default-deny base ruleset on the session (never empty), so +// every ContainerSpec-created session boots armed even with no allowlist set. +func TestCreateRecordsDefaultDenyScript(t *testing.T) { + m := NewMicroVMRuntime(MicroVMConfig{RunRoot: t.TempDir()}) + id, err := m.Create(t.Context(), ContainerSpec{Name: "agent-1", UID: 1000}) + if err != nil { + t.Fatalf("Create: %v", err) + } + session, err := m.session(id) + if err != nil { + t.Fatalf("session after Create: %v", err) + } + if session.nftScript == "" { + t.Fatal("Create recorded an empty nftScript; the zero-value policy must emit the default-deny base ruleset (§(e))") + } + if session.nftScript != (EgressPolicy{}).NftScript() { + t.Errorf("recorded nftScript does not equal the zero-value default-deny script") + } +} + +// TestStartDeliversScriptVerbatim: via the launchFunc + newGuestClient seams, +// Start's Provision RPC carries the exact script Create recorded — the host→guest +// egress delivery contract (§(a)), hermetic with no real VMM or vsock dial. +func TestStartDeliversScriptVerbatim(t *testing.T) { + spec := ContainerSpec{Name: "agent-1", UID: 1000, Egress: MustAllowEgress("github.com")} + m, id, _, client := seamStart(t, spec, nil) + + want := spec.Egress.NftScript() + if err := m.Start(t.Context(), id); err != nil { + t.Fatalf("Start: %v", err) + } + req := client.recorded() + if req == nil { + t.Fatal("Start did not issue a Provision RPC") + } + if req.GetNftScript() != want { + t.Errorf("ProvisionRequest.NftScript = %q, want the recorded script %q", req.GetNftScript(), want) + } + if req.GetDefaultExecUid() != 1000 { + t.Errorf("ProvisionRequest.DefaultExecUid = %d, want 1000", req.GetDefaultExecUid()) + } +} + +// TestStartProvisionErrorFailsAndTearsDown: a Provision error fails Start and the +// fake handle's Shutdown is called (the fail-closed teardown, §(b)/(d)) — the VM +// booted by launchFunc must not be left running when the arm/provision fails. +func TestStartProvisionErrorFailsAndTearsDown(t *testing.T) { + provErr := connect.NewError(connect.CodeInternal, errors.New("arm failed")) + spec := ContainerSpec{Name: "agent-1", UID: 1000} + m, id, vm, _ := seamStart(t, spec, provErr) + + err := m.Start(t.Context(), id) + if err == nil { + t.Fatal("Start must fail when Provision errors") + } + if !vm.wasShutdown() { + t.Error("a failed Provision must tear the booted VM down (Shutdown), fail-closed") + } + // The session must not retain a VM handle after a failed Start. + session, sessErr := m.session(id) + if sessErr != nil { + t.Fatalf("session after failed Start: %v", sessErr) + } + if session.vm != nil { + t.Error("a failed Start must not store the VM handle on the session") + } +} diff --git a/go/internal/runtime/podman.go b/go/internal/runtime/podman.go index 4a1126c8..ed21827e 100644 --- a/go/internal/runtime/podman.go +++ b/go/internal/runtime/podman.go @@ -111,6 +111,14 @@ type ContainerSpec struct { // $HOME as (the T1/T2 baked-agent-uid invariant; see // docs/designs/infra/runtime/compass-runner-arbitrary-uid/design.md). UID uint32 + // Egress is the default-deny egress policy for this container. The podman + // backend IGNORES this field: createArgs is untouched (the podman argv stays + // byte-identical) and the podman path arms via AgentRuntime.armEgress's + // post-start exec. The microVM backend delivers it to guestd instead — + // MicroVMRuntime.Create records spec.Egress.NftScript() and Start's + // Provision RPC carries it, so the guest arms in-VM before the exec gate + // opens (design §(a)/(c)). + Egress EgressPolicy } // ExecSpec is how to run a command inside a container. @@ -388,6 +396,15 @@ type ContainerRuntime interface { Resize(ctx context.Context, id ContainerID, limits ResourceLimits) error } +// ContainerRuntime is frozen (the Resize reservation above): a backend that +// self-arms egress does NOT grow a verb here. Instead MicroVMRuntime carries an +// off-interface marker method, EgressArmedInGuest(), and AgentRuntime.provision +// type-asserts the unexported inGuestEgressArmer (agent.go) to skip armEgress on +// such a backend (design §(c)). A future backend — or any ContainerRuntime +// decorator, which would otherwise swallow the marker and silently re-enable +// armEgress on the microVM backend — must re-expose EgressArmedInGuest to keep +// the probe working. + // defaultCommandTimeout is the default per-command wall-clock cap. A hung podman // (stalled pull, wedged userns/cgroup setup, hung exec) must surface as an // error, never block the calling task forever. From a3c68f5c24361ac2b686583e8d5a1db367b4a8a4 Mon Sep 17 00:00:00 2001 From: mintaka Date: Sun, 30 Aug 2026 17:04:34 -0400 Subject: [PATCH 2/4] test(runtime): assert Start success-path handle ownership; sharpen guestVM doc (V3 W2 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold the two low findings from the W2 review pass: - TestStartDeliversScriptVerbatim now asserts the SUCCESS path stores the VM handle on the session and does NOT run the deferred teardown — the mirror of the error-path teardown assertion, closing a coverage gap where a dropped ownership transfer would leave a dead handle on a live session. - The guestVM interface doc-comment enumerates the Shutdown call sites precisely (Start defer, Stop, Remove) and attributes Health to awaitHealthy's poll, so the next maintainer can re-verify exhaustiveness. Refs RIG-3019 Co-authored-by: Matt Wilkinson --- go/internal/runtime/microvm_lifecycle.go | 9 +++++---- go/internal/runtime/microvm_start_test.go | 14 +++++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/go/internal/runtime/microvm_lifecycle.go b/go/internal/runtime/microvm_lifecycle.go index 7d0f642a..9320d8b3 100644 --- a/go/internal/runtime/microvm_lifecycle.go +++ b/go/internal/runtime/microvm_lifecycle.go @@ -85,10 +85,11 @@ const execDefaultTimeout = 120 * time.Second // *microvm.VM so Start is hermetically testable behind a fake handle. Its method // set is NOT merely what Start calls — it retypes the shared microvmSession.vm // field, so it must cover EVERY method invoked on that field anywhere in the -// package across ALL build tags: Health/Shutdown (Start/awaitHealthy and Stop, -// launch.go), WaitVMMExit (Stop, microvm_lifecycle.go), and PSS (the Q-budget -// contract test's session.vm.PSS(), contract_microvm_test.go, //go:build microvm -// && unix). *microvm.VM satisfies all four as-is (design §W2 seams). +// package across ALL build tags: Health (awaitHealthy's poll in Start), +// Shutdown (Start's defer, Stop, and Remove), WaitVMMExit (Stop, +// microvm_lifecycle.go), and PSS (the Q-budget contract test's session.vm.PSS(), +// contract_microvm_test.go, //go:build microvm && unix). All in launch.go except +// where noted. *microvm.VM satisfies all four as-is (design §W2 seams). type guestVM interface { Health(ctx context.Context) (*compassv1.HealthResponse, error) Shutdown(ctx context.Context) error diff --git a/go/internal/runtime/microvm_start_test.go b/go/internal/runtime/microvm_start_test.go index 5fc8066e..3e334e3f 100644 --- a/go/internal/runtime/microvm_start_test.go +++ b/go/internal/runtime/microvm_start_test.go @@ -162,7 +162,7 @@ func TestCreateRecordsDefaultDenyScript(t *testing.T) { // egress delivery contract (§(a)), hermetic with no real VMM or vsock dial. func TestStartDeliversScriptVerbatim(t *testing.T) { spec := ContainerSpec{Name: "agent-1", UID: 1000, Egress: MustAllowEgress("github.com")} - m, id, _, client := seamStart(t, spec, nil) + m, id, vm, client := seamStart(t, spec, nil) want := spec.Egress.NftScript() if err := m.Start(t.Context(), id); err != nil { @@ -178,6 +178,18 @@ func TestStartDeliversScriptVerbatim(t *testing.T) { if req.GetDefaultExecUid() != 1000 { t.Errorf("ProvisionRequest.DefaultExecUid = %d, want 1000", req.GetDefaultExecUid()) } + // The success path transfers handle ownership to the session and must NOT + // run the deferred teardown (mirror of the error-path teardown assertion). + session, err := m.session(id) + if err != nil { + t.Fatalf("session after successful Start: %v", err) + } + if session.vm == nil { + t.Error("a successful Start must store the booted VM handle on the session") + } + if vm.wasShutdown() { + t.Error("a successful Start must not tear the booted VM down") + } } // TestStartProvisionErrorFailsAndTearsDown: a Provision error fails Start and the From 7ff48d37734559e9d03cf8321804225692524fd5 Mon Sep 17 00:00:00 2001 From: mintaka Date: Mon, 31 Aug 2026 00:23:00 -0400 Subject: [PATCH 3/4] test(runtime): KVM-gated in-guest egress integration suite (V3 W3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The V3 milestone acceptance gate: prove the egress firewall where it runs — inside the guest netns, armed by guestd as guest root before the exec gate opens — driving a real MicroVMRuntime through Create→Start→Exec on live hardware. Test-only; no production code. Two files, each `//go:build microvm && unix`, gated on microvmtest.Require(t): - runtime/egress_inguest_microvm_test.go — the runtime-API scenarios: - W3(1) allowlisted raw-IPv4 reachable / non-allowlisted blocked (the podman lifecycle firewall proof, inside the guest netns); - W3(3) agent-uid `nft flush` refused (empty capability set) and the allow/deny behavior still holds after; - W3(4) a zero-value policy still boots armed default-deny (the §(e)/OQ-3 always-arm verification) — external egress blocked with no allowlist set. - runtime/microvm/egress_arm_microvm_test.go — W3(2): drive guestd directly with a failing arm script; Provision fails CodeInternal and the exec gate stays closed (a follow-up Exec is refused), the fail-closed §(b)/§(d) contract on real hardware. Probe mechanism, grounded on hardware: the guest ships bash + sh + nft + coreutils only, so reachability rides bash `/dev/tcp` against a RAW IP — never a DNS name (getent over the harness resolver stalls to the exec timeout and destabilizes the guest). A default-drop policy drops the SYN, so a blocked connect hangs until a guest-side `timeout` fires (exit 124); an allowed connect completes (exit 0). IPv6: the microVM guest network (passt) and the CI runners provide no IPv6 route, so a live v6 connect fails ENETUNREACH regardless of the firewall — a live v6 assertion would pass vacuously. The dual-stack property (every allowlisted host populates both the allow4 and allow6 nft sets) is proven hermetically at the script level (egress_test.go), so it is not re-asserted live here. Flagged in the PR for review. V8 alignment (record §W3(5)): this suite satisfies V8 row (2) — egress fail-closed inside the guest netns under the full backend — and row (8)'s re-arm half (agent-uid cannot alter the ruleset). No new V8 scope; the suite header carries the pointer. Refs RIG-3020 Co-authored-by: Matt Wilkinson --- .../runtime/egress_inguest_microvm_test.go | 216 ++++++++++++++++++ .../microvm/egress_arm_microvm_test.go | 102 +++++++++ 2 files changed, 318 insertions(+) create mode 100644 go/internal/runtime/egress_inguest_microvm_test.go create mode 100644 go/internal/runtime/microvm/egress_arm_microvm_test.go diff --git a/go/internal/runtime/egress_inguest_microvm_test.go b/go/internal/runtime/egress_inguest_microvm_test.go new file mode 100644 index 00000000..999ce8ab --- /dev/null +++ b/go/internal/runtime/egress_inguest_microvm_test.go @@ -0,0 +1,216 @@ +//go:build microvm && unix + +package runtime + +// The RIG-3020 (V3 W3) KVM-gated in-guest egress integration suite: the +// milestone acceptance gate for egress-in-guest, driving a real MicroVMRuntime +// through Create→Start→Exec on live hardware so the egress firewall is proven +// where it actually runs — inside the guest netns, armed by guestd as guest +// root before the exec gate opens (design §(a)/(c)/(e)). Every test opens with +// microvmtest.Require(t): on a KVM-less box it SKIPS (unless +// COMPASS_REQUIRE_MICROVM=1 forces a hard fail), so the suite is only real where +// /dev/kvm is openable and the guest images are exported into the env. +// +// It complements, not duplicates, the hermetic W2 suite (microvm_start_test.go) +// and the direct-guestd arm proofs (microvm/boot_microvm_test.go +// TestInGuestEgressArmAutoloadsNetfilter, microvm/egress_arm_microvm_test.go): +// those prove script delivery, netfilter autoload, and the fail-closed arm in +// isolation; this proves the whole path end-to-end at the runtime API the +// Runner calls, with real agent-uid execs hitting the armed ruleset. +// +// Probe mechanism (grounded on hardware): the guest ships bash + sh + nft + +// coreutils only — no wget/curl/nc/ip — so a reachability probe rides bash's +// /dev/tcp against a RAW IP (never a DNS name: getent over the harness resolver +// stalls to the 120s exec timeout and destabilizes the guest). A default-drop +// nft policy silently drops the SYN, so a blocked connect hangs until a +// guest-side `timeout` fires (exit 124); an allowed connect completes (exit 0, +// "connected"). This mirrors the podman lifecycle proof's raw-IPv4 allow/deny +// (lifecycle_test.go:137-166) inside the guest. +// +// IPv6 / dual-stack: the microVM guest network (passt) and the CI runners +// provide NO IPv6 route, so a live v6 connect fails ENETUNREACH regardless of +// the firewall — a live v6 assertion would pass vacuously (it cannot fail on a +// removed v6 rule), which is worse than none. The dual-stack property (every +// allowlisted host populates BOTH the allow4 and allow6 nft sets) is proven +// hermetically at the script level (egress_test.go +// TestAllowlistedHostPopulatesBothFamilies / TestEveryAllowlistedHostIsResolved), +// so it is not re-asserted live here. See the PR's Open Questions. +// +// V8 alignment (record §W3(5), microvm-runner.md:609-621): this suite satisfies +// V8 row (2) — "egress fail-closed asserted inside the guest netns" is exactly +// what runs here under the full backend. V8 row (8)'s re-arm half (an agent-uid +// process cannot re-arm/alter the ruleset) is covered by +// TestInGuestEgressAgentCannotAlterRuleset below plus W1's already-provisioned +// refusal and the peer-CID listener V2b built. No new V8 scope lands here. + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/RigelBuild/compass/go/internal/microvmtest" +) + +const ( + // egressProbeTimeout bounds one in-guest reachability exec (the host-side + // ctx). It must exceed the guest-side `timeout` below so the guest's own + // bounded connect reports (exit 124) rather than the host ctx firing first. + egressProbeTimeout = 25 * time.Second + // guestConnectTimeout is the guest-side `timeout N` wrapping the /dev/tcp + // connect: a dropped SYN hangs, so this is what turns a blocked host into a + // bounded exit-124 rather than an indefinite hang. + guestConnectTimeout = 10 + // allowedIP is the raw IPv4 the firewall must let through; deniedIP is a + // different globally-reachable raw IPv4 the default-deny policy must block. + // Both are stable anycast hosts reachable from CI when NOT firewalled, so a + // blocked deniedIP proves the firewall (not an unreachable host). Raw IPs, + // never names — DNS resolution stalls the harness (see the file header). + allowedIP = "1.1.1.1" + deniedIP = "8.8.8.8" +) + +// startEgressSession boots a real microVM session with the given egress policy +// and a throwaway workspace, returning the runtime + id ready for agent-uid +// execs. It registers teardown. The session is armed in-guest by guestd during +// Start (the Provision RPC carries the recorded nft_script), so by the time this +// returns the firewall is live. +func startEgressSession(t *testing.T, egress EgressPolicy, name string) (*MicroVMRuntime, ContainerID) { + t.Helper() + env := microvmtest.Require(t) + m := NewMicroVMRuntime(e2eConfig(t, env)) + + workspace := t.TempDir() + id, err := m.Create(t.Context(), ContainerSpec{ + Name: name, + UID: 1000, + Egress: egress, + Mounts: []Mount{{HostPath: workspace, ContainerPath: "/workspace"}}, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + t.Cleanup(func() { + _ = m.Stop(context.WithoutCancel(t.Context()), id, 5*time.Second) + _ = m.Remove(context.WithoutCancel(t.Context()), id) + }) + + // Start with t.Context(), NOT a WithTimeout/defer-cancel ctx: microvm.Launch + // spawns the VMM (+virtiofsd/passt) via exec.CommandContext bound to this + // ctx, so the Start ctx is the VM's LIFETIME — cancelling it kills the guest. + // A helper-local `defer cancel()` fires on return, before any exec, and would + // tear the VM down mid-session (guestd dies → vsock reset). The boot is + // already bounded internally (bootDeadline) and the whole run by the KVM + // -timeout, so t.Context() is both correct and sufficient (the contract-suite + // pattern, contract_suite_test.go). + if startErr := m.Start(t.Context(), id); startErr != nil { + t.Fatalf("Start (in-guest egress arm must succeed): %v", startErr) + } + return m, id +} + +// canReachIPv4 execs an agent-uid bash /dev/tcp connect to ip:443 inside the +// guest, bounded by a guest-side `timeout`, and reports whether the handshake +// completed. A blocked host's SYN is dropped by the default-deny policy, so the +// connect hangs until the guest `timeout` fires (exit 124) — reported as +// unreachable; an allowed host completes (exit 0, "connected"). A non-zero exit +// with no "connected" is unreachable; any transport/exec error fails the test +// (that is a harness fault, not a firewall verdict). +func canReachIPv4(t *testing.T, m *MicroVMRuntime, id ContainerID, ip string) bool { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), egressProbeTimeout) + defer cancel() + script := "timeout " + itoa(guestConnectTimeout) + + " bash -c 'exec 3<>/dev/tcp/" + ip + "/443 && echo connected'" + out, err := m.Exec(ctx, id, NewExecSpec("sh", "-c", script).AsUser("1000")) + if err != nil { + t.Fatalf("in-guest connect probe to %s errored (harness fault, not a firewall verdict): %v", ip, err) + } + reached := out.ExitCode == 0 && strings.Contains(out.Stdout, "connected") + t.Logf("connect %s:443 -> reached=%v (exit=%d stdout=%q stderr=%q)", ip, reached, out.ExitCode, out.Stdout, out.Stderr) + return reached +} + +// TestInGuestEgressAllowlistAndDeny is W3(1): an allowlisted raw IPv4 connects +// from inside the guest and a non-allowlisted raw IPv4 is blocked — the live +// firewall proof at the runtime API, armed in-guest by guestd. This is the +// microVM analog of the podman lifecycle firewall proof (lifecycle_test.go: +// 137-166), run inside the guest netns instead of the container netns. +func TestInGuestEgressAllowlistAndDeny(t *testing.T) { + m, id := startEgressSession(t, MustAllowEgress(allowedIP), "w3-allowdeny") + + if !canReachIPv4(t, m, id, allowedIP) { + t.Errorf("allowlisted host %s must be reachable through the in-guest firewall", allowedIP) + } + if canReachIPv4(t, m, id, deniedIP) { + t.Errorf("non-allowlisted host %s must be blocked by the default-deny in-guest firewall", deniedIP) + } +} + +// TestInGuestEgressAgentCannotAlterRuleset is W3(3) (and the V8 row-(8) re-arm +// half): after the arm, an agent-uid exec cannot tear down or alter the ruleset +// — `nft flush ruleset` fails (the agent runs with an empty capability set, +// guestd never runs an exec as root, supervisor.go resolveUID), and the +// allow/deny behavior still holds afterward. A regression that ran the agent +// privileged, or armed with a flushable ruleset, fails here. +func TestInGuestEgressAgentCannotAlterRuleset(t *testing.T) { + m, id := startEgressSession(t, MustAllowEgress(allowedIP), "w3-integrity") + + // Run `nft flush ruleset` BARE (no `; echo rc=$?` wrapper): the exec's + // ExitCode is then nft's own exit, so a refusal surfaces directly as a + // non-zero ExitCode. A trailing `echo` would make the shell exit 0 and mask + // the refusal. A non-zero exit here is a SUCCESSFUL exec call carrying a + // failed command (guest exec model), never a transport error. + ctx, cancel := context.WithTimeout(t.Context(), egressProbeTimeout) + defer cancel() + flush, err := m.Exec(ctx, id, NewExecSpec("nft", "flush", "ruleset").AsUser("1000")) + if err != nil { + t.Fatalf("nft flush probe errored (harness fault, not a firewall verdict): %v", err) + } + if flush.ExitCode == 0 { + t.Fatalf("agent uid must NOT be able to flush the in-guest firewall; got exit=0 stdout=%q stderr=%q", + flush.Stdout, flush.Stderr) + } + t.Logf("nft flush refused for agent uid: exit=%d stderr=%q", flush.ExitCode, flush.Stderr) + + // The firewall must still hold after the refused flush attempt. + if !canReachIPv4(t, m, id, allowedIP) { + t.Errorf("allowlisted host %s must stay reachable after a refused flush", allowedIP) + } + if canReachIPv4(t, m, id, deniedIP) { + t.Errorf("non-allowlisted host %s must stay blocked after a refused flush", deniedIP) + } +} + +// TestInGuestEgressAlwaysArmedDefaultDeny is W3(4) / the §(e)/OQ-3 always-arm +// verification: a session created with the ZERO-VALUE egress policy still boots +// armed default-deny, so external egress is blocked even though no allowlist was +// set. This is the load-bearing always-arm claim — every ContainerSpec-created +// microVM session is firewalled at Start whether or not a caller set Egress — +// proven live. A regression that skipped the arm on an empty policy (a silent +// open-egress VM) fails here: the deniedIP would become reachable. +func TestInGuestEgressAlwaysArmedDefaultDeny(t *testing.T) { + // Zero-value EgressPolicy: no allowlist, pure default-deny. + m, id := startEgressSession(t, EgressPolicy{}, "w3-defaultdeny") + + if canReachIPv4(t, m, id, allowedIP) { + t.Errorf("default-deny session must block %s: an always-armed empty policy allows no external egress", allowedIP) + } + if canReachIPv4(t, m, id, deniedIP) { + t.Errorf("default-deny session must block %s: an always-armed empty policy allows no external egress", deniedIP) + } +} + +// itoa renders a small non-negative int without pulling strconv into this test +// file's imports for a single call. +func itoa(n int) string { + if n == 0 { + return "0" + } + var b []byte + for n > 0 { + b = append([]byte{byte('0' + n%10)}, b...) + n /= 10 + } + return string(b) +} diff --git a/go/internal/runtime/microvm/egress_arm_microvm_test.go b/go/internal/runtime/microvm/egress_arm_microvm_test.go new file mode 100644 index 00000000..51e75e54 --- /dev/null +++ b/go/internal/runtime/microvm/egress_arm_microvm_test.go @@ -0,0 +1,102 @@ +//go:build microvm && unix + +package microvm + +// The RIG-3020 (V3 W3) direct-guestd arm-failure proof, run KVM-backed beside +// the boot spike and the netfilter-autoload proof (boot_microvm_test.go). Where +// TestInGuestEgressArmAutoloadsNetfilter proves a VALID arm succeeds, this proves +// the fail-closed half on REAL hardware: an arm script that exits non-zero fails +// Provision with CodeInternal AND leaves the exec gate closed, so a follow-up +// Exec is refused. That fail-closed guestd path (supervisor.go: a failed armFunc +// returns before the ready->provisioned transition) is only fully exercised +// against the real guestd running as PID 1 in the guest — the hermetic W2 seam +// test (runtime.TestStartProvisionErrorFailsAndTearsDown) proves the host +// tears the VM down on a Provision error, but only a real boot proves guestd +// itself refuses the exec after a real arm failure. + +import ( + "context" + "testing" + "time" + + "connectrpc.com/connect" + + compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1" + "github.com/RigelBuild/compass/go/internal/microvmtest" +) + +// TestInGuestArmFailureKeepsExecGateClosed drives guestd directly (Launch + +// GuestClient) with a Provision whose nft_script exits non-zero. The arm runs +// `/bin/sh -c