diff --git a/go/internal/runtime/agent.go b/go/internal/runtime/agent.go index 520f6206..e8aaf85f 100644 --- a/go/internal/runtime/agent.go +++ b/go/internal/runtime/agent.go @@ -269,6 +269,9 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont // Keep the container alive so the Runner can exec into it; the agent is // driven via exec, not as the container's main process. Command: []string{"sleep", "infinity"}, + // The microVM backend delivers this to guestd for an in-guest arm; the + // podman backend ignores it and arms via provision's armEgress exec. + Egress: spec.Egress, } id, err := r.runtime.Create(ctx, container) @@ -285,11 +288,27 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont return id, nil } +// inGuestEgressArmer is a backend that arms the egress firewall itself, inside +// its isolation boundary (as guest root, before the exec gate opens), so the +// host-side armEgress exec must be skipped. It is a marker, deliberately NOT a +// verb on the frozen ContainerRuntime interface (podman.go): AgentRuntime probes +// for it and skips arming when a backend self-arms (design §(c)). Only +// MicroVMRuntime implements it; PodmanCLI and the test fakes do not, so the +// host-side arm runs byte-identically for them. +type inGuestEgressArmer interface { + EgressArmedInGuest() bool +} + // provision runs the post-start steps, all inside the running container: -// firewall (root), credentials (agent user), checkout dir (agent user). +// firewall (root), credentials (agent user), checkout dir (agent user). A +// backend that self-arms egress in-guest (inGuestEgressArmer, the microVM +// backend) has already armed by Start, so the host-side armEgress exec — which +// on that backend would run capability-less and fail — is skipped. func (r *AgentRuntime) provision(ctx context.Context, id ContainerID, spec AgentSpec) error { - if err := r.armEgress(ctx, id, spec.Egress); err != nil { - return err + if armer, ok := r.runtime.(inGuestEgressArmer); !ok || !armer.EgressArmedInGuest() { + if err := r.armEgress(ctx, id, spec.Egress); err != nil { + return err + } } if err := r.installCredentials(ctx, id, spec.Workspace); err != nil { return err diff --git a/go/internal/runtime/agent_test.go b/go/internal/runtime/agent_test.go index 052a8ca2..18b27702 100644 --- a/go/internal/runtime/agent_test.go +++ b/go/internal/runtime/agent_test.go @@ -294,3 +294,61 @@ func TestTeardownStopsThenRemoves(t *testing.T) { t.Errorf("stop (%d) must precede remove (%d)", stop, remove) } } + +// inGuestArmingFakeRuntime is a fakeRuntime that self-arms egress in-guest: it +// implements the inGuestEgressArmer marker (EgressArmedInGuest), so +// AgentRuntime.provision must skip the host-side armEgress exec — mirroring the +// microVM backend without booting a VM. +type inGuestArmingFakeRuntime struct { + *fakeRuntime +} + +func (f *inGuestArmingFakeRuntime) EgressArmedInGuest() bool { return true } + +// TestInGuestArmerSkipsHostArmEgress: a backend that self-arms in-guest receives +// NO nft exec, and provision still proceeds to credentials + checkout dir. This +// is the (c) probe-and-skip contract — the microVM path arms inside Start, so +// the host-side capability-less arm exec must not run. +func TestInGuestArmerSkipsHostArmEgress(t *testing.T) { + fake := &inGuestArmingFakeRuntime{fakeRuntime: newFakeRuntime(t)} + rt := NewAgentRuntime(fake) + + if _, err := rt.Launch(t.Context(), specWithCreds(true)); err != nil { + t.Fatalf("Launch error = %v", err) + } + + // No exec may carry the egress ruleset: the backend armed in-guest. + for _, e := range fake.execsSnapshot() { + if slices.ContainsFunc(e.Command, func(tok string) bool { return strings.Contains(tok, "compass_egress") }) { + t.Errorf("a self-arming backend must not run the host armEgress exec; command = %v", e.Command) + } + } + // Provision still proceeds: credentials install + checkout dir are made. + calls := fake.callsSnapshot() + mkdir := slices.IndexFunc(calls, func(c string) bool { return strings.Contains(c, "mkdir") }) + if mkdir < 0 { + t.Errorf("provision must still create the checkout dir after skipping the arm; calls = %v", calls) + } + creds := slices.ContainsFunc(fake.execsSnapshot(), func(e ExecSpec) bool { + return e.Stdin != nil && strings.Contains(*e.Stdin, "git-credentials") + }) + if !creds { + t.Errorf("provision must still install credentials after skipping the arm; execs = %v", fake.execsSnapshot()) + } +} + +// TestCreateArgsIgnoresEgress pins the podman byte-identical constraint: setting +// ContainerSpec.Egress must not change the `podman create` argv at all. The +// podman backend arms via AgentRuntime.armEgress, never from the spec field, so +// createArgs output for a spec with Egress set equals its output without. +func TestCreateArgsIgnoresEgress(t *testing.T) { + base := ContainerSpec{Name: "c", Image: "img", UID: 1000, CapAdd: []string{capNetAdmin}} + withEgress := base + withEgress.Egress = MustAllowEgress("github.com", "example.com") + + got := createArgs(withEgress) + want := createArgs(base) + if !slices.Equal(got, want) { + t.Errorf("createArgs changed when Egress was set:\n with = %q\n without = %q", got, want) + } +} diff --git a/go/internal/runtime/microvm.go b/go/internal/runtime/microvm.go index 4cb56390..9ffec7bd 100644 --- a/go/internal/runtime/microvm.go +++ b/go/internal/runtime/microvm.go @@ -74,15 +74,24 @@ type MicroVMRuntime struct { // this map for a matching spec.Name — a scan is cheap at one-VM-per-session // scale and keeps a single source of truth. sessions map[ContainerID]*microvmSession + // launchFunc boots a session guest behind the guestVM seam; newGuestClient + // dials the guest control plane. Both default to the real microvm + // implementations (installSeamDefaults, //go:build unix) and are overridden + // in hermetic Start tests so no real VMM boots and no real vsock is dialed + // (design §W2 seams, mirroring the supervisor's newCredential seam). + launchFunc guestLaunchFunc + newGuestClient guestClientFunc } // NewMicroVMRuntime builds a MicroVMRuntime from the supplied config, mirroring // NewPodmanCLI's shape, with an empty session table ready for Create. func NewMicroVMRuntime(cfg MicroVMConfig) *MicroVMRuntime { - return &MicroVMRuntime{ + m := &MicroVMRuntime{ config: cfg, sessions: make(map[ContainerID]*microvmSession), } + m.installSeamDefaults() + return m } // SelectBackend chooses the container runtime backend from cfg. An empty or diff --git a/go/internal/runtime/microvm_lifecycle.go b/go/internal/runtime/microvm_lifecycle.go index 7e4c60cc..9320d8b3 100644 --- a/go/internal/runtime/microvm_lifecycle.go +++ b/go/internal/runtime/microvm_lifecycle.go @@ -81,6 +81,48 @@ const healthPollInterval = 200 * time.Millisecond // timeout error, never block the calling task forever. const execDefaultTimeout = 120 * time.Second +// guestVM is the running-guest handle MicroVMRuntime drives, an interface over +// *microvm.VM so Start is hermetically testable behind a fake handle. Its method +// set is NOT merely what Start calls — it retypes the shared microvmSession.vm +// field, so it must cover EVERY method invoked on that field anywhere in the +// package across ALL build tags: Health (awaitHealthy's poll in Start), +// Shutdown (Start's defer, Stop, and Remove), WaitVMMExit (Stop, +// microvm_lifecycle.go), and PSS (the Q-budget contract test's session.vm.PSS(), +// contract_microvm_test.go, //go:build microvm && unix). All in launch.go except +// where noted. *microvm.VM satisfies all four as-is (design §W2 seams). +type guestVM interface { + Health(ctx context.Context) (*compassv1.HealthResponse, error) + Shutdown(ctx context.Context) error + WaitVMMExit(timeout time.Duration) bool + PSS() (map[string]int64, error) +} + +// guestLaunchFunc boots a session guest, returning it behind the guestVM seam. +// It defaults to a thin adapter over microvm.Launch (installSeamDefaults) and is +// overridden in hermetic Start tests so no real VMM boots. +type guestLaunchFunc func(context.Context, microvm.BootConfig) (guestVM, error) + +// guestClientFunc dials the guest control plane, returning a GuestControlClient. +// It defaults to microvm.GuestClient (installSeamDefaults) and is overridden in +// hermetic Start tests so a fake client answers Provision with no real vsock +// dial. It seams Start's Provision client ONLY; Stop's own dial stays direct. +type guestClientFunc func(socket string, port uint32) compassv1internalconnect.GuestControlClient + +// installSeamDefaults wires the production launch + client implementations onto a +// freshly constructed MicroVMRuntime. It is //go:build unix (like the microvm +// package it names) and is called by NewMicroVMRuntime, so hermetic tests can +// override the seams after construction. +func (m *MicroVMRuntime) installSeamDefaults() { + m.launchFunc = func(ctx context.Context, cfg microvm.BootConfig) (guestVM, error) { + vm, err := microvm.Launch(ctx, cfg) + if err != nil { + return nil, err + } + return vm, nil + } + m.newGuestClient = microvm.GuestClient +} + // microvmSession is one allocated microVM session's state. Created by Create // (not yet booted), populated with the running VM handle and exec client by // Start, and dropped by Remove. All fields are read/written under @@ -100,11 +142,20 @@ type microvmSession struct { // nonce is the per-session boot nonce (raw bytes); its hex encoding rides // the cmdline, and Start verifies guestd echoes it before opening the gate. nonce []byte + // nftScript is the egress ruleset delivered to guestd on Start's Provision + // RPC (as ProvisionRequest.nft_script). Recorded at Create from + // spec.Egress.NftScript(); NEVER empty for a ContainerSpec-created session, + // since the zero-value EgressPolicy still emits the full default-deny base + // ruleset (design §(e), egress.go). guestd arms it as guest root before the + // exec gate opens. + nftScript string // runtimeDir is /microvm//, holding the session's sockets. runtimeDir string // vm and guestExec are nil until Start boots the guest; Start sets both - // under the lock once the boot + Provision succeed. - vm *microvm.VM + // under the lock once the boot + Provision succeed. vm is typed as the + // unexported guestVM interface (not *microvm.VM directly) so Start is + // hermetically testable behind a fake handle (design §W2 seams). + vm guestVM guestExec *microvm.GuestExec } @@ -162,12 +213,15 @@ func (m *MicroVMRuntime) Create(_ context.Context, spec ContainerSpec) (Containe } session := µvmSession{ - id: id, - name: spec.Name, - cfg: m.bootConfig(runtimeDir, nonce, shared), - uid: spec.UID, - env: spec.Env, - nonce: nonce, + id: id, + name: spec.Name, + cfg: m.bootConfig(runtimeDir, nonce, shared), + uid: spec.UID, + env: spec.Env, + nonce: nonce, + // Never empty: the zero-value EgressPolicy still emits the default-deny + // base ruleset, so every ContainerSpec-created session boots armed (§(e)). + nftScript: spec.Egress.NftScript(), runtimeDir: runtimeDir, } @@ -279,7 +333,7 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error { return err } - vm, err := microvm.Launch(ctx, session.cfg) + vm, err := m.launchFunc(ctx, session.cfg) if err != nil { return fmt.Errorf("microvm: launching session %s: %w", id, err) } @@ -301,8 +355,9 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error { if session.uid == 0 { return fmt.Errorf("microvm: session %s has a zero exec uid; Provision requires a non-zero default_exec_uid", id) } - client := microvm.GuestClient(session.cfg.VsockSocket, session.cfg.VsockPort) + client := m.newGuestClient(session.cfg.VsockSocket, session.cfg.VsockPort) if _, err := client.Provision(ctx, connect.NewRequest(&compassv1.ProvisionRequest{ + NftScript: session.nftScript, DefaultExecUid: session.uid, BaseEnv: session.env, })); err != nil { @@ -326,12 +381,20 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error { return nil } +// EgressArmedInGuest marks this backend as self-arming egress in-guest: the +// Provision RPC Start issues carries nft_script, so guestd arms as guest root +// before the exec gate opens (§(b)/(c)). AgentRuntime.provision probes for this +// marker (the unexported inGuestEgressArmer, agent.go) and skips its host-side +// armEgress exec — which on this backend would run capability-less and fail. +// Deliberately NOT a verb on the frozen ContainerRuntime interface (podman.go). +func (m *MicroVMRuntime) EgressArmedInGuest() bool { return true } + // awaitHealthy polls the guest's Health until it reports net_provisioned && // workspace_mounted (the V2a fail-closed readiness proof) within the boot // deadline, then verifies the echoed boot_nonce equals the minted nonce before // returning — a mismatch is an error (§(e) identity binding). The deadline // derives from ctx when it carries one, else bootDeadline. -func (m *MicroVMRuntime) awaitHealthy(ctx context.Context, vm *microvm.VM, nonce []byte) error { +func (m *MicroVMRuntime) awaitHealthy(ctx context.Context, vm guestVM, nonce []byte) error { pollCtx, cancel := bootPollContext(ctx) defer cancel() diff --git a/go/internal/runtime/microvm_start_test.go b/go/internal/runtime/microvm_start_test.go new file mode 100644 index 00000000..3e334e3f --- /dev/null +++ b/go/internal/runtime/microvm_start_test.go @@ -0,0 +1,218 @@ +//go:build unix + +package runtime + +// The hermetic MicroVMRuntime.Start suite: it drives Start behind the launchFunc +// + newGuestClient seams so no real cloud-hypervisor boots and no real vsock is +// dialed. A fake guestVM answers Health (with the minted boot nonce) and records +// Shutdown; a fake GuestControlClient records the ProvisionRequest. Together they +// prove the script-delivery contract (§(a)/(e)) and the fail-closed teardown +// (§(b)/(d)) with no KVM. It is //go:build unix because the seams and the +// guestVM interface it fakes are unix-only. +// +// This is deliberately NOT modelled on serveFakeGuest, which returns a +// *GuestExec over a plain unix listener that cannot speak the vsock CONNECT +// preamble and cannot answer Health. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "connectrpc.com/connect" + + compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1" + "github.com/RigelBuild/compass/go/internal/gen/compass/v1/compassv1internalconnect" + "github.com/RigelBuild/compass/go/internal/runtime/microvm" +) + +// fakeGuestVM is a guestVM handle that answers Health ready (echoing the minted +// nonce so awaitHealthy's identity binding passes) and records whether Shutdown +// was called (the fail-closed teardown assertion). WaitVMMExit/PSS satisfy the +// interface's remaining method set but are unexercised here. +type fakeGuestVM struct { + nonce []byte + mu sync.Mutex + shutdown bool + healthCalls int +} + +func (f *fakeGuestVM) Health(context.Context) (*compassv1.HealthResponse, error) { + f.mu.Lock() + f.healthCalls++ + f.mu.Unlock() + return &compassv1.HealthResponse{ + NetProvisioned: true, + WorkspaceMounted: true, + BootNonce: f.nonce, + }, nil +} + +func (f *fakeGuestVM) Shutdown(context.Context) error { + f.mu.Lock() + f.shutdown = true + f.mu.Unlock() + return nil +} + +func (f *fakeGuestVM) WaitVMMExit(_ time.Duration) bool { return true } + +// PSS returns an empty map: it satisfies the guestVM interface's method set (the +// //go:build microvm contract test calls session.vm.PSS()) but is never invoked +// on the hermetic Start path this fake serves. +func (f *fakeGuestVM) PSS() (map[string]int64, error) { return map[string]int64{}, nil } + +func (f *fakeGuestVM) wasShutdown() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.shutdown +} + +// fakeGuestClient is a GuestControlClient that records the ProvisionRequest and +// optionally fails it. Only Provision is exercised by Start; the other verbs +// satisfy the interface but are never called on this hermetic path. +type fakeGuestClient struct { + mu sync.Mutex + provisioned *compassv1.ProvisionRequest + provErr error +} + +func (c *fakeGuestClient) Provision(_ context.Context, req *connect.Request[compassv1.ProvisionRequest]) (*connect.Response[compassv1.ProvisionResponse], error) { + c.mu.Lock() + c.provisioned = req.Msg + c.mu.Unlock() + if c.provErr != nil { + return nil, c.provErr + } + return connect.NewResponse(&compassv1.ProvisionResponse{}), nil +} + +func (c *fakeGuestClient) Health(context.Context, *connect.Request[compassv1.HealthRequest]) (*connect.Response[compassv1.HealthResponse], error) { + return nil, errors.New("fakeGuestClient: Health not used on the hermetic Start path") +} + +func (c *fakeGuestClient) Exec(context.Context, *connect.Request[compassv1.ExecRequest]) (*connect.Response[compassv1.ExecResponse], error) { + return nil, errors.New("fakeGuestClient: Exec not used on the hermetic Start path") +} + +func (c *fakeGuestClient) ExecStream(context.Context) *connect.BidiStreamForClient[compassv1.ExecStreamRequest, compassv1.ExecStreamResponse] { + return nil +} + +func (c *fakeGuestClient) Signal(context.Context, *connect.Request[compassv1.SignalRequest]) (*connect.Response[compassv1.SignalResponse], error) { + return nil, errors.New("fakeGuestClient: Signal not used on the hermetic Start path") +} + +func (c *fakeGuestClient) recorded() *compassv1.ProvisionRequest { + c.mu.Lock() + defer c.mu.Unlock() + return c.provisioned +} + +var _ guestVM = (*fakeGuestVM)(nil) +var _ compassv1internalconnect.GuestControlClient = (*fakeGuestClient)(nil) + +// seamStart wires a MicroVMRuntime's launch + client seams to the supplied fakes +// and creates one session, returning the runtime, the created id, and the fakes. +// The Create records the zero-value default-deny script (unless spec overrides), +// which Start must then deliver verbatim. +func seamStart(t *testing.T, spec ContainerSpec, provErr error) (*MicroVMRuntime, ContainerID, *fakeGuestVM, *fakeGuestClient) { + t.Helper() + m := NewMicroVMRuntime(MicroVMConfig{RunRoot: t.TempDir()}) + id, err := m.Create(t.Context(), spec) + if err != nil { + t.Fatalf("Create: %v", err) + } + session, err := m.session(id) + if err != nil { + t.Fatalf("session after Create: %v", err) + } + vm := &fakeGuestVM{nonce: session.nonce} + client := &fakeGuestClient{provErr: provErr} + m.launchFunc = func(context.Context, microvm.BootConfig) (guestVM, error) { return vm, nil } + m.newGuestClient = func(string, uint32) compassv1internalconnect.GuestControlClient { return client } + return m, id, vm, client +} + +// TestCreateRecordsDefaultDenyScript pins §(e): Create records the zero-value +// EgressPolicy's full default-deny base ruleset on the session (never empty), so +// every ContainerSpec-created session boots armed even with no allowlist set. +func TestCreateRecordsDefaultDenyScript(t *testing.T) { + m := NewMicroVMRuntime(MicroVMConfig{RunRoot: t.TempDir()}) + id, err := m.Create(t.Context(), ContainerSpec{Name: "agent-1", UID: 1000}) + if err != nil { + t.Fatalf("Create: %v", err) + } + session, err := m.session(id) + if err != nil { + t.Fatalf("session after Create: %v", err) + } + if session.nftScript == "" { + t.Fatal("Create recorded an empty nftScript; the zero-value policy must emit the default-deny base ruleset (§(e))") + } + if session.nftScript != (EgressPolicy{}).NftScript() { + t.Errorf("recorded nftScript does not equal the zero-value default-deny script") + } +} + +// TestStartDeliversScriptVerbatim: via the launchFunc + newGuestClient seams, +// Start's Provision RPC carries the exact script Create recorded — the host→guest +// egress delivery contract (§(a)), hermetic with no real VMM or vsock dial. +func TestStartDeliversScriptVerbatim(t *testing.T) { + spec := ContainerSpec{Name: "agent-1", UID: 1000, Egress: MustAllowEgress("github.com")} + m, id, vm, client := seamStart(t, spec, nil) + + want := spec.Egress.NftScript() + if err := m.Start(t.Context(), id); err != nil { + t.Fatalf("Start: %v", err) + } + req := client.recorded() + if req == nil { + t.Fatal("Start did not issue a Provision RPC") + } + if req.GetNftScript() != want { + t.Errorf("ProvisionRequest.NftScript = %q, want the recorded script %q", req.GetNftScript(), want) + } + if req.GetDefaultExecUid() != 1000 { + t.Errorf("ProvisionRequest.DefaultExecUid = %d, want 1000", req.GetDefaultExecUid()) + } + // The success path transfers handle ownership to the session and must NOT + // run the deferred teardown (mirror of the error-path teardown assertion). + session, err := m.session(id) + if err != nil { + t.Fatalf("session after successful Start: %v", err) + } + if session.vm == nil { + t.Error("a successful Start must store the booted VM handle on the session") + } + if vm.wasShutdown() { + t.Error("a successful Start must not tear the booted VM down") + } +} + +// TestStartProvisionErrorFailsAndTearsDown: a Provision error fails Start and the +// fake handle's Shutdown is called (the fail-closed teardown, §(b)/(d)) — the VM +// booted by launchFunc must not be left running when the arm/provision fails. +func TestStartProvisionErrorFailsAndTearsDown(t *testing.T) { + provErr := connect.NewError(connect.CodeInternal, errors.New("arm failed")) + spec := ContainerSpec{Name: "agent-1", UID: 1000} + m, id, vm, _ := seamStart(t, spec, provErr) + + err := m.Start(t.Context(), id) + if err == nil { + t.Fatal("Start must fail when Provision errors") + } + if !vm.wasShutdown() { + t.Error("a failed Provision must tear the booted VM down (Shutdown), fail-closed") + } + // The session must not retain a VM handle after a failed Start. + session, sessErr := m.session(id) + if sessErr != nil { + t.Fatalf("session after failed Start: %v", sessErr) + } + if session.vm != nil { + t.Error("a failed Start must not store the VM handle on the session") + } +} diff --git a/go/internal/runtime/podman.go b/go/internal/runtime/podman.go index 4a1126c8..ed21827e 100644 --- a/go/internal/runtime/podman.go +++ b/go/internal/runtime/podman.go @@ -111,6 +111,14 @@ type ContainerSpec struct { // $HOME as (the T1/T2 baked-agent-uid invariant; see // docs/designs/infra/runtime/compass-runner-arbitrary-uid/design.md). UID uint32 + // Egress is the default-deny egress policy for this container. The podman + // backend IGNORES this field: createArgs is untouched (the podman argv stays + // byte-identical) and the podman path arms via AgentRuntime.armEgress's + // post-start exec. The microVM backend delivers it to guestd instead — + // MicroVMRuntime.Create records spec.Egress.NftScript() and Start's + // Provision RPC carries it, so the guest arms in-VM before the exec gate + // opens (design §(a)/(c)). + Egress EgressPolicy } // ExecSpec is how to run a command inside a container. @@ -388,6 +396,15 @@ type ContainerRuntime interface { Resize(ctx context.Context, id ContainerID, limits ResourceLimits) error } +// ContainerRuntime is frozen (the Resize reservation above): a backend that +// self-arms egress does NOT grow a verb here. Instead MicroVMRuntime carries an +// off-interface marker method, EgressArmedInGuest(), and AgentRuntime.provision +// type-asserts the unexported inGuestEgressArmer (agent.go) to skip armEgress on +// such a backend (design §(c)). A future backend — or any ContainerRuntime +// decorator, which would otherwise swallow the marker and silently re-enable +// armEgress on the microVM backend — must re-expose EgressArmedInGuest to keep +// the probe working. + // defaultCommandTimeout is the default per-command wall-clock cap. A hung podman // (stalled pull, wedged userns/cgroup setup, hung exec) must surface as an // error, never block the calling task forever.