diff --git a/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go b/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go index 21e0c6ce..be6264f4 100644 --- a/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go +++ b/go/internal/gen/compass/v1/compassv1internalconnect/guest_control.connect.go @@ -96,8 +96,10 @@ type GuestControlClient interface { Signal(context.Context, *connect.Request[v1.SignalRequest]) (*connect.Response[v1.SignalResponse], error) // Provision transitions the supervisor ready -> provisioned: it records the // session's default exec uid and base env and, when nft_script is non-empty, - // arms egress (V3; a non-empty script is unimplemented in V2b). Exec and - // ExecStream are refused until Provision succeeds. + // arms egress in-guest (V3) — the script runs as guest root before the exec + // gate opens, and a failed arm fails Provision with the gate left closed. An + // empty nft_script skips the arm (a hermetic test seam). Exec and ExecStream + // are refused until Provision succeeds. Provision(context.Context, *connect.Request[v1.ProvisionRequest]) (*connect.Response[v1.ProvisionResponse], error) } @@ -201,8 +203,10 @@ type GuestControlHandler interface { Signal(context.Context, *connect.Request[v1.SignalRequest]) (*connect.Response[v1.SignalResponse], error) // Provision transitions the supervisor ready -> provisioned: it records the // session's default exec uid and base env and, when nft_script is non-empty, - // arms egress (V3; a non-empty script is unimplemented in V2b). Exec and - // ExecStream are refused until Provision succeeds. + // arms egress in-guest (V3) — the script runs as guest root before the exec + // gate opens, and a failed arm fails Provision with the gate left closed. An + // empty nft_script skips the arm (a hermetic test seam). Exec and ExecStream + // are refused until Provision succeeds. Provision(context.Context, *connect.Request[v1.ProvisionRequest]) (*connect.Response[v1.ProvisionResponse], error) } diff --git a/go/internal/gen/compass/v1/guest_control.pb.go b/go/internal/gen/compass/v1/guest_control.pb.go index 912499d7..3442c840 100644 --- a/go/internal/gen/compass/v1/guest_control.pb.go +++ b/go/internal/gen/compass/v1/guest_control.pb.go @@ -828,8 +828,10 @@ func (*SignalResponse) Descriptor() ([]byte, []int) { type ProvisionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // nft_script is the egress ruleset (V3, EgressPolicy.NftScript()); empty in - // V2b, where a non-empty value is unimplemented. + // nft_script is the egress ruleset (V3, EgressPolicy.NftScript()). A non-empty + // script is armed in-guest as root before the exec gate opens; an empty script + // skips the arm and is a hermetic test seam (the host production path always + // sends a non-empty default-deny ruleset). NftScript string `protobuf:"bytes,1,opt,name=nft_script,json=nftScript,proto3" json:"nft_script,omitempty"` // default_exec_uid is the session's agent uid (ContainerSpec.UID), the // default for an exec with no uid. Validated non-zero. diff --git a/go/internal/guestd/guestd.go b/go/internal/guestd/guestd.go index f72c3c56..61265247 100644 --- a/go/internal/guestd/guestd.go +++ b/go/internal/guestd/guestd.go @@ -150,6 +150,7 @@ func run(ctx context.Context, cfg config, steps bootSteps) error { workspaceMounted: true, bootNonce: bootNonce, newCredential: linuxCredential, + armFunc: runNftScript, stopServing: stopServing, state: stateReady, execs: make(map[string]*childExec), diff --git a/go/internal/guestd/supervisor.go b/go/internal/guestd/supervisor.go index 33b73684..8e50f7bc 100644 --- a/go/internal/guestd/supervisor.go +++ b/go/internal/guestd/supervisor.go @@ -60,6 +60,45 @@ func linuxCredential(uid uint32) *syscall.Credential { return &syscall.Credential{Uid: uid, Gid: uid} } +// armTimeout bounds the in-guest egress arm (§(d), OQ-5), mirroring podman's +// per-command defaultCommandTimeout that bounds the same script on that backend. +const armTimeout = 120 * time.Second + +// armStderrTail caps the bytes of the script's combined output carried in a +// failed-arm error so a runaway script cannot balloon the RPC error. +const armStderrTail = 4 << 10 + +// runNftScript is the production armFunc (§(d)): it spawns the egress script as +// guestd's own root — a spawn path deliberately SEPARATE from exec children, +// with NO syscall.Credential (it never passes through resolveUID/newCredential) +// and never entered in the exec table. The arm is bounded by armTimeout and the +// caller's ctx. On a non-zero exit, timeout, or spawn failure it returns an +// error carrying the exit status and a bounded tail of the combined output. +func runNftScript(ctx context.Context, script string) error { + ctx, cancel := context.WithTimeout(ctx, armTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script) //nolint:gosec // script is the host-delivered egress ruleset run as guest root by design — this IS the arm surface (§(d)) + // guestd is PID 1 with no PATH, and the arm is a spawn path SEPARATE from + // exec children (§(d)), so it never inherits mergeEnv's PATH floor. Set the + // guest rootfs PATH explicitly so the script's bare nft/getent/awk (linked + // under /bin, guest-image/default.nix) resolve; without it every microVM + // Start fails "nft: command not found" — the §(e) total-backend outage. + cmd.Env = []string{"PATH=" + defaultGuestPATH} + out, err := cmd.CombinedOutput() + if err != nil { + if len(out) > armStderrTail { + out = out[len(out)-armStderrTail:] + } + // Only append the output tail when there is one, so a silent failure + // (e.g. a bare non-zero exit) reads "exit status N", not "exit status N: ". + if tail := strings.TrimSpace(string(out)); tail != "" { + return fmt.Errorf("%w: %s", err, tail) + } + return err + } + return nil +} + // childExec is one running exec: a direct child of guestd (guest PID 1) in its // own process group. Signal targets the group; the reap that feeds the exit // frame is the owning ExecStream handler's cmd.Wait. @@ -97,6 +136,12 @@ type supervisor struct { // spawn as their own uid. newCredential credentialFunc + // armFunc arms egress from a non-empty nft_script (§(d)); a seam so + // hermetic tests inject a fake arm. Production is runNftScript, which + // spawns the script as guestd's own root, a spawn path deliberately + // separate from exec children (never through resolveUID/newCredential). + armFunc func(ctx context.Context, script string) error + // stopServing cancels the serving context on an RPC-driven Stop // (Signal("", ...)); run wires it and observes rpcStop to drive poweroff. stopServing context.CancelFunc @@ -135,18 +180,18 @@ func (s *supervisor) Health( } // Provision transitions ready -> provisioned (§(b)): it records the session's -// default exec uid (validated non-zero) and base env, opening the exec gate. A -// non-empty nft_script is V3's egress arm — unimplemented in V2b, so it is a -// hard error that leaves the gate closed (the host tears the VM down). +// default exec uid (validated non-zero) and base env, opening the exec gate. +// When nft_script is non-empty it arms egress in-guest (§(d)): the script runs +// as guestd's own root via armFunc BEFORE the state transition, under s.mu, so +// no exec is served until the arm succeeds. A failed arm returns CodeInternal +// and leaves the gate closed at stateReady (the host tears the VM down). An +// empty nft_script skips the arm (the §(e) hermetic test seam) and opens the +// gate as before. func (s *supervisor) Provision( - _ context.Context, + ctx context.Context, req *connect.Request[compassv1internal.ProvisionRequest], ) (*connect.Response[compassv1internal.ProvisionResponse], error) { m := req.Msg - if m.GetNftScript() != "" { - return nil, connect.NewError(connect.CodeUnimplemented, - errors.New("nft egress arm is V3; a non-empty nft_script is not supported in V2b")) - } if m.GetDefaultExecUid() == 0 { return nil, connect.NewError(connect.CodeInvalidArgument, errors.New("default_exec_uid must be non-zero: the guest supervisor never runs an exec as root")) @@ -162,6 +207,15 @@ func (s *supervisor) Provision( return nil, connect.NewError(connect.CodeFailedPrecondition, errors.New("not ready: net/mount bringup incomplete")) } + // Arm egress before opening the gate (§(d), OQ-4: under s.mu). A non-empty + // script that fails to arm leaves the state at stateReady so requireProvisioned + // keeps refusing every exec; a retried Provision may run. + if script := m.GetNftScript(); script != "" { + if err := s.armFunc(ctx, script); err != nil { + return nil, connect.NewError(connect.CodeInternal, + fmt.Errorf("arming nft egress: %w", err)) + } + } s.state = stateProvisioned s.defaultExecUID = m.GetDefaultExecUid() s.baseEnv = m.GetBaseEnv() diff --git a/go/internal/guestd/supervisor_test.go b/go/internal/guestd/supervisor_test.go index 8d0289a1..ef9d6b49 100644 --- a/go/internal/guestd/supervisor_test.go +++ b/go/internal/guestd/supervisor_test.go @@ -5,6 +5,7 @@ package guestd import ( "context" "crypto/tls" + "errors" "net" "net/http" "os" @@ -111,14 +112,9 @@ func TestProvisionOpensGateAndRejectsRootAndNft(t *testing.T) { t.Fatalf("Provision with uid 0 = %v, want InvalidArgument", err) } - // Non-empty nft_script is unimplemented in V2b. - _, err = client.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ - DefaultExecUid: 1000, - NftScript: "table inet filter {}", - })) - if err == nil || connect.CodeOf(err) != connect.CodeUnimplemented { - t.Fatalf("Provision with nft_script = %v, want Unimplemented", err) - } + // A non-empty nft_script now arms egress via the injected armFunc, not a + // CodeUnimplemented refusal (V3). Detailed arm coverage lives in the tests + // below; here we only confirm the clean uid path still opens the gate. // A clean Provision opens the gate. _, err = client.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ @@ -136,6 +132,176 @@ func TestProvisionOpensGateAndRejectsRootAndNft(t *testing.T) { } } +// newArmSupervisor builds a stateReady supervisor with an injected armFunc for +// the hermetic arm tests, mirroring the &supervisor{} literal pattern used +// elsewhere in this suite. Provision is exercised by direct method call. +func newArmSupervisor(arm func(ctx context.Context, script string) error) *supervisor { + return &supervisor{ + version: "v-test", + newCredential: testCredential, + armFunc: arm, + state: stateReady, + execs: map[string]*childExec{}, + } +} + +func TestProvisionArmsNonEmptyScript(t *testing.T) { + const script = "table inet filter { chain c {} }" + var got string + var calls int + svc := newArmSupervisor(func(_ context.Context, s string) error { + calls++ + got = s + return nil + }) + _, err := svc.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + NftScript: script, + })) + if err != nil { + t.Fatalf("Provision with arm: %v", err) + } + if calls != 1 || got != script { + t.Fatalf("armFunc calls=%d arg=%q, want 1 call with exact script %q", calls, got, script) + } + svc.mu.Lock() + state := svc.state + svc.mu.Unlock() + if state != stateProvisioned { + t.Fatalf("state after successful arm = %d, want provisioned", state) + } +} + +func TestProvisionArmFailureLeavesGateClosed(t *testing.T) { + svc := newArmSupervisor(func(_ context.Context, _ string) error { + return errors.New("nft: exit status 1") + }) + _, err := svc.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + NftScript: "bad script", + })) + if err == nil || connect.CodeOf(err) != connect.CodeInternal { + t.Fatalf("Provision with failing arm = %v, want CodeInternal", err) + } + svc.mu.Lock() + state := svc.state + svc.mu.Unlock() + if state != stateReady { + t.Fatalf("state after failed arm = %d, want stateReady", state) + } + // The exec gate is still closed. + if gerr := svc.requireProvisioned(); gerr == nil { + t.Fatal("requireProvisioned after failed arm = nil, want refused") + } + // A retried Provision may run (state still stateReady). + svc.armFunc = func(_ context.Context, _ string) error { return nil } + if _, rerr := svc.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + NftScript: "good script", + })); rerr != nil { + t.Fatalf("retried Provision after failed arm: %v", rerr) + } + svc.mu.Lock() + state = svc.state + svc.mu.Unlock() + if state != stateProvisioned { + t.Fatalf("state after retried arm = %d, want provisioned", state) + } +} + +func TestProvisionEmptyScriptSkipsArm(t *testing.T) { + var calls int + svc := newArmSupervisor(func(_ context.Context, _ string) error { + calls++ + return nil + }) + _, err := svc.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + })) + if err != nil { + t.Fatalf("Provision with empty script: %v", err) + } + if calls != 0 { + t.Fatalf("armFunc called %d times for empty script, want 0", calls) + } + svc.mu.Lock() + state := svc.state + svc.mu.Unlock() + if state != stateProvisioned { + t.Fatalf("state after empty-script Provision = %d, want provisioned (gate open)", state) + } +} + +func TestProvisionAlreadyProvisionedAfterArm(t *testing.T) { + var calls int + svc := newArmSupervisor(func(_ context.Context, _ string) error { + calls++ + return nil + }) + req := connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + NftScript: "table inet filter {}", + }) + if _, err := svc.Provision(t.Context(), req); err != nil { + t.Fatalf("first Provision: %v", err) + } + // A second Provision is refused before any re-arm (no wire re-arm surface). + _, err := svc.Provision(t.Context(), req) + if err == nil || connect.CodeOf(err) != connect.CodeFailedPrecondition { + t.Fatalf("re-Provision = %v, want FailedPrecondition", err) + } + if calls != 1 { + t.Fatalf("armFunc called %d times, want 1 (no re-arm)", calls) + } +} + +func TestRunNftScriptCarriesExitStatus(t *testing.T) { + // This exercises the HOST /bin/sh only — it proves runNftScript surfaces a + // non-zero exit status without root or nft. The GUEST /bin/sh link is proven + // by the W3/KVM suite, not here. + svc := newArmSupervisor(runNftScript) + _, err := svc.Provision(t.Context(), connect.NewRequest(&compassv1internal.ProvisionRequest{ + DefaultExecUid: 1000, + NftScript: "exit 7", + })) + if err == nil || connect.CodeOf(err) != connect.CodeInternal { + t.Fatalf("Provision with failing real script = %v, want CodeInternal", err) + } + if !strings.Contains(err.Error(), "exit status 7") { + t.Fatalf("error %q does not carry exit status 7", err.Error()) + } + svc.mu.Lock() + state := svc.state + svc.mu.Unlock() + if state != stateReady { + t.Fatalf("state after failed real arm = %d, want stateReady", state) + } +} + +func TestRunNftScriptFloorsGuestPATH(t *testing.T) { + // The §(e) total-backend-outage regression: guestd is PID 1 with no PATH, + // and the arm is a spawn path SEPARATE from exec children, so it never + // inherits mergeEnv's PATH floor. Without an explicit floor the script's + // bare nft/getent/awk resolve against an empty PATH and fail + // "command not found", failing EVERY microVM Start. Prove the arm child + // runs with PATH == defaultGuestPATH: echo/redirection are shell builtins + // (no PATH needed), so the captured value reflects only the env the arm + // sets. With the bug the child inherits the test process's ambient PATH, so + // this mismatches (red); with the floor it matches (green). + dir := t.TempDir() + out := dir + "/path" + if err := runNftScript(t.Context(), `echo "$PATH" > `+out); err != nil { + t.Fatalf("runNftScript(capture PATH) = %v", err) + } + got, err := os.ReadFile(out) + if err != nil { + t.Fatalf("reading captured PATH: %v", err) + } + if strings.TrimSpace(string(got)) != defaultGuestPATH { + t.Fatalf("arm child PATH = %q, want %q", strings.TrimSpace(string(got)), defaultGuestPATH) + } +} + func TestExecUIDZeroRefused(t *testing.T) { client, _ := newTestSupervisor(t, true, 1000) _, err := client.Exec(t.Context(), connect.NewRequest(&compassv1internal.ExecRequest{ diff --git a/guest-image/default.nix b/guest-image/default.nix index b8983c29..31057a78 100644 --- a/guest-image/default.nix +++ b/guest-image/default.nix @@ -313,14 +313,19 @@ let chmod -R u+w $out/etc fi - # The egress prerequisites (microvm-runner.md:446-449). Already present via - # the toolchain closure above (agent-image/toolchain.nix:144-147), linked - # again here explicitly so the guest's contract does not depend on the + # The egress prerequisites (microvm-runner.md:446-449) plus /bin/sh. Already + # present via the toolchain closure above (agent-image/toolchain.nix:144-147), + # linked again here explicitly so the guest's contract does not depend on the # toolchain's internal package list. `ln -sf` because the toolchain loop may - # already have created these names. + # already have created these names. /bin/sh is load-bearing under always-arm + # (record §(e)): every microVM Start spawns `/bin/sh -c