Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions go/internal/runtime/agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,9 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont
// Keep the container alive so the Runner can exec into it; the agent is
// driven via exec, not as the container's main process.
Command: []string{"sleep", "infinity"},
// The microVM backend delivers this to guestd for an in-guest arm; the
// podman backend ignores it and arms via provision's armEgress exec.
Egress: spec.Egress,
}

id, err := r.runtime.Create(ctx, container)
Expand All @@ -285,11 +288,27 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont
return id, nil
}

// inGuestEgressArmer is a backend that arms the egress firewall itself, inside
// its isolation boundary (as guest root, before the exec gate opens), so the
// host-side armEgress exec must be skipped. It is a marker, deliberately NOT a
// verb on the frozen ContainerRuntime interface (podman.go): AgentRuntime probes
// for it and skips arming when a backend self-arms (design §(c)). Only
// MicroVMRuntime implements it; PodmanCLI and the test fakes do not, so the
// host-side arm runs byte-identically for them.
type inGuestEgressArmer interface {
EgressArmedInGuest() bool
}

// provision runs the post-start steps, all inside the running container:
// firewall (root), credentials (agent user), checkout dir (agent user).
// firewall (root), credentials (agent user), checkout dir (agent user). A
// backend that self-arms egress in-guest (inGuestEgressArmer, the microVM
// backend) has already armed by Start, so the host-side armEgress exec — which
// on that backend would run capability-less and fail — is skipped.
func (r *AgentRuntime) provision(ctx context.Context, id ContainerID, spec AgentSpec) error {
if err := r.armEgress(ctx, id, spec.Egress); err != nil {
return err
if armer, ok := r.runtime.(inGuestEgressArmer); !ok || !armer.EgressArmedInGuest() {
if err := r.armEgress(ctx, id, spec.Egress); err != nil {
return err
}
}
if err := r.installCredentials(ctx, id, spec.Workspace); err != nil {
return err
Expand Down
58 changes: 58 additions & 0 deletions go/internal/runtime/agent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -294,3 +294,61 @@ func TestTeardownStopsThenRemoves(t *testing.T) {
t.Errorf("stop (%d) must precede remove (%d)", stop, remove)
}
}

// inGuestArmingFakeRuntime is a fakeRuntime that self-arms egress in-guest: it
// implements the inGuestEgressArmer marker (EgressArmedInGuest), so
// AgentRuntime.provision must skip the host-side armEgress exec — mirroring the
// microVM backend without booting a VM.
type inGuestArmingFakeRuntime struct {
*fakeRuntime
}

func (f *inGuestArmingFakeRuntime) EgressArmedInGuest() bool { return true }

// TestInGuestArmerSkipsHostArmEgress: a backend that self-arms in-guest receives
// NO nft exec, and provision still proceeds to credentials + checkout dir. This
// is the (c) probe-and-skip contract — the microVM path arms inside Start, so
// the host-side capability-less arm exec must not run.
func TestInGuestArmerSkipsHostArmEgress(t *testing.T) {
fake := &inGuestArmingFakeRuntime{fakeRuntime: newFakeRuntime(t)}
rt := NewAgentRuntime(fake)

if _, err := rt.Launch(t.Context(), specWithCreds(true)); err != nil {
t.Fatalf("Launch error = %v", err)
}

// No exec may carry the egress ruleset: the backend armed in-guest.
for _, e := range fake.execsSnapshot() {
if slices.ContainsFunc(e.Command, func(tok string) bool { return strings.Contains(tok, "compass_egress") }) {
t.Errorf("a self-arming backend must not run the host armEgress exec; command = %v", e.Command)
}
}
// Provision still proceeds: credentials install + checkout dir are made.
calls := fake.callsSnapshot()
mkdir := slices.IndexFunc(calls, func(c string) bool { return strings.Contains(c, "mkdir") })
if mkdir < 0 {
t.Errorf("provision must still create the checkout dir after skipping the arm; calls = %v", calls)
}
creds := slices.ContainsFunc(fake.execsSnapshot(), func(e ExecSpec) bool {
return e.Stdin != nil && strings.Contains(*e.Stdin, "git-credentials")
})
if !creds {
t.Errorf("provision must still install credentials after skipping the arm; execs = %v", fake.execsSnapshot())
}
}

// TestCreateArgsIgnoresEgress pins the podman byte-identical constraint: setting
// ContainerSpec.Egress must not change the `podman create` argv at all. The
// podman backend arms via AgentRuntime.armEgress, never from the spec field, so
// createArgs output for a spec with Egress set equals its output without.
func TestCreateArgsIgnoresEgress(t *testing.T) {
base := ContainerSpec{Name: "c", Image: "img", UID: 1000, CapAdd: []string{capNetAdmin}}
withEgress := base
withEgress.Egress = MustAllowEgress("github.com", "example.com")

got := createArgs(withEgress)
want := createArgs(base)
if !slices.Equal(got, want) {
t.Errorf("createArgs changed when Egress was set:\n with = %q\n without = %q", got, want)
}
}
11 changes: 10 additions & 1 deletion go/internal/runtime/microvm.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,24 @@ type MicroVMRuntime struct {
// this map for a matching spec.Name — a scan is cheap at one-VM-per-session
// scale and keeps a single source of truth.
sessions map[ContainerID]*microvmSession
// launchFunc boots a session guest behind the guestVM seam; newGuestClient
// dials the guest control plane. Both default to the real microvm
// implementations (installSeamDefaults, //go:build unix) and are overridden
// in hermetic Start tests so no real VMM boots and no real vsock is dialed
// (design §W2 seams, mirroring the supervisor's newCredential seam).
launchFunc guestLaunchFunc
newGuestClient guestClientFunc
}

// NewMicroVMRuntime builds a MicroVMRuntime from the supplied config, mirroring
// NewPodmanCLI's shape, with an empty session table ready for Create.
func NewMicroVMRuntime(cfg MicroVMConfig) *MicroVMRuntime {
return &MicroVMRuntime{
m := &MicroVMRuntime{
config: cfg,
sessions: make(map[ContainerID]*microvmSession),
}
m.installSeamDefaults()
return m
}

// SelectBackend chooses the container runtime backend from cfg. An empty or
Expand Down
85 changes: 74 additions & 11 deletions go/internal/runtime/microvm_lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,48 @@ const healthPollInterval = 200 * time.Millisecond
// timeout error, never block the calling task forever.
const execDefaultTimeout = 120 * time.Second

// guestVM is the running-guest handle MicroVMRuntime drives, an interface over
// *microvm.VM so Start is hermetically testable behind a fake handle. Its method
// set is NOT merely what Start calls — it retypes the shared microvmSession.vm
// field, so it must cover EVERY method invoked on that field anywhere in the
// package across ALL build tags: Health (awaitHealthy's poll in Start),
// Shutdown (Start's defer, Stop, and Remove), WaitVMMExit (Stop,
// microvm_lifecycle.go), and PSS (the Q-budget contract test's session.vm.PSS(),
// contract_microvm_test.go, //go:build microvm && unix). All in launch.go except
// where noted. *microvm.VM satisfies all four as-is (design §W2 seams).
type guestVM interface {
Health(ctx context.Context) (*compassv1.HealthResponse, error)
Shutdown(ctx context.Context) error
WaitVMMExit(timeout time.Duration) bool
PSS() (map[string]int64, error)
}

// guestLaunchFunc boots a session guest, returning it behind the guestVM seam.
// It defaults to a thin adapter over microvm.Launch (installSeamDefaults) and is
// overridden in hermetic Start tests so no real VMM boots.
type guestLaunchFunc func(context.Context, microvm.BootConfig) (guestVM, error)

// guestClientFunc dials the guest control plane, returning a GuestControlClient.
// It defaults to microvm.GuestClient (installSeamDefaults) and is overridden in
// hermetic Start tests so a fake client answers Provision with no real vsock
// dial. It seams Start's Provision client ONLY; Stop's own dial stays direct.
type guestClientFunc func(socket string, port uint32) compassv1internalconnect.GuestControlClient

// installSeamDefaults wires the production launch + client implementations onto a
// freshly constructed MicroVMRuntime. It is //go:build unix (like the microvm
// package it names) and is called by NewMicroVMRuntime, so hermetic tests can
// override the seams after construction.
func (m *MicroVMRuntime) installSeamDefaults() {
m.launchFunc = func(ctx context.Context, cfg microvm.BootConfig) (guestVM, error) {
vm, err := microvm.Launch(ctx, cfg)
if err != nil {
return nil, err
}
return vm, nil
}
m.newGuestClient = microvm.GuestClient
}

// microvmSession is one allocated microVM session's state. Created by Create
// (not yet booted), populated with the running VM handle and exec client by
// Start, and dropped by Remove. All fields are read/written under
Expand All @@ -100,11 +142,20 @@ type microvmSession struct {
// nonce is the per-session boot nonce (raw bytes); its hex encoding rides
// the cmdline, and Start verifies guestd echoes it before opening the gate.
nonce []byte
// nftScript is the egress ruleset delivered to guestd on Start's Provision
// RPC (as ProvisionRequest.nft_script). Recorded at Create from
// spec.Egress.NftScript(); NEVER empty for a ContainerSpec-created session,
// since the zero-value EgressPolicy still emits the full default-deny base
// ruleset (design §(e), egress.go). guestd arms it as guest root before the
// exec gate opens.
nftScript string
// runtimeDir is <RunRoot>/microvm/<id>/, holding the session's sockets.
runtimeDir string
// vm and guestExec are nil until Start boots the guest; Start sets both
// under the lock once the boot + Provision succeed.
vm *microvm.VM
// under the lock once the boot + Provision succeed. vm is typed as the
// unexported guestVM interface (not *microvm.VM directly) so Start is
// hermetically testable behind a fake handle (design §W2 seams).
vm guestVM
guestExec *microvm.GuestExec
}

Expand Down Expand Up @@ -162,12 +213,15 @@ func (m *MicroVMRuntime) Create(_ context.Context, spec ContainerSpec) (Containe
}

session := &microvmSession{
id: id,
name: spec.Name,
cfg: m.bootConfig(runtimeDir, nonce, shared),
uid: spec.UID,
env: spec.Env,
nonce: nonce,
id: id,
name: spec.Name,
cfg: m.bootConfig(runtimeDir, nonce, shared),
uid: spec.UID,
env: spec.Env,
nonce: nonce,
// Never empty: the zero-value EgressPolicy still emits the default-deny
// base ruleset, so every ContainerSpec-created session boots armed (§(e)).
nftScript: spec.Egress.NftScript(),
runtimeDir: runtimeDir,
}

Expand Down Expand Up @@ -279,7 +333,7 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error {
return err
}

vm, err := microvm.Launch(ctx, session.cfg)
vm, err := m.launchFunc(ctx, session.cfg)
if err != nil {
return fmt.Errorf("microvm: launching session %s: %w", id, err)
}
Expand All @@ -301,8 +355,9 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error {
if session.uid == 0 {
return fmt.Errorf("microvm: session %s has a zero exec uid; Provision requires a non-zero default_exec_uid", id)
}
client := microvm.GuestClient(session.cfg.VsockSocket, session.cfg.VsockPort)
client := m.newGuestClient(session.cfg.VsockSocket, session.cfg.VsockPort)
if _, err := client.Provision(ctx, connect.NewRequest(&compassv1.ProvisionRequest{
NftScript: session.nftScript,
DefaultExecUid: session.uid,
BaseEnv: session.env,
})); err != nil {
Expand All @@ -326,12 +381,20 @@ func (m *MicroVMRuntime) Start(ctx context.Context, id ContainerID) error {
return nil
}

// EgressArmedInGuest marks this backend as self-arming egress in-guest: the
// Provision RPC Start issues carries nft_script, so guestd arms as guest root
// before the exec gate opens (§(b)/(c)). AgentRuntime.provision probes for this
// marker (the unexported inGuestEgressArmer, agent.go) and skips its host-side
// armEgress exec — which on this backend would run capability-less and fail.
// Deliberately NOT a verb on the frozen ContainerRuntime interface (podman.go).
func (m *MicroVMRuntime) EgressArmedInGuest() bool { return true }

// awaitHealthy polls the guest's Health until it reports net_provisioned &&
// workspace_mounted (the V2a fail-closed readiness proof) within the boot
// deadline, then verifies the echoed boot_nonce equals the minted nonce before
// returning — a mismatch is an error (§(e) identity binding). The deadline
// derives from ctx when it carries one, else bootDeadline.
func (m *MicroVMRuntime) awaitHealthy(ctx context.Context, vm *microvm.VM, nonce []byte) error {
func (m *MicroVMRuntime) awaitHealthy(ctx context.Context, vm guestVM, nonce []byte) error {
pollCtx, cancel := bootPollContext(ctx)
defer cancel()

Expand Down
Loading
Loading