fix(guest-image): stage /sbin/modprobe so in-guest egress arm autoloads netfilter (RIG-3028)

go/internal/runtime/microvm/boot_microvm_test.go

@@ -24,6 +24,9 @@ import (
 	"testing"
 	"time"
 
+	"connectrpc.com/connect"
+
+	compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1"
 	"github.com/RigelBuild/compass/go/internal/microvmtest"
 )
 
@@ -182,6 +185,88 @@ func TestFullBoot(t *testing.T) {
 	}
 }
 
+// armRuleset is a minimal but representative in-guest egress arm: it creates the
+// inet table + a conntrack-stateful output rule, forcing a representative slice
+// of the netfilter autoload chain — the NETLINK_NETFILTER socket (nfnetlink),
+// the nf_tables subsystem, and the `ct state` expression (nf_conntrack +
+// nft_ct). Once the usermode helper is staged, request_module resolves ANY
+// module from the full /lib/modules tree, so this slice proves the mechanism the
+// real base ruleset relies on (its interval sets and DNS rules pull further
+// modules the same way). It is `set -eu` so any nft failing aborts non-zero,
+// like EgressPolicy.NftScript()'s base ruleset. Kept as a local literal, not a
+// runtime.EgressPolicy call, because this package must not import
+// internal/runtime (config.go: no runtime dep, no cycle).
+const armRuleset = `set -eu
+nft add table inet compass_egress
+nft add chain inet compass_egress output '{ type filter hook output priority 0 ; policy drop ; }'
+nft add rule inet compass_egress output ct state established,related accept`
+
+// TestInGuestEgressArmAutoloadsNetfilter is the RIG-3028 proof: on a real guest
+// boot, Provision with a NON-EMPTY nft_script (the §(d) in-guest arm W1 landed)
+// must succeed. The arm runs `/bin/sh -c <script>` as guest root, and the first
+// `nft` opens a NETLINK_NETFILTER socket — which requires the guest kernel to
+// autoload nfnetlink/nf_tables on demand. The guest has no udev/systemd-modules-
+// load (guestd is PID 1), so that autoload rides ENTIRELY on the kernel
+// usermode-helper: /sbin/modprobe resolving modules from the shipped /lib/modules
+// tree (guest-image/default.nix). Before that helper was staged, this arm failed
+// with `mnl.c:66: Unable to initialize Netlink socket: Protocol not supported`
+// (EPROTONOSUPPORT) and §(e) always-arm reddened every microVM Start; this test
+// is the red-green guard for that fix. The arm script is `set -eu` and adds a
+// table, a chain, and a `ct state` rule, so Provision succeeding proves the
+// whole chain (nfnetlink, nf_tables, nf_conntrack, nft_ct) autoloaded — any
+// missing module aborts an `nft add` non-zero and surfaces as a Provision error.
+func TestInGuestEgressArmAutoloadsNetfilter(t *testing.T) {
+	env := microvmtest.Require(t)
+	cfg := bootConfig(t, env, 2, 1024)
+
+	vm, err := Launch(t.Context(), cfg)
+	if err != nil {
+		t.Fatalf("Launch failed: %v", err)
+	}
+	t.Cleanup(func() {
+		if shutErr := vm.Shutdown(context.WithoutCancel(t.Context())); shutErr != nil {
+			t.Errorf("Shutdown: %v", shutErr)
+		}
+	})
+
+	// Wait for the guest to report ready (net + workspace), same gate as the
+	// full-boot test: Provision is only valid once guestd is serving.
+	ready := waitFor(t, fullBootDeadline, func() bool {
+		ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
+		defer cancel()
+		resp, healthErr := vm.Health(ctx)
+		if healthErr != nil {
+			return false
+		}
+		return resp.GetNetProvisioned() && resp.GetWorkspaceMounted()
+	})
+	if !ready {
+		t.Fatalf("guest did not reach Health{net_provisioned && workspace_mounted} within %s.\n%s",
+			fullBootDeadline, vm.Diagnostics())
+	}
+
+	client := GuestClient(vm.vsockSocket, vm.vsockPort)
+
+	// The load-bearing assertion: Provision with a non-empty nft_script arms
+	// egress in-guest. This is the exact path that failed EPROTONOSUPPORT before
+	// /sbin/modprobe was staged. A non-nil error here (especially one carrying
+	// mnl.c:66 / "Protocol not supported") means the netfilter autoload chain is
+	// broken again.
+	// 30s is a deliberately generous CLIENT bound: the arm (module autoload + a
+	// few nft adds) is sub-second, and the real ceiling is the server's 120s
+	// armTimeout (guestd/supervisor.go), not this deadline.
+	provCtx, provCancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer provCancel()
+	if _, provErr := client.Provision(provCtx, connect.NewRequest(&compassv1.ProvisionRequest{
+		NftScript:      armRuleset,
+		DefaultExecUid: 1000,
+	})); provErr != nil {
+		t.Fatalf("Provision with non-empty nft_script failed — in-guest netfilter "+
+			"autoload broken (RIG-3028): %v\n%s", provErr, vm.Diagnostics())
+	}
+	t.Logf("in-guest egress arm succeeded: nfnetlink/nf_tables/nf_conntrack autoloaded on demand")
+}
+
 // TestCorruptRootfsFailsClosed is the fail-closed negative: boot with a
 // nonexistent rootfs path and assert the boot fails inside the deadline with the
 // cause visible in the serial console (the initrd's erofs-mount failure), not as

guest-image/default.nix

@@ -335,12 +335,36 @@ let
     # the guest's net bringup can rewrite it through the tmpfs overlay at boot.
     install -Dm644 ${resolvConf} $out/etc/resolv.conf
 
-    # The kernel's FULL /lib/modules tree (record §(a)): so guestd's
-    # virtio_net/virtiofs/vsock transport and V3's in-guest netfilter arm
-    # autoload on demand post-switch_root. Already in the closure (the kernel is
-    # substituted) — zero extra build. A symlink into the modules output; the
-    # erofs packing below dereferences it into the image.
+    # The kernel's FULL /lib/modules tree (record §(a)), depmod metadata and
+    # all: guestd's virtio_net/virtiofs/vsock transport and V3's in-guest
+    # netfilter arm resolve their modules from here. Already in the closure (the
+    # kernel is substituted) — zero extra build. A symlink into the modules
+    # output; the erofs packing below dereferences it into the image.
     ln -s ${kernel.modules}/lib/modules $out/lib/modules
+
+    # The kernel module-autoload usermode helper. The guest has no
+    # udev/systemd-modules-load (guestd is PID 1, §(d)), so post-switch_root the
+    # ONLY on-demand module loader is the kernel's request_module() path: when
+    # in-kernel code needs an unloaded module it execs the binary named by
+    # /proc/sys/kernel/modprobe (CONFIG_MODPROBE_PATH is unset in the pinned
+    # kernel, so this defaults to /sbin/modprobe). Nothing staged that binary,
+    # so request_module was a silent no-op and the /lib/modules tree above was
+    # necessary but NOT sufficient (the false OQ-3 assumption, RIG-3028): the
+    # first `nft` of the egress arm opened a NETLINK_NETFILTER socket, the
+    # kernel fired request_module("net-pf-16-proto-12") -> nfnetlink, found no
+    # helper, and nf_tables never registered -> EPROTONOSUPPORT (mnl.c:66), so
+    # §(e) always-arm failed EVERY microVM Start. Staging kmod's modprobe here
+    # (NOT busybox's: modules are .ko.xz and CONFIG_MODULE_DECOMPRESS is unset,
+    # so the helper must decompress in userspace — the same reason the initrd
+    # uses kmod at line 244) closes that: any module the guest asks for
+    # autoloads on demand from the shipped tree via its depmod alias/dep
+    # metadata. This is the general mechanism (not a fixed preload), so it also
+    # covers egress rulesets beyond the base one — a future user-defined rule
+    # pulling a new nft expression module autoloads with no guest-image change.
+    # The ${pkgs.kmod} reference pulls kmod into the rootfs closure; the erofs
+    # packing below materializes it into the image. Plain `ln -s` (no -f): no
+    # prior modprobe name exists to overwrite, matching /sbin/init above.
+    ln -s ${pkgs.kmod}/bin/modprobe $out/sbin/modprobe
   '';
 
   # The store closure the rootfs symlink farm points into. Materialized into the