From 963497bce473147a224a43ab76c6c7dae8fc5a86 Mon Sep 17 00:00:00 2001
From: mintaka <mintaka@rigel.build>
Date: Sun, 30 Aug 2026 19:01:54 -0400
Subject: [PATCH 1/2] fix(guest-image): stage /sbin/modprobe so in-guest egress
 arm autoloads netfilter (RIG-3028)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The V3 in-guest egress arm (W1, #754) runs `nft` as guest root, whose first
NETLINK_NETFILTER socket open requires the kernel to autoload nfnetlink /
nf_tables on demand. The guest has no udev / systemd-modules-load (guestd is
PID 1, record §(d)), so on-demand loading rides entirely on the kernel's
request_module() usermode-helper path — which execs the binary named by
/proc/sys/kernel/modprobe (default /sbin/modprobe, CONFIG_MODPROBE_PATH unset in
the pinned kernel). Nothing staged that binary, so request_module was a silent
no-op: the arm failed `mnl.c:66: Unable to initialize Netlink socket: Protocol
not supported` (EPROTONOSUPPORT), and under §(e) always-arm this reddened every
microVM Start. The shipped /lib/modules tree was necessary but not sufficient —
the false OQ-3 autoload assumption CI surfaced on the W2 microvm leg.

Fix: symlink /sbin/modprobe to kmod's modprobe in the rootfs (kmod, not busybox
— modules are .ko.xz and CONFIG_MODULE_DECOMPRESS is unset, so the helper must
decompress in userspace, the same reason the initrd already uses kmod). With the
helper staged, any module the guest asks for autoloads on demand from the shipped
tree via its depmod alias/dep metadata. This is the general request_module
mechanism, not a fixed preload, so it also covers egress rulesets beyond the base
one: a future user-defined rule pulling a new nft expression module autoloads
with no guest-image change.

Adds a KVM-gated red-green guard, TestInGuestEgressArmAutoloadsNetfilter: it
boots a real guest and Provisions a non-empty nft_script (table + chain + `ct
state` rule under `set -eu`), so Provision succeeding proves the whole chain
(nfnetlink, nf_tables, nf_conntrack, nft_ct) autoloaded. Verified red on a
post-W1/pre-fix rootfs (reproduces `mnl.c:66 EPROTONOSUPPORT`) and green with the
fix.

Refs RIG-3028

Co-authored-by: Matt Wilkinson <matt@rigel.build>
---
 .../runtime/microvm/boot_microvm_test.go      | 79 +++++++++++++++++++
 guest-image/default.nix                       | 32 ++++++--
 2 files changed, 106 insertions(+), 5 deletions(-)

diff --git a/go/internal/runtime/microvm/boot_microvm_test.go b/go/internal/runtime/microvm/boot_microvm_test.go
index 4fba5b181..342418b21 100644
--- a/go/internal/runtime/microvm/boot_microvm_test.go
+++ b/go/internal/runtime/microvm/boot_microvm_test.go
@@ -24,6 +24,9 @@ import (
 	"testing"
 	"time"
 
+	"connectrpc.com/connect"
+
+	compassv1 "github.com/RigelBuild/compass/go/internal/gen/compass/v1"
 	"github.com/RigelBuild/compass/go/internal/microvmtest"
 )
 
@@ -182,6 +185,82 @@ func TestFullBoot(t *testing.T) {
 	}
 }
 
+// armRuleset is a minimal but representative in-guest egress arm: it creates the
+// inet table + a conntrack-stateful output rule, forcing exactly the netfilter
+// autoload chain the real base ruleset needs — the NETLINK_NETFILTER socket
+// (nfnetlink), the nf_tables subsystem, and the `ct state` expression
+// (nf_conntrack + nft_ct). It is `set -eu` so any nft failing aborts non-zero,
+// exactly as EgressPolicy.NftScript()'s base ruleset does. Kept as a local
+// literal, not a runtime.EgressPolicy call, because this package must not import
+// internal/runtime (config.go: no runtime dep, no cycle).
+const armRuleset = `set -eu
+nft add table inet compass_egress
+nft add chain inet compass_egress output '{ type filter hook output priority 0 ; policy drop ; }'
+nft add rule inet compass_egress output ct state established,related accept`
+
+// TestInGuestEgressArmAutoloadsNetfilter is the RIG-3028 proof: on a real guest
+// boot, Provision with a NON-EMPTY nft_script (the §(d) in-guest arm W1 landed)
+// must succeed. The arm runs `/bin/sh -c <script>` as guest root, and the first
+// `nft` opens a NETLINK_NETFILTER socket — which requires the guest kernel to
+// autoload nfnetlink/nf_tables on demand. The guest has no udev/systemd-modules-
+// load (guestd is PID 1), so that autoload rides ENTIRELY on the kernel
+// usermode-helper: /sbin/modprobe resolving modules from the shipped /lib/modules
+// tree (guest-image/default.nix). Before that helper was staged, this arm failed
+// with `mnl.c:66: Unable to initialize Netlink socket: Protocol not supported`
+// (EPROTONOSUPPORT) and §(e) always-arm reddened every microVM Start; this test
+// is the red-green guard for that fix. The arm script is `set -eu` and adds a
+// table, a chain, and a `ct state` rule, so Provision succeeding proves the
+// whole chain (nfnetlink, nf_tables, nf_conntrack, nft_ct) autoloaded — any
+// missing module aborts an `nft add` non-zero and surfaces as a Provision error.
+func TestInGuestEgressArmAutoloadsNetfilter(t *testing.T) {
+	env := microvmtest.Require(t)
+	cfg := bootConfig(t, env, 2, 1024)
+
+	vm, err := Launch(t.Context(), cfg)
+	if err != nil {
+		t.Fatalf("Launch failed: %v", err)
+	}
+	t.Cleanup(func() {
+		if shutErr := vm.Shutdown(context.WithoutCancel(t.Context())); shutErr != nil {
+			t.Errorf("Shutdown: %v", shutErr)
+		}
+	})
+
+	// Wait for the guest to report ready (net + workspace), same gate as the
+	// full-boot test: Provision is only valid once guestd is serving.
+	ready := waitFor(t, fullBootDeadline, func() bool {
+		ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second)
+		defer cancel()
+		resp, healthErr := vm.Health(ctx)
+		if healthErr != nil {
+			return false
+		}
+		return resp.GetNetProvisioned() && resp.GetWorkspaceMounted()
+	})
+	if !ready {
+		t.Fatalf("guest did not reach Health{net_provisioned && workspace_mounted} within %s.\n%s",
+			fullBootDeadline, vm.Diagnostics())
+	}
+
+	client := GuestClient(vm.vsockSocket, vm.vsockPort)
+
+	// The load-bearing assertion: Provision with a non-empty nft_script arms
+	// egress in-guest. This is the exact path that failed EPROTONOSUPPORT before
+	// /sbin/modprobe was staged. A non-nil error here (especially one carrying
+	// mnl.c:66 / "Protocol not supported") means the netfilter autoload chain is
+	// broken again.
+	provCtx, provCancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer provCancel()
+	if _, provErr := client.Provision(provCtx, connect.NewRequest(&compassv1.ProvisionRequest{
+		NftScript:      armRuleset,
+		DefaultExecUid: 1000,
+	})); provErr != nil {
+		t.Fatalf("Provision with non-empty nft_script failed — in-guest netfilter "+
+			"autoload broken (RIG-3028): %v\n%s", provErr, vm.Diagnostics())
+	}
+	t.Logf("in-guest egress arm succeeded: nfnetlink/nf_tables/nf_conntrack autoloaded on demand")
+}
+
 // TestCorruptRootfsFailsClosed is the fail-closed negative: boot with a
 // nonexistent rootfs path and assert the boot fails inside the deadline with the
 // cause visible in the serial console (the initrd's erofs-mount failure), not as
diff --git a/guest-image/default.nix b/guest-image/default.nix
index 31057a780..3d1a6f326 100644
--- a/guest-image/default.nix
+++ b/guest-image/default.nix
@@ -335,12 +335,34 @@ let
     # the guest's net bringup can rewrite it through the tmpfs overlay at boot.
     install -Dm644 ${resolvConf} $out/etc/resolv.conf
 
-    # The kernel's FULL /lib/modules tree (record §(a)): so guestd's
-    # virtio_net/virtiofs/vsock transport and V3's in-guest netfilter arm
-    # autoload on demand post-switch_root. Already in the closure (the kernel is
-    # substituted) — zero extra build. A symlink into the modules output; the
-    # erofs packing below dereferences it into the image.
+    # The kernel's FULL /lib/modules tree (record §(a)), depmod metadata and
+    # all: guestd's virtio_net/virtiofs/vsock transport and V3's in-guest
+    # netfilter arm resolve their modules from here. Already in the closure (the
+    # kernel is substituted) — zero extra build. A symlink into the modules
+    # output; the erofs packing below dereferences it into the image.
     ln -s ${kernel.modules}/lib/modules $out/lib/modules
+
+    # The kernel module-autoload usermode helper. The guest has no
+    # udev/systemd-modules-load (guestd is PID 1, §(d)), so post-switch_root the
+    # ONLY on-demand module loader is the kernel's request_module() path: when
+    # in-kernel code needs an unloaded module it execs the binary named by
+    # /proc/sys/kernel/modprobe (CONFIG_MODPROBE_PATH is unset in the pinned
+    # kernel, so this defaults to /sbin/modprobe). Nothing staged that binary,
+    # so request_module was a silent no-op and the /lib/modules tree above was
+    # necessary but NOT sufficient (the false OQ-3 assumption, RIG-3028): the
+    # first `nft` of the egress arm opened a NETLINK_NETFILTER socket, the
+    # kernel fired request_module("net-pf-16-proto-12") -> nfnetlink, found no
+    # helper, and nf_tables never registered -> EPROTONOSUPPORT (mnl.c:66), so
+    # §(e) always-arm failed EVERY microVM Start. Staging kmod's modprobe here
+    # (NOT busybox's: modules are .ko.xz and CONFIG_MODULE_DECOMPRESS is unset,
+    # so the helper must decompress in userspace — the same reason the initrd
+    # uses kmod at line 244) closes that: any module the guest asks for
+    # autoloads on demand from the shipped tree via its depmod alias/dep
+    # metadata. This is the general mechanism (not a fixed preload), so it also
+    # covers egress rulesets beyond the base one — a future user-defined rule
+    # pulling a new nft expression module autoloads with no guest-image change.
+    # `ln -sf` pulls kmod into the rootfs closure via the reference.
+    ln -sf ${pkgs.kmod}/bin/modprobe $out/sbin/modprobe
   '';
 
   # The store closure the rootfs symlink farm points into. Materialized into the

From 0e4b2151326daf01f2ef00d3cbdd5b48b3ddb1f5 Mon Sep 17 00:00:00 2001
From: mintaka <mintaka@rigel.build>
Date: Sun, 30 Aug 2026 19:17:32 -0400
Subject: [PATCH 2/2] =?UTF-8?q?fix(guest-image):=20address=20review=20nits?=
 =?UTF-8?q?=20=E2=80=94=20comment=20precision=20+=20drop=20inert=20-f=20(R?=
 =?UTF-8?q?IG-3028)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Review-round low findings on PR #759, all comment/precision (no behavior change):

- boot_microvm_test.go: soften the armRuleset doc-comment — it proves a
  representative slice of the netfilter autoload chain (nfnetlink, nf_tables,
  nf_conntrack, nft_ct), not "exactly" every module the base ruleset needs (the
  base ruleset's interval sets + DNS rules pull further modules the same way via
  request_module).
- boot_microvm_test.go: note the 30s Provision context is a deliberately generous
  CLIENT bound; the real ceiling is the server's 120s armTimeout.
- guest-image/default.nix: reword the modprobe symlink comment to attribute
  closure inclusion to the ${pkgs.kmod} reference, not the ln flag; drop the inert
  -f (no prior modprobe name exists to overwrite) to match the sibling `ln -s`
  for /sbin/init and /lib/modules.

Re-verified GREEN on real KVM after the change (rootfs still packs
/sbin/modprobe -> kmod; TestInGuestEgressArmAutoloadsNetfilter passes).

Refs RIG-3028

Co-authored-by: Matt Wilkinson <matt@rigel.build>
---
 .../runtime/microvm/boot_microvm_test.go       | 18 ++++++++++++------
 guest-image/default.nix                        |  6 ++++--
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/go/internal/runtime/microvm/boot_microvm_test.go b/go/internal/runtime/microvm/boot_microvm_test.go
index 342418b21..e334c4b45 100644
--- a/go/internal/runtime/microvm/boot_microvm_test.go
+++ b/go/internal/runtime/microvm/boot_microvm_test.go
@@ -186,12 +186,15 @@ func TestFullBoot(t *testing.T) {
 }
 
 // armRuleset is a minimal but representative in-guest egress arm: it creates the
-// inet table + a conntrack-stateful output rule, forcing exactly the netfilter
-// autoload chain the real base ruleset needs — the NETLINK_NETFILTER socket
-// (nfnetlink), the nf_tables subsystem, and the `ct state` expression
-// (nf_conntrack + nft_ct). It is `set -eu` so any nft failing aborts non-zero,
-// exactly as EgressPolicy.NftScript()'s base ruleset does. Kept as a local
-// literal, not a runtime.EgressPolicy call, because this package must not import
+// inet table + a conntrack-stateful output rule, forcing a representative slice
+// of the netfilter autoload chain — the NETLINK_NETFILTER socket (nfnetlink),
+// the nf_tables subsystem, and the `ct state` expression (nf_conntrack +
+// nft_ct). Once the usermode helper is staged, request_module resolves ANY
+// module from the full /lib/modules tree, so this slice proves the mechanism the
+// real base ruleset relies on (its interval sets and DNS rules pull further
+// modules the same way). It is `set -eu` so any nft failing aborts non-zero,
+// like EgressPolicy.NftScript()'s base ruleset. Kept as a local literal, not a
+// runtime.EgressPolicy call, because this package must not import
 // internal/runtime (config.go: no runtime dep, no cycle).
 const armRuleset = `set -eu
 nft add table inet compass_egress
@@ -249,6 +252,9 @@ func TestInGuestEgressArmAutoloadsNetfilter(t *testing.T) {
 	// /sbin/modprobe was staged. A non-nil error here (especially one carrying
 	// mnl.c:66 / "Protocol not supported") means the netfilter autoload chain is
 	// broken again.
+	// 30s is a deliberately generous CLIENT bound: the arm (module autoload + a
+	// few nft adds) is sub-second, and the real ceiling is the server's 120s
+	// armTimeout (guestd/supervisor.go), not this deadline.
 	provCtx, provCancel := context.WithTimeout(t.Context(), 30*time.Second)
 	defer provCancel()
 	if _, provErr := client.Provision(provCtx, connect.NewRequest(&compassv1.ProvisionRequest{
diff --git a/guest-image/default.nix b/guest-image/default.nix
index 3d1a6f326..d6716ad8c 100644
--- a/guest-image/default.nix
+++ b/guest-image/default.nix
@@ -361,8 +361,10 @@ let
     # metadata. This is the general mechanism (not a fixed preload), so it also
     # covers egress rulesets beyond the base one — a future user-defined rule
     # pulling a new nft expression module autoloads with no guest-image change.
-    # `ln -sf` pulls kmod into the rootfs closure via the reference.
-    ln -sf ${pkgs.kmod}/bin/modprobe $out/sbin/modprobe
+    # The ${pkgs.kmod} reference pulls kmod into the rootfs closure; the erofs
+    # packing below materializes it into the image. Plain `ln -s` (no -f): no
+    # prior modprobe name exists to overwrite, matching /sbin/init above.
+    ln -s ${pkgs.kmod}/bin/modprobe $out/sbin/modprobe
   '';
 
   # The store closure the rootfs symlink farm points into. Materialized into the