From 8d252bd92ef0d1696ae69302536521263a392274 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Mon, 10 Aug 2026 13:19:44 -0700 Subject: [PATCH 1/6] hack: make the kind cluster's IP family configurable IP_FAMILY selects ipv4, ipv6 or dual and becomes networking.ipFamily, leaving kind's per-family subnet defaults alone. The script also recreates a pre-IPv6 "kind" Docker network, fails fast if the daemon has IPv6 off, sets proxy_ndp alongside proxy_arp for gVisor pod-to-pod traffic, and repoints an ipv6 kubeconfig from [::1] at localhost so a client outside the Docker host can still reach the apiserver. Tested on kind with all three families: node InternalIPs, Service ClusterIPs and pod IPs land in the requested families, pod-to-pod and CoreDNS work on them, and pods still pull through the local registry. --- README.md | 2 +- hack/create-kind-cluster.sh | 87 ++++++++++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 56151941a..4ab8ed56e 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ To quickly set up the complete environment: 2. Run the following steps: ```shell -# create cluster and local registry +# create cluster and local registry (IPv4; IP_FAMILY=dual|ipv6 overrides) hack/create-kind-cluster.sh # install ate, valkey, rustfs diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index 26548f2b4..b6c90dc86 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -18,9 +18,36 @@ set -o errexit -o nounset -o pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" +KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="5001" +if [[ $# -gt 0 ]]; then + case "$1" in + -h|--help) + echo "Usage: $0" + echo "Creates the kind cluster '${KIND_CLUSTER_NAME}' and a local registry container on port ${reg_port}." + echo + echo "Configured through the environment:" + echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." + echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + exit 0 + ;; + esac +fi + +# Only ipFamily is set; kind's per-family podSubnet/serviceSubnet defaults are +# already what we want. +IP_FAMILY="${IP_FAMILY:-ipv4}" +case "${IP_FAMILY}" in + ipv4|ipv6|dual) + ;; + *) + echo "error: IP_FAMILY must be one of ipv4, ipv6, dual (got '${IP_FAMILY}')" >&2 + exit 1 + ;; +esac + mkdir -p "${ROOT}/bin" # 1. Create registry container unless it already exists @@ -33,6 +60,9 @@ if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true fi if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != "true" ]; then + # Published on both loopback families so `ko` reaches localhost:5001 whichever + # one its resolver picks. The node side is separate: it goes over the "kind" + # network in step 4. docker run \ -d --restart=always \ --label created-by=agent-substrate \ @@ -57,7 +87,7 @@ else echo "/dev/kvm not available: micro-VM support disabled (gVisor still works)." fi -echo "Creating kind configuration for cluster '${KIND_CLUSTER_NAME}'..." +echo "Creating kind configuration for cluster '${KIND_CLUSTER_NAME}' (ipFamily=${IP_FAMILY})..." cat < "${ROOT}/bin/kind-config.yaml" kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 @@ -83,18 +113,65 @@ featureGates: PodCertificateRequest: true runtimeConfig: "certificates.k8s.io/v1beta1": "true" +networking: + ipFamily: ${IP_FAMILY} EOF echo "Deleting existing kind cluster '${KIND_CLUSTER_NAME}' if it exists..." "${ROOT}"/hack/kind.sh delete cluster --name "${KIND_CLUSTER_NAME}" || true +# kind reuses an existing "kind" network as-is, so one created by an older kind +# or while the daemon had IPv6 off (kind falls back to a v4-only network rather +# than failing) leaves the nodes with no v6 address — seen much later as pods +# stuck at ContainerCreating. Deleting the cluster does not drop the network +# either: the registry is still attached to it. Step 4 reconnects the registry. +if [[ "${IP_FAMILY}" != "ipv4" && + "$(docker network inspect kind --format '{{.EnableIPv6}}' 2>/dev/null || echo absent)" == "false" ]]; then + echo "The 'kind' Docker network exists without IPv6; recreating it..." + docker network disconnect kind "${reg_name}" 2>/dev/null || true + if ! docker network rm kind >/dev/null; then + echo "error: could not remove the 'kind' Docker network. Something else is still" >&2 + echo " attached to it; disconnect it and re-run:" >&2 + echo " docker network inspect kind --format '{{json .Containers}}'" >&2 + exit 1 + fi +fi + echo "Creating kind cluster '${KIND_CLUSTER_NAME}'..." "${ROOT}"/hack/kind.sh create cluster --name "${KIND_CLUSTER_NAME}" --config "${ROOT}/bin/kind-config.yaml" -# 2.5 Enable Proxy ARP on kind nodes for gVisor loopback pod-to-pod networking -echo "Enabling Proxy ARP on kind nodes..." +# A daemon with IPv6 off hands kind a v4-only network whatever it asked for. +if [[ "${IP_FAMILY}" != "ipv4" && + "$(docker network inspect kind --format '{{.EnableIPv6}}')" != "true" ]]; then + echo "error: the 'kind' Docker network has no IPv6, so the nodes have no v6 address." >&2 + echo " Enable IPv6 in the Docker daemon and re-run. On Linux, add to" >&2 + echo " /etc/docker/daemon.json and restart dockerd:" >&2 + echo ' {"ipv6": true, "ip6tables": true}' >&2 + exit 1 +fi + +# For ipv6 kind writes a kubeconfig pointing at [::1], the address it published +# the apiserver on, which only works for a client on the Docker host itself: a +# VM-hosted daemon (Lima on macOS) forwards the port to the *v4* loopback, so +# every kubectl below fails at connect. localhost is a SAN on the apiserver +# cert and lets the client pick a family that works from either side. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + server="$(kubectl config view \ + -o jsonpath="{.clusters[?(@.name==\"${KUBECTL_CONTEXT}\")].cluster.server}")" + if [[ "${server}" == "https://[::1]:"* ]]; then + echo "Repointing the kubeconfig for '${KUBECTL_CONTEXT}' at localhost..." + kubectl config set-cluster "${KUBECTL_CONTEXT}" \ + --server="https://localhost:${server##*:}" >/dev/null + fi +fi + +# 2.5 Enable Proxy ARP/NDP on kind nodes for gVisor loopback pod-to-pod networking +echo "Enabling Proxy ARP/NDP on kind nodes..." for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do + # Unconditional: harmless on a v6-only cluster, where the nodes still carry + # IPv4 on the Docker bridge, and proxy_ndp just supports IPv6 if configured. docker exec "${node}" sysctl net.ipv4.conf.all.proxy_arp=1 + docker exec "${node}" sysctl net.ipv6.conf.all.proxy_ndp=1 done # 2.6 When KVM is available: make /dev/kvm usable inside the node and label @@ -103,7 +180,7 @@ if [ "${HAS_KVM}" = "1" ]; then echo "Preparing kind nodes for micro-VM (kata + cloud-hypervisor) runtime..." for node in $("${ROOT}"/hack/kind.sh get nodes --name "${KIND_CLUSTER_NAME}"); do docker exec "${node}" chmod 666 /dev/kvm - kubectl label node "${node}" ate.dev/sandboxClass=microvm --overwrite + kubectl --context="${KUBECTL_CONTEXT}" label node "${node}" ate.dev/sandboxClass=microvm --overwrite done fi @@ -125,7 +202,7 @@ fi # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." -cat < Date: Thu, 13 Aug 2026 10:43:50 -0700 Subject: [PATCH 2/6] hack/create-kind-cluster.sh: fix DNS on IPv6-only clusters CoreDNS runs dnsPolicy: Default and inherits the node's Docker-generated /etc/resolv.conf, which always names an IPv4 resolver -- unreachable from a v6-only pod, so every external lookup dies at "connect: network is unreachable". Behind that sits a second failure: this script wires the registry into containerd on the node, but atelet pulls actor images from its own pod netns, where kind-registry does not resolve at all. Rewrite the Corefile on IPv6-only clusters: a hosts block mapping kind-registry to the registry's GlobalIPv6Address, with fallthrough so non-registry names still reach the new IPv6 forwarder. Hard-fail if the substitution was a no-op, and probe both names from a pod rather than the node, which is dual-stack and would resolve them either way. --- hack/create-kind-cluster.sh | 64 +++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/hack/create-kind-cluster.sh b/hack/create-kind-cluster.sh index b6c90dc86..2eb458977 100755 --- a/hack/create-kind-cluster.sh +++ b/hack/create-kind-cluster.sh @@ -21,6 +21,7 @@ KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind}" KUBECTL_CONTEXT="kind-${KIND_CLUSTER_NAME}" reg_name="kind-registry" reg_port="5001" +IPV6_DNS_UPSTREAM="${IPV6_DNS_UPSTREAM:-2001:4860:4860::8888 2001:4860:4860::8844}" if [[ $# -gt 0 ]]; then case "$1" in @@ -31,6 +32,8 @@ if [[ $# -gt 0 ]]; then echo "Configured through the environment:" echo " KIND_CLUSTER_NAME Name of the cluster to create (default: kind)." echo " IP_FAMILY Address families for pods and Services: ipv4, ipv6 or dual (default: ipv4)." + echo " IPV6_DNS_UPSTREAM Space-separated IPv6 resolvers CoreDNS forwards to when IP_FAMILY=ipv6" + echo " (default: Google Public DNS). Override where those are unreachable." exit 0 ;; esac @@ -200,6 +203,67 @@ if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name} docker network connect "kind" "${reg_name}" fi +# 4.5. Give CoreDNS an IPv6 forwarder and a registry entry (ipv6 only) +# +# CoreDNS runs dnsPolicy: Default, so it inherits the node's Docker-generated +# /etc/resolv.conf, which always names an IPv4 resolver. Pods here have no IPv4 +# address, so without this every external lookup SERVFAILs and anything that +# fetches at runtime -- atelet pulling the gVisor tarball, for one -- never +# starts. Step 3 wired the registry into containerd on the *node*, which does +# not help a pod: atelet pulls actor images from its own netns, where +# "kind-registry" NXDOMAINs. Two Corefile clauses fix both. +if [[ "${IP_FAMILY}" == "ipv6" ]]; then + echo "Repointing CoreDNS at an IPv6 resolver and teaching it '${reg_name}'..." + reg_v6="$(docker inspect "${reg_name}" \ + --format '{{.NetworkSettings.Networks.kind.GlobalIPv6Address}}')" + if [[ -z "${reg_v6}" ]]; then + echo "error: '${reg_name}' has no IPv6 address on the 'kind' network" >&2 + exit 1 + fi + + corefile="$(kubectl --context="${KUBECTL_CONTEXT}" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}')" + # fallthrough is load-bearing: without it every name that is not the registry + # NXDOMAINs, trading one outage for a worse one. Both sides are left unquoted + # -- bash 3.2 would splice the quotes in literally. + search="forward . /etc/resolv.conf" + replace="hosts { + ${reg_v6} ${reg_name} + fallthrough + } + forward . ${IPV6_DNS_UPSTREAM}" + patched="${corefile/$search/$replace}" + if [[ "${patched}" == "${corefile}" ]]; then + echo "error: '${search}' not found in the CoreDNS Corefile" >&2 + echo " a silent no-op here is the whole failure mode; inspect it by hand" >&2 + exit 1 + fi + + # A YAML patch file avoids escaping the Corefile's newlines into JSON. + { printf 'data:\n Corefile: |\n'; printf '%s\n' "${patched}" | sed 's/^/ /'; } \ + > "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system patch cm coredns \ + --type=merge --patch-file "${ROOT}/bin/coredns-patch.yaml" + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout restart deploy/coredns + kubectl --context="${KUBECTL_CONTEXT}" -n kube-system rollout status deploy/coredns \ + --timeout=120s + + # Probe from a pod, never from the node: the node is dual-stack and resolves + # both names either way, so a node-side check proves nothing. The registry leg + # fetches rather than resolves, because the hosts entry above is AAAA-only and + # `nslookup kind-registry` fails on its A query even though every real client + # (getaddrinfo, and so containerd and atelet) is satisfied by the AAAA. + echo "Verifying DNS from a pod..." + if ! kubectl --context="${KUBECTL_CONTEXT}" run "coredns-probe-$$" \ + --rm --attach --quiet --restart=Never --image=busybox:1.36 --command -- \ + sh -c "nslookup storage.googleapis.com >/dev/null && + wget -q -T10 -O/dev/null http://${reg_name}:5000/v2/"; then + echo "error: a pod cannot resolve an external name and reach '${reg_name}'" >&2 + echo " IPV6_DNS_UPSTREAM is '${IPV6_DNS_UPSTREAM}'; set it to a reachable resolver" >&2 + exit 1 + fi +fi + # 5. Document the local registry in kube-public ConfigMap echo "Documenting local registry in cluster..." cat < Date: Sat, 8 Aug 2026 10:58:33 -0700 Subject: [PATCH 3/6] atenet/router: bind the Envoy ingress listeners dual-stack The HTTP and HTTPS ingress listeners bound 0.0.0.0 only, so on a dual-stack cluster Envoy answered on the router Service's IPv4 ClusterIP and nothing at all on its IPv6 one. Pair each primary socket with an additional "::" address on the same port. Ipv4Compat is false on the additional address. Setting it would clear IPV6_V6ONLY and collide with the primary IPv4 wildcard already bound to that port, and Envoy rejects the whole listener when an additional address fails to bind -- that would take down all ingress, not just the IPv6 half. Hoisting the literal into a helper keeps the two listeners from drifting. No behaviour change on an IPv4-only cluster: the primary address is untouched, and a host without IPv6 simply has no second socket to bind. --- cmd/atenet/internal/router/xds.go | 28 +++++++++++++++++++++ cmd/atenet/internal/router/xds_test.go | 35 ++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/cmd/atenet/internal/router/xds.go b/cmd/atenet/internal/router/xds.go index d2ec58dec..bc2c0e9d5 100644 --- a/cmd/atenet/internal/router/xds.go +++ b/cmd/atenet/internal/router/xds.go @@ -797,6 +797,32 @@ func (x *XdsServer) buildTracing() *hcmv3.HttpConnectionManager_Tracing { } } +// dualStackAdditionalAddresses returns the IPv6 half of a dual-stack ingress +// listener, to pair with a primary 0.0.0.0 socket on the same port. +// +// Ipv4Compat must stay false here: it would clear IPV6_V6ONLY and collide with +// the primary IPv4 wildcard on the same port, and Envoy rejects the whole +// listener when an additional address fails to bind. The admin socket in +// manifests/ate-install/atenet-router.yaml is a single socket, so it sets the +// opposite. +func dualStackAdditionalAddresses(port uint32) []*listenerv3.AdditionalAddress { + return []*listenerv3.AdditionalAddress{ + { + Address: &corev3.Address{ + Address: &corev3.Address_SocketAddress{ + SocketAddress: &corev3.SocketAddress{ + Address: "::", + Ipv4Compat: false, + PortSpecifier: &corev3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + }, + }, + } +} + func (x *XdsServer) buildListener() *listenerv3.Listener { hcm := x.buildHcm("ingress_http") @@ -812,6 +838,7 @@ func (x *XdsServer) buildListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.ingressPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ @@ -859,6 +886,7 @@ func (x *XdsServer) buildHttpsListener() *listenerv3.Listener { }, }, }, + AdditionalAddresses: dualStackAdditionalAddresses(uint32(x.httpsPort)), FilterChains: []*listenerv3.FilterChain{ { Filters: []*listenerv3.Filter{ diff --git a/cmd/atenet/internal/router/xds_test.go b/cmd/atenet/internal/router/xds_test.go index 98287eae8..b041124be 100644 --- a/cmd/atenet/internal/router/xds_test.go +++ b/cmd/atenet/internal/router/xds_test.go @@ -157,6 +157,22 @@ func TestXdsServer_UpdateSnapshot(t *testing.T) { if sa.GetAddress() != "0.0.0.0" { t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8081 { + t.Errorf("Expected additional port 8081, got %d", asa.GetPortValue()) + } } } @@ -195,6 +211,25 @@ func TestXdsServer_UpdateSnapshot_WithHttps(t *testing.T) { if sa.GetPortValue() != 8443 { t.Errorf("Expected port 8443, got %d", sa.GetPortValue()) } + if sa.GetAddress() != "0.0.0.0" { + t.Errorf("Expected address '0.0.0.0', got %s", sa.GetAddress()) + } + + addrs := l.GetAdditionalAddresses() + if len(addrs) == 0 { + t.Fatalf("Expected an additional address on %s, got none", IngressHTTPSListener) + } + + asa := addrs[0].GetAddress().GetSocketAddress() + if asa.GetAddress() != "::" { + t.Errorf("Expected additional address '::', got %s", asa.GetAddress()) + } + if asa.GetIpv4Compat() { + t.Errorf("Expected additional address Ipv4Compat to be false") + } + if asa.GetPortValue() != 8443 { + t.Errorf("Expected additional port 8443, got %d", asa.GetPortValue()) + } // Verify the TLS config references the serving cert via SDS rather // than embedding it: inline filename DataSources are read only once From 5c14a3b81bc55444f1ccf655a9af927330f975e6 Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Wed, 12 Aug 2026 21:41:31 -0700 Subject: [PATCH 4/6] atenet/router: serve the admin socket and Service on both families The Envoy admin socket bound 0.0.0.0, leaving it reachable over IPv4 only. It binds "::" with ipv4_compat now -- one socket for both families. ipv4_compat is required rather than incidental here: Envoy sets IPV6_V6ONLY without it, and dataplane.go health-checks the listener over http://127.0.0.1:9901/ready, so dropping it would take the dataplane component of /statusz unhealthy. The atenet-router Service carried no ipFamilyPolicy, which the API server defaults to SingleStack -- an IPv4 ClusterIP and nothing else, which leaves the listeners above with no IPv6 address to answer on. Prefer, not Require, so this stays valid on a single-stack cluster, where it is a no-op. spec.ipFamilies is deliberately left alone: the primary family is immutable and the API server appends the secondary one itself. --- manifests/ate-install/atenet-router.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/manifests/ate-install/atenet-router.yaml b/manifests/ate-install/atenet-router.yaml index 9d73ecb14..0ee109b8e 100644 --- a/manifests/ate-install/atenet-router.yaml +++ b/manifests/ate-install/atenet-router.yaml @@ -86,7 +86,10 @@ data: admin: address: socket_address: - address: 0.0.0.0 + # ipv4_compat clears IPV6_V6ONLY, so this one socket serves both + # families; dataplane.go probes /ready over the IPv4 loopback. + address: "::" + ipv4_compat: true port_value: 9901 node: @@ -339,6 +342,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-router ports: From 8cc4aca8cd49cf62b0c2cfba4397fe65e0d748ee Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Thu, 13 Aug 2026 07:20:18 -0700 Subject: [PATCH 5/6] atenet/egress: serve the admin socket, listener, and Service on both families The gateway's two Envoy sockets bound 0.0.0.0, so on an IPv6-primary cluster the kubelet's startup probe against the admin port was refused and atenet-egress crashlooped while Envoy itself started fine and logged "admin address: 0.0.0.0:15000". The :443 listener had the same gap, leaving no v6 path for an actor's CONNECT. Both are single sockets, so they bind "::" with ipv4_compat rather than taking the additional-address pairing the ingress listeners use. On the admin socket ipv4_compat is load-bearing: the ext-proc sidecar's drainer reaches it at 127.0.0.1:15000 and envoydrain.go reads a refusal as "Envoy already exited", so a bare "::" would silently skip the drain. The Service gets PreferDualStack for the same reason the router's does -- without it a dual-stack cluster hands out one ClusterIP and the new v6 bind is unreachable. --- manifests/ate-install/atenet-egress.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/manifests/ate-install/atenet-egress.yaml b/manifests/ate-install/atenet-egress.yaml index 591ef31fc..808a828e7 100644 --- a/manifests/ate-install/atenet-egress.yaml +++ b/manifests/ate-install/atenet-egress.yaml @@ -37,12 +37,13 @@ data: envoy.yaml: | admin: address: - socket_address: { address: 0.0.0.0, port_value: 15000 } + # ipv4_compat: the drainer dials this on IPv4 loopback (--envoy-admin-address, below). + socket_address: { address: "::", ipv4_compat: true, port_value: 15000 } static_resources: listeners: - name: egress address: - socket_address: { address: 0.0.0.0, port_value: 443 } + socket_address: { address: "::", ipv4_compat: true, port_value: 443 } filter_chains: # Named so ext_proc can read it back as xds.filter_chain_name. Must # match EgressFilterChainName in @@ -379,6 +380,8 @@ metadata: namespace: ate-system spec: type: ClusterIP + # Prefer, not Require: Require fails Service creation on a single-stack cluster. + ipFamilyPolicy: PreferDualStack selector: app: atenet-egress ports: From 68972e5e221112ba31ae175d1955ff1e694f8f9d Mon Sep 17 00:00:00 2001 From: Yuan Gao Date: Fri, 14 Aug 2026 11:03:44 -0700 Subject: [PATCH 6/6] ci: add an IPv6-only kind e2e job Runs the full install plus the demo and networking e2e suites against a single-stack IPv6-only kind cluster, and asserts the cluster really is v6-only so a green run cannot quietly become a second IPv4 run. It stays out of the e2e-test merge gate, so it reports IPv6 status without being able to block a PR, and it runs on every PR for now so the results are visible; the TODO on the trigger records the intended ci/ipv6 label gate. ubuntu-latest has no IPv6 egress, so the job stands up tayga for NAT64 and points CoreDNS at an upstream resolver through the well-known prefix. DNS64 is scoped to a catch-all server block: synthesizing AAAA over the cluster zones destroys the v6-only ClusterIP answers and the control plane never comes up. --- .github/workflows/e2e-ipv6.yaml | 370 ++++++++++++++++++++++++++++++++ 1 file changed, 370 insertions(+) create mode 100644 .github/workflows/e2e-ipv6.yaml diff --git a/.github/workflows/e2e-ipv6.yaml b/.github/workflows/e2e-ipv6.yaml new file mode 100644 index 000000000..308f12360 --- /dev/null +++ b/.github/workflows/e2e-ipv6.yaml @@ -0,0 +1,370 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e-ipv6 +# Separate from pr-workflow.yaml so this can be gated independently -- and so a +# 40-minute IPv6 run never delays that workflow's merge-gating jobs. +# +# TODO(246): gate this before merging. It runs on every PR today so the IPv6-only +# results are visible without a maintainer having to act first; the intended +# steady state is a `ci/ipv6` label, which needs the label created upstream: +# +# on: {pull_request: {types: [labeled, opened, synchronize, reopened]}} +# if: contains(github.event.pull_request.labels.*.name, 'ci/ipv6') +# +# Either way this job stays out of the `e2e-test` gate, so it never blocks a PR. +on: + pull_request: +permissions: + contents: read +jobs: + e2e-test-ipv6: + runs-on: ubuntu-latest + # Nothing else in this workflow sets a timeout, so jobs inherit GitHub's + # 6-hour default. A broken IPv6 cluster does not crash, it misses + # 10-minute ActorTemplate deadlines, so an uncapped job burns hours. + timeout-minutes: 40 + env: + # Non-default name so these steps can be replayed locally without + # touching an existing cluster. install-ate-kind.sh does not derive + # KUBECTL_CONTEXT from the cluster name the way run-e2e-kind.sh does, + # so both have to be set here. + KIND_CLUSTER_NAME: ate-ipv6 + KUBECTL_CONTEXT: kind-ate-ipv6 + # 8.8.8.8 reached through the well-known NAT64 prefix. CoreDNS is a + # v6-only pod on a runner with no IPv6 egress of its own, so this is the + # only shape of upstream resolver it can reach. See "Set up NAT64". + IPV6_DNS_UPSTREAM: 64:ff9b::808:808 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: 'go.mod' + - name: Free disk space + # kind node image + control-plane images + snapshots are tight on the + # ~14GB runner disk even without the micro-VM assets. + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - name: Enable IPv6 in the Docker daemon + # ubuntu-latest ships dockerd with IPv6 off, so kind would create its + # network v4-only and create-kind-cluster.sh would reject the cluster. + # Merge the two keys into whatever daemon.json the runner image ships + # rather than replacing the file. + run: | + sudo mkdir -p /etc/docker + [ -s /etc/docker/daemon.json ] || echo '{}' | sudo tee /etc/docker/daemon.json >/dev/null + sudo cat /etc/docker/daemon.json \ + | jq '. + {"ipv6": true, "ip6tables": true}' \ + | sudo tee /etc/docker/daemon.json.new >/dev/null + sudo mv /etc/docker/daemon.json.new /etc/docker/daemon.json + sudo systemctl restart docker + docker network inspect bridge --format 'bridge EnableIPv6={{.EnableIPv6}}' + - name: Set up NAT64 on the runner + # ubuntu-latest has no IPv6 egress whatsoever -- measured, not assumed: + # every curl -6 fails in ~2ms. A v6-only cluster still has to reach real + # v4 destinations (atelet fetches the gVisor tarball from GCS, + # TestActorEgress fetches example.com), so the runner translates for it. + # Ordered after the dockerd restart, which rebuilds the iptables chains + # these rules live in. + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq tayga dnsutils + # tayga answers to .1/::1; the tun holds .2/::2 so host-originated + # traffic is not sourced from tayga's own address, which is + # self-addressed rather than translatable. The pool avoids both. + sudo tee /etc/tayga.conf >/dev/null <<'EOF' + tun-device nat64 + ipv4-addr 192.168.255.1 + # tayga refuses the well-known prefix with an RFC1918 pool unless it + # also holds a v6 address of its own, outside that prefix. + ipv6-addr 2001:db8:64::1 + prefix 64:ff9b::/96 + dynamic-pool 192.168.255.128/25 + data-dir /var/spool/tayga + EOF + sudo mkdir -p /var/spool/tayga + sudo tayga --mktun + sudo ip link set nat64 up + sudo ip addr add 192.168.255.2/24 dev nat64 + sudo ip -6 addr add 2001:db8:64::2/128 dev nat64 + sudo ip -6 route add 64:ff9b::/96 dev nat64 src 2001:db8:64::2 + sudo sysctl -qw net.ipv4.ip_forward=1 + sudo sysctl -qw net.ipv6.conf.all.forwarding=1 + sudo iptables -t nat -A POSTROUTING -s 192.168.255.0/24 -j MASQUERADE + # Insert, not append: docker sets the FORWARD policy to DROP. + sudo iptables -I FORWARD 1 -i nat64 -j ACCEPT + sudo iptables -I FORWARD 1 -o nat64 -j ACCEPT + sudo ip6tables -I FORWARD 1 -i nat64 -j ACCEPT + sudo ip6tables -I FORWARD 1 -o nat64 -j ACCEPT + # -d keeps tayga in the foreground and logs every dropped packet with a + # reason; detaching hides exactly the failures worth diagnosing. + sudo sh -c 'nohup tayga -d --config /etc/tayga.conf >/tmp/tayga.log 2>&1 &' + sleep 3 + pgrep -a tayga || { + echo "::error::tayga is not running"; sudo cat /tmp/tayga.log; exit 1; + } + - name: Verify NAT64 before building anything on it + # Hard gate. The cluster takes ~4 minutes and every step after it depends + # on translation working, so a broken translator should fail here with + # one clear message rather than as a rollout timeout ten minutes later. + run: | + # Map a live A record rather than hardcoding one: example.com's old + # 93.184.216.34 is retired and would fail for the wrong reason. + v4=$(getent ahostsv4 storage.googleapis.com | awk 'NR==1{print $1}') + # shellcheck disable=SC2086 + set -- ${v4//./ } + v6=$(printf '64:ff9b::%02x%02x:%02x%02x' "$1" "$2" "$3" "$4") + echo "NAT64 maps ${v4} -> ${v6}" + dig +timeout=5 +tries=1 @"${IPV6_DNS_UPSTREAM}" storage.googleapis.com A +short + code=$(curl -6 -sS -m 15 -o /dev/null -w '%{http_code}' \ + --resolve "storage.googleapis.com:443:[${v6}]" \ + https://storage.googleapis.com/ || echo 000) + echo "NAT64 HTTPS probe returned ${code}" + case "${code}" in + # Any HTTP status proves the translator carried a TCP stream; GCS + # answers a bare / with 400. ICMP is separately blocked, so a ping + # test here would report a failure that does not matter. + 2*|3*|4*) ;; + *) echo "::error::NAT64 is not translating; the cluster cannot egress" + sudo cat /tmp/tayga.log || true + exit 1 ;; + esac + - name: Create cluster + env: + IP_FAMILY: ipv6 + run: hack/create-kind-cluster.sh + - name: Assert the cluster is single-stack IPv6 + # This job is worthless if the cluster is not actually v6-only, and a + # green run leaves no evidence either way -- the diagnostics dump below + # only runs on failure. A kind default change or an IP_FAMILY regression + # would otherwise turn this into a second IPv4 run that reports success. + # Checked here rather than later so it fails as itself. + # + # PreferDualStack Services resolving to a single clusterIP is the + # positive signal: on a dual-stack cluster they would get two. + run: | + k() { kubectl --context="$KUBECTL_CONTEXT" "$@"; } + pod_cidrs=$(k get nodes -o jsonpath='{.items[*].spec.podCIDRs[*]}') + svc_ips=$(k -n default get svc kubernetes -o jsonpath='{.spec.clusterIPs[*]}') + node_ips=$(k get nodes -o jsonpath='{.items[*].status.addresses[?(@.type=="InternalIP")].address}') + for pair in "podCIDRs=${pod_cidrs}" "kubernetes.clusterIPs=${svc_ips}" "node.InternalIP=${node_ips}"; do + case "${pair#*=}" in + *.*) echo "::error::not single-stack IPv6 -- ${pair}"; exit 1 ;; + "") echo "::error::empty, cannot confirm IP family -- ${pair}"; exit 1 ;; + esac + echo " ${pair}" + done + echo "single-stack IPv6 confirmed" + - name: Apply DNS64 to external names only + # create-kind-cluster.sh already points CoreDNS at IPV6_DNS_UPSTREAM, so + # names resolve -- but the answers are unusable. Plain DNS64 synthesizes + # only for names with no AAAA, and the external names this job needs + # (storage.googleapis.com, example.com) do have AAAA records, pointing at + # real IPv6 addresses the runner cannot reach. Only translate_all forces + # them through the prefix. + # + # translate_all cannot go in the same server block as the cluster zones. + # dns64 wraps the whole plugin chain below it, and it answers a AAAA query + # by synthesizing from A -- so for an AAAA-only name it synthesizes from + # nothing and returns an empty answer. Every ClusterIP on a v6-only + # cluster is AAAA-only, so a single-block Corefile takes out all + # in-cluster service discovery: ate-api-server cannot find + # valkey-cluster.ate-system.svc and the install times out. + # + # So: cluster zones keep the chain kind shipped, the registry gets its own + # block, and dns64 sits in the catch-all with the forwarder. + run: | + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns \ + -o jsonpath='{.data.Corefile}' > /tmp/Corefile + # Read the registry address out of the Corefile create-kind-cluster.sh + # wrote; docker inspect would need it picked out of the several + # networks the registry is attached to. + reg_v6=$(awk '$2 == "kind-registry" {print $1; exit}' /tmp/Corefile) + if [ -z "${reg_v6}" ]; then + echo "::error::no kind-registry hosts entry in the Corefile" + cat /tmp/Corefile; exit 1 + fi + # Re-zone the block kind shipped and lift out the two directives that + # move elsewhere, keeping health/ready/kubernetes/cache/etc. as-is. + awk ' + NR == 1 && /^\.:53[[:space:]]*\{/ { + print "cluster.local:53 in-addr.arpa:53 ip6.arpa:53 {"; next + } + /^ (hosts|forward)([[:space:]].*)?\{$/ { skip = 1; next } + skip && /^ \}$/ { skip = 0; next } + skip { next } + { print } + ' /tmp/Corefile > /tmp/Corefile.new + if ! grep -q '^cluster.local:53' /tmp/Corefile.new; then + echo "::error::Corefile did not start with the .:53 block kind ships" + cat /tmp/Corefile; exit 1 + fi + if grep -q 'forward' /tmp/Corefile.new; then + echo "::error::the forward block survived the split" + cat /tmp/Corefile.new; exit 1 + fi + cat >>/tmp/Corefile.new < /tmp/coredns-dns64.yaml + kubectl --context="$KUBECTL_CONTEXT" -n kube-system patch cm coredns \ + --type=merge --patch-file /tmp/coredns-dns64.yaml + kubectl --context="$KUBECTL_CONTEXT" -n kube-system rollout restart deploy/coredns + kubectl --context="$KUBECTL_CONTEXT" -n kube-system rollout status deploy/coredns --timeout=120s + cat /tmp/Corefile.new + - name: Verify cluster DNS answers both internal and external names + # The install is the next step and it takes ten minutes to fail. A DNS + # regression is the failure this Corefile is most likely to cause, so + # assert all three cases here where the message is unambiguous. + run: | + set -o pipefail + kubectl --context="$KUBECTL_CONTEXT" run dnscheck --rm --attach --quiet \ + --restart=Never --image=busybox:1.36 --command -- \ + sh -c ' + nslookup kubernetes.default.svc.cluster.local >/dev/null 2>&1 \ + || { echo "FAIL: an in-cluster Service does not resolve"; exit 1; } + nslookup storage.googleapis.com 2>/dev/null | grep -q "64:ff9b" \ + || { echo "FAIL: external names are not synthesized through NAT64"; exit 1; } + # Informational. Nothing downstream resolves this from a pod -- + # containerd pulls images on the node, and create-kind-cluster.sh + # runs its own registry probe before DNS64 is applied -- so a miss + # here is not a reason to fail the job. Printed because a change + # here would still be worth seeing. + echo "--- kind-registry, informational" + nslookup kind-registry 2>&1 | tail -4 + echo "DNS-OK" + ' | tee /tmp/dnscheck.log + grep -q DNS-OK /tmp/dnscheck.log + - name: Install Agent Substrate + run: hack/install-ate-kind.sh --deploy-ate-system --ateapi-client-auth=cert + - name: Assert the control plane is up + # install-ate.sh runs under pipefail, so a failed apply does propagate. + # What it would not catch is a Deployment that rolls out and then + # crash-loops. Re-check everything deploy_ate_system waits on -- + # atenet-egress included, since a non-dual-stack Envoy listener fails + # there first, by way of a readiness probe the kubelet cannot reach. + run: | + for r in deployment/ate-api-server deployment/ate-controller \ + deployment/atenet-router deployment/atenet-egress \ + statefulset/valkey-cluster daemonset/atelet; do + kubectl --context="$KUBECTL_CONTEXT" -n ate-system rollout status "$r" --timeout=120s + done + kubectl --context="$KUBECTL_CONTEXT" -n podcertificate-controller-system \ + rollout status deployment/podcertificate-controller --timeout=120s + if kubectl --context="$KUBECTL_CONTEXT" -n ate-system get pods \ + -o jsonpath='{.items[*].status.containerStatuses[*].state.waiting.reason}' \ + | grep -q CrashLoopBackOff; then + echo "::error::a pod in ate-system is in CrashLoopBackOff" + exit 1 + fi + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Assert the demo fixtures exist + # A failed demo deploy exits 0: install-ate.sh dispatches demos through + # `if "${demo}_cmdline" "$1"`, which suspends errexit, and _cmdline ends + # in an unconditional `return 0`. Without this the suites fail later with + # "ActorTemplate not found", pointing at the tests instead of the install. + run: | + for ns_tmpl in ate-demo-counter/counter ate-demo-egress/egress; do + ns=${ns_tmpl%/*}; tmpl=${ns_tmpl#*/} + kubectl --context="$KUBECTL_CONTEXT" -n "${ns}" get actortemplate "${tmpl}" \ + || { echo "::error::${ns_tmpl} was not created -- the demo deploy failed silently"; exit 1; } + done + # One suite per step: run-e2e.sh takes exactly one target path, and this + # way a failure names the suite that produced it. + - name: Run E2E tests (demo) + id: e2e-demo + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color 2>&1 \ + | tee /tmp/e2e-demo.log + - name: Run E2E tests (networking) + # Runs even when demo failed -- networking is the half most likely to + # expose a single-family bug -- but stays skipped when an earlier step + # left no cluster to test against. always() is required, not decorative: + # an if: without a status function is implicitly ANDed with success(), + # which skips this step on exactly the failure it is meant to survive. + if: always() && steps.e2e-demo.outcome != 'skipped' + run: | + set -o pipefail + hack/run-e2e-kind.sh ./internal/e2e/suites/networking -v -args --no-color 2>&1 \ + | tee /tmp/e2e-networking.log + - name: Guard against a vacuously green run + # A suite that gates on a dual-stack Service and skips itself on v6-only + # exits 0, so a suite that only skipped would otherwise read as a pass. + # The bar is one real PASS per suite, not zero skips: demo legitimately + # skips the micro-VM-only Golden resume and the CSI volume tests on any + # family. Skips are printed so a growing list gets noticed. + # Skipped when the suites never ran: with no logs to count, this step + # would otherwise report a reassuring zero on a job that failed earlier. + if: always() && steps.e2e-demo.outcome != 'skipped' + run: | + for f in /tmp/e2e-demo.log /tmp/e2e-networking.log; do + [ -s "$f" ] || { echo "::error::${f} is missing or empty"; exit 1; } + passed=$(grep -c -- '--- PASS' "$f" || true) + echo "${f}: ${passed} passed, $(grep -c -- '--- SKIP' "$f" || true) skipped" + grep -h -- '--- SKIP' "$f" || true + if [ "${passed}" -eq 0 ]; then + echo "::error::${f} has no passing tests -- a suite that only skips proves nothing" + exit 1 + fi + done + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context="$KUBECTL_CONTEXT" get actortemplate,workerpool,pods -A -o wide || true + dump() { + echo "=== logs: $1/$2 ===" + kubectl --context="$KUBECTL_CONTEXT" logs -n "$1" "$2" --all-containers --tail=300 2>/dev/null || true + } + for p in $(kubectl --context="$KUBECTL_CONTEXT" get pods -n ate-system -o name 2>/dev/null); do + dump ate-system "$p" + done + # Every worker pod in any namespace: the demo pools plus the e2e suites' + # randomly-named per-test namespaces, which the suites keep on failure. + kubectl --context="$KUBECTL_CONTEXT" get pods -A -l ate.dev/worker-pool \ + -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ + | while read -r ns name; do dump "$ns" "$name"; done + # IPv6-specific: the rewritten Corefile, and what each Service actually + # got assigned, are the two things that differ from the IPv4 job. + kubectl --context="$KUBECTL_CONTEXT" -n kube-system logs -l k8s-app=kube-dns --tail=100 || true + kubectl --context="$KUBECTL_CONTEXT" -n kube-system get cm coredns -o jsonpath='{.data.Corefile}' || true + kubectl --context="$KUBECTL_CONTEXT" get svc -A \ + -o custom-columns=NS:.metadata.namespace,NAME:.metadata.name,POLICY:.spec.ipFamilyPolicy,IPS:.spec.clusterIPs || true + # tayga logs a reason for every packet it declines to translate, which + # is the only view of an egress failure that is not a bare timeout. + echo "=== tayga ===" + sudo tail -100 /tmp/tayga.log || true