diff --git a/Makefile b/Makefile index 493b488..08255df 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # SourceOS Continuum — lifecycle entry points. # Control-plane targets delegate to Makefile.porter (the rehomed Porter control plane). -.PHONY: validate onboard dev-up dev-down shim-test test tools-test rollout promotion-gate portal compute mesh-demo grant commons mcp spine run loop verify lease +.PHONY: validate onboard dev-up dev-down shim-test test tools-test rollout promotion-gate portal compute mesh-demo grant commons mcp spine run loop verify lease sphere validate: ## repo hygiene + CapD validity python3 tools/validate.py @@ -42,6 +42,9 @@ verify: ## volunteer-mesh verification demo: redundant quorum over untrusted wor lease: ## pull/lease scheduler demo: workers pull WUs, crash-stop re-lending, ordered re-merge cd tools && python3 lease_scheduler.py +sphere: ## data-sphere demo: immutable dm-verity sphere, construction-tenancy, intent x link x durability + cd tools && python3 data_sphere.py + onboard: ## bring up a workstation: local sovereign forge + local cluster + sourceosctl @echo "[continuum] onboard — scaffold: wires Gitea bring-up + kind/k3s + sourceos-devtools/sourceosctl" diff --git a/capd/data-spheres.mesh.capd.json b/capd/data-spheres.mesh.capd.json new file mode 100644 index 0000000..31a096d --- /dev/null +++ b/capd/data-spheres.mesh.capd.json @@ -0,0 +1,30 @@ +{ + "capability_id": "caps.data.spheres@0.1.0", + "kind": "data.governed-sphere", + "status": "experimental", + "name": "Data Spheres — ring-fenced, immutable, provenance-tracked data", + "description": "A data sphere is the trusted-data-sphere pattern (immutable + persistent + token-gated + provenance) fused with our commons record + grant + Needs-firewall, and hardened by the live-mount-table analysis: integrity by construction (squashfs/erofs + dm-verity with a pinned root hash, not readOnly:true), tenancy by construction (the isolation key in the mount source ref, session-scoped and unforgeable, not a policy), direction as a first-class axis (ingress|egress|bidirectional|none) with an at-most-one-egress chokepoint invariant, and backend as a function of intent x link_availability x durability (reference-mount over reliable links; copy+reconcile only when an intermittent link forces divergence). Reading a sphere requires a Grant with effect:read bound to the sphere id + content digest; access is fail-closed.", + "links": { + "engine": "tools/data_sphere.py", + "grant_authority": "tools/mcp_a2a_grant.py", + "commons": "tools/commons.py", + "placement": "tools/compute_plane.py", + "spec_witness": "docs/MOUNT_TAXONOMY.md", + "reference_pattern": "Trusted Data Spheres (token-gated immutable provenance) + the live-mount-table analysis (dm-verity integrity, construction-tenancy, direction axis + egress chokepoint, intent x link x durability lattice)" + }, + "composes_with": { + "commons": "caps.knowledge.commons@0.1.0", + "compute_plane": "caps.compute.mesh-plane@0.1.0", + "control_plane": "caps.infra.paas.continuum-local@0.1.0", + "scales_up_to": "caps.infra.cluster-scaleup.hyperswarm@0.1.0" + }, + "policy": { + "immutable": true, + "integrity_by_construction": true, + "tenancy_by_construction": true, + "grant_gated_read": true, + "single_egress_chokepoint": true, + "residency_ring_fenced": true, + "evidence_emitting": true + } +} diff --git a/docs/MOUNT_TAXONOMY.md b/docs/MOUNT_TAXONOMY.md new file mode 100644 index 0000000..c685e9a --- /dev/null +++ b/docs/MOUNT_TAXONOMY.md @@ -0,0 +1,84 @@ +# Mount-intent taxonomy — sharpened by the live mount table + +The mount-intent → backend diagram was scored against the sandbox's own live mount table (now a real +VM: `/dev/vda` ext4 root, `/dev/vdb–vdd` squashfs read-only, four `rclone` FUSE mounts projecting a +remote object store as a filesystem). Four of the eight intents don't instantiate there — not a gap, +just evidence that the taxonomy is scoped to a **stateful fog node** while that environment is a +**disposable single-session actor**. The taxonomy survives; only a subgraph lights up. But the +comparison surfaced four real upgrades, now codified in `tools/data_sphere.py`. + +## 1. Integrity by construction, not by flag + +`readOnly: true` (a ro bind mount, a ro PVC) is a *mount-flag* assertion — the underlying inode stays +writable through any other view (a second mount, the host, an rw sidecar holding the same PV). For a +corpus that must be provably unmutated between crawl-time and query-time (SP-SCOPE-CRAWL-001), that's +a trust assertion, not a guarantee. + +**A squashfs/erofs image + dm-verity with the root hash pinned in the deployment manifest** has no +write path at any layer, and integrity becomes a **signature check**. `data_sphere.mint_sphere` +carries `integrity: dm-verity` + a pinned `root_hash`; `access_check` **refuses a sphere with no +pinned integrity root**. This is the correct sovereign form of `type=image mount` — wire that +otherwise-dangling node to `curated_corpus` as **T0**. + +## 2. Tenancy by construction, not by policy + +The sandbox binds isolation in the mount **source** string (`rclone-filestore::/mnt/…`) — +session-scoped and unforgeable from inside the guest. Contrast K8s: a PVC is namespace-scoped *by +policy*, and a bad manifest can bind a pod to a volume it shouldn't. Putting the identity in the +remote **name**, not a path prefix or an admission rule, moves isolation from *policy-enforced* to +*construction-enforced*. `data_sphere.tenancy_binding` returns `sphere-store::`. + +## 3. The missing axis — DIRECTION + +The intents encoded **lifecycle** (canonical/derived/scratch/cache) and **sensitivity** +(secrets/config_ro) but not **direction**. The sandbox's entire security model *is* directional: +three `ro` ingress channels, exactly one `rw` egress channel, nothing bidirectional except the +disposable root. That yields a **single durable-write chokepoint** — exactly what a chain-of-custody +wants, because egress attestation then has one place to live. + +`direction ∈ {ingress, egress, bidirectional, none}` is now a first-class attribute on every sphere, +with the invariant enforced by `data_sphere.check_egress_invariant`: + +> **At most one egress mount per workload, named — and it is the only mount whose contents survive +> the workload.** + +Our current diagram permits `canonical_data → PVC | Docker volume | Podman volume`, all `rw`: three +durable write paths, three attestation points, three ways to get it wrong. The invariant collapses +that to one. + +## 4. Backend is a function of the product, not the data + +The edge/fog↔cloud-twin diagram's three link modes (LAN/WAN/sneakernet) all have **copy semantics** +(snapshot, export, reconcile). The sandbox's rclone FUSE has **reference semantics** — no second +copy, therefore no divergence, therefore no reconciliation, therefore no conflict algebra. The +`(S3-compatible store)` box and rclone are *the same object*; "how does the edge reach the store" is +answered *mount it*, not *sync to it* (price: zero offline capability — cut the link and every channel +fails at once). + +So the two diagrams compose into one lattice. The real signature is not `intent → backend` but: + +``` +intent × link_availability × durability_requirement → backend +``` + +`data_sphere.backend_for` implements it: + +| link | durability | backend | copy? | reconcile? | +|---|---|---|---|---| +| reliable | any | **reference-mount** | no | no | +| intermittent | canonical | copy + reconcile | yes | **yes** (SP-EVAL-CRF-001 earns its keep here) | +| intermittent | derived/scratch | copy | yes | no | + +The reconciliation burden is a **function of the product**, not a property of the data — so the +Mellumwork ternary / conflict-resolution-faithfulness machinery is paid *only* on the +`intermittent × canonical` cell, and every other mount class gets correctness for free from the link. +Add `reference mount (no copy)` as a fourth column on the intermittent-link box with the availability +precondition on the edge, and the two diagrams are one. + +## The honest counterexample + +The sandbox's root ext4 is simultaneously runtime image, scratch, cache, and working directory — the +exact collapse this taxonomy exists to prevent. Harmless when the whole thing is disposable per +session; **wrong for a fog node** where retention policy differs per class. It's the counterexample +that motivates the taxonomy, and it's why the direction axis + the egress chokepoint matter: they +make the collapse structurally impossible to express. diff --git a/tools/data_sphere.py b/tools/data_sphere.py new file mode 100644 index 0000000..81ebe67 --- /dev/null +++ b/tools/data_sphere.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Data Spheres — ring-fenced, immutable, provenance-tracked data as a first-class governed unit. + +A data sphere is the "trusted data sphere" (immutable + persistent + token-gated + provenance) fused +with our commons record + grant + Needs-firewall — and hardened by four lessons read straight off +the sandbox's own live mount table: + + * INTEGRITY by construction, not by flag. `readOnly: true` is a mount-flag assertion — the inode + stays writable through any other view (a second mount, the host, an rw sidecar). A squashfs/erofs + image + dm-verity with the root hash PINNED in the manifest makes immutability a signature check, + not a trust assertion. + * TENANCY by construction, not by policy. The isolation key lives in the mount SOURCE reference + (session-scoped, unforgeable from inside the guest) — not a path prefix or an admission rule a + bad manifest can get wrong. + * DIRECTION is a first-class axis: ingress | egress | bidirectional | none. Invariant: AT MOST ONE + egress mount per workload, named — the single durable-write chokepoint where egress attestation + lives, and the only mount whose contents survive the workload. + * BACKEND is a function of intent x link_availability x durability, not of the data. Over a reliable + link a sphere is REFERENCE-mounted (no copy, no divergence, no reconciliation); only an + intermittent link forces copy + divergence + conflict-resolution. +""" +from __future__ import annotations + +import hashlib +import json + +DIRECTIONS = ("ingress", "egress", "bidirectional", "none") + + +def _digest(content) -> str: + return hashlib.sha256(json.dumps(content, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def mint_sphere(*, name: str, version: str, content, residency: str = "any", + direction: str = "ingress", provenance: dict | None = None, + integrity: str = "dm-verity") -> dict: + """A content-addressed, immutable data sphere. `root_hash` is the pinned dm-verity root — integrity + becomes a signature check, not a `readOnly` assertion.""" + assert direction in DIRECTIONS, f"direction must be one of {DIRECTIONS}" + d = _digest(content) + return {"sphere_id": f"sphere:{name}@{version}+{d[:12]}", + "content_digest": "sha256:" + d, + "residency": residency, "direction": direction, + "integrity": integrity, "root_hash": "sha256:" + d, # dm-verity root, pinned + "immutable": True, "provenance": provenance or {}} + + +def tenancy_binding(sphere: dict, *, session: str) -> str: + """Construction-enforced tenancy: the isolation key is in the mount SOURCE ref, session-scoped and + unforgeable from inside the guest (cf. `rclone-filestore::/path`).""" + return f"sphere-store:{session}:{sphere['sphere_id']}" + + +def read_grant_capability(sphere: dict) -> dict: + """The Grant capability required to read this sphere (the 'read token'): effect=read, ref bound to + the sphere id + its content digest.""" + return {"kind": "mcp_tool", "capability_ref": "capd://" + sphere["sphere_id"], + "capability_digest": sphere["content_digest"], "effect": "read"} + + +def access_check(sphere: dict, grant: dict, *, requested_effect: str = "read", + residency_ok: bool = True) -> dict: + """Fail-closed access: the Grant must be for THIS sphere, the right effect, integrity pinned, and + residency satisfied. Returns {authorized, reason}.""" + cap = grant.get("capability", {}) + if not sphere.get("root_hash"): + return {"authorized": False, "reason": "sphere has no pinned integrity root — refuse"} + if cap.get("capability_ref") != "capd://" + sphere["sphere_id"]: + return {"authorized": False, "reason": "grant is not bound to this sphere"} + if cap.get("capability_digest") != sphere["content_digest"]: + return {"authorized": False, "reason": "content digest mismatch — sphere mutated or wrong grant"} + if cap.get("effect") != requested_effect: + return {"authorized": False, "reason": f"effect {requested_effect!r} not granted (granted {cap.get('effect')!r})"} + if not residency_ok: + return {"authorized": False, "reason": f"residency {sphere['residency']!r} not satisfied at this node"} + return {"authorized": True, "reason": "grant bound, integrity pinned, effect + residency satisfied"} + + +def check_egress_invariant(mounts: list) -> dict: + """At most ONE egress mount per workload (the single durable-write chokepoint). mounts: + [{name, direction}]. Fail-closed on >1.""" + egress = [m["name"] for m in mounts if m.get("direction") == "egress"] + return {"ok": len(egress) <= 1, "egress_mounts": egress, + "reason": ("ok" if len(egress) <= 1 + else f"{len(egress)} egress mounts ({egress}); the invariant is at most one, named")} + + +def backend_for(*, intent: str, link_availability: str, durability: str) -> dict: + """intent x link_availability x durability -> backend. The reconciliation burden is a function of + this product, not a property of the data. Over a reliable link: REFERENCE-mount (no copy).""" + if link_availability == "reliable": + return {"backend": "reference-mount", "copy": False, "reconciliation": False, + "note": "mount it, don't sync it — no second copy, no divergence, no CRF machinery"} + # intermittent link -> forced into copy semantics + if durability == "canonical": + return {"backend": "copy+reconcile", "copy": True, "reconciliation": True, + "note": "intermittent + canonical: copy forces divergence -> conflict-resolution (SP-EVAL-CRF-001)"} + return {"backend": "copy", "copy": True, "reconciliation": False, + "note": "snapshot/export; derived/scratch does not need reconciliation"} + + +if __name__ == "__main__": + sphere = mint_sphere(name="crawl-corpus", version="2026.08", content={"docs": 1_000_000}, + residency="eu", direction="ingress", + provenance={"crawl": "SP-SCOPE-CRAWL-001"}) + grant = {"capability": read_grant_capability(sphere)} + print(json.dumps({ + "sphere_id": sphere["sphere_id"], + "integrity": f"{sphere['integrity']} root {sphere['root_hash'][:20]}…", + "tenancy": tenancy_binding(sphere, session="sess_abc"), + "access": access_check(sphere, grant)["authorized"], + "reliable_link": backend_for(intent="canonical", link_availability="reliable", durability="canonical")["backend"], + "intermittent_link": backend_for(intent="canonical", link_availability="intermittent", durability="canonical")["backend"], + "egress_invariant_2": check_egress_invariant([{"name": "a", "direction": "egress"}, + {"name": "b", "direction": "egress"}])["ok"], + }, indent=2)) diff --git a/tools/test_data_sphere.py b/tools/test_data_sphere.py new file mode 100644 index 0000000..ddb133f --- /dev/null +++ b/tools/test_data_sphere.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Tests for Data Spheres. Load-bearing: access is fail-closed and integrity-pinned (a mutated +sphere or a wrong-effect grant is denied), the at-most-one-egress invariant holds, and the +intent×link×durability lattice picks reference-mount over reliable links (no reconciliation).""" +import data_sphere as ds + + +def test_sphere_is_content_addressed_immutable_and_integrity_pinned(): + s = ds.mint_sphere(name="corpus", version="1", content={"n": 10}, direction="ingress") + assert s["sphere_id"].startswith("sphere:corpus@1+") and s["immutable"] is True + assert s["root_hash"].startswith("sha256:") and s["integrity"] == "dm-verity" + # same content -> same id (content-addressed); different content -> different + assert ds.mint_sphere(name="corpus", version="1", content={"n": 10})["sphere_id"] == s["sphere_id"] + assert ds.mint_sphere(name="corpus", version="1", content={"n": 11})["sphere_id"] != s["sphere_id"] + + +def test_tenancy_is_bound_in_the_mount_source_not_the_path(): + s = ds.mint_sphere(name="c", version="1", content={}) + binding = ds.tenancy_binding(s, session="sess_xyz") + assert binding.startswith("sphere-store:sess_xyz:") # session in the source ref, unforgeable + + +def test_access_check_authorizes_a_bound_read_grant(): + s = ds.mint_sphere(name="c", version="1", content={"x": 1}) + grant = {"capability": ds.read_grant_capability(s)} + assert ds.access_check(s, grant)["authorized"] is True + + +def test_access_denies_wrong_effect_mutation_and_bad_residency(): + s = ds.mint_sphere(name="c", version="1", content={"x": 1}, residency="eu") + grant = {"capability": ds.read_grant_capability(s)} + # wrong effect + assert ds.access_check(s, grant, requested_effect="write")["authorized"] is False + # residency not satisfied at this node + assert ds.access_check(s, grant, residency_ok=False)["authorized"] is False + # content-digest mismatch = sphere mutated or wrong grant + tampered = {"capability": {**ds.read_grant_capability(s), "capability_digest": "sha256:" + "0" * 64}} + assert ds.access_check(s, tampered)["authorized"] is False + + +def test_access_refuses_a_sphere_without_pinned_integrity(): + s = ds.mint_sphere(name="c", version="1", content={}) + s = {**s, "root_hash": None} # integrity not pinned + grant = {"capability": ds.read_grant_capability(s)} + assert ds.access_check(s, grant)["authorized"] is False + + +def test_at_most_one_egress_mount_invariant(): + ok = ds.check_egress_invariant([{"name": "out", "direction": "egress"}, + {"name": "in", "direction": "ingress"}]) + bad = ds.check_egress_invariant([{"name": "out1", "direction": "egress"}, + {"name": "out2", "direction": "egress"}]) + assert ok["ok"] is True and bad["ok"] is False + assert bad["egress_mounts"] == ["out1", "out2"] + + +def test_backend_lattice_reference_mounts_over_reliable_links(): + reliable = ds.backend_for(intent="canonical", link_availability="reliable", durability="canonical") + assert reliable["backend"] == "reference-mount" and reliable["copy"] is False and reliable["reconciliation"] is False + # intermittent + canonical -> the reconciliation burden appears (not before) + inter = ds.backend_for(intent="canonical", link_availability="intermittent", durability="canonical") + assert inter["copy"] is True and inter["reconciliation"] is True + # intermittent + derived -> copy but no reconciliation + derived = ds.backend_for(intent="derived", link_availability="intermittent", durability="derived") + assert derived["copy"] is True and derived["reconciliation"] is False + + +if __name__ == "__main__": + import sys + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + for fn in fns: + fn() + print(f"ok: {len(fns)} data-sphere tests passed") + sys.exit(0) diff --git a/tools/validate.py b/tools/validate.py index 550674e..936dbcd 100644 --- a/tools/validate.py +++ b/tools/validate.py @@ -24,6 +24,7 @@ "capd/knowledge-commons.mesh.capd.json", "capd/self-healing-loop.mesh.capd.json", "capd/volunteer-mesh-verification.mesh.capd.json", + "capd/data-spheres.mesh.capd.json", "tools/promotion_gate.py", "tools/portal_server.py", "tools/compute_plane.py", @@ -39,6 +40,7 @@ "tools/work_unit.py", "tools/lease_scheduler.py", "tools/devmode.py", + "tools/data_sphere.py", ] CAPD_KEYS = ("capability_id", "kind", "status", "links", "composes_with", "policy") # Every CapD in capd/ must carry the core keys and parse — not just the flagship control-plane one. @@ -49,6 +51,7 @@ "capd/knowledge-commons.mesh.capd.json": "caps.knowledge.commons", "capd/self-healing-loop.mesh.capd.json": "caps.compute.self-healing-loop", "capd/volunteer-mesh-verification.mesh.capd.json": "caps.compute.volunteer-mesh-verification", + "capd/data-spheres.mesh.capd.json": "caps.data.spheres", } errors: list[str] = []