fix+feat(memory): roadmap phases 1-4 — tails, smartness, proof, operability

AGENTS.md

@@ -235,6 +235,16 @@ ODEK_E2E=true go test -v -count=1 ./cmd/odek/ -run "TestMCPE2E_"
 
 # Sandbox integration tests (requires Docker)
 go test -v -count=1 ./cmd/odek/ -run "TestSandbox"
+
+# PIGuard sidecar E2E (local only — too heavy for CI; provisions the
+# docker stack, runs the env-gated test, tears down. Use --linux on macOS
+# for full socket-mode coverage.)
+docker/piguard-e2e.sh
+
+# Fuzz soaks (extractJSON, SKILL.md parsing, session loading)
+go test -fuzz=FuzzExtractJSON -fuzztime=30s ./internal/memory/extended/
+go test -fuzz=FuzzParseSkillContent -fuzztime=30s ./internal/skills/
+go test -fuzz=FuzzSessionLoad -fuzztime=30s ./internal/session/
 ```
 
 Note: MCP client E2E tests build the fakeserver from `internal/mcpclient/testdata/main.go` at test time (no pre-compiled binary). macOS temp dirs are classified as `LocalWrite` (not `SystemWrite`), and the Docker availability check verifies daemon reachability before running sandbox tests.

cmd/odek/memory_cmd.go

@@ -1,11 +1,16 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
+	"time"
 
+	"github.com/BackendStack21/odek/internal/config"
+	"github.com/BackendStack21/odek/internal/llm"
 	"github.com/BackendStack21/odek/internal/memory"
 	"github.com/BackendStack21/odek/internal/memory/extended"
 )
@@ -70,10 +75,10 @@ func memoryCmd(args []string) error {
 	}
 }
 
-// extendedMemoryCmd handles `odek memory extended forget|quarantine|compact`.
+// extendedMemoryCmd handles `odek memory extended <subcommand>`.
 func extendedMemoryCmd(dir string, args []string) error {
 	if len(args) == 0 {
-		fmt.Fprintf(os.Stderr, "Usage: odek memory extended <forget|promote|pin|quarantine|compact|pending|confirm|reject> [args]\n")
+		fmt.Fprintf(os.Stderr, "Usage: odek memory extended <forget|promote|pin|quarantine|compact|stats|consolidate|pending|confirm|reject> [args]\n")
 		return nil
 	}
 
@@ -144,6 +149,48 @@ func extendedMemoryCmd(dir string, args []string) error {
 		fmt.Println("odek: Extended Memory vector index compaction triggered in the background")
 		return nil
 
+	case "stats":
+		st := em.Stats()
+		fmt.Println("Extended Memory stats:")
+		fmt.Printf("  live atoms:        %d\n", st.LiveAtoms)
+		fmt.Printf("  quarantined atoms: %d\n", st.QuarantinedAtoms)
+		if len(st.QuarantineReasons) > 0 {
+			reasons := make([]string, 0, len(st.QuarantineReasons))
+			for r := range st.QuarantineReasons {
+				reasons = append(reasons, r)
+			}
+			sort.Strings(reasons)
+			for _, r := range reasons {
+				fmt.Printf("    %-16s %d\n", r+":", st.QuarantineReasons[r])
+			}
+		}
+		fmt.Printf("  index vectors:     %d (dirty: %v)\n", st.IndexVectors, st.IndexDirty)
+		fmt.Printf("  store size:        %.1f MiB\n", float64(st.StoreSizeBytes)/(1<<20))
+		fmt.Printf("  recall timeouts:   %d\n", st.RecallTimeouts)
+		fmt.Printf("  recall failures:   %d\n", st.RecallFailures)
+		if st.RecallTimeouts+st.RecallFailures > 0 {
+			fmt.Println("  warning: recall degraded this process — check LLM/embedding backend latency")
+		}
+		return nil
+
+	case "consolidate":
+		// Merging near-duplicate atoms needs an LLM; resolve the operator
+		// backend the same way the agent does.
+		resolved := config.LoadConfig(config.CLIFlags{})
+		if resolved.APIKey == "" {
+			return fmt.Errorf("memory extended consolidate requires an LLM backend (no API key resolved)")
+		}
+		llmClient := llm.New(resolved.BaseURL, resolved.APIKey, resolved.Model, "", 0, 120*time.Second)
+		emLLM := extended.New(extDir, llmClient, cfg)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+		defer cancel()
+		merged, err := emLLM.ConsolidateAtoms(ctx)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("odek: consolidation complete — %d atom(s) merged into existing or new entries\n", merged)
+		return nil
+
 	case "pending":
 		pending, err := em.ListPendingReview()
 		if err != nil {
@@ -186,6 +233,6 @@ func extendedMemoryCmd(dir string, args []string) error {
 		return nil
 
 	default:
-		return fmt.Errorf("unknown extended memory subcommand %q (expected: forget, promote, pin, quarantine, compact, pending, confirm, reject)", sub)
+		return fmt.Errorf("unknown extended memory subcommand %q (expected: forget, promote, pin, quarantine, compact, stats, consolidate, pending, confirm, reject)", sub)
 	}
 }

docker/piguard-e2e.sh

@@ -0,0 +1,114 @@
+#!/usr/bin/env bash
+# Local E2E runner for the PIGuard prompt-injection sidecar.
+#
+# Builds (if needed) and starts the daemon + gateway exactly like the
+# docker-compose stack, then runs the env-gated E2E test in
+# internal/guard/piguard_e2e_test.go against them, and tears everything
+# down afterwards.
+#
+# Why a script and not CI: the stack is heavy (a ~735 MB ONNX model plus
+# two image builds), which made the GitHub Actions job too slow to keep.
+# Run this before merging changes that touch internal/guard or the
+# docker piguard stack.
+#
+# Requirements: Docker, Go. First run needs the model exported once:
+#   docker/piguard/download-model.sh
+#
+# Usage:
+#   docker/piguard-e2e.sh            build images if missing, run E2E, tear down
+#   docker/piguard-e2e.sh --build    force image rebuild
+#   docker/piguard-e2e.sh --linux    also run the test binary inside a Linux
+#                                    container (full socket-mode coverage;
+#                                    on macOS the host socket subtest skips
+#                                    because unix sockets do not cross the
+#                                    Docker Desktop VM boundary)
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+DOCKER_DIR="${REPO_ROOT}/docker"
+MODEL_DIR="${DOCKER_DIR}/piguard/models"
+SOCK_DIR="/tmp/piguard-e2e"
+NETWORK="piguard-e2e"
+BUILD=0
+LINUX=0
+for arg in "$@"; do
+  case "$arg" in
+    --build) BUILD=1 ;;
+    --linux) LINUX=1 ;;
+    *) echo "unknown flag: $arg" >&2; exit 2 ;;
+  esac
+done
+
+cleanup() {
+  docker rm -f piguard-e2e-gateway piguard-e2e-daemon >/dev/null 2>&1 || true
+  docker network rm "${NETWORK}" >/dev/null 2>&1 || true
+  rm -rf "${SOCK_DIR}"
+}
+trap cleanup EXIT
+
+docker info >/dev/null 2>&1 || { echo "docker daemon is not running" >&2; exit 1; }
+
+if [ ! -f "${MODEL_DIR}/model.onnx" ]; then
+  echo "PIGuard model not found in ${MODEL_DIR}." >&2
+  echo "Run ${DOCKER_DIR}/piguard/download-model.sh first (one-time, ~735 MB)." >&2
+  exit 1
+fi
+
+if [ "${BUILD}" = 1 ] || ! docker image inspect piguard:local >/dev/null 2>&1; then
+  (cd "${DOCKER_DIR}" && docker compose --profile restricted build piguard)
+fi
+if [ "${BUILD}" = 1 ] || ! docker image inspect piguard-gateway:local >/dev/null 2>&1; then
+  (cd "${DOCKER_DIR}" && docker compose --profile restricted build piguard-gateway)
+fi
+
+cleanup # remove leftovers from a previous run
+mkdir -p "${SOCK_DIR}"
+docker network create "${NETWORK}" >/dev/null
+docker run -d --name piguard-e2e-daemon --network "${NETWORK}" \
+  -v "${MODEL_DIR}:/models:ro" \
+  -v "${SOCK_DIR}:/run/piguard" \
+  piguard:local \
+  --socket=/run/piguard/piguard.sock --model-dir=/models \
+  --max-batch=32 --batch-wait=5ms >/dev/null
+docker run -d --name piguard-e2e-gateway --network "${NETWORK}" \
+  -p 127.0.0.1:18080:8080 \
+  -v "${SOCK_DIR}:/run/piguard" \
+  piguard-gateway:local \
+  --addr=:8080 --socket=/run/piguard/piguard.sock >/dev/null
+
+echo "Waiting for gateway health..."
+healthy=0
+for _ in $(seq 1 90); do
+  if curl -fsS http://127.0.0.1:18080/healthz >/dev/null 2>&1; then
+    healthy=1
+    break
+  fi
+  sleep 2
+done
+if [ "${healthy}" != 1 ]; then
+  echo "=== daemon logs ===" >&2; docker logs piguard-e2e-daemon >&2 || true
+  echo "=== gateway logs ===" >&2; docker logs piguard-e2e-gateway >&2 || true
+  echo "gateway never became healthy" >&2
+  exit 1
+fi
+
+export ODEK_E2E_GUARD=1
+export PIGUARD_URL="http://127.0.0.1:18080/detect"
+export PIGUARD_SOCKET="${SOCK_DIR}/piguard.sock"
+
+echo "Running guard E2E (host)..."
+(cd "${REPO_ROOT}" && go test ./internal/guard -run E2E -count=1 -v)
+
+if [ "${LINUX}" = 1 ]; then
+  echo "Running guard E2E inside a Linux container (socket mode covered)..."
+  (cd "${REPO_ROOT}" && CGO_ENABLED=0 GOOS=linux go test -c -o /tmp/piguard-e2e/guard.test ./internal/guard)
+  docker run --rm --network "${NETWORK}" \
+    -e ODEK_E2E_GUARD=1 \
+    -e PIGUARD_URL="http://piguard-e2e-gateway:8080/detect" \
+    -e PIGUARD_SOCKET=/run/piguard/piguard.sock \
+    -v "${SOCK_DIR}:/run/piguard" \
+    -v /tmp/piguard-e2e:/out \
+    debian:bookworm-slim /out/guard.test -test.run E2E -test.v
+fi
+
+echo "PIGuard E2E passed."

docs/CONFIG.md

@@ -375,6 +375,8 @@ The `memory` section controls the persistent memory system (see [docs/MEMORY.md]
       "semantic_search_overfetch": 4,
       "semantic_search_min_score": 0.55,
       "semantic_search_rerank": true,
+      "semantic_dedup_threshold": 0.92,
+      "consolidate_similarity_threshold": 0.9,
       "atom_max_chars": 300,
       "memory_budget_chars": 2000,
       "decay_half_life_days": 30,
@@ -409,6 +411,8 @@ The `memory` section controls the persistent memory system (see [docs/MEMORY.md]
 | `semantic_search_overfetch` | `4` | — | — | Candidate multiplier before filtering and reranking. |
 | `semantic_search_min_score` | `0.55` | — | — | Minimum cosine similarity for a candidate to be considered. |
 | `semantic_search_rerank` | `true` | — | — | Use the memory LLM to rerank candidates. |
+| `semantic_dedup_threshold` | `0.92` | — | — | Cosine similarity at or above which an incoming atom is treated as a paraphrase of an existing live atom and refreshes it instead of appending. `0` disables the semantic tier (exact-match dedup always runs). |
+| `consolidate_similarity_threshold` | `0.9` | — | — | Pairwise cosine similarity at or above which live atoms are grouped for LLM merging by `odek memory extended consolidate` / `ConsolidateAtoms`. |
 | `atom_max_chars` | `300` | `ODEK_MEMORY_EXTENDED_ATOM_MAX_CHARS` | `--memory-extended-atom-max-chars` | Maximum stored text length per atom. |
 | `memory_budget_chars` | `2000` | `ODEK_MEMORY_EXTENDED_MEMORY_BUDGET_CHARS` | `--memory-extended-memory-budget-chars` | Maximum injected Extended Memory context per turn. |
 | `decay_half_life_days` | `30` | — | — | Days until an atom's recall/eviction weight halves. |

docs/EXTENDED_MEMORY.md

@@ -99,13 +99,13 @@ The current extraction prompt and tool schema accept all nine primary types. Unk
     │   └── <atom-id>.md
     ├── vectors.gob              # persisted go-vector store
     ├── vectors.gob.emb          # persisted embedder state (RP vocabulary / HTTP fingerprint)
-    ├── vectors_meta.json        # embedding-space fingerprint for invalidation
+    ├── vectors_meta.json        # embedding-space + corpus fingerprints for invalidation/staleness
     ├── quarantine.json          # tainted atoms awaiting promotion
     ├── user_model.json          # persisted inferred user state + pending review queue
     └── associations.json        # bidirectional atom association links
 ```
 
-All files are written atomically and use `0600` permissions. The vector store and metadata are rebuilt from the chunk files if they are ever corrupted. The user model and associations are loaded at startup and persisted on every change.
+All files are written atomically and use `0600` permissions. The vector store and metadata are rebuilt from the chunk files if they are ever corrupted. `vectors_meta.json` records both the embedding-space fingerprint (which embedding backend produced the vectors) and a corpus fingerprint (atom count plus a hash of the sorted atom IDs); a mismatch on either — including a missing corpus fingerprint in files written before it was tracked — marks the persisted index stale so it is rebuilt once instead of silently serving an outdated corpus. The user model and associations are loaded at startup and persisted on every change.
 
 ## Dedicated Memory LLM
 
@@ -163,6 +163,23 @@ Extracted atoms are immediately:
 4. Embedded and written to the atom store.
 5. Checked against the 100 MB size cap; low-retention atoms are evicted if needed.
 
+Explicit LLM-provided `confidence` values in (0, 1] are kept. A missing or out-of-range confidence defaults to **0.7** for extraction-origin atoms, so unqualified extracted memories do not inflate their retention score.
+
+### Write-Path Deduplication
+
+Atoms are deduplicated in two tiers at persistence time:
+
+1. **Exact match**: an atom whose normalized text (lowercased, whitespace-collapsed) equals an existing live atom refreshes it — new `CreatedAt`, the higher confidence wins, the original ID is kept — instead of appending a duplicate.
+2. **Semantic match**: if no exact match exists and `semantic_dedup_threshold` is non-zero, the incoming atom is compared against the live corpus via the vector index (top-1 similarity); atoms stored earlier in the same batch, which the index does not cover yet, are compared by embedding both texts directly. A match at or above the threshold is refreshed the same way. The dedup search uses the pre-batch index state and never triggers extra index rebuilds: the single index invalidation after the batch is preserved.
+
+### Atom Consolidation
+
+`ExtendedMemory.ConsolidateAtoms(ctx)` — exposed as `odek memory extended consolidate` — merges groups of live atoms that are near-duplicates (pairwise cosine similarity at or above `consolidate_similarity_threshold`). For each group it asks the memory LLM — one call per group — to merge the texts into a single concise atom, stores the merged atom through the normal add path (scan and size cap apply) keeping the group's highest confidence, a refreshed `CreatedAt`, and the union of the group's outward associations, then removes the originals. Quarantined atoms are never consolidated. On any failure for a group (LLM error, empty response, scan rejection) the originals are kept untouched.
+
+### Observability
+
+`ExtendedMemory.Stats()` returns a `Stats` snapshot — surfaced as `odek memory extended stats` — for operators and monitoring: live/quarantined atom counts, quarantine entries grouped by reason prefix (`tainted`, `scan_rejected`), index vector count and dirty flag, on-disk store size, and recall error counters (`RecallTimeouts` for context-deadline-exceeded errors, `RecallFailures` for all other recall errors). The CLI prints a degradation warning when either counter is non-zero.
+
 ## Semantic Search
 
 Extended Memory replaces episode-based recall with semantic search over atom vectors.
@@ -200,6 +217,8 @@ The final recall result is also bounded by `memory_budget_chars`. Tainted atoms
 
 When `follow_up_anticipation_enabled` is true (default), the memory LLM receives the current user message, the last several user messages, and the current user-state model. It returns a JSON array of up to `predictive_intents` likely follow-up intents. Each intent is embedded and searched, and the union of literal-query matches and predicted-intent matches is injected into the main agent's context. For each predicted intent, the system also performs a type-targeted recall for `convention`, `file`, and `error` atoms so the agent can pre-load relevant conventions, references, and known failure modes.
 
+Intents carry a `confidence` (0.0-1.0). Intents below 0.3 are skipped entirely; otherwise the composite score of every atom found via a predicted intent is multiplied by that intent's confidence, so a speculative intent cannot outrank literal-query matches.
+
 Example:
 
 - User: "Refactor the auth package to remove JWT."
@@ -355,6 +374,8 @@ Extended Memory is configured under the `memory.extended` section.
       "user_state_max_pending": 20,
       "associations_enabled": true,
       "association_semantic_top_k": 3,
+      "semantic_dedup_threshold": 0.92,
+      "consolidate_similarity_threshold": 0.9,
       "proactive_return_after_break": true,
       "style_mirroring_enabled": true,
       "anaphora_resolution_enabled": true,
@@ -401,6 +422,8 @@ Extended Memory is configured under the `memory.extended` section.
 | `user_state_max_pending` | `20` | Maximum pending-review entries kept in the user model. |
 | `associations_enabled` | `true` | Enable bidirectional atom associations (temporal, task, semantic). |
 | `association_semantic_top_k` | `3` | Number of semantic neighbours linked when building associations. |
+| `semantic_dedup_threshold` | `0.92` | Cosine similarity at or above which an incoming atom is treated as a paraphrase of an existing live atom and refreshes it instead of appending. `0` disables the semantic tier (exact-match dedup always runs). |
+| `consolidate_similarity_threshold` | `0.9` | Cosine similarity at or above which live atoms are grouped as near-duplicates for `ConsolidateAtoms` merging. |
 | `proactive_return_after_break` | `true` | On session resume, inject a "where you left off" summary. |
 | `style_mirroring_enabled` | `true` | Inject a style-guidance directive based on the inferred user model. |
 | `anaphora_resolution_enabled` | `true` | Resolve the first pronoun in a user message against recent trusted atoms when the top atom's score is high enough. |
@@ -455,6 +478,8 @@ Additional CLI commands:
 odek memory extended forget <atom-id>
 odek memory extended quarantine
 odek memory extended compact
+odek memory extended stats
+odek memory extended consolidate
 odek memory extended promote <atom-id>
 odek memory extended pin <atom-id>
 odek memory extended pending
@@ -467,6 +492,8 @@ odek memory extended reject <pending-id>
 | `forget` | Removes an atom from the live store or quarantine. |
 | `quarantine` | Lists tainted atoms awaiting promotion. |
 | `compact` | Triggers a background rebuild of the vector index to reclaim space. |
+| `stats` | Shows store/index sizes, quarantine reason breakdown, and recall degradation counters (timeouts/failures). |
+| `consolidate` | Merges near-duplicate live atoms via the LLM (requires a configured backend). |
 | `promote` | Moves a quarantined atom to the live store as `user_approved`. |
 | `pin` | Pins a live atom so it is never evicted. |
 | `pending` | Lists pending user-model inferences. |