Replace DuckLake with authoritative Parquet storage

README.md

@@ -22,27 +22,36 @@ executable, including the React client.
 
 ![Fanout architecture](docs/diagrams/architecture.svg)
 
-Telemetry lands over OTLP/gRPC or OTLP/HTTP, is batched into DuckLake/Parquet,
-and is read back through a DuckDB query kernel that also maintains service,
-endpoint, and edge rollups. The browser client, an in-process agent, and any
+Telemetry lands over OTLP/gRPC or OTLP/HTTP. Concurrent small requests may
+share a group-commit batch, while up to four workers independently encode and
+durably publish atomic Parquet directories with persistent trace indexes.
+Targeted trace reads go through those indexes; DuckDB scans the same Parquet
+for SQL and maintains rebuildable service, endpoint, and edge rollups. The
+browser client, an in-process agent, and any
 external MCP host all reach the same typed observability contract rather than
 issuing raw SQL.
 
-Every write to the telemetry catalog — ingest flush, rollups, and background
-maintenance alike — passes through a single write gate that holds one catalog
-write in flight at a time:
+Parquet is the telemetry source of truth, DuckDB query state is disposable,
+and SQLite is reserved for transactional product state. Native compaction
+prepares replacements while reads continue and briefly gates readers only for
+the crash-safe namespace swap:
 
 ![Fanout persistence](docs/diagrams/persistence.svg)
 
 Application state (users, sessions, dashboards, alert rules, agent threads)
-lives in a separate SQLite database and never sits on the telemetry write path.
+lives in that separate SQLite database and never sits on the telemetry write
+path. There is no Iceberg, DuckLake, external catalog, or telemetry server
+database.
 
 ## Performance
 
 The bundled [`cmd/bench`](cmd/bench) driver measures authenticated ingest and
-optional dashboard read load against your hardware. Fanout does not publish a
-throughput headline until the raw reports and exact driver revision can ship
-with it; see [the benchmark publication standard](docs/benchmarking.md).
+optional dashboard read load against your hardware. Ingest, DuckDB queries,
+and native Parquet maintenance have separate coordination paths but still
+compete for the same CPU, memory bandwidth, filesystem cache, and disk. Fanout
+does not publish a throughput headline until the raw reports and exact driver
+revision can ship with it; see [the benchmark publication
+standard](docs/benchmarking.md).
 
 ## How it compares
 
@@ -53,7 +62,7 @@ what separates it from its neighbours.
 | If you use | Where Fanout differs |
 | --- | --- |
 | **Grafana with Loki, Tempo, and Mimir** | That stack keeps a service and a query language per signal, plus object storage underneath. Fanout keeps one process, one data directory, and one typed contract across all three signals, at the cost of the horizontal scale those components are built for. |
-| **SigNoz** | Both are OTLP-native and self-hosted. SigNoz composes a collector, ClickHouse, and query services; Fanout compiles ingest, storage, query, alerting, and the browser client into one binary, with DuckLake/Parquet on local disk instead of a database cluster. |
+| **SigNoz** | Both are OTLP-native and self-hosted. SigNoz composes a collector, ClickHouse, and query services; Fanout compiles ingest, indexed storage, DuckDB queries, alerting, and the browser client into one binary, with open Parquet on local disk instead of a database cluster. |
 | **Jaeger** | Jaeger covers traces and expects a storage backend you run separately. Fanout ingests traces, logs, and metrics into the same store, with nothing else to deploy. |
 | **Prometheus with Grafana** | Prometheus pulls metrics and is excellent at them. Fanout accepts pushed OTLP for all three signals and is built around investigating a specific incident rather than maintaining long-range metric series. |
 | **Datadog**, **Honeycomb**, **Grafana Cloud** | Those are managed services: someone else runs the storage, the scaling, and the upgrades, and your telemetry leaves your network to get there. Fanout is a binary you run, on data that stays on your disk. |

cmd/bench/main.go

@@ -81,8 +81,7 @@ type config struct {
 	maxQueryP95  float64
 	// backfillHours, when >0, spreads each event's timestamp uniformly over the
 	// last N hours (instead of "now"). Used to PRE-SEED a multi-hour dataset so
-	// the lake spans several hour partitions — required to exercise within-day
-	// (hour-partition) pruning, which a same-hour run can't.
+	// time-window queries and rollups cover more than the current hour.
 	backfillHours float64
 	// seed makes the synthetic workload reproducible: same seed, same services,
 	// endpoints, attributes, and error placement. Two runs are only comparable if
@@ -557,7 +556,7 @@ func (g *generator) outCtx(ctx context.Context) context.Context {
 
 // eventTime returns the timestamp for an emitted event: now(), or — when
 // backfillHours>0 — a time spread uniformly over the last N hours so a pre-seed
-// run populates multiple hour partitions (to exercise within-day pruning).
+// run exercises multi-hour time-window queries and rollups.
 func (g *generator) eventTime(rng *rand.Rand) time.Time {
 	if g.cfg.backfillHours <= 0 {
 		return time.Now()
@@ -910,43 +909,43 @@ type latencyReport struct {
 }
 
 type serverReport struct {
-	BaselineAvailable     bool                                 `json:"baseline_available"`
-	ProcessStartTime      float64                              `json:"process_start_time_seconds"`
-	ProcessRestarted      bool                                 `json:"process_restarted"`
-	IngestRowsStart       float64                              `json:"ingest_rows_start"`
-	IngestRowsEnd         float64                              `json:"ingest_rows_end"`
-	IngestRowsDelta       float64                              `json:"ingest_rows_delta"`
-	RowsDroppedStart      float64                              `json:"rows_dropped_start"`
-	RowsDroppedEnd        float64                              `json:"rows_dropped_end"`
-	RowsDroppedDelta      float64                              `json:"rows_dropped_delta"`
-	LakePartitionsStart   float64                              `json:"lake_partitions_start"`
-	LakePartitions        float64                              `json:"lake_partitions"`
-	LakePartitionsDelta   float64                              `json:"lake_partitions_delta"`
-	LakeSizeBytesStart    float64                              `json:"lake_size_bytes_start"`
-	LakeSizeBytes         float64                              `json:"lake_size_bytes"`
-	LakeSizeBytesDelta    float64                              `json:"lake_size_bytes_delta"`
-	LakeGrowthBytesPerSec float64                              `json:"lake_growth_bytes_per_sec"`
-	IngestQueueDepth      float64                              `json:"ingest_queue_depth"`
-	AvgRollupMs           float64                              `json:"avg_rollup_ms"`
-	AvgFlushMs            float64                              `json:"avg_flush_ms"`
-	AvgQueryMs            float64                              `json:"avg_query_ms"`
-	CPUSecondsStart       float64                              `json:"cpu_seconds_start"`
-	CPUSecondsEnd         float64                              `json:"cpu_seconds_end"`
-	CPUSecondsDelta       float64                              `json:"cpu_seconds_delta"`
-	CPUCores              float64                              `json:"cpu_cores"`
-	RSSBytes              float64                              `json:"rss_bytes"`
-	HeapAllocBytes        float64                              `json:"heap_alloc_bytes"`
-	AllocBytesStart       float64                              `json:"alloc_bytes_start"`
-	AllocBytesEnd         float64                              `json:"alloc_bytes_end"`
-	AllocBytesDelta       float64                              `json:"alloc_bytes_delta"`
-	AllocBytesPerSec      float64                              `json:"alloc_bytes_per_sec"`
-	GCPauseSecondsStart   float64                              `json:"gc_pause_seconds_start"`
-	GCPauseSecondsEnd     float64                              `json:"gc_pause_seconds_end"`
-	GCPauseSecondsDelta   float64                              `json:"gc_pause_seconds_delta"`
-	WriteGateWaitMs       map[string]distributionReport        `json:"write_gate_wait_ms,omitempty"`
-	WriteGateHoldMs       map[string]distributionReport        `json:"write_gate_hold_ms,omitempty"`
-	DuckLakeOperations    map[string]backgroundOperationReport `json:"ducklake_operations,omitempty"`
-	Rollups               map[string]rollupReport              `json:"rollups,omitempty"`
+	BaselineAvailable        bool                                 `json:"baseline_available"`
+	ProcessStartTime         float64                              `json:"process_start_time_seconds"`
+	ProcessRestarted         bool                                 `json:"process_restarted"`
+	IngestRowsStart          float64                              `json:"ingest_rows_start"`
+	IngestRowsEnd            float64                              `json:"ingest_rows_end"`
+	IngestRowsDelta          float64                              `json:"ingest_rows_delta"`
+	RowsDroppedStart         float64                              `json:"rows_dropped_start"`
+	RowsDroppedEnd           float64                              `json:"rows_dropped_end"`
+	RowsDroppedDelta         float64                              `json:"rows_dropped_delta"`
+	ParquetFilesStart        float64                              `json:"parquet_files_start"`
+	ParquetFiles             float64                              `json:"parquet_files"`
+	ParquetFilesDelta        float64                              `json:"parquet_files_delta"`
+	ParquetSizeBytesStart    float64                              `json:"parquet_size_bytes_start"`
+	ParquetSizeBytes         float64                              `json:"parquet_size_bytes"`
+	ParquetSizeBytesDelta    float64                              `json:"parquet_size_bytes_delta"`
+	ParquetGrowthBytesPerSec float64                              `json:"parquet_growth_bytes_per_sec"`
+	IngestQueueDepth         float64                              `json:"ingest_queue_depth"`
+	AvgRollupMs              float64                              `json:"avg_rollup_ms"`
+	AvgFlushMs               float64                              `json:"avg_flush_ms"`
+	AvgQueryMs               float64                              `json:"avg_query_ms"`
+	CPUSecondsStart          float64                              `json:"cpu_seconds_start"`
+	CPUSecondsEnd            float64                              `json:"cpu_seconds_end"`
+	CPUSecondsDelta          float64                              `json:"cpu_seconds_delta"`
+	CPUCores                 float64                              `json:"cpu_cores"`
+	RSSBytes                 float64                              `json:"rss_bytes"`
+	HeapAllocBytes           float64                              `json:"heap_alloc_bytes"`
+	AllocBytesStart          float64                              `json:"alloc_bytes_start"`
+	AllocBytesEnd            float64                              `json:"alloc_bytes_end"`
+	AllocBytesDelta          float64                              `json:"alloc_bytes_delta"`
+	AllocBytesPerSec         float64                              `json:"alloc_bytes_per_sec"`
+	GCPauseSecondsStart      float64                              `json:"gc_pause_seconds_start"`
+	GCPauseSecondsEnd        float64                              `json:"gc_pause_seconds_end"`
+	GCPauseSecondsDelta      float64                              `json:"gc_pause_seconds_delta"`
+	WriteGateWaitMs          map[string]distributionReport        `json:"write_gate_wait_ms,omitempty"`
+	WriteGateHoldMs          map[string]distributionReport        `json:"write_gate_hold_ms,omitempty"`
+	TelemetryOperations      map[string]backgroundOperationReport `json:"telemetry_operations,omitempty"`
+	Rollups                  map[string]rollupReport              `json:"rollups,omitempty"`
 }
 
 func printReport(r report) {
@@ -982,11 +981,11 @@ func printReport(r report) {
 		s := r.Server
 		fmt.Printf("server (Δ over run):\n")
 		fmt.Printf("  rows accepted=%.0f  dropped=%.0f\n", s.IngestRowsDelta, s.RowsDroppedDelta)
-		fmt.Printf("  lake_partitions=%.0f  lake_size=%.1fMB  ingest_queue_depth=%.0f\n",
-			s.LakePartitions, s.LakeSizeBytes/(1<<20), s.IngestQueueDepth)
+		fmt.Printf("  parquet_files=%.0f  parquet_size=%.1fMB  ingest_queue_depth=%.0f\n",
+			s.ParquetFiles, s.ParquetSizeBytes/(1<<20), s.IngestQueueDepth)
 		fmt.Printf("  avg rollup=%.1fms  flush=%.1fms  query=%.1fms\n", s.AvgRollupMs, s.AvgFlushMs, s.AvgQueryMs)
-		fmt.Printf("  cpu=%.2f core(s)  rss=%.1fMB  alloc=%.1fMB/s  lake_growth=%.1fMB\n",
-			s.CPUCores, s.RSSBytes/(1<<20), s.AllocBytesPerSec/(1<<20), s.LakeSizeBytesDelta/(1<<20))
+		fmt.Printf("  cpu=%.2f core(s)  rss=%.1fMB  alloc=%.1fMB/s  parquet_growth=%.1fMB\n",
+			s.CPUCores, s.RSSBytes/(1<<20), s.AllocBytesPerSec/(1<<20), s.ParquetSizeBytesDelta/(1<<20))
 	}
 	if r.Passed {
 		fmt.Printf("verdict        PASS\n")