diff --git a/internal/app/feed.go b/internal/app/feed.go index c08b496..582316b 100644 --- a/internal/app/feed.go +++ b/internal/app/feed.go @@ -3,7 +3,11 @@ package app import ( "compress/gzip" "context" + "crypto/md5" + "encoding/hex" "fmt" + "hash" + "hash/crc32" "io" "os" "strings" @@ -61,11 +65,12 @@ func newFeedStatusCmd(of *outputFlags, cf *configFlags) *cobra.Command { return &cobra.Command{ Use: "status ", Aliases: []string{"metadata"}, - Short: "Show a feed's current metadata (date, timestamps, filesize)", + Short: "Show a feed's current metadata (date, timestamps, filesize, checksums)", Long: "Fetch the metadata document for a Spur data feed and report the " + "current file's date, when it was generated and made available, its " + - "relative location, and its size. Run `spur feed list` for the feed " + - "types you can pass.\n\n" + + "relative location, its size, and its checksums (CRC32C/MD5) as " + + "reported by the CDN. Run `spur feed list` for the feed types you can " + + "pass.\n\n" + "Output is a styled report at a terminal and JSON when piped or " + "redirected; use --format to override.\n\n" + "Authentication resolves a token from SPUR_TOKEN, then the config file, " + @@ -87,6 +92,7 @@ func newFeedStatusCmd(of *outputFlags, cf *configFlags) *cobra.Command { if err != nil { return err } + attachFeedDigests(cmd.Context(), client, feedType.Name, &md) return of.emit(cmd, md) }, } @@ -107,6 +113,7 @@ func newFeedDownloadCmd(of *outputFlags, cf *configFlags) *cobra.Command { asJSON bool decompress bool silent bool + verify bool ) cmd := &cobra.Command{ Use: "download ", @@ -131,6 +138,10 @@ func newFeedDownloadCmd(of *outputFlags, cf *configFlags) *cobra.Command { "Download progress is shown on stderr at a terminal and suppressed with " + "--silent or when stderr is not a terminal, so stdout stays a clean data " + "channel.\n\n" + + "After the download, a size and checksum (CRC32C/MD5) summary as reported " + + "by the CDN is printed to stderr, unless --silent. Pass --verify to hash " + + "the downloaded bytes as they arrive and compare them to that reported " + + "checksum, failing the command on a mismatch.\n\n" + "Authentication resolves a token from SPUR_TOKEN, then the config file, " + "then the OS keychain; run `spur auth` to store one.", Args: cobra.ExactArgs(1), @@ -166,15 +177,31 @@ func newFeedDownloadCmd(of *outputFlags, cf *configFlags) *cobra.Command { // never gzip.) expand := of.toStdout() || decompress + // When the feed data itself is streaming to an interactive terminal + // (no -o), the self-overwriting progress bar collides with the data + // scrolling past on the same screen. Suppress it in that case; the + // data is the output the user asked for. Piping or -o keeps stdout + // clean, so the bar is safe and useful there. + dataToTerminal := of.toStdout() && output.IsTerminal(cmd.OutOrStdout()) + // Progress is a diagnostic: stderr only, and only when a human is - // watching. It never touches the stdout data channel. + // watching and the data channel isn't the same terminal. It never + // touches the stdout data channel. var progress *progressLine - if !silent && output.IsTerminal(os.Stderr) { + if showDownloadProgress(silent, dataToTerminal, output.IsTerminal(os.Stderr)) { w := cmd.ErrOrStderr() progress = newProgressLine(w, of.colorEnabled(w), "Downloading "+slug) } - return downloadFeed(cmd.Context(), client, slug, date, format, of.sink(cmd), expand, progress) + // The integrity summary is a diagnostic like progress, but unlike + // progress it is shown whenever a human isn't opting out with + // --silent, even when stderr is piped (CI logs still want it). + var summary io.Writer + if !silent { + summary = cmd.ErrOrStderr() + } + + return downloadFeed(cmd.Context(), client, slug, date, format, of.sink(cmd), expand, progress, verify, summary) }, } cmd.Flags().StringVar(&date, "date", "", "download a historical release for this date (YYYYMMDD) instead of the latest") @@ -184,9 +211,20 @@ func newFeedDownloadCmd(of *outputFlags, cf *configFlags) *cobra.Command { cmd.Flags().BoolVar(&asJSON, "json", false, "download the newline-JSON gzip artifact (overrides the ipgeo MMDB default)") cmd.Flags().BoolVar(&decompress, "decompress", false, "decompress the gzip when writing JSON to a file (-o); stdout JSON is always decompressed") cmd.Flags().BoolVar(&silent, "silent", false, "suppress the download progress indicator") + cmd.Flags().BoolVar(&verify, "verify", false, "hash the downloaded bytes and verify them against the CDN-reported checksum (crc32c/md5); a mismatch fails the command but a partially-written -o file is left in place") return cmd } +// showDownloadProgress decides whether to draw the download progress bar. It is +// shown only when a human is watching stderr (stderrIsTerminal), the user has +// not opted out (--silent), and the feed data is not itself streaming to the +// same interactive terminal (dataToTerminal) — where the self-overwriting bar +// would collide with the data scrolling past. Piping the data or writing it to +// a file (-o) keeps stdout clean, so the bar is safe and useful there. +func showDownloadProgress(silent, dataToTerminal, stderrIsTerminal bool) bool { + return !silent && !dataToTerminal && stderrIsTerminal +} + // resolveDownloadFormat picks the artifact format from the explicit flags, // falling back to a per-feed default: ipgeo is a geolocation database whose // primary form is MMDB, so it defaults to MMDB; every other feed defaults to the @@ -246,7 +284,20 @@ func resolveDownloadSlug(arg string, ipv6, realtime bool) (string, error) { // progress, when non-nil, receives a self-overwriting progress bar; the // counter wraps the transfer stream so it tracks bytes downloaded against the // CDN's Content-Length. -func downloadFeed(ctx context.Context, client *spur.Client, feedType, date string, format spur.FeedFormat, openSink func() (io.Writer, func() error, error), expand bool, progress *progressLine) error { +// +// When verify is true the RAW bytes as received from the CDN (before any +// progress wrapping or gzip decompression) are hashed with CRC32C and MD5 and +// compared against the CDN-reported digests (x-goog-hash, surfaced as +// integ.CRC32C/integ.MD5) once the transfer completes; a mismatch, or a +// request to verify against a response that reported no digest at all, fails +// the command. summary, when non-nil, receives a one-line size/checksum +// report (as reported by the CDN, regardless of verify) plus a second line +// noting which digests were verified when verify is true. +func downloadFeed( + ctx context.Context, client *spur.Client, feedType, date string, format spur.FeedFormat, + openSink func() (io.Writer, func() error, error), expand bool, progress *progressLine, + verify bool, summary io.Writer, +) error { if format == spur.FeedMMDB { md, err := fetchFeedMetadata(ctx, client, feedType) if err != nil { @@ -257,7 +308,7 @@ func downloadFeed(ctx context.Context, client *spur.Client, feedType, date strin } } - body, size, err := client.DownloadFeed(ctx, feedType, date, format) + body, integ, err := client.DownloadFeed(ctx, feedType, date, format) if err != nil { return collapseAPIError(err, map[error]string{ spur.ErrForbidden: fmt.Sprintf("access to the %q feed is denied: your token's subscription does not include it", feedType), @@ -266,9 +317,22 @@ func downloadFeed(ctx context.Context, client *spur.Client, feedType, date strin defer func() { _ = body.Close() }() var src io.Reader = body + + // Hash the RAW bytes as received from the CDN: x-goog-hash is computed + // over the stored object (.json.gz/.mmdb), so this tee must sit before + // both the progress wrapper and any gzip decompression, or it would hash + // the wrong bytes. + var crc32cHash hash.Hash32 + var md5Hash hash.Hash + if verify { + crc32cHash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + md5Hash = md5.New() + src = io.TeeReader(src, io.MultiWriter(crc32cHash, md5Hash)) + } + var prog *progressReader if progress != nil { - prog = newProgressReader(src, progress, size) + prog = newProgressReader(src, progress, integ.Size) src = prog } @@ -296,9 +360,87 @@ func downloadFeed(ctx context.Context, client *spur.Client, feedType, date strin if prog != nil { prog.Finish() } + + if verify { + if err := verifyFeedIntegrity(integ, crc32cHash, md5Hash); err != nil { + return err + } + } + + if summary != nil { + writeFeedIntegritySummary(summary, integ, verify) + } + return nil +} + +// verifyFeedIntegrity compares the locally-computed CRC32C/MD5 digests +// (lowercase hex, matching the stored/displayed FeedIntegrity representation) +// against the CDN-reported ones in integ, checking only the digests the CDN +// actually reported. A caller that asked to verify against a response +// reporting neither digest gets a clear error rather than a silent no-op. +func verifyFeedIntegrity(integ spur.FeedIntegrity, crc32cHash hash.Hash32, md5Hash hash.Hash) error { + if integ.CRC32C == "" && integ.MD5 == "" { + return fmt.Errorf("--verify requested but the CDN did not report a checksum to verify against") + } + if integ.CRC32C != "" { + if got := fmt.Sprintf("%08x", crc32cHash.Sum32()); got != integ.CRC32C { + return fmt.Errorf("feed integrity check failed: crc32c mismatch (expected %s, got %s)", integ.CRC32C, got) + } + } + if integ.MD5 != "" { + if got := hex.EncodeToString(md5Hash.Sum(nil)); got != integ.MD5 { + return fmt.Errorf("feed integrity check failed: md5 mismatch (expected %s, got %s)", integ.MD5, got) + } + } return nil } +// writeFeedIntegritySummary prints a one-line size/checksum report as +// reported by the CDN (regardless of whether verify ran), and, when verified +// is true, a second line naming which digests were checked. +func writeFeedIntegritySummary(w io.Writer, integ spur.FeedIntegrity, verified bool) { + parts := []string{sizeSummary(integ.Size)} + var checked []string + if integ.CRC32C != "" { + parts = append(parts, "crc32c="+integ.CRC32C) + checked = append(checked, "crc32c") + } + if integ.MD5 != "" { + parts = append(parts, "md5="+integ.MD5) + checked = append(checked, "md5") + } + fmt.Fprintln(w, strings.Join(parts, " ")) + if verified { + fmt.Fprintf(w, "integrity verified (%s)\n", strings.Join(checked, ", ")) + } +} + +// sizeSummary renders integ.Size for the summary line: "size=unknown" when +// the CDN did not report a size (-1), else the byte count. +func sizeSummary(size int64) string { + if size < 0 { + return "size=unknown" + } + return fmt.Sprintf("size=%d bytes", size) +} + +// attachFeedDigests best-effort populates the CRC32C/MD5 of each artifact in +// md by probing the CDN with ProbeFeedIntegrity. The metadata document itself +// never carries these digests; this is a client-side probe of the artifact +// (accepted tradeoff: one extra request per artifact). A probe failure leaves +// that artifact's digests empty and is otherwise ignored — it never fails the +// status command. +func attachFeedDigests(ctx context.Context, client *spur.Client, feedType string, md *spur.FeedMetadata) { + if integ, err := client.ProbeFeedIntegrity(ctx, feedType, "", spur.FeedJSONGzip); err == nil { + md.JSON.CRC32C, md.JSON.MD5 = integ.CRC32C, integ.MD5 + } + if md.MMDB != nil { + if integ, err := client.ProbeFeedIntegrity(ctx, feedType, "", spur.FeedMMDB); err == nil { + md.MMDB.CRC32C, md.MMDB.MD5 = integ.CRC32C, integ.MD5 + } + } +} + // fetchFeedMetadata calls the client and collapses an API failure to a clear // cause. A forbidden status becomes a feed-specific, actionable message (the // token's subscription doesn't include this feed); the other classes surface diff --git a/internal/app/feed_test.go b/internal/app/feed_test.go index 553b4c6..7051a2b 100644 --- a/internal/app/feed_test.go +++ b/internal/app/feed_test.go @@ -4,6 +4,11 @@ import ( "bytes" "compress/gzip" "context" + "crypto/md5" + "encoding/base64" + "encoding/binary" + "fmt" + "hash/crc32" "net/http" "net/http/httptest" "strings" @@ -12,6 +17,22 @@ import ( "github.com/spurintel/cli/internal/spur" ) +// googHash computes the base64 crc32c/md5 digests X-Goog-Hash would carry for +// raw, and formats them as a single X-Goog-Hash header value the way GCS does +// (comma-joined). +func googHash(raw []byte) string { + c := crc32.New(crc32.MakeTable(crc32.Castagnoli)) + _, _ = c.Write(raw) + var sum [4]byte + binary.BigEndian.PutUint32(sum[:], c.Sum32()) + crc32c := base64.StdEncoding.EncodeToString(sum[:]) + + m := md5.Sum(raw) + md5b64 := base64.StdEncoding.EncodeToString(m[:]) + + return "crc32c=" + crc32c + ",md5=" + md5b64 +} + // gzipString returns plain compressed with gzip, for feeding a fake feed body. func gzipString(t *testing.T, plain string) []byte { t.Helper() @@ -83,6 +104,96 @@ func TestFetchFeedMetadataForbiddenIsClear(t *testing.T) { } } +// attachFeedDigests populates JSON.CRC32C/MD5 from a successful probe of the +// JSON artifact (AC: "populate them via a best-effort ProbeFeedIntegrity per +// artifact"). +func TestAttachFeedDigestsPopulatesJSON(t *testing.T) { + t.Parallel() + + raw := []byte("feed bytes") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasSuffix(r.URL.Path, "/latest.json.gz") { + t.Errorf("unexpected probe path %q", r.URL.Path) + } + w.Header().Set("X-Goog-Hash", googHash(raw)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-0/%d", len(raw))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(raw[:1]) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + md := &spur.FeedMetadata{JSON: spur.FeedFile{Date: "20250324"}} + attachFeedDigests(context.Background(), client, "anonymous", md) + + if md.JSON.CRC32C == "" || md.JSON.MD5 == "" { + t.Errorf("JSON digests not populated: %+v", md.JSON) + } +} + +// attachFeedDigests also probes the MMDB artifact when the metadata reports +// one, populating its digests independently of the JSON artifact's. +func TestAttachFeedDigestsPopulatesMMDB(t *testing.T) { + t.Parallel() + + jsonRaw := []byte("json bytes") + mmdbRaw := []byte("mmdb bytes") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + raw := jsonRaw + if strings.HasSuffix(r.URL.Path, "/latest.mmdb") { + raw = mmdbRaw + } + w.Header().Set("X-Goog-Hash", googHash(raw)) + w.Header().Set("Content-Range", fmt.Sprintf("bytes 0-0/%d", len(raw))) + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(raw[:1]) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + md := &spur.FeedMetadata{ + JSON: spur.FeedFile{Date: "20250324"}, + MMDB: &spur.FeedFile{Date: "20250324"}, + } + attachFeedDigests(context.Background(), client, "ipgeo", md) + + if md.JSON.CRC32C == "" || md.JSON.MD5 == "" { + t.Errorf("JSON digests not populated: %+v", md.JSON) + } + if md.MMDB.CRC32C == "" || md.MMDB.MD5 == "" { + t.Errorf("MMDB digests not populated: %+v", md.MMDB) + } + if md.JSON.CRC32C == md.MMDB.CRC32C { + t.Errorf("JSON and MMDB digests should differ (different bytes), both = %q", md.JSON.CRC32C) + } +} + +// A probe failure is best-effort: it leaves the artifact's digests empty and +// never fails the caller (AC: "if the probe fails, render metadata as before +// with empty digests"). +func TestAttachFeedDigestsProbeFailureLeavesEmpty(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + md := &spur.FeedMetadata{JSON: spur.FeedFile{Date: "20250324"}} + attachFeedDigests(context.Background(), client, "anonymous", md) + + if md.JSON.CRC32C != "" || md.JSON.MD5 != "" { + t.Errorf("digests should stay empty on probe failure, got %+v", md.JSON) + } +} + // stdout (expand=true) yields decompressed newline-JSON — the default so the // output pipes straight into a JSON tool (AC: "streams decompressed newline-JSON // to stdout"). @@ -99,7 +210,7 @@ func TestDownloadFeedDecompressesToStdout(t *testing.T) { client.FeedsBase = srv.URL var dst bytes.Buffer - if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), true, nil); err != nil { + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), true, nil, false, nil); err != nil { t.Fatalf("downloadFeed: %v", err) } if dst.String() != ndjson { @@ -122,7 +233,7 @@ func TestDownloadFeedRawToFile(t *testing.T) { client.FeedsBase = srv.URL var dst bytes.Buffer - if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil); err != nil { + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, false, nil); err != nil { t.Fatalf("downloadFeed: %v", err) } if !bytes.Equal(dst.Bytes(), raw) { @@ -144,7 +255,7 @@ func TestDownloadFeedForbiddenIsClear(t *testing.T) { client.FeedsBase = srv.URL var dst bytes.Buffer - err := downloadFeed(context.Background(), client, "dch", "", spur.FeedJSONGzip, bufSink(&dst), true, nil) + err := downloadFeed(context.Background(), client, "dch", "", spur.FeedJSONGzip, bufSink(&dst), true, nil, false, nil) if err == nil { t.Fatal("expected an error, got nil") } @@ -179,7 +290,7 @@ func TestDownloadMMDBNotOfferedIsClear(t *testing.T) { client.FeedsBase = srv.URL var dst bytes.Buffer - err := downloadFeed(context.Background(), client, "dch", "", spur.FeedMMDB, bufSink(&dst), false, nil) + err := downloadFeed(context.Background(), client, "dch", "", spur.FeedMMDB, bufSink(&dst), false, nil, false, nil) if err == nil { t.Fatal("expected an error, got nil") } @@ -213,7 +324,7 @@ func TestDownloadMMDBStreamsRaw(t *testing.T) { client.FeedsBase = srv.URL var dst bytes.Buffer - if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedMMDB, bufSink(&dst), true, nil); err != nil { + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedMMDB, bufSink(&dst), true, nil, false, nil); err != nil { t.Fatalf("downloadFeed: %v", err) } if dst.String() != mmdb { @@ -221,6 +332,39 @@ func TestDownloadMMDBStreamsRaw(t *testing.T) { } } +func TestShowDownloadProgress(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + silent bool + dataToTerminal bool + stderrIsTerminal bool + want bool + }{ + // The reported bug: data streaming to the terminal must NOT also draw + // the progress bar, or the two collide on the same screen. + {name: "data to terminal suppresses bar", dataToTerminal: true, stderrIsTerminal: true, want: false}, + // Piped/redirected data (stdout not a terminal) keeps stdout clean, so + // the bar on stderr is safe and useful. + {name: "piped data, stderr terminal shows bar", dataToTerminal: false, stderrIsTerminal: true, want: true}, + // --silent always wins. + {name: "silent suppresses bar", silent: true, dataToTerminal: false, stderrIsTerminal: true, want: false}, + // No human watching stderr (piped/CI): no bar. + {name: "stderr not a terminal, no bar", dataToTerminal: false, stderrIsTerminal: false, want: false}, + {name: "data to terminal but stderr not terminal", dataToTerminal: true, stderrIsTerminal: false, want: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := showDownloadProgress(tt.silent, tt.dataToTerminal, tt.stderrIsTerminal); got != tt.want { + t.Errorf("showDownloadProgress(silent=%v, dataToTerminal=%v, stderrIsTerminal=%v) = %v, want %v", + tt.silent, tt.dataToTerminal, tt.stderrIsTerminal, got, tt.want) + } + }) + } +} + func TestResolveDownloadFormat(t *testing.T) { t.Parallel() @@ -300,3 +444,208 @@ func TestResolveDownloadSlug(t *testing.T) { }) } } + +// A successful download always reports the CDN's size and both digests to +// the summary writer, with no --verify involved. +func TestDownloadFeedSummaryReportsIntegrity(t *testing.T) { + t.Parallel() + + raw := gzipString(t, `{"ip":"1.2.3.4"}`+"\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Goog-Hash", googHash(raw)) + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst, summary bytes.Buffer + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, false, &summary); err != nil { + t.Fatalf("downloadFeed: %v", err) + } + got := summary.String() + if !strings.Contains(got, fmt.Sprintf("size=%d bytes", len(raw))) { + t.Errorf("summary %q should report the size", got) + } + if !strings.Contains(got, "crc32c=") { + t.Errorf("summary %q should report crc32c", got) + } + if !strings.Contains(got, "md5=") { + t.Errorf("summary %q should report md5", got) + } + if strings.Contains(got, "verified") { + t.Errorf("summary %q should not claim verification when --verify was not used", got) + } +} + +// A nil summary writer (the --silent path) means nothing is written, and the +// download still succeeds and produces the expected bytes. +func TestDownloadFeedNilSummarySuppressed(t *testing.T) { + t.Parallel() + + ndjson := `{"ip":"1.2.3.4"}` + "\n" + raw := gzipString(t, ndjson) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst bytes.Buffer + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), true, nil, false, nil); err != nil { + t.Fatalf("downloadFeed: %v", err) + } + if dst.String() != ndjson { + t.Errorf("output = %q, want %q", dst.String(), ndjson) + } +} + +// --verify with matching digests succeeds and the summary notes verification. +func TestDownloadFeedVerifyMatchSucceeds(t *testing.T) { + t.Parallel() + + raw := gzipString(t, `{"ip":"1.2.3.4"}`+"\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Goog-Hash", googHash(raw)) + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst, summary bytes.Buffer + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, true, &summary); err != nil { + t.Fatalf("downloadFeed: %v", err) + } + if !strings.Contains(summary.String(), "verified") { + t.Errorf("summary %q should note the digests were verified", summary.String()) + } +} + +// --verify against a served body whose reported crc32c does not match the raw +// bytes fails clearly, naming crc32c. +func TestDownloadFeedVerifyMismatchCRC32C(t *testing.T) { + t.Parallel() + + raw := gzipString(t, `{"ip":"1.2.3.4"}`+"\n") + good := googHash(raw) + // Corrupt only the crc32c half of the header, keep a valid-looking md5. + bad := strings.Replace(good, "crc32c=", "crc32c=AAAAAA==,ignored=", 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Goog-Hash", bad) + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst bytes.Buffer + err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, true, nil) + if err == nil { + t.Fatal("expected an error, got nil") + } + if !strings.Contains(err.Error(), "crc32c") { + t.Errorf("error %q should name crc32c", err.Error()) + } +} + +// --verify against a served body whose reported md5 does not match the raw +// bytes fails clearly, naming md5. +func TestDownloadFeedVerifyMismatchMD5(t *testing.T) { + t.Parallel() + + raw := gzipString(t, `{"ip":"1.2.3.4"}`+"\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Goog-Hash", "md5=AAAAAAAAAAAAAAAAAAAAAA==") + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst bytes.Buffer + err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, true, nil) + if err == nil { + t.Fatal("expected an error, got nil") + } + if !strings.Contains(err.Error(), "md5") { + t.Errorf("error %q should name md5", err.Error()) + } +} + +// --verify hashes the RAW, pre-decompress bytes: with expand=true (the stdout +// decompress path) the digest served must still match the COMPRESSED bytes, +// proving the hash covers the gzip stream, not the expanded JSON. +func TestDownloadFeedVerifyHashesRawGzipBytes(t *testing.T) { + t.Parallel() + + const ndjson = `{"ip":"1.2.3.4"}` + "\n" + raw := gzipString(t, ndjson) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Goog-Hash", googHash(raw)) + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst bytes.Buffer + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), true, nil, true, nil); err != nil { + t.Fatalf("downloadFeed: %v", err) + } + if dst.String() != ndjson { + t.Errorf("decompressed output = %q, want %q", dst.String(), ndjson) + } +} + +// --verify with no X-Goog-Hash at all from the server is a clear error; the +// same response without --verify succeeds and reports the size. +func TestDownloadFeedVerifyNoChecksumIsError(t *testing.T) { + t.Parallel() + + raw := gzipString(t, `{"ip":"1.2.3.4"}`+"\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst bytes.Buffer + err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, true, nil) + if err == nil { + t.Fatal("expected an error, got nil") + } + if !strings.Contains(err.Error(), "checksum") { + t.Errorf("error %q should say no checksum was available", err.Error()) + } +} + +func TestDownloadFeedNoChecksumWithoutVerifySucceeds(t *testing.T) { + t.Parallel() + + raw := gzipString(t, `{"ip":"1.2.3.4"}`+"\n") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(raw) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + var dst, summary bytes.Buffer + if err := downloadFeed(context.Background(), client, "anonymous", "", spur.FeedJSONGzip, bufSink(&dst), false, nil, false, &summary); err != nil { + t.Fatalf("downloadFeed: %v", err) + } + if !strings.Contains(summary.String(), fmt.Sprintf("size=%d bytes", len(raw))) { + t.Errorf("summary %q should report the size", summary.String()) + } +} diff --git a/internal/output/csv.go b/internal/output/csv.go index a9a3b96..2608bba 100644 --- a/internal/output/csv.go +++ b/internal/output/csv.go @@ -142,6 +142,8 @@ func writeFeedTypesCSV(cw *csv.Writer, types []spur.FeedType) error { // requested feed type, then the JSON and MMDB file fields. MMDB columns are // empty when the feed is not offered in that format, and filesize is empty when // absent (the realtime feed) so a scripted reader can tell "unknown" from zero. +// crc32c/md5 are populated only when the status command's best-effort CDN +// probe succeeded; they are empty otherwise. var feedMetadataCSVHeader = []string{ "feedType", "json.date", @@ -149,11 +151,15 @@ var feedMetadataCSVHeader = []string{ "json.generatedAt", "json.availableAt", "json.filesize", + "json.crc32c", + "json.md5", "mmdb.date", "mmdb.location", "mmdb.generatedAt", "mmdb.availableAt", "mmdb.filesize", + "mmdb.crc32c", + "mmdb.md5", } func writeFeedMetadataCSV(cw *csv.Writer, md spur.FeedMetadata) error { @@ -173,15 +179,16 @@ func writeFeedMetadataCSV(cw *csv.Writer, md spur.FeedMetadata) error { return cw.Error() } -// feedFileCSVFields renders one FeedFile as its five columns. A zero filesize +// feedFileCSVFields renders one FeedFile as its seven columns. A zero filesize // (absent on the realtime feed, and on an empty MMDB container) emits an empty -// cell rather than "0". +// cell rather than "0"; CRC32C/MD5 are empty unless the CDN probe populated +// them. func feedFileCSVFields(f spur.FeedFile) []string { filesize := "" if f.Filesize > 0 { filesize = strconv.FormatInt(f.Filesize, 10) } - return []string{f.Date, f.Location, f.GeneratedAt, f.AvailableAt, filesize} + return []string{f.Date, f.Location, f.GeneratedAt, f.AvailableAt, filesize, f.CRC32C, f.MD5} } // ipContextCSVHeader is the column order shared by the single-value and batch diff --git a/internal/output/output_test.go b/internal/output/output_test.go index d87c6d4..2aad05a 100644 --- a/internal/output/output_test.go +++ b/internal/output/output_test.go @@ -427,11 +427,15 @@ func sampleFeedMetadata() spur.FeedMetadata { GeneratedAt: "2025-03-24T00:59:41Z", AvailableAt: "2025-03-24T00:59:56Z", Filesize: 89553131, + CRC32C: "9f6804a9", + MD5: "5d41402abc4b2a76b9719d911017c592", }, MMDB: &spur.FeedFile{ Date: "20250324", Location: "20250324/feed.mmdb", Filesize: 42, + CRC32C: "00000000", + MD5: "d41d8cd98f00b204e9800998ecf8427e", }, } } @@ -457,6 +461,9 @@ func TestRenderFeedMetadataJSON(t *testing.T) { if got.Type != "anonymous" || got.JSON.Filesize != 89553131 { t.Errorf("decoded feed metadata = %+v", got) } + if got.JSON.CRC32C != "9f6804a9" || got.JSON.MD5 != "5d41402abc4b2a76b9719d911017c592" { + t.Errorf("decoded feed metadata digests = %+v", got.JSON) + } // Filesize round-trips as a bare number, not a quoted string. if !strings.Contains(out, `"filesize":89553131`) { t.Errorf("filesize not emitted as bare number: %s", out) @@ -483,12 +490,21 @@ func TestRenderFeedMetadataCSV(t *testing.T) { if !slices.Contains(header, "json.filesize") { t.Errorf("feed csv header missing json.filesize: %v", header) } + if !slices.Contains(header, "json.crc32c") || !slices.Contains(header, "json.md5") { + t.Errorf("feed csv header missing json digest columns: %v", header) + } + if !slices.Contains(header, "mmdb.crc32c") || !slices.Contains(header, "mmdb.md5") { + t.Errorf("feed csv header missing mmdb digest columns: %v", header) + } if !slices.Contains(row, "89553131") { t.Errorf("feed csv row missing filesize: %v", row) } if !slices.Contains(row, "anonymous") { t.Errorf("feed csv row missing feed type: %v", row) } + if !slices.Contains(row, "9f6804a9") || !slices.Contains(row, "5d41402abc4b2a76b9719d911017c592") { + t.Errorf("feed csv row missing json digests: %v", row) + } } // A realtime document has no MMDB container and no filesize; the MMDB columns @@ -517,6 +533,16 @@ func TestRenderFeedMetadataCSVRealtime(t *testing.T) { if row[idx] != "" { t.Errorf("realtime json.filesize = %q, want empty", row[idx]) } + // No probe ran for this document, so the digest cells stay empty too. + for _, col := range []string{"json.crc32c", "json.md5", "mmdb.crc32c", "mmdb.md5"} { + idx := slices.Index(records[0], col) + if idx < 0 { + t.Fatalf("no %s column: %v", col, records[0]) + } + if row[idx] != "" { + t.Errorf("realtime %s = %q, want empty", col, row[idx]) + } + } } func TestRenderFeedMetadataTextNoColor(t *testing.T) { @@ -529,7 +555,7 @@ func TestRenderFeedMetadataTextNoColor(t *testing.T) { if strings.Contains(out, esc) { t.Errorf("--no-color feed text contains ANSI escape codes: %q", out) } - for _, want := range []string{"Feed", "anonymous", "20250324", "2025-03-24T00:59:41Z", "MMDB", "MiB"} { + for _, want := range []string{"Feed", "anonymous", "20250324", "2025-03-24T00:59:41Z", "MMDB", "MiB", "CRC32C", "9f6804a9", "MD5", "5d41402abc4b2a76b9719d911017c592"} { if !strings.Contains(out, want) { t.Errorf("feed text missing %q\n%s", want, out) } diff --git a/internal/output/text.go b/internal/output/text.go index ec850e5..492b3ce 100644 --- a/internal/output/text.go +++ b/internal/output/text.go @@ -249,6 +249,8 @@ func renderFeedMetadata(w io.Writer, s Styles, width int, md spur.FeedMetadata) {"Generated At", f.GeneratedAt}, {"Available At", f.AvailableAt}, {"Filesize", filesize}, + {"CRC32C", f.CRC32C}, + {"MD5", f.MD5}, }} } diff --git a/internal/spur/exports.go b/internal/spur/exports.go index bd9f4ca..8e22f97 100644 --- a/internal/spur/exports.go +++ b/internal/spur/exports.go @@ -292,5 +292,6 @@ func (c *Client) ExportFeed(ctx context.Context, slug string, q url.Values) (io. if enc := q.Encode(); enc != "" { endpoint += "?" + enc } - return c.doStream(ctx, endpoint) + body, integ, err := c.doStream(ctx, endpoint) + return body, integ.Size, err } diff --git a/internal/spur/feeds.go b/internal/spur/feeds.go index a542a06..4b6ca75 100644 --- a/internal/spur/feeds.go +++ b/internal/spur/feeds.go @@ -2,11 +2,14 @@ package spur import ( "context" + "encoding/base64" + "encoding/hex" "errors" "fmt" "io" "net/http" "net/url" + "strconv" "strings" ) @@ -136,38 +139,162 @@ func (c *Client) GetFeedMetadata(ctx context.Context, feedType string) (FeedMeta // third-party CDN (Go's default policy strips Authorization/Cookie but not a // custom header). The returned body is the raw artifact stream (gzip for JSON, // an uncompressed MaxMind DB for MMDB); the caller owns it and must Close it. -// size is the CDN's Content-Length, or -1 when unknown. +// integ is the CDN's size/CRC32C/MD5, per integrityFromResponse. // // Unlike the JSON methods this does not buffer or size-limit the body: feeds are // hundreds of megabytes and must stream. A slow or stuck transfer is bounded by // ctx, not a client timeout. DownloadFeed does not check that feedType offers // the requested format — callers that need a clear "not offered" error should // consult GetFeedMetadata (its MMDB container) first. -func (c *Client) DownloadFeed(ctx context.Context, feedType, date string, format FeedFormat) (io.ReadCloser, int64, error) { +func (c *Client) DownloadFeed(ctx context.Context, feedType, date string, format FeedFormat) (io.ReadCloser, FeedIntegrity, error) { feedType = strings.Trim(strings.TrimSpace(feedType), "/") if feedType == "" { - return nil, 0, fmt.Errorf("feed type is required") + return nil, FeedIntegrity{}, fmt.Errorf("feed type is required") } return c.doStream(ctx, c.FeedArtifactURL(feedType, date, format)) } +// ProbeFeedIntegrity fetches a feed artifact's CDN object-integrity values +// (size + x-goog-hash CRC32C/MD5) WITHOUT downloading it, via a +// "Range: bytes=0-0" GET that the CDN answers with a 206 carrying the hash +// headers and the full size in Content-Range. Uses the same download client +// and cross-host Token-stripping redirect policy as DownloadFeed. On 206/200 +// it returns integrity from the response headers; a non-2xx status is an +// error. The (<=1 byte) body is always drained and closed. Callers treat any +// error as "integrity unavailable" and omit it — this never fails a tool. +func (c *Client) ProbeFeedIntegrity(ctx context.Context, feedType, date string, format FeedFormat) (FeedIntegrity, error) { + feedType = strings.Trim(strings.TrimSpace(feedType), "/") + if feedType == "" { + return FeedIntegrity{}, fmt.Errorf("feed type is required") + } + + endpoint := c.FeedArtifactURL(feedType, date, format) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) + if err != nil { + return FeedIntegrity{}, err + } + req.Header.Set("Token", c.Token) + req.Header.Set("User-Agent", c.userAgent()) + req.Header.Set("Range", "bytes=0-0") + + resp, err := c.downloadHTTPClient().Do(req) + if err != nil { + return FeedIntegrity{}, err + } + defer func() { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxResponseBodyBytes)) + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + return FeedIntegrity{}, newAPIError(resp.StatusCode) + } + + return integrityFromResponse(resp), nil +} + +// parseGoogHash extracts the crc32c and md5 digests from the raw values of an +// X-Goog-Hash response header. Each element of values may itself be a +// comma-joined list (crc32c=..,md5=..), which is how GCS emits the header when +// both digests are present; multiple separate header lines are also accepted. +// Unknown keys are ignored; a missing digest returns "" for that return value. +func parseGoogHash(values []string) (crc32c, md5 string) { + for _, v := range values { + for _, part := range strings.Split(v, ",") { + part = strings.TrimSpace(part) + switch { + case strings.HasPrefix(part, "crc32c="): + crc32c = strings.TrimSpace(strings.TrimPrefix(part, "crc32c=")) + case strings.HasPrefix(part, "md5="): + md5 = strings.TrimSpace(strings.TrimPrefix(part, "md5=")) + } + } + } + return crc32c, md5 +} + +// integrityFromResponse derives a FeedIntegrity from a download response's +// headers. Size prefers the total from a Content-Range header (present on +// partial-content responses, where Content-Length reflects only the returned +// range) and falls back to Content-Length, else -1 when neither is available. +// CRC32C/MD5 come from X-Goog-Hash via parseGoogHash, base64-decoded and +// re-encoded as lowercase hex — the header itself is always base64 (that's +// what GCS sends), but everything downstream of this function stores/displays +// the digest as hex for readability. +func integrityFromResponse(resp *http.Response) FeedIntegrity { + crc32c, md5 := parseGoogHash(resp.Header.Values("X-Goog-Hash")) + return FeedIntegrity{ + Size: sizeFromResponse(resp), + CRC32C: b64ToHex(crc32c), + MD5: b64ToHex(md5), + } +} + +// b64ToHex decodes a standard-base64 digest (as X-Goog-Hash carries it) to +// lowercase hex. An empty input or a decode failure returns "" — best-effort, +// never an error — so a malformed or absent digest just yields an absent one. +func b64ToHex(s string) string { + if s == "" { + return "" + } + raw, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "" + } + return hex.EncodeToString(raw) +} + +// sizeFromResponse resolves the object's total size: the Content-Range total +// when present and parseable, else resp.ContentLength when known, else the raw +// Content-Length header, else -1. +func sizeFromResponse(resp *http.Response) int64 { + if total, ok := contentRangeTotal(resp.Header.Get("Content-Range")); ok { + return total + } + if resp.ContentLength >= 0 { + return resp.ContentLength + } + if cl := resp.Header.Get("Content-Length"); cl != "" { + if n, err := strconv.ParseInt(cl, 10, 64); err == nil { + return n + } + } + return -1 +} + +// contentRangeTotal parses the total-length component of a Content-Range +// header value (bytes 0-0/12345 -> 12345, true). A missing or non-numeric +// total (e.g. the "*" GCS uses when the size is unknown) reports ok=false. +func contentRangeTotal(headerValue string) (int64, bool) { + idx := strings.LastIndex(headerValue, "/") + if idx == -1 || idx == len(headerValue)-1 { + return 0, false + } + total, err := strconv.ParseInt(headerValue[idx+1:], 10, 64) + if err != nil { + return 0, false + } + return total, true +} + // doStream issues an authenticated GET for a large artifact on the download // client (no overall timeout, Token stripped on cross-host redirects) and -// returns the raw body stream plus the Content-Length (-1 when unknown). The -// caller owns the body and must Close it. It is the shared request path for -// DownloadFeed and ExportFeed. -func (c *Client) doStream(ctx context.Context, endpoint string) (io.ReadCloser, int64, error) { +// returns the raw body stream plus its FeedIntegrity (size, CRC32C, MD5) per +// integrityFromResponse. The caller owns the body and must Close it. It is the +// shared request path for DownloadFeed and ExportFeed. +func (c *Client) doStream(ctx context.Context, endpoint string) (io.ReadCloser, FeedIntegrity, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil) if err != nil { - return nil, 0, err + return nil, FeedIntegrity{}, err } req.Header.Set("Token", c.Token) req.Header.Set("User-Agent", c.userAgent()) resp, err := c.downloadHTTPClient().Do(req) if err != nil { - return nil, 0, err + return nil, FeedIntegrity{}, err } if resp.StatusCode < 200 || resp.StatusCode >= 300 { @@ -175,10 +302,10 @@ func (c *Client) doStream(ctx context.Context, endpoint string) (io.ReadCloser, // upstream error page we deliberately do not surface. _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxResponseBodyBytes)) _ = resp.Body.Close() - return nil, 0, newAPIError(resp.StatusCode) + return nil, FeedIntegrity{}, newAPIError(resp.StatusCode) } - return resp.Body, resp.ContentLength, nil + return resp.Body, integrityFromResponse(resp), nil } // downloadHTTPClient returns an HTTP client tuned for large streaming downloads: diff --git a/internal/spur/feeds_test.go b/internal/spur/feeds_test.go index 31dba45..9e2f720 100644 --- a/internal/spur/feeds_test.go +++ b/internal/spur/feeds_test.go @@ -168,7 +168,8 @@ func gzipBytes(t *testing.T, plain string) []byte { } // DownloadFeed hits the latest .json.gz path with the Token header and streams -// the (compressed) body back verbatim, along with the CDN Content-Length. +// the (compressed) body back verbatim, along with the CDN's integrity (size +// and X-Goog-Hash digests). func TestDownloadFeedStreamsBody(t *testing.T) { t.Parallel() @@ -182,6 +183,7 @@ func TestDownloadFeedStreamsBody(t *testing.T) { t.Errorf("Token header = %q, want secret", got) } w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("X-Goog-Hash", "crc32c=n03x6A==,md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc=") _, _ = w.Write(payload) })) defer srv.Close() @@ -189,7 +191,7 @@ func TestDownloadFeedStreamsBody(t *testing.T) { client := NewClient("secret") client.FeedsBase = srv.URL - body, size, err := client.DownloadFeed(context.Background(), "anonymous", "", FeedJSONGzip) + body, integ, err := client.DownloadFeed(context.Background(), "anonymous", "", FeedJSONGzip) if err != nil { t.Fatalf("DownloadFeed: %v", err) } @@ -202,8 +204,52 @@ func TestDownloadFeedStreamsBody(t *testing.T) { if !bytes.Equal(got, payload) { t.Errorf("streamed body did not match the served bytes") } - if size != int64(len(payload)) { - t.Errorf("size = %d, want %d", size, len(payload)) + if integ.Size != int64(len(payload)) { + t.Errorf("integ.Size = %d, want %d", integ.Size, len(payload)) + } + // The header (as GCS actually sends it) is base64; the stored/displayed + // FeedIntegrity value is the lowercase-hex equivalent. + if integ.CRC32C != "9f4df1e8" { + t.Errorf("integ.CRC32C = %q, want 9f4df1e8", integ.CRC32C) + } + if integ.MD5 != "3a393d737761726520796f75206b696464696e67" { + t.Errorf("integ.MD5 = %q, want 3a393d737761726520796f75206b696464696e67", integ.MD5) + } +} + +// When the server omits X-Goog-Hash, DownloadFeed still returns the size but +// leaves both digest fields empty rather than erroring. +func TestDownloadFeedNoHashHeader(t *testing.T) { + t.Parallel() + + payload := gzipBytes(t, `{"ip":"1.2.3.4"}`+"\n") + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(payload) + })) + defer srv.Close() + + client := NewClient("secret") + client.FeedsBase = srv.URL + + body, integ, err := client.DownloadFeed(context.Background(), "anonymous", "", FeedJSONGzip) + if err != nil { + t.Fatalf("DownloadFeed: %v", err) + } + defer func() { _ = body.Close() }() + if _, err := io.ReadAll(body); err != nil { + t.Fatalf("read body: %v", err) + } + + if integ.Size != int64(len(payload)) { + t.Errorf("integ.Size = %d, want %d", integ.Size, len(payload)) + } + if integ.CRC32C != "" { + t.Errorf("integ.CRC32C = %q, want empty", integ.CRC32C) + } + if integ.MD5 != "" { + t.Errorf("integ.MD5 = %q, want empty", integ.MD5) } } @@ -339,6 +385,266 @@ func TestDownloadFeedStripsTokenOnRedirect(t *testing.T) { _ = body.Close() } +// x-goog-hash may pack both digests into one comma-joined header value; +// parseGoogHash must split on the comma and recover both. +func TestParseGoogHashSingleCommaJoinedLine(t *testing.T) { + t.Parallel() + + crc32c, md5 := parseGoogHash([]string{"crc32c=n03x6A==,md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc="}) + if crc32c != "n03x6A==" { + t.Errorf("crc32c = %q, want n03x6A==", crc32c) + } + if md5 != "Ojk9c3dhcmUgeW91IGtpZGRpbmc=" { + t.Errorf("md5 = %q, want Ojk9c3dhcmUgeW91IGtpZGRpbmc=", md5) + } +} + +// x-goog-hash may also arrive as multiple separate header lines, one digest +// per line; parseGoogHash must recover both regardless of ordering. +func TestParseGoogHashMultipleLines(t *testing.T) { + t.Parallel() + + crc32c, md5 := parseGoogHash([]string{"md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc=", "crc32c=n03x6A=="}) + if crc32c != "n03x6A==" { + t.Errorf("crc32c = %q, want n03x6A==", crc32c) + } + if md5 != "Ojk9c3dhcmUgeW91IGtpZGRpbmc=" { + t.Errorf("md5 = %q, want Ojk9c3dhcmUgeW91IGtpZGRpbmc=", md5) + } +} + +// Edge cases: only one digest present, an unknown key mixed in, and empty/nil +// input must all resolve without panicking or leaking the unknown key. +func TestParseGoogHashEdgeCases(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + values []string + wantCRC32C string + wantMD5 string + }{ + {"only crc32c", []string{"crc32c=n03x6A=="}, "n03x6A==", ""}, + {"only md5", []string{"md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc="}, "", "Ojk9c3dhcmUgeW91IGtpZGRpbmc="}, + {"unknown key ignored", []string{"crc32c=n03x6A==,goog-generation=17"}, "n03x6A==", ""}, + {"nil input", nil, "", ""}, + {"empty slice", []string{}, "", ""}, + {"empty string element", []string{""}, "", ""}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + crc32c, md5 := parseGoogHash(tc.values) + if crc32c != tc.wantCRC32C { + t.Errorf("crc32c = %q, want %q", crc32c, tc.wantCRC32C) + } + if md5 != tc.wantMD5 { + t.Errorf("md5 = %q, want %q", md5, tc.wantMD5) + } + }) + } +} + +// A 206 partial-content response carries the true object size in the +// Content-Range total, not the Content-Length of the returned range; +// integrityFromResponse must prefer the range total. +func TestIntegrityFromResponsePrefersContentRangeTotal(t *testing.T) { + t.Parallel() + + resp := &http.Response{ + StatusCode: http.StatusPartialContent, + Header: http.Header{"Content-Range": []string{"bytes 0-0/12345"}, "Content-Length": []string{"1"}}, + ContentLength: 1, + } + + got := integrityFromResponse(resp) + if got.Size != 12345 { + t.Errorf("Size = %d, want 12345", got.Size) + } +} + +// Without a Content-Range header, Size falls back to Content-Length; when +// neither is usable it reports -1 rather than a misleading zero. +func TestIntegrityFromResponseSizeFallback(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + header http.Header + contentLength int64 + wantSize int64 + }{ + {"content-length header only", http.Header{"Content-Length": []string{"500"}}, -1, 500}, + {"resp.ContentLength known", http.Header{}, 500, 500}, + {"garbage content-length header", http.Header{"Content-Length": []string{"nope"}}, -1, -1}, + {"missing everything", http.Header{}, -1, -1}, + {"content-range with unknown total", http.Header{"Content-Range": []string{"bytes 0-0/*"}}, -1, -1}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + resp := &http.Response{Header: tc.header, ContentLength: tc.contentLength} + got := integrityFromResponse(resp) + if got.Size != tc.wantSize { + t.Errorf("Size = %d, want %d", got.Size, tc.wantSize) + } + }) + } +} + +// integrityFromResponse must populate CRC32C/MD5 from X-Goog-Hash alongside +// Size, not just Size alone. The header is base64 (real GCS behavior); +// integrityFromResponse decodes it to lowercase hex. +func TestIntegrityFromResponsePopulatesHashes(t *testing.T) { + t.Parallel() + + resp := &http.Response{ + Header: http.Header{"X-Goog-Hash": []string{"crc32c=n03x6A==,md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc="}, "Content-Length": []string{"42"}}, + ContentLength: 42, + } + + got := integrityFromResponse(resp) + if got.CRC32C != "9f4df1e8" { + t.Errorf("CRC32C = %q, want 9f4df1e8", got.CRC32C) + } + if got.MD5 != "3a393d737761726520796f75206b696464696e67" { + t.Errorf("MD5 = %q, want 3a393d737761726520796f75206b696464696e67", got.MD5) + } + if got.Size != 42 { + t.Errorf("Size = %d, want 42", got.Size) + } +} + +// A 206 partial-content response carries the true object size in the +// Content-Range total plus the X-Goog-Hash digests; ProbeFeedIntegrity must +// surface both without erroring. +func TestProbeFeedIntegrity206(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Range", "bytes 0-0/999") + w.Header().Set("X-Goog-Hash", "crc32c=n03x6A==,md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc=") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("x")) + })) + defer srv.Close() + + client := NewClient("secret") + client.FeedsBase = srv.URL + + integ, err := client.ProbeFeedIntegrity(context.Background(), "anonymous", "", FeedJSONGzip) + if err != nil { + t.Fatalf("ProbeFeedIntegrity: %v", err) + } + if integ.Size != 999 { + t.Errorf("Size = %d, want 999", integ.Size) + } + if integ.CRC32C != "9f4df1e8" { + t.Errorf("CRC32C = %q, want 9f4df1e8", integ.CRC32C) + } + if integ.MD5 != "3a393d737761726520796f75206b696464696e67" { + t.Errorf("MD5 = %q, want 3a393d737761726520796f75206b696464696e67", integ.MD5) + } +} + +// Some CDNs/objects ignore the Range header and answer 200 with the full +// Content-Length; ProbeFeedIntegrity must still resolve integrity from that. +func TestProbeFeedIntegrity200IgnoresRange(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Length", "1234") + w.Header().Set("X-Goog-Hash", "crc32c=n03x6A==,md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc=") + _, _ = w.Write([]byte("full body ignored by probe")) + })) + defer srv.Close() + + client := NewClient("secret") + client.FeedsBase = srv.URL + + integ, err := client.ProbeFeedIntegrity(context.Background(), "anonymous", "", FeedJSONGzip) + if err != nil { + t.Fatalf("ProbeFeedIntegrity: %v", err) + } + if integ.Size != 1234 { + t.Errorf("Size = %d, want 1234", integ.Size) + } + if integ.CRC32C != "9f4df1e8" { + t.Errorf("CRC32C = %q, want 9f4df1e8", integ.CRC32C) + } +} + +// A non-2xx status (e.g. 403) is an error; the returned integrity must be the +// zero value. +func TestProbeFeedIntegrityNon2xxIsError(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "forbidden", http.StatusForbidden) + })) + defer srv.Close() + + client := NewClient("secret") + client.FeedsBase = srv.URL + + integ, err := client.ProbeFeedIntegrity(context.Background(), "anonymous", "", FeedJSONGzip) + if err == nil { + t.Fatal("expected error") + } + if integ != (FeedIntegrity{}) { + t.Errorf("integ = %+v, want zero value", integ) + } +} + +// The probe request must carry Range: bytes=0-0 and the Token on the first +// hop, and the tiny response body must be fully drained/closed (verified by +// the server not blocking on a second read). +func TestProbeFeedIntegrityRequestHeaders(t *testing.T) { + t.Parallel() + + var gotRange, gotToken string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotRange = r.Header.Get("Range") + gotToken = r.Header.Get("Token") + w.Header().Set("Content-Range", "bytes 0-0/1") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("x")) + })) + defer srv.Close() + + client := NewClient("secret") + client.FeedsBase = srv.URL + + if _, err := client.ProbeFeedIntegrity(context.Background(), "anonymous", "", FeedJSONGzip); err != nil { + t.Fatalf("ProbeFeedIntegrity: %v", err) + } + if gotRange != "bytes=0-0" { + t.Errorf("Range header = %q, want bytes=0-0", gotRange) + } + if gotToken != "secret" { + t.Errorf("Token header = %q, want secret", gotToken) + } +} + +// An empty feed type must error before any network call. +func TestProbeFeedIntegrityRequiresType(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatal("network call made for empty feed type") + })) + defer srv.Close() + + client := NewClient("secret") + client.FeedsBase = srv.URL + + if _, err := client.ProbeFeedIntegrity(context.Background(), " ", "", FeedJSONGzip); err == nil { + t.Fatal("empty feed type = nil error, want error") + } +} + func TestLookupFeedType(t *testing.T) { t.Parallel() diff --git a/internal/spur/types.go b/internal/spur/types.go index af9b24e..d802179 100644 --- a/internal/spur/types.go +++ b/internal/spur/types.go @@ -143,13 +143,26 @@ type FeedMetadata struct { // FeedFile describes one downloadable feed artifact: where it lives relative to // the feed root and when it was produced. GeneratedAt, AvailableAt, and Filesize // are absent on the realtime feed, which carries only Date and Location, so they -// are optional. +// are optional. CRC32C and MD5 do not come from this metadata document at all; +// they are populated by a best-effort CDN probe of the artifact itself and are +// empty when that probe was not run or failed. type FeedFile struct { Date string `json:"date" jsonschema:"The feed's date. YYYYMMDD for daily feeds; an ISO8601 timestamp for the realtime feed."` Location string `json:"location" jsonschema:"Path to the feed file relative to the feed root, e.g. 20250324/feed.json.gz."` GeneratedAt string `json:"generated_at,omitempty" jsonschema:"When the feed was generated, in ISO8601 format."` AvailableAt string `json:"available_at,omitempty" jsonschema:"When the feed was made available, in ISO8601 format."` Filesize int64 `json:"filesize,omitempty" jsonschema:"Size of the feed file in bytes."` + CRC32C string `json:"crc32c,omitempty" jsonschema:"Lowercase hex CRC32C (Castagnoli) digest of the feed file, derived from the CDN's x-goog-hash. Populated by a CDN probe, not the metadata document; empty when unavailable."` + MD5 string `json:"md5,omitempty" jsonschema:"Lowercase hex MD5 digest of the feed file, derived from the CDN's x-goog-hash. Populated by a CDN probe, not the metadata document; empty when unavailable or for composite objects GCS does not MD5."` +} + +// FeedIntegrity carries the size and content-hash information a feed download +// response exposes, so callers can verify a streamed artifact against what the +// CDN claims to have sent without re-deriving header parsing themselves. +type FeedIntegrity struct { + Size int64 `json:"size" jsonschema:"Object size in bytes from the CDN response (Content-Range total, else Content-Length). -1 when unknown."` + CRC32C string `json:"crc32c,omitempty" jsonschema:"Lowercase hex CRC32C (Castagnoli) digest, derived from x-goog-hash."` + MD5 string `json:"md5,omitempty" jsonschema:"Lowercase hex MD5 digest, derived from x-goog-hash. Absent for composite/large objects GCS does not MD5."` } // Meta carries the response headers that accompany a Context API call. It is diff --git a/internal/tools/feeds.go b/internal/tools/feeds.go index 7da91dd..67c5e9f 100644 --- a/internal/tools/feeds.go +++ b/internal/tools/feeds.go @@ -42,7 +42,8 @@ type FeedStatusInput struct { // FeedStatusOutput wraps the feed's metadata document. type FeedStatusOutput struct { - Metadata spur.FeedMetadata `json:"metadata" jsonschema:"The feed's current metadata: date, generation/availability timestamps, relative location, and filesize for the JSON (and, where offered, MMDB) artifact."` + Metadata spur.FeedMetadata `json:"metadata" jsonschema:"The feed's current metadata: date, generation/availability timestamps, relative location, and filesize for the JSON (and, where offered, MMDB) artifact."` + Integrity *spur.FeedIntegrity `json:"integrity,omitempty" jsonschema:"CDN object-integrity for the latest JSON artifact (size + x-goog-hash CRC32C/MD5), probed from the CDN. Omitted if the probe was unavailable."` } // registerFeedStatus wires the feed_status tool, backed by the same @@ -56,20 +57,34 @@ func registerFeedStatus(s *mcp.Server, c *spur.Client) { Description: "Fetches the current metadata for a Spur data feed: the file's date, " + "when it was generated and made available, its relative location, and its " + "size (for the JSON artifact and, where offered, the MMDB). Use list_feeds " + - "for the valid type identifiers. Cheap metadata — no bulk data is returned.", + "for the valid type identifiers. Also does a cheap ranged probe of the CDN " + + "for the latest JSON artifact's integrity headers (size, CRC32C, MD5); no " + + "bulk feed data is downloaded.", }, func(ctx context.Context, req *mcp.CallToolRequest, in FeedStatusInput) (*mcp.CallToolResult, FeedStatusOutput, error) { - feedType, ok := spur.LookupFeedType(strings.TrimSpace(in.Type)) - if !ok { - return toolError(fmt.Errorf("unknown feed type %q", in.Type), "Call list_feeds for the supported feed types."), FeedStatusOutput{}, nil - } - md, err := c.GetFeedMetadata(ctx, feedType.Name) - if err != nil { - return toolError(err, statusHint(err)), FeedStatusOutput{}, nil - } - return nil, FeedStatusOutput{Metadata: md}, nil + return feedStatus(ctx, c, in) }) } +// feedStatus implements the feed_status tool body. It is factored out of the +// mcp.AddTool closure so it can be exercised directly in tests. The integrity +// probe is best-effort: a probe failure never fails the tool, it just omits +// Integrity from the result. +func feedStatus(ctx context.Context, c *spur.Client, in FeedStatusInput) (*mcp.CallToolResult, FeedStatusOutput, error) { + feedType, ok := spur.LookupFeedType(strings.TrimSpace(in.Type)) + if !ok { + return toolError(fmt.Errorf("unknown feed type %q", in.Type), "Call list_feeds for the supported feed types."), FeedStatusOutput{}, nil + } + md, err := c.GetFeedMetadata(ctx, feedType.Name) + if err != nil { + return toolError(err, statusHint(err)), FeedStatusOutput{}, nil + } + out := FeedStatusOutput{Metadata: md} + if integ, err := c.ProbeFeedIntegrity(ctx, feedType.Name, "", spur.FeedJSONGzip); err == nil { + out.Integrity = &integ + } + return nil, out, nil +} + // FeedDownloadInput is the argument object for feed_download. type FeedDownloadInput struct { Type string `json:"type" jsonschema:"Feed type identifier from list_feeds, e.g. anonymous or anonymous-residential/realtime."` @@ -80,12 +95,13 @@ type FeedDownloadInput struct { // FeedDownloadOutput is a download pointer for a feed's data artifact: enough // metadata to decide whether to fetch it, plus the URL and command that do. type FeedDownloadOutput struct { - Type string `json:"type" jsonschema:"The feed type this pointer describes."` - Format string `json:"format" jsonschema:"The artifact format described: json or mmdb."` - File *spur.FeedFile `json:"file,omitempty" jsonschema:"Metadata for the latest artifact: date, relative location, generation/availability timestamps, and filesize in bytes. Omitted when a historical date is requested — the metadata document only describes the latest release."` - DownloadURL string `json:"download_url" jsonschema:"Absolute feeds-host URL of the artifact. It is authenticated with the Token header and 302-redirects to the CDN; it is not returned inline here."` - Command string `json:"command" jsonschema:"The spur CLI command that downloads this artifact."` - Note string `json:"note" jsonschema:"Why the bytes are not returned inline."` + Type string `json:"type" jsonschema:"The feed type this pointer describes."` + Format string `json:"format" jsonschema:"The artifact format described: json or mmdb."` + File *spur.FeedFile `json:"file,omitempty" jsonschema:"Metadata for the latest artifact: date, relative location, generation/availability timestamps, and filesize in bytes. Omitted when a historical date is requested — the metadata document only describes the latest release."` + DownloadURL string `json:"download_url" jsonschema:"Absolute feeds-host URL of the artifact. It is authenticated with the Token header and 302-redirects to the CDN; it is not returned inline here."` + Command string `json:"command" jsonschema:"The spur CLI command that downloads this artifact."` + Note string `json:"note" jsonschema:"Why the bytes are not returned inline."` + Integrity *spur.FeedIntegrity `json:"integrity,omitempty" jsonschema:"CDN object-integrity for the resolved artifact (size + x-goog-hash CRC32C/MD5), probed from the CDN. Present even for historical dates where 'file' is omitted. Omitted if the probe was unavailable."` } // registerFeedDownload wires the feed_download tool. Feeds are hundreds of @@ -105,58 +121,71 @@ func registerFeedDownload(s *mcp.Server, c *spur.Client) { "Use list_feeds for the valid type identifiers; 'format' selects json " + "(default) or mmdb (only where offered).", }, func(ctx context.Context, req *mcp.CallToolRequest, in FeedDownloadInput) (*mcp.CallToolResult, FeedDownloadOutput, error) { - feedType, ok := spur.LookupFeedType(strings.TrimSpace(in.Type)) - if !ok { - return toolError(fmt.Errorf("unknown feed type %q", in.Type), "Call list_feeds for the supported feed types."), FeedDownloadOutput{}, nil - } - - format := spur.FeedJSONGzip - formatName := "json" - switch strings.ToLower(strings.TrimSpace(in.Format)) { - case "", "json": - // default - case "mmdb": - format, formatName = spur.FeedMMDB, "mmdb" - default: - return toolError(fmt.Errorf("invalid format %q", in.Format), "Use json (default) or mmdb."), FeedDownloadOutput{}, nil - } - - date, err := spur.ValidateHistoricalDate(in.Date) - if err != nil { - return toolError(err, "Use YYYYMMDD for a historical release, or omit 'date' for the latest."), FeedDownloadOutput{}, nil - } - if date != "" && strings.HasSuffix(feedType.Name, "/realtime") { - return toolError(fmt.Errorf("the realtime feed does not support a historical date"), "Omit 'date' for the realtime feed."), FeedDownloadOutput{}, nil - } - - md, err := c.GetFeedMetadata(ctx, feedType.Name) - if err != nil { - return toolError(err, statusHint(err)), FeedDownloadOutput{}, nil - } + return feedDownload(ctx, c, in) + }) +} - file := &md.JSON - command := "spur feed download " + feedType.Name - if format == spur.FeedMMDB { - if md.MMDB == nil { - return toolError(fmt.Errorf("the %q feed is not offered in MMDB format", feedType.Name), "Request format json instead."), FeedDownloadOutput{}, nil - } - file = md.MMDB - command += " --mmdb" - } - if date != "" { - // The metadata document only describes the latest release; its - // dates and sizes would be wrong for a historical artifact. - file = nil - command += " --date " + date +// feedDownload implements the feed_download tool body. It is factored out of +// the mcp.AddTool closure so it can be exercised directly in tests. The +// integrity probe runs only after every validation/metadata step succeeds and +// is best-effort: a probe failure never fails the tool, it just omits +// Integrity from the result. +func feedDownload(ctx context.Context, c *spur.Client, in FeedDownloadInput) (*mcp.CallToolResult, FeedDownloadOutput, error) { + feedType, ok := spur.LookupFeedType(strings.TrimSpace(in.Type)) + if !ok { + return toolError(fmt.Errorf("unknown feed type %q", in.Type), "Call list_feeds for the supported feed types."), FeedDownloadOutput{}, nil + } + + format := spur.FeedJSONGzip + formatName := "json" + switch strings.ToLower(strings.TrimSpace(in.Format)) { + case "", "json": + // default + case "mmdb": + format, formatName = spur.FeedMMDB, "mmdb" + default: + return toolError(fmt.Errorf("invalid format %q", in.Format), "Use json (default) or mmdb."), FeedDownloadOutput{}, nil + } + + date, err := spur.ValidateHistoricalDate(in.Date) + if err != nil { + return toolError(err, "Use YYYYMMDD for a historical release, or omit 'date' for the latest."), FeedDownloadOutput{}, nil + } + if date != "" && strings.HasSuffix(feedType.Name, "/realtime") { + return toolError(fmt.Errorf("the realtime feed does not support a historical date"), "Omit 'date' for the realtime feed."), FeedDownloadOutput{}, nil + } + + md, err := c.GetFeedMetadata(ctx, feedType.Name) + if err != nil { + return toolError(err, statusHint(err)), FeedDownloadOutput{}, nil + } + + file := &md.JSON + command := "spur feed download " + feedType.Name + if format == spur.FeedMMDB { + if md.MMDB == nil { + return toolError(fmt.Errorf("the %q feed is not offered in MMDB format", feedType.Name), "Request format json instead."), FeedDownloadOutput{}, nil } - - return nil, FeedDownloadOutput{ - Type: feedType.Name, - Format: formatName, - File: file, - DownloadURL: c.FeedArtifactURL(feedType.Name, date, format), - Command: command, - Note: "Metadata only. Spur feeds are large bulk files and are not returned inline over MCP; fetch the artifact with the command or a GET to download_url carrying the Token header.", - }, nil - }) + file = md.MMDB + command += " --mmdb" + } + if date != "" { + // The metadata document only describes the latest release; its + // dates and sizes would be wrong for a historical artifact. + file = nil + command += " --date " + date + } + + out := FeedDownloadOutput{ + Type: feedType.Name, + Format: formatName, + File: file, + DownloadURL: c.FeedArtifactURL(feedType.Name, date, format), + Command: command, + Note: "Metadata only. Spur feeds are large bulk files and are not returned inline over MCP; fetch the artifact with the command or a GET to download_url carrying the Token header.", + } + if integ, err := c.ProbeFeedIntegrity(ctx, feedType.Name, date, format); err == nil { + out.Integrity = &integ + } + return nil, out, nil } diff --git a/internal/tools/feeds_test.go b/internal/tools/feeds_test.go new file mode 100644 index 0000000..d939a60 --- /dev/null +++ b/internal/tools/feeds_test.go @@ -0,0 +1,283 @@ +package tools + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/spurintel/cli/internal/spur" +) + +const metadataBody = `{ + "json": { + "location": "20250324/feed.json.gz", + "date": "20250324", + "generated_at": "2025-03-24T00:59:41Z", + "available_at": "2025-03-24T00:59:56Z", + "filesize": 89553131 + } +}` + +const metadataBodyWithMMDB = `{ + "json": { + "location": "20250324/feed.json.gz", + "date": "20250324", + "filesize": 89553131 + }, + "mmdb": { + "location": "20250324/feed.mmdb", + "date": "20250324", + "filesize": 42 + } +}` + +// feedTestServer routes the metadata endpoint plus one or more artifact +// probe endpoints, each answering with the given status/headers. probes is +// keyed by URL path. +func feedTestServer(t *testing.T, metadata string, probes map[string]http.HandlerFunc) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v2/anonymous/latest" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(metadata)) + return + } + if h, ok := probes[r.URL.Path]; ok { + h(w, r) + return + } + t.Fatalf("unexpected request path %q", r.URL.Path) + })) +} + +func okProbeHandler(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Range", "bytes 0-0/999") + w.Header().Set("X-Goog-Hash", "crc32c=n03x6A==,md5=Ojk9c3dhcmUgeW91IGtpZGRpbmc=") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("x")) +} + +// 1. feed_status: metadata OK + artifact probe succeeds -> Integrity populated, +// Metadata unchanged. +func TestFeedStatusPopulatesIntegrityOnSuccessfulProbe(t *testing.T) { + t.Parallel() + + srv := feedTestServer(t, metadataBody, map[string]http.HandlerFunc{ + "/v2/anonymous/latest.json.gz": okProbeHandler, + }) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, out, err := feedStatus(context.Background(), client, FeedStatusInput{Type: "anonymous"}) + if err != nil { + t.Fatalf("feedStatus: %v", err) + } + if res != nil { + t.Fatalf("expected success result, got error result: %+v", res) + } + if out.Metadata.JSON.Date != "20250324" { + t.Errorf("Metadata.JSON.Date = %q, want 20250324", out.Metadata.JSON.Date) + } + if out.Integrity == nil { + t.Fatal("Integrity = nil, want populated") + } + if out.Integrity.Size != 999 { + t.Errorf("Integrity.Size = %d, want 999", out.Integrity.Size) + } + if out.Integrity.CRC32C != "9f4df1e8" { + t.Errorf("Integrity.CRC32C = %q, want 9f4df1e8", out.Integrity.CRC32C) + } +} + +// 2. feed_status: probe endpoint returns 500 -> Integrity nil, tool result +// still success (best-effort, no mcp error). +func TestFeedStatusBestEffortWhenProbeFails(t *testing.T) { + t.Parallel() + + srv := feedTestServer(t, metadataBody, map[string]http.HandlerFunc{ + "/v2/anonymous/latest.json.gz": func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + }, + }) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, out, err := feedStatus(context.Background(), client, FeedStatusInput{Type: "anonymous"}) + if err != nil { + t.Fatalf("feedStatus: %v", err) + } + if res != nil { + t.Fatalf("expected success result despite probe failure, got: %+v", res) + } + if out.Integrity != nil { + t.Errorf("Integrity = %+v, want nil", out.Integrity) + } + if out.Metadata.JSON.Date != "20250324" { + t.Errorf("Metadata.JSON.Date = %q, want 20250324", out.Metadata.JSON.Date) + } +} + +// 3. feed_download json default -> Integrity reflects the JSON artifact; +// File/DownloadURL/Command unchanged. +func TestFeedDownloadJSONPopulatesIntegrity(t *testing.T) { + t.Parallel() + + srv := feedTestServer(t, metadataBody, map[string]http.HandlerFunc{ + "/v2/anonymous/latest.json.gz": okProbeHandler, + }) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, out, err := feedDownload(context.Background(), client, FeedDownloadInput{Type: "anonymous"}) + if err != nil { + t.Fatalf("feedDownload: %v", err) + } + if res != nil { + t.Fatalf("expected success result, got: %+v", res) + } + if out.File == nil || out.File.Date != "20250324" { + t.Errorf("File = %+v, want date 20250324", out.File) + } + if out.Command != "spur feed download anonymous" { + t.Errorf("Command = %q", out.Command) + } + if out.Integrity == nil || out.Integrity.Size != 999 { + t.Errorf("Integrity = %+v, want size 999", out.Integrity) + } +} + +// 4a. feed_download --mmdb when metadata offers MMDB -> probes the mmdb +// artifact. +func TestFeedDownloadMMDBOfferedProbesMMDBArtifact(t *testing.T) { + t.Parallel() + + srv := feedTestServer(t, metadataBodyWithMMDB, map[string]http.HandlerFunc{ + "/v2/anonymous/latest.mmdb": okProbeHandler, + }) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, out, err := feedDownload(context.Background(), client, FeedDownloadInput{Type: "anonymous", Format: "mmdb"}) + if err != nil { + t.Fatalf("feedDownload: %v", err) + } + if res != nil { + t.Fatalf("expected success result, got: %+v", res) + } + if out.Integrity == nil || out.Integrity.Size != 999 { + t.Errorf("Integrity = %+v, want size 999", out.Integrity) + } +} + +// 4b. feed_download --mmdb when NOT offered -> existing MMDB error still +// fires, with no probe attempted. +func TestFeedDownloadMMDBNotOfferedErrorsWithoutProbe(t *testing.T) { + t.Parallel() + + srv := feedTestServer(t, metadataBody, map[string]http.HandlerFunc{ + "/v2/anonymous/latest.mmdb": func(w http.ResponseWriter, r *http.Request) { + t.Fatal("probe should not be attempted when mmdb is not offered") + }, + }) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, out, err := feedDownload(context.Background(), client, FeedDownloadInput{Type: "anonymous", Format: "mmdb"}) + if err != nil { + t.Fatalf("feedDownload: %v", err) + } + if res == nil || !res.IsError { + t.Fatalf("expected mcp error result, got %+v (out=%+v)", res, out) + } + if out.Integrity != nil { + t.Errorf("Integrity = %+v, want nil", out.Integrity) + } +} + +// 5. feed_download historical date -> File still nil, Integrity populated +// from the dated-artifact probe. +func TestFeedDownloadHistoricalDatePopulatesIntegrity(t *testing.T) { + t.Parallel() + + srv := feedTestServer(t, metadataBody, map[string]http.HandlerFunc{ + "/v2/anonymous/20250101/feed.json.gz": okProbeHandler, + }) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, out, err := feedDownload(context.Background(), client, FeedDownloadInput{Type: "anonymous", Date: "20250101"}) + if err != nil { + t.Fatalf("feedDownload: %v", err) + } + if res != nil { + t.Fatalf("expected success result, got: %+v", res) + } + if out.File != nil { + t.Errorf("File = %+v, want nil for historical date", out.File) + } + if out.Integrity == nil || out.Integrity.Size != 999 { + t.Errorf("Integrity = %+v, want size 999", out.Integrity) + } +} + +// 6. Unknown feed type / invalid format / bad date -> unchanged existing +// errors, no probe attempted (server fails the test if hit at all). +func TestFeedDownloadValidationErrorsSkipProbe(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected request to %q on a validation error path", r.URL.Path) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + cases := []FeedDownloadInput{ + {Type: "nope-such-feed"}, + {Type: "anonymous", Format: "bogus"}, + {Type: "anonymous", Date: "not-a-date"}, + } + for _, in := range cases { + res, _, err := feedDownload(context.Background(), client, in) + if err != nil { + t.Fatalf("feedDownload(%+v): %v", in, err) + } + if res == nil || !res.IsError { + t.Errorf("feedDownload(%+v) = %+v, want mcp error result", in, res) + } + } +} + +func TestFeedStatusUnknownTypeSkipsProbe(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Fatalf("unexpected request to %q on a validation error path", r.URL.Path) + })) + defer srv.Close() + + client := spur.NewClient("secret") + client.FeedsBase = srv.URL + + res, _, err := feedStatus(context.Background(), client, FeedStatusInput{Type: "nope-such-feed"}) + if err != nil { + t.Fatalf("feedStatus: %v", err) + } + if res == nil || !res.IsError { + t.Fatalf("expected mcp error result, got %+v", res) + } +}