diff --git a/frontend/src/__tests__/mocks/omezarrHelper.ts b/frontend/src/__tests__/mocks/omezarrHelper.ts index 92422468..ce7addc6 100644 --- a/frontend/src/__tests__/mocks/omezarrHelper.ts +++ b/frontend/src/__tests__/mocks/omezarrHelper.ts @@ -48,5 +48,6 @@ export const omezarrHelperMock = { generateNeuroglancerStateForOmeZarr: vi.fn(() => 'mock-state-ome-zarr'), determineLayerType: vi.fn(async () => 'image'), translateUnitToNeuroglancer: vi.fn((unit: string) => unit), - getResolvedScales: vi.fn(() => [1.0, 0.5, 0.5]) + getResolvedScales: vi.fn(() => [1.0, 0.5, 0.5]), + getDatasetWarnings: vi.fn(() => []) }; diff --git a/frontend/src/__tests__/unitTests/datasetWarnings.test.ts b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts new file mode 100644 index 00000000..e01e7686 --- /dev/null +++ b/frontend/src/__tests__/unitTests/datasetWarnings.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect } from 'vitest'; +import { getDatasetWarnings } from '@/omezarr-helper'; +import type { Metadata } from '@/omezarr-helper'; + +// Minimal stand-in for the parts of Metadata the checks read. Codec info is +// left out by default, which the chunk check treats as compressed. +const createMetadata = ( + chunks: number[], + dtype = 'uint16', + extra: Partial = {}, + shape: number[] = [8, 8, 8] +): Metadata => + ({ + arr: { chunks, dtype, shape }, + ...extra + }) as unknown as Metadata; + +const levels = (count: number): Partial => ({ + multiscales: [ + { datasets: Array.from({ length: count }, () => ({})) } + ] as unknown as Metadata['multiscales'] +}); + +// zstd nested inside a sharding_indexed pipeline, as a sharded v3 array stores it. +const SHARDED_ZSTD: Partial = { + codecs: [ + { + name: 'sharding_indexed', + configuration: { codecs: [{ name: 'bytes' }, { name: 'zstd' }] } + } + ] +}; +const UNCOMPRESSED_V3: Partial = { + codecs: [{ name: 'bytes' }, { name: 'crc32c' }] +}; + +describe('getDatasetWarnings: chunk size', () => { + it('says nothing about reasonable chunks', () => { + expect(getDatasetWarnings(createMetadata([64, 64, 64]))).toEqual([]); + }); + + it('does not warn about a compressed 48 MB chunk', () => { + // 48 MB inner chunks that zstd takes to well under the 32 MB guidance. + expect( + getDatasetWarnings( + createMetadata([24, 128, 128, 128], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([]); + }); + + it('holds an uncompressed array to the stricter limit', () => { + // The same 48 MB chunks, but stored raw, so 48 MB is what transfers. + for (const raw of [UNCOMPRESSED_V3, { compressor: null }]) { + expect( + getDatasetWarnings(createMetadata([24, 128, 128, 128], 'uint8', raw)) + ).toEqual([ + { + case: 'zarr-large-chunks', + size: '48 MB', + compressed: false, + sharded: false + } + ]); + } + }); + + it('finds a compressor nested inside a sharding codec', () => { + // sharding_indexed is structural, so a flat scan would call this + // uncompressed and warn at 48 MB. + expect( + getDatasetWarnings( + createMetadata([24, 128, 128, 128], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([]); + }); + + it('assumes compressed when codec metadata was never fetched', () => { + // Unknown lands on the permissive limit: a missed warning beats a false one. + expect( + getDatasetWarnings(createMetadata([24, 128, 128, 128], 'uint8')) + ).toEqual([]); + }); + + it('warns above the compressed limit', () => { + // seed151 img: 128 MB chunks. + expect( + getDatasetWarnings(createMetadata([256, 256, 256, 8], 'uint8')) + ).toEqual([ + { + case: 'zarr-large-chunks', + size: '128 MB', + compressed: true, + sharded: false + } + ]); + }); + + it('calls out that a sharded array is measured by its inner chunks', () => { + // zarrita resolves the sharding codec, so arr.chunks is the inner chunk + // shape - the shard around it is never what we size. + expect( + getDatasetWarnings( + createMetadata([256, 256, 256, 8], 'uint8', SHARDED_ZSTD) + ) + ).toEqual([ + { + case: 'zarr-large-chunks', + size: '128 MB', + compressed: true, + sharded: true + } + ]); + }); + + it('accounts for the dtype width', () => { + expect(getDatasetWarnings(createMetadata([256, 256, 256]))).toEqual([]); + expect( + getDatasetWarnings(createMetadata([256, 256, 256], 'float64')) + ).toHaveLength(1); + }); +}); + +describe('getDatasetWarnings: resolution levels', () => { + const BIG = [3000, 3000, 1350, 8]; // 91 GB of uint8, the seed151 img extent + + it('warns when multiscales declares a single level for a large image', () => { + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', levels(1), BIG)) + ).toEqual([{ case: 'zarr-single-level', size: '91 GB' }]); + }); + + it('says nothing when the pyramid has levels', () => { + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', levels(5), BIG)) + ).toEqual([]); + }); + + it('says nothing about a small single-level image', () => { + expect( + getDatasetWarnings( + createMetadata([64, 64, 64], 'uint8', levels(1), [256, 256, 256]) + ) + ).toEqual([]); + }); + + it('never fires on a plain array, however large', () => { + // The bug that made this warn on raw/s2: a plain array also has one shape, + // but it declares no multiscales and so claims nothing. + expect( + getDatasetWarnings(createMetadata([64, 64, 64], 'uint8', {}, BIG)) + ).toEqual([]); + }); +}); diff --git a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx index 13fadf2f..888061c0 100644 --- a/frontend/src/components/ui/BrowsePage/MetadataHint.tsx +++ b/frontend/src/components/ui/BrowsePage/MetadataHint.tsx @@ -12,6 +12,14 @@ type MetadataHintVariant = | { case: 'zarr-v2-no-multiscales' } | { case: 'zarr-v3-no-multiscales' } | { case: 'zarr-query-error'; errorMessage?: string } + // Zarr - metadata is valid, but the layout will make viewing awkward + | { case: 'zarr-single-level'; size: string } + | { + case: 'zarr-large-chunks'; + size: string; + compressed: boolean; + sharded: boolean; + } // N5 - query never fired | { case: 'n5-has-s0-no-attrs' } | { case: 'n5-has-attrs-no-s0' } @@ -72,6 +80,18 @@ function getHintConfig(variant: MetadataHintVariant): HintConfig { ? `Could not read Zarr metadata. ${variant.errorMessage}` : 'Could not read Zarr metadata.' }; + case 'zarr-single-level': + return { + kind: 'warning', + title: 'Only one resolution level', + description: `This dataset declares multiscales but only supplies a single level, for ${variant.size} of data. Without multiple levels, viewers read the full-resolution data at every zoom level, making viewing slow. Generating a multiscale pyramid fixes this.` + }; + case 'zarr-large-chunks': + return { + kind: 'warning', + title: 'Chunks may be too large for efficient viewing', + description: `This dataset uses ${variant.size} ${variant.sharded ? 'inner chunks' : 'chunks'} ${variant.compressed ? '(before compression)' : '(without compression)'}. Very large chunks make viewing slow, because a viewer must fetch a whole chunk to show any part of it. A stored chunk size of 1-32 MB works best.` + }; case 'n5-has-s0-no-attrs': logger.info( 'This folder has a .n5 extension but does not contain an attributes.json file required for N5 metadata preview.' diff --git a/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx b/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx index fac230b7..d7d0aa8f 100644 --- a/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx +++ b/frontend/src/components/ui/BrowsePage/ZarrPreview.tsx @@ -6,13 +6,14 @@ import zarrLogo from '@/assets/zarr.jpg'; import ZarrMetadataTable from '@/components/ui/BrowsePage/ZarrMetadataTable'; import DataLinkDialog from '@/components/ui/Dialogs/DataLink'; import DataToolLinks from './DataToolLinks'; +import MetadataHint from './MetadataHint'; import type { OpenWithToolUrls, ZarrMetadata, PendingToolKey } from '@/hooks/useZarrMetadata'; import useDataToolLinks from '@/hooks/useDataToolLinks'; -import { Metadata } from '@/omezarr-helper'; +import { Metadata, getDatasetWarnings } from '@/omezarr-helper'; type ZarrPreviewProps = { readonly fspName: string; @@ -41,6 +42,12 @@ export default function ZarrPreview({ const [showDataLinkDialog, setShowDataLinkDialog] = useState(false); const [pendingToolKey, setPendingToolKey] = useState(null); + const metadata = zarrMetadataQuery.data?.metadata; + const warnings = + metadata && 'arr' in metadata + ? getDatasetWarnings(metadata as Metadata) + : []; + const { handleToolClick, handleDialogConfirm, @@ -55,6 +62,13 @@ export default function ZarrPreview({ return (
+ {warnings.length > 0 ? ( +
+ {warnings.map(warning => ( + + ))} +
+ ) : null}
diff --git a/frontend/src/omezarr-helper.ts b/frontend/src/omezarr-helper.ts index c34369b2..09c43926 100644 --- a/frontend/src/omezarr-helper.ts +++ b/frontend/src/omezarr-helper.ts @@ -1,6 +1,8 @@ import { default as log } from '@/logger'; +import { formatFileSize } from '@/utils'; import * as zarr from 'zarrita'; import * as omezarr from 'ome-zarr.js'; +import { classifyCodec } from '@bioimagetools/capability-manifest'; import type { OmeZarrMetadata, MultiscaleMetadata, @@ -22,6 +24,154 @@ export type Metadata = OmeZarrMetadata & { zarrVersion: 2 | 3; }; +/** + * Something about the dataset's layout that will make it awkward to view. + * Purely advisory - nothing is withheld on account of these. + */ +export type DatasetWarning = + | { case: 'zarr-single-level'; size: string } + | { + case: 'zarr-large-chunks'; + size: string; + compressed: boolean; + sharded: boolean; + }; + +/** + * Chunks above this are large enough to slow viewing down: a viewer has to + * fetch a whole chunk to show any part of it. Applies when the array is stored + * without compression, so the size we compute is the size that transfers. + * + * Published guidance puts the useful range well below this - the image.sc + * discussion of OME-Zarr chunk sizes lands on 1-10 MB [1], AWS gives 8-16 MB as + * typical for S3 byte-range reads [2], and webknossos recommends 32^3 to 128^3 + * voxel inner chunks [3]. 32 MB is the top of what anyone recommends, so a chunk + * past it is outside the range rather than merely on the large side of it. + * + * [1] https://forum.image.sc/t/should-compression-play-a-role-in-selecting-chunk-sizes-for-ome-zarr-v0-4-datasets/117877 + * [2] https://d1.awsstatic.com/whitepapers/AmazonS3BestPractices.pdf + * [3] https://docs.webknossos.org/webknossos/data/zarr.html + */ +export const MAX_CHUNK_BYTES = 32 * 1024 ** 2; +/** + * The same limit for compressed arrays, where all we can compute is the logical + * (uncompressed) extent and the real transfer is some unknowable fraction of it. + * Doubled, so a chunk has to be outside the recommended range even at a + * conservative 2x ratio before we say anything. + */ +export const MAX_LOGICAL_CHUNK_BYTES = 64 * 1024 ** 2; +/** + * Below this, a single-level image is small enough that the missing pyramid + * costs nothing worth mentioning. + */ +export const MAX_SINGLE_LEVEL_BYTES = 1024 ** 3; + +/** + * Whether the array's chunks are compressed on disk. + * + * A codec pipeline can nest: a sharded v3 array lists only `sharding_indexed` + * at the top level and carries the real compressor in its configuration, so the + * pipeline has to be walked rather than scanned. Anything `classifyCodec` does + * not recognize counts as compression, and metadata we never fetched counts as + * compression too - both keep us on the permissive threshold, where the cost of + * being wrong is a missed warning instead of a false one. + */ +function hasCompressionCodec(codecs: NonNullable): boolean { + return codecs.some(codec => { + const nested = codec.configuration?.codecs; + if (Array.isArray(nested) && hasCompressionCodec(nested)) { + return true; + } + return classifyCodec(codec.name) !== 'structural'; + }); +} + +/** + * Whether the array is sharded. Worth knowing because zarrita resolves the + * sharding codec when it opens the array - `arr.chunks` is then the inner chunk + * shape, the unit a viewer actually fetches, and not the shard around it. The + * warning says so, since "chunk" alone is ambiguous once sharding is in play. + */ +function isSharded(metadata: Metadata): boolean { + return ( + metadata.codecs?.some(codec => codec.name === 'sharding_indexed') ?? false + ); +} + +function isStoredCompressed(metadata: Metadata): boolean { + if (metadata.codecs) { + return hasCompressionCodec(metadata.codecs); + } + if (metadata.compressor !== undefined) { + return metadata.compressor !== null; + } + return true; +} + +/** + * Bytes per element for a zarrita dtype. Only numeric dtypes are sized; bool, + * string and object dtypes fall back to 1, which under-estimates rather than + * over-warns (they don't occur in imaging data). + */ +function getBytesPerElement(dtype: string): number { + const bits = Number(/^(?:u?int|float)(\d+)$/.exec(dtype)?.[1]); + return Number.isFinite(bits) ? bits / 8 : 1; +} + +function product(dims: number[]): number { + return dims.reduce((total, dim) => total * dim, 1); +} + +/** + * Flag layout choices that make a dataset awkward to view. + * + * A multiscales group with a single dataset provides no downsampled data, so + * every zoom level reads full-resolution chunks - the root cause of the incident + * that prompted these checks. + * + * Chunks far above the recommended range slow viewing down, because a viewer has + * to fetch a whole chunk to show any part of it. Sizes are computed from the + * shape, so they are logical: exact for an uncompressed array and an upper bound + * for a compressed one, which is why the limit depends on whether a compressor + * is in play. + */ +export function getDatasetWarnings(metadata: Metadata): DatasetWarning[] { + const { arr } = metadata; + if (!arr) { + return []; + } + + const bytesPerElement = getBytesPerElement(arr.dtype); + const warnings: DatasetWarning[] = []; + + // Declaring multiscales with one dataset is declaring a pyramid and supplying + // none. Keyed on the dataset count rather than the number of shapes, because + // a plain zarr array also has exactly one shape and is not making any such + // claim - `arr` is level 0, so its shape is the full resolution. + const levels = metadata.multiscales?.[0]?.datasets?.length; + const fullResBytes = product(arr.shape) * bytesPerElement; + if (levels === 1 && fullResBytes > MAX_SINGLE_LEVEL_BYTES) { + warnings.push({ + case: 'zarr-single-level', + size: formatFileSize(fullResBytes) + }); + } + + const compressed = isStoredCompressed(metadata); + const chunkBytes = product(arr.chunks) * bytesPerElement; + const chunkLimit = compressed ? MAX_LOGICAL_CHUNK_BYTES : MAX_CHUNK_BYTES; + if (chunkBytes > chunkLimit) { + warnings.push({ + case: 'zarr-large-chunks', + size: formatFileSize(chunkBytes), + compressed, + sharded: isSharded(metadata) + }); + } + + return warnings; +} + type OmeZarrChannel = { name: string; color: string; diff --git a/frontend/src/queries/zarrQueries.ts b/frontend/src/queries/zarrQueries.ts index a574ffc0..d449a10a 100644 --- a/frontend/src/queries/zarrQueries.ts +++ b/frontend/src/queries/zarrQueries.ts @@ -196,7 +196,10 @@ async function fetchZarrMetadata({ scales: undefined, omero: undefined, labels: undefined, - zarrVersion: effectiveVersion + zarrVersion: effectiveVersion, + // This zarr.json is the array metadata, so the codec pipeline is + // already in hand - no second fetch needed. + codecs: (attrs as ZarrV3ArrayMetadata).codecs }, omeZarrUrl: null, availableZarrVersions, @@ -367,6 +370,21 @@ async function fetchZarrMetadata({ log.info('Getting Zarr array for', imageUrl, 'with Zarr version', 2); const arr = await getZarrArray(imageUrl, 2); const shapes = [arr.shape]; + + // Read the compressor so chunk-size warnings know whether the logical + // size is also the stored size. Left undefined on failure, which callers + // treat as compressed. + let compressor: ZarrV2ArrayMetadata['compressor']; + try { + const arrayMeta = (await fetchFileAsJson( + fspName, + zarrayFile.path + )) as ZarrV2ArrayMetadata; + compressor = arrayMeta.compressor; + } catch (error) { + log.trace('Could not fetch .zarray for compressor:', error); + } + return { metadata: { arr, @@ -375,7 +393,8 @@ async function fetchZarrMetadata({ scales: undefined, omero: undefined, labels: undefined, - zarrVersion: 2 + zarrVersion: 2, + compressor }, omeZarrUrl: null, availableZarrVersions,