diff --git a/CHANGELOG.md b/CHANGELOG.md index 4695f0c2..1ec99785 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed - Nullable Utf8/Binary columns are now compressed through the full cascade instead of stored as raw `vortex.varbin`. `MaskedEncodingEncoder` previously encoded its non-null values with a fixed first-match encoder (primitive/varbin), so a nullable low-cardinality string column never reached Dict or FSST — producing files 10–47× larger than the Rust reference on the string-heavy Raincloud corpus (e.g. `uci-mushroom` 22×, `uci-online-retail-ii` 25×). The masked values now run through the same `CascadingCompressor` as dense columns, dropping those ratios to ~2–3×. ([#258](https://github.com/dfa1/vortex-java/issues/258)) +- `ScanIterator` can now slice a shared `ListArray`/`FixedSizeListArray` whose covering flat chunk spans several scan windows (misaligned per-column chunk grids). Previously any scan over such a column threw `VortexException("scan: cannot slice shared array of type ...")` — a real reading gap, not only an export-side one. ([#265](https://github.com/dfa1/vortex-java/issues/265)) +- `CsvExporter` now reads `vortex.list` offsets of any integer width (`I8`/`U8`/`I16`/`U16`, matching `VarBinArray`'s offsets), not only `I32`/`I64`. A narrower-offset list column (e.g. Raincloud's `osm-germany-relations`, whose offsets are `I16`) previously threw `VortexException("unexpected list offsets type: ...")` on export. ([#263](https://github.com/dfa1/vortex-java/issues/263)) ## [0.12.2] — 2026-07-10 diff --git a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java index 573ea2fe..0d5f1a4b 100644 --- a/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java +++ b/csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java @@ -291,7 +291,9 @@ private static String jsonArray(Array elements, long start, long end) { } /// Reads offset `idx` from `offsets` as a non-negative long. - /// The encoder always writes I64 offsets; I32 is included for forward compatibility. + /// The encoder picks the narrowest ptype that fits the max offset value (mirroring + /// `VarBinArray`'s offsets), so any integer width can appear on the wire; Byte/ShortArray's + /// `getInt` already zero-extends U8/U16 per their dtype, so widening to long is exact. /// /// @param offsets the offsets array /// @param idx the index to read @@ -300,6 +302,8 @@ private static long offsetAt(Array offsets, long idx) { return switch (offsets) { case LongArray la -> la.getLong(idx); case IntArray ia -> Integer.toUnsignedLong(ia.getInt(idx)); + case ShortArray sa -> sa.getInt(idx); + case ByteArray ba -> ba.getInt(idx); default -> throw new VortexException("unexpected list offsets type: " + offsets.getClass().getSimpleName()); }; } diff --git a/docs/compatibility.md b/docs/compatibility.md index b8071b55..6ed33dac 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -62,12 +62,18 @@ Per-slug status lives in `integration/src/test/resources/raincloud/expected-stat (`ok` must pass; `gap:` must still fail, so a fix flips the entry in the same change; `untriaged` runs and reports without failing the build). A scheduled workflow (`raincloud-conformance.yml`) hydrates a size-capped subset weekly. Current triage — -40 `ok`, 0 known gaps; 207 slugs untriaged. Every gap found so far is fixed +117 `ok`, 0 known gaps; 130 slugs untriaged. Every gap found so far is fixed ([#206](https://github.com/dfa1/vortex-java/issues/206)–[#211](https://github.com/dfa1/vortex-java/issues/211), [#215](https://github.com/dfa1/vortex-java/issues/215)–[#217](https://github.com/dfa1/vortex-java/issues/217), [#221](https://github.com/dfa1/vortex-java/issues/221), [#225](https://github.com/dfa1/vortex-java/issues/225) RunEnd run-values validity, -[#226](https://github.com/dfa1/vortex-java/issues/226) Sparse null fill / nullable patches). +[#226](https://github.com/dfa1/vortex-java/issues/226) Sparse null fill / nullable patches, +[#257](https://github.com/dfa1/vortex-java/issues/257) CsvExporter FixedSizeList/List support, +[#259](https://github.com/dfa1/vortex-java/issues/259) conformance-test pipe deadlock, +[#261](https://github.com/dfa1/vortex-java/issues/261) oracle repeated/list column support, plus +two bugs the widened oracle then caught: `CsvExporter` list offsets assumed I32/I64 only (any +integer width is wire-legal, mirroring `VarBinArray`), and `ScanIterator` could not slice a shared +`ListArray`/`FixedSizeListArray` spanning several split windows). ## Encodings diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java index d349559d..9c2da330 100644 --- a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java +++ b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java @@ -3,10 +3,11 @@ import de.siegmar.fastcsv.writer.CsvWriter; import dev.hardwood.InputFile; import dev.hardwood.metadata.LogicalType; -import dev.hardwood.metadata.RepetitionType; import dev.hardwood.reader.ParquetFileReader; import dev.hardwood.reader.RowReader; -import dev.hardwood.schema.ColumnSchema; +import dev.hardwood.row.PqList; +import dev.hardwood.row.PqStruct; +import dev.hardwood.schema.SchemaNode; import io.github.dfa1.vortex.core.error.VortexException; import io.github.dfa1.vortex.csv.CsvExporter; import io.github.dfa1.vortex.csv.ExportOptions; @@ -280,7 +281,7 @@ private static void assertFilesMatch(Path result, Path oracle) throws IOExceptio } /// Writes the parquet oracle to a file using the same cell rules as `CsvExporter`. - /// An oracle-side failure (nested columns, unsupported physical type) aborts the + /// An oracle-side failure (unsupported physical type, MAP column) aborts the /// slug via [TestAbortedException] rather than failing it — it says nothing about /// vortex-java. An `ok` slug whose parquet cannot be read stops being verified /// (visibly, as skipped) — widen the oracle rather than let unverifiable entries @@ -307,81 +308,172 @@ private static void writeOracleCsv(Path parquet, Writer out) throws IOException RowReader rows = pfr.rowReader(); CsvWriter csv = CsvWriter.builder().fieldSeparator(',').build(out)) { - List cols = pfr.getFileSchema().getColumns(); - // Abort before writing anything if the schema has repeated (list/map) columns. - // Such columns cannot be read as scalar values, and leaving the oracle running - // would create a header-column-count mismatch that would deadlock the pipe. - for (ColumnSchema col : cols) { - if (col.maxRepetitionLevel() > 0) { - throw new TestAbortedException( - "oracle cannot format repeated list column: " + col.fieldPath().topLevelName()); - } - } + // One CSV cell per top-level field, mirroring CsvExporter's one-cell-per-Vortex-column + // model — not one per leaf column, which for a LIST/STRUCT field would emit multiple + // cells for what CsvExporter renders as a single JSON array/object cell (#261). + List topLevel = pfr.getFileSchema().getRootNode().children(); + // De-duplicate duplicate column names with the Rust Vortex writer's algorithm: // the Nth (N >= 1) occurrence of a base name gets a " [N]" suffix, matching // the de-duplicated names in the Vortex file (#256). - // Use the top-level logical name (e.g. "vector") rather than the leaf name - // (e.g. "element" inside a LIST group) so the header matches the Vortex column name. Map seen = new LinkedHashMap<>(); - List header = new ArrayList<>(cols.size()); - for (ColumnSchema col : cols) { - String logicalName = col.fieldPath().topLevelName(); - int count = seen.merge(logicalName, 1, Integer::sum) - 1; - header.add(count == 0 ? logicalName : logicalName + " [" + count + "]"); + List header = new ArrayList<>(topLevel.size()); + for (SchemaNode node : topLevel) { + int count = seen.merge(node.name(), 1, Integer::sum) - 1; + header.add(count == 0 ? node.name() : node.name() + " [" + count + "]"); } csv.writeRecord(header); - String[] row = new String[cols.size()]; + String[] row = new String[topLevel.size()]; while (rows.hasNext()) { rows.next(); - for (int c = 0; c < cols.size(); c++) { - row[c] = oracleCell(cols.get(c), rows); + for (int c = 0; c < topLevel.size(); c++) { + row[c] = oracleCell(topLevel.get(c), rows, c); } csv.writeRecord(row); } } } - /// Formats a parquet cell with the exact rules of `CsvExporter.cellValue`: - /// null rows export as an empty field, valid rows use the JDK canonical - /// `toString` of the value. + /// Formats one top-level field with the exact rules of `CsvExporter.cellValue`: a null field + /// renders as an empty CSV field, a scalar leaf uses the JDK canonical `toString` of the + /// value, and a LIST/STRUCT field renders as a JSON array/object cell via + /// [#oracleJsonValue(SchemaNode, Object)]. /// - /// Row access uses column index rather than name so that files with duplicate - /// column names (#256) read the right column. + /// Field access is by index — the field's position among top-level schema children, per + /// `dev.hardwood.row.StructAccessor`'s "projected schema order" — rather than by name, so + /// that files with duplicate column names (#256) read the right field. /// - /// INT32/INT64 columns with a `UINT_32`/`UINT_64` logical-type annotation are treated - /// as unsigned so their string representation matches the U32/U64 Vortex columns that - /// carry the same bits (#253). + /// INT32/INT64 leaves with a `UINT_32`/`UINT_64` logical-type annotation are treated as + /// unsigned so their string representation matches the U32/U64 Vortex columns that carry the + /// same bits (#253); this applies at any nesting depth, not only top-level scalars. /// - /// @param col the column schema - /// @param rows the row reader positioned at the current row + /// @param node the top-level field's schema node + /// @param rows the row reader positioned at the current row + /// @param fieldIndex the field's position among top-level schema children /// @return the formatted cell string - private static String oracleCell(ColumnSchema col, RowReader rows) { - int idx = col.columnIndex(); - // Repeated list columns (maxRepetitionLevel > 0) cannot be read as a single scalar value. - // Abort rather than silently reading only the first element. - if (col.maxRepetitionLevel() > 0) { - throw new TestAbortedException( - "oracle cannot format repeated list column: " + col.fieldPath().topLevelName()); - } - if (col.repetitionType() == RepetitionType.OPTIONAL && rows.isNull(idx)) { + private static String oracleCell(SchemaNode node, RowReader rows, int fieldIndex) { + if (rows.isNull(fieldIndex)) { return ""; } - boolean unsignedInt = col.logicalType() instanceof LogicalType.IntType lt && !lt.isSigned(); - return switch (col.type()) { - case INT32 -> unsignedInt ? Integer.toUnsignedString(rows.getInt(idx)) : Integer.toString(rows.getInt(idx)); - case INT64 -> unsignedInt ? Long.toUnsignedString(rows.getLong(idx)) : Long.toString(rows.getLong(idx)); - case FLOAT -> Float.toString(rows.getFloat(idx)); - case DOUBLE -> Double.toString(rows.getDouble(idx)); - case BOOLEAN -> Boolean.toString(rows.getBoolean(idx)); - case BYTE_ARRAY -> rows.getString(idx); + if (node instanceof SchemaNode.GroupNode group) { + if (group.isList()) { + return oracleJsonArray(group, rows.getList(fieldIndex)); + } + if (group.isMap()) { + throw new TestAbortedException("oracle cannot format MAP column: " + group.name()); + } + return oracleJsonObject(group, rows.getStruct(fieldIndex)); + } + SchemaNode.PrimitiveNode prim = (SchemaNode.PrimitiveNode) node; + boolean unsignedInt = isUnsignedInt(prim); + return switch (prim.type()) { + case INT32 -> unsignedInt ? Integer.toUnsignedString(rows.getInt(fieldIndex)) : Integer.toString(rows.getInt(fieldIndex)); + case INT64 -> unsignedInt ? Long.toUnsignedString(rows.getLong(fieldIndex)) : Long.toString(rows.getLong(fieldIndex)); + case FLOAT -> Float.toString(rows.getFloat(fieldIndex)); + case DOUBLE -> Double.toString(rows.getDouble(fieldIndex)); + case BOOLEAN -> Boolean.toString(rows.getBoolean(fieldIndex)); + case BYTE_ARRAY -> rows.getString(fieldIndex); // aborts (not fails) the slug: the oracle can't format this physical type // yet, which is an oracle limitation rather than a vortex-java gap default -> throw new TestAbortedException( - "oracle cannot format parquet type " + col.type() + " (column: " + col.name() + ")"); + "oracle cannot format parquet type " + prim.type() + " (column: " + prim.name() + ")"); }; } + /// Renders a nested struct value as a JSON object `{"field":value,...}` with fields in + /// schema order, mirroring `CsvExporter.jsonObject`. + private static String oracleJsonObject(SchemaNode.GroupNode structNode, PqStruct struct) { + List fields = structNode.children(); + StringBuilder sb = new StringBuilder("{"); + for (int i = 0; i < fields.size(); i++) { + if (i > 0) { + sb.append(','); + } + oracleJsonString(sb, fields.get(i).name()); + sb.append(':').append(oracleJsonValue(fields.get(i), struct.getValue(i))); + } + return sb.append('}').toString(); + } + + /// Renders a nested list value as a JSON array `[v0,v1,...]`, mirroring `CsvExporter.jsonArray`. + private static String oracleJsonArray(SchemaNode.GroupNode listNode, PqList list) { + SchemaNode element = listNode.getListElement(); + int size = list.size(); + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < size; i++) { + if (i > 0) { + sb.append(','); + } + sb.append(oracleJsonValue(element, list.get(i))); + } + return sb.append(']').toString(); + } + + /// Renders one decoded struct field / list element as JSON text, mirroring + /// `CsvExporter.jsonValue`: a nested object/array for STRUCT/LIST, a quoted escaped string for + /// STRING, JSON `null` for a null value, and the bare JDK `toString` for numbers/booleans — + /// unsigned for an INT32/INT64 `node` with a `UINT_*` logical-type annotation. + /// + /// @param node the value's schema node (its element/field type) + /// @param value the decoded value, from [PqStruct#getValue(int)] or [PqList#get(int)] + /// @return the rendered JSON text + private static String oracleJsonValue(SchemaNode node, Object value) { + if (value == null) { + return "null"; + } + return switch (value) { + case PqStruct struct -> oracleJsonObject((SchemaNode.GroupNode) node, struct); + case PqList list -> oracleJsonArray((SchemaNode.GroupNode) node, list); + case String s -> { + StringBuilder sb = new StringBuilder(); + oracleJsonString(sb, s); + yield sb.toString(); + } + case Integer i -> isUnsignedInt(node) ? Integer.toUnsignedString(i) : Integer.toString(i); + case Long l -> isUnsignedInt(node) ? Long.toUnsignedString(l) : Long.toString(l); + case Float f -> Float.toString(f); + case Double d -> Double.toString(d); + case Boolean b -> Boolean.toString(b); + default -> throw new TestAbortedException( + "oracle cannot format nested value of type " + value.getClass().getSimpleName()); + }; + } + + /// Whether `node` is an INT32/INT64 leaf with a `UINT_*` logical-type annotation (#253): + /// `false` for any other node, including `null` (an unresolvable list element schema). + private static boolean isUnsignedInt(SchemaNode node) { + return node instanceof SchemaNode.PrimitiveNode prim + && prim.logicalType() instanceof LogicalType.IntType lt && !lt.isSigned(); + } + + /// Appends `value` to `sb` as a JSON string literal, mirroring `CsvExporter.jsonString`: + /// wrapped in double quotes with `"`, backslash, the standard short escapes, and any other + /// control character below `U+0020` escaped as `\\uXXXX`. + private static void oracleJsonString(StringBuilder sb, String value) { + sb.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"' -> sb.append("\\\""); + case '\\' -> sb.append("\\\\"); + case '\n' -> sb.append("\\n"); + case '\r' -> sb.append("\\r"); + case '\t' -> sb.append("\\t"); + case '\b' -> sb.append("\\b"); + case '\f' -> sb.append("\\f"); + default -> { + if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + } + } + sb.append('"'); + } + private static Path manifestPath() { String env = System.getenv("RAINCLOUD_CORPUS_MANIFEST"); return env != null ? Path.of(env) : DEFAULT_MANIFEST; diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv index a6d7da2d..7c43ce85 100644 --- a/integration/src/test/resources/raincloud/expected-status.csv +++ b/integration/src/test/resources/raincloud/expected-status.csv @@ -105,9 +105,9 @@ frames-benchmark,untriaged ghcn-daily,untriaged glass,ok global-fossil-co2-emissions-by-country-2002-2022,untriaged -glove-6b-100d,gap:261 -glove-6b-200d,gap:261 -glove-6b-50d,gap:261 +glove-6b-100d,ok +glove-6b-200d,ok +glove-6b-50d,ok goodbooks-10k,ok google-cluster-trace-2011-machine-events,ok green_tripdata_2025,ok @@ -156,7 +156,7 @@ openlibrary-works,untriaged openorca,untriaged openpowerlifting,ok osm-germany-nodes,untriaged -osm-germany-relations,gap:261 +osm-germany-relations,ok osm-germany-ways,untriaged osmi-mental-health-in-tech-2016,untriaged osmi-mental-health-in-tech-2017,untriaged diff --git a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java index 9c4d3af5..a5e6df8a 100644 --- a/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java +++ b/reader/src/main/java/io/github/dfa1/vortex/reader/ScanIterator.java @@ -12,8 +12,10 @@ import io.github.dfa1.vortex.reader.array.BoolArray; import io.github.dfa1.vortex.reader.array.ByteArray; import io.github.dfa1.vortex.reader.array.DoubleArray; +import io.github.dfa1.vortex.reader.array.FixedSizeListArray; import io.github.dfa1.vortex.reader.array.FloatArray; import io.github.dfa1.vortex.reader.array.IntArray; +import io.github.dfa1.vortex.reader.array.ListArray; import io.github.dfa1.vortex.reader.array.LongArray; import io.github.dfa1.vortex.reader.array.MaskedArray; import io.github.dfa1.vortex.reader.array.NullArray; @@ -775,6 +777,24 @@ private static Array sliceArray(Array full, long offset, long length, DType dtyp } yield new StructArray(sd, length, slicedFields); } + case ListArray a -> { + // Offsets store absolute positions into the shared elements array, so only the + // offsets sub-range needs slicing (length+1 entries for the window's row + // boundaries); elements stay shared unchanged, same as ListArray#limited. + DType.List ld = (DType.List) dtype; + Array offsetsSlice = sliceArray(a.offsets(), offset, length + 1, a.offsets().dtype()); + yield new ListArray(ld, length, a.elements(), offsetsSlice); + } + case FixedSizeListArray a -> { + // Unlike ListArray, element position is computed from the local row index + // (row * fixedSize), so the elements sub-range itself must be sliced to the + // window's absolute position rather than shared unchanged. + DType.FixedSizeList fd = (DType.FixedSizeList) dtype; + int fixedSize = a.fixedSize(); + Array elementsSlice = sliceArray(a.elements(), offset * (long) fixedSize, + length * (long) fixedSize, fd.elementType()); + yield new FixedSizeListArray(fd, length, elementsSlice); + } default -> throw new VortexException( "scan: cannot slice shared array of type " + full.getClass().getSimpleName()); };