From 3e2dfc4f0db99ba107683577968b60844909c458 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Sun, 26 Jul 2026 21:31:35 +0200 Subject: [PATCH] Revert "GH-3574: Statistics.toParquetStatistics always set null_count(#3575)" This reverts commit 63aebcc0cf3684ba6c9dc2b7243c6007d3a1269f. --- .../format/converter/ParquetMetadataConverter.java | 10 ++++------ .../converter/TestParquetMetadataConverter.java | 11 +++++------ 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java index 8600b2ced4..465516e48f 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/format/converter/ParquetMetadataConverter.java @@ -815,17 +815,15 @@ public static Statistics toParquetStatistics(org.apache.parquet.column.statistic public static Statistics toParquetStatistics( org.apache.parquet.column.statistics.Statistics stats, int truncateLength) { Statistics formatStats = new Statistics(); - if (!stats.isEmpty()) { - formatStats.setNull_count(stats.getNumNulls()); - if (stats.isNanCountSet()) { - formatStats.setNan_count(stats.getNanCount()); - } - } // Don't write stats larger than the max size rather than truncating. The // rationale is that some engines may use the minimum value in the page as // the true minimum for aggregations and there is no way to mark that a // value has been truncated and is a lower bound and not in the page. if (!stats.isEmpty() && withinLimit(stats, truncateLength)) { + formatStats.setNull_count(stats.getNumNulls()); + if (stats.isNanCountSet()) { + formatStats.setNan_count(stats.getNanCount()); + } if (stats.hasNonNullValue()) { byte[] min; byte[] max; diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java index c6deee2855..4d361d6aa0 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/format/converter/TestParquetMetadataConverter.java @@ -878,7 +878,7 @@ private void testBinaryStats(StatsHelper helper) { } assertThat(formatStats.getNull_count()).as("Num nulls should match").isEqualTo(3004); - // min/max are not written because the values are too large, but null count is always written + // convert to empty stats because the values are too large stats.setMinMaxFromBytes(max, max); formatStats = helper.toParquetStatistics(stats); @@ -891,7 +891,9 @@ private void testBinaryStats(StatsHelper helper) { assertThat(formatStats.isSetMax_value()) .as("Max_value should not be set") .isFalse(); - assertThat(formatStats.getNull_count()).as("Num nulls should match").isEqualTo(3004); + assertThat(formatStats.isSetNull_count()) + .as("Num nulls should not be set") + .isFalse(); Statistics roundTripStats = ParquetMetadataConverter.fromParquetStatisticsInternal( Version.FULL_VERSION, @@ -901,10 +903,7 @@ private void testBinaryStats(StatsHelper helper) { assertThat(roundTripStats.isEmpty()) .as("Round-trip stats should not be empty (null count is set)") - .isFalse(); - assertThat(roundTripStats.getNumNulls()) - .as("Round-trip null count should match") - .isEqualTo(3004); + .isTrue(); } @Test