diff --git a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java index edb01d7e77ef..5572b6869708 100644 --- a/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java +++ b/data/src/test/java/org/apache/iceberg/data/TestMetricsRowGroupFilter.java @@ -200,7 +200,9 @@ public void createOrcInputFile() throws IOException { record.setField("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0 record.setField( "_no_stats_parquet", - TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats + (i % 10 == 0) + ? null + : TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats // in Parquet, but will produce stats for ORC record.setField("_required", "req"); // required, always non-null record.setField("_all_nulls", null); // never non-null @@ -240,7 +242,9 @@ private void createParquetInputFile() throws IOException { builder.setField("_id", INT_MIN_VALUE + i); // min=30, max=79, num-nulls=0 builder.setField( "_no_stats_parquet", - TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats + (i % 10 == 0) + ? null + : TOO_LONG_FOR_STATS_PARQUET); // value longer than 4k will produce no stats // in Parquet builder.setField("_required", "req"); // required, always non-null builder.setField("_all_nulls", null); // never non-null @@ -341,7 +345,7 @@ public void testFloatWithNan() { shouldRead = shouldRead(lessThan("some_nans", 3.0)); assertThat(shouldRead).isTrue(); - shouldRead = shouldRead(lessThanOrEqual("some_nans", 1.0)); + shouldRead = shouldRead(lessThanOrEqual("some_nans", 2.0)); assertThat(shouldRead).isTrue(); shouldRead = shouldRead(equal("some_nans", 2.0)); @@ -361,7 +365,7 @@ public void testDoubleWithNan() { shouldRead = shouldRead(lessThan("some_double_nans", 3.0)); assertThat(shouldRead).as("Should read: column with some nans contains target value").isTrue(); - shouldRead = shouldRead(lessThanOrEqual("some_double_nans", 1.0)); + shouldRead = shouldRead(lessThanOrEqual("some_double_nans", 2.0)); assertThat(shouldRead).as("Should read: column with some nans contains target value").isTrue(); shouldRead = shouldRead(equal("some_double_nans", 2.0)); diff --git a/flink/v1.20/flink-runtime/runtime-deps.txt b/flink/v1.20/flink-runtime/runtime-deps.txt index 224e19d09f7a..ced50d5a7e8b 100644 --- a/flink/v1.20/flink-runtime/runtime-deps.txt +++ b/flink/v1.20/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/flink/v2.1/flink-runtime/runtime-deps.txt b/flink/v2.1/flink-runtime/runtime-deps.txt index 224e19d09f7a..ced50d5a7e8b 100644 --- a/flink/v2.1/flink-runtime/runtime-deps.txt +++ b/flink/v2.1/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/flink/v2.2/flink-runtime/runtime-deps.txt b/flink/v2.2/flink-runtime/runtime-deps.txt index 224e19d09f7a..ced50d5a7e8b 100644 --- a/flink/v2.2/flink-runtime/runtime-deps.txt +++ b/flink/v2.2/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/flink/v2.3/flink-runtime/runtime-deps.txt b/flink/v2.3/flink-runtime/runtime-deps.txt index 224e19d09f7a..ced50d5a7e8b 100644 --- a/flink/v2.3/flink-runtime/runtime-deps.txt +++ b/flink/v2.3/flink-runtime/runtime-deps.txt @@ -14,14 +14,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 org.jspecify:jspecify:1.0 org.locationtech.jts:jts-core:1.20 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index dbf74bc538f4..14e9438b584c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -82,7 +82,7 @@ nessie = "0.108.4" netty-buffer = "4.2.17.Final" object-client-bundle = "3.3.2" orc = "1.9.9" -parquet = "1.17.1" +parquet = "1.18.1" roaringbitmap = "1.6.20" scala-collection-compat = "2.14.0" slf4j = "2.0.18" diff --git a/kafka-connect/kafka-connect-runtime/runtime-deps.txt b/kafka-connect/kafka-connect-runtime/runtime-deps.txt index 7b71108c2644..869cd2835f77 100644 --- a/kafka-connect/kafka-connect-runtime/runtime-deps.txt +++ b/kafka-connect/kafka-connect-runtime/runtime-deps.txt @@ -171,14 +171,14 @@ org.apache.httpcomponents:httpclient:4.5 org.apache.httpcomponents:httpcore:4.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.bouncycastle:bcprov-jdk18on:1.85 org.checkerframework:checker-compat-qual:2.5 org.checkerframework:checker-qual:3.49 diff --git a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java index 6667109197da..b9ea8f23176f 100644 --- a/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java +++ b/parquet/src/test/java/org/apache/iceberg/parquet/TestVariantMetrics.java @@ -480,12 +480,21 @@ public void testVariantFloatNaN() throws IOException { assertThat(metrics.nullValueCounts()).isEqualTo(Map.of(1, 0L, 2, 0L)); assertThat(metrics.nanValueCounts()).isEqualTo(Map.of()); - assertThat(metrics.lowerBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 0L))); - assertThat(metrics.upperBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L))); + // NaN is ignored when computing the shredded value bounds, so both bounds are the float value + assertThat(metrics.lowerBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 0L)); + assertThat(metrics.lowerBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0F)); + assertThat(metrics.upperBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 1L)); + assertThat(metrics.upperBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0F)); - assertThat(metrics).extracting("originalTypes").isEqualTo(Map.of(1, Types.LongType.get())); + assertThat(metrics) + .extracting("originalTypes") + .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); } @Test @@ -502,12 +511,21 @@ public void testVariantDoubleNaN() throws IOException { assertThat(metrics.nullValueCounts()).isEqualTo(Map.of(1, 0L, 2, 0L)); assertThat(metrics.nanValueCounts()).isEqualTo(Map.of()); - assertThat(metrics.lowerBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 0L))); - assertThat(metrics.upperBounds()) - .isEqualTo(Map.of(1, Conversions.toByteBuffer(Type.TypeID.LONG, 1L))); + // NaN is ignored when computing the shredded value bounds, so both bounds are the double value + assertThat(metrics.lowerBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 0L)); + assertThat(metrics.lowerBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0D)); + assertThat(metrics.upperBounds().get(1)) + .isEqualTo(Conversions.toByteBuffer(Type.TypeID.LONG, 1L)); + assertThat(metrics.upperBounds().get(2)) + .extracting(b -> Variant.from(b).value().asObject().get(ROOT_FIELD)) + .isEqualTo(Variants.of(1.0D)); - assertThat(metrics).extracting("originalTypes").isEqualTo(Map.of(1, Types.LongType.get())); + assertThat(metrics) + .extracting("originalTypes") + .isEqualTo(Map.of(1, Types.LongType.get(), 2, Types.VariantType.get())); } @Test diff --git a/spark/v3.5/spark-runtime/runtime-deps.txt b/spark/v3.5/spark-runtime/runtime-deps.txt index 3bc4bcaed680..da04115c9cd7 100644 --- a/spark/v3.5/spark-runtime/runtime-deps.txt +++ b/spark/v3.5/spark-runtime/runtime-deps.txt @@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 diff --git a/spark/v4.0/spark-runtime/runtime-deps.txt b/spark/v4.0/spark-runtime/runtime-deps.txt index 3bc4bcaed680..da04115c9cd7 100644 --- a/spark/v4.0/spark-runtime/runtime-deps.txt +++ b/spark/v4.0/spark-runtime/runtime-deps.txt @@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1 diff --git a/spark/v4.1/spark-runtime/runtime-deps.txt b/spark/v4.1/spark-runtime/runtime-deps.txt index b81c39446bb7..8736db54f6be 100644 --- a/spark/v4.1/spark-runtime/runtime-deps.txt +++ b/spark/v4.1/spark-runtime/runtime-deps.txt @@ -21,14 +21,14 @@ org.apache.httpcomponents.core5:httpcore5-h2:5.4 org.apache.httpcomponents.core5:httpcore5:5.4 org.apache.orc:orc-core:1.9 org.apache.orc:orc-shims:1.9 -org.apache.parquet:parquet-avro:1.17 -org.apache.parquet:parquet-column:1.17 -org.apache.parquet:parquet-common:1.17 -org.apache.parquet:parquet-encoding:1.17 -org.apache.parquet:parquet-format-structures:1.17 -org.apache.parquet:parquet-hadoop:1.17 -org.apache.parquet:parquet-jackson:1.17 -org.apache.parquet:parquet-variant:1.17 +org.apache.parquet:parquet-avro:1.18 +org.apache.parquet:parquet-column:1.18 +org.apache.parquet:parquet-common:1.18 +org.apache.parquet:parquet-encoding:1.18 +org.apache.parquet:parquet-format-structures:1.18 +org.apache.parquet:parquet-hadoop:1.18 +org.apache.parquet:parquet-jackson:1.18 +org.apache.parquet:parquet-variant:1.18 org.eclipse.collections:eclipse-collections-api:11.1 org.eclipse.collections:eclipse-collections:11.1 org.eclipse.microprofile.openapi:microprofile-openapi-api:4.1