From 90f5de3717d49d10f9948a401d4f09e87409a227 Mon Sep 17 00:00:00 2001 From: Fokko Date: Wed, 16 Apr 2025 09:18:01 +0200 Subject: [PATCH 1/6] Ignore partition fields that are dropped from the current-schema --- .../org/apache/iceberg/PartitionSpec.java | 10 +++-- .../java/org/apache/iceberg/Partitioning.java | 8 ++-- .../TestAlterTablePartitionFields.java | 44 +++++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index 9b74893f1831..871d49459155 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -131,6 +131,12 @@ public StructType partitionType() { for (PartitionField field : fields) { Type sourceType = schema.findType(field.sourceId()); Type resultType = field.transform().getResultType(sourceType); + + // When the source field has been dropped we cannot determine the type + if (resultType == null) { + resultType = Types.UnknownType.get(); + } + structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType)); } @@ -632,9 +638,7 @@ static void checkCompatibility(PartitionSpec spec, Schema schema) { // https://iceberg.apache.org/spec/#partition-transforms // We don't care about the source type since a VoidTransform is always compatible and skip the // checks - if (!transform.equals(Transforms.alwaysNull())) { - ValidationException.check( - sourceType != null, "Cannot find source column for partition field: %s", field); + if (sourceType != null && !transform.equals(Transforms.alwaysNull())) { ValidationException.check( sourceType.isPrimitiveType(), "Cannot partition by non-primitive source field: %s", diff --git a/core/src/main/java/org/apache/iceberg/Partitioning.java b/core/src/main/java/org/apache/iceberg/Partitioning.java index 832e0b59fe50..c708d39f523e 100644 --- a/core/src/main/java/org/apache/iceberg/Partitioning.java +++ b/core/src/main/java/org/apache/iceberg/Partitioning.java @@ -239,7 +239,8 @@ public static StructType groupingKeyType(Schema schema, Collection specs = table.specs().values(); - return buildPartitionProjectionType("table partition", specs, allFieldIds(specs)); + return buildPartitionProjectionType( + "table partition", specs, allActiveFieldIds(table.schema(), specs)); } /** @@ -346,10 +347,11 @@ private static boolean compatibleTransforms(Transform t1, Transform || t2.equals(Transforms.alwaysNull()); } - // collects IDs of all partition field used across specs - private static Set allFieldIds(Collection specs) { + // collects IDs of all partition field used across specs that are in the current schema + private static Set allActiveFieldIds(Schema schema, Collection specs) { return FluentIterable.from(specs) .transformAndConcat(PartitionSpec::fields) + .filter(field -> schema.findField(field.sourceId()) != null) .transform(PartitionField::fieldId) .toSet(); } diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index dd49d8a254c3..54d58efb5273 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -20,6 +20,9 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.List; import org.apache.iceberg.Parameter; import org.apache.iceberg.ParameterizedTestExtension; import org.apache.iceberg.Parameters; @@ -27,6 +30,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableProperties; import org.apache.iceberg.TestHelpers; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.spark.SparkCatalogConfig; import org.apache.iceberg.spark.source.SparkTable; import org.apache.spark.sql.connector.catalog.CatalogManager; @@ -583,4 +587,44 @@ private void createTable(String schema, String spec) { tableName, schema, spec, TableProperties.FORMAT_VERSION, formatVersion); } } + + private void runCreateAndDropPartitionField( + String column, String partitionType, List expected, String predicate) { + sql("DROP TABLE IF EXISTS %s", tableName); + sql( + "CREATE TABLE %s (col_int INTEGER, col_ts TIMESTAMP_NTZ, col_long BIGINT) USING ICEBERG TBLPROPERTIES ('format-version' = %d)", + tableName, formatVersion); + sql("INSERT INTO %s VALUES (1000, CAST('2024-03-01 19:25:00' as TIMESTAMP), 2100)", tableName); + sql("ALTER TABLE %s ADD PARTITION FIELD %s AS col2_partition", tableName, partitionType); + sql("INSERT INTO %s VALUES (2000, CAST('2024-04-01 19:25:00' as TIMESTAMP), 2200)", tableName); + sql("ALTER TABLE %s DROP PARTITION FIELD col2_partition", tableName); + sql("INSERT INTO %s VALUES (3000, CAST('2024-05-01 19:25:00' as TIMESTAMP), 2300)", tableName); + sql("ALTER TABLE %s DROP COLUMN %s", tableName, column); + + assertEquals( + "Should return correct data", + expected, + sql("SELECT * FROM %s WHERE %s ORDER BY col_int", tableName, predicate)); + } + + @TestTemplate + public void testDropPartitionAndUnderlyingField() { + String predicateLong = "col_ts >= '2024-04-01 19:25:00'"; + List expectedLong = + Lists.newArrayList( + new Object[] {2000, LocalDateTime.ofEpochSecond(1711999500, 0, ZoneOffset.UTC)}, + new Object[] {3000, LocalDateTime.ofEpochSecond(1714591500, 0, ZoneOffset.UTC)}); + runCreateAndDropPartitionField("col_long", "col_long", expectedLong, predicateLong); + runCreateAndDropPartitionField( + "col_long", "truncate(2, col_long)", expectedLong, predicateLong); + runCreateAndDropPartitionField("col_long", "bucket(16, col_long)", expectedLong, predicateLong); + + String predicateTs = "col_long >= 2200"; + List expectedTs = + Lists.newArrayList(new Object[] {2000, 2200L}, new Object[] {3000, 2300L}); + runCreateAndDropPartitionField("col_ts", "col_ts", expectedTs, predicateTs); + runCreateAndDropPartitionField("col_ts", "year(col_ts)", expectedTs, predicateTs); + runCreateAndDropPartitionField("col_ts", "month(col_ts)", expectedTs, predicateTs); + runCreateAndDropPartitionField("col_ts", "day(col_ts)", expectedTs, predicateTs); + } } From 9e998ccacb2f616a09d63ffc490043342a74f352 Mon Sep 17 00:00:00 2001 From: Nynke Gaikema Date: Thu, 26 Jun 2025 09:23:02 +0200 Subject: [PATCH 2/6] Build unchecked when parsing a spec from JSON --- api/src/main/java/org/apache/iceberg/PartitionSpec.java | 4 +++- .../src/main/java/org/apache/iceberg/PartitionSpecParser.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index efed652b2da5..f856e251cc30 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -640,7 +640,9 @@ static void checkCompatibility(PartitionSpec spec, Schema schema) { // https://iceberg.apache.org/spec/#partition-transforms // We don't care about the source type since a VoidTransform is always compatible and skip the // checks - if (sourceType != null && !transform.equals(Transforms.alwaysNull())) { + if (!transform.equals(Transforms.alwaysNull())) { + ValidationException.check( + sourceType != null, "Cannot find source column for partition field: %s", field); ValidationException.check( sourceType.isPrimitiveType(), "Cannot partition by non-primitive source field: %s", diff --git a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java index a51b03c8f015..c0653855a0a5 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java +++ b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java @@ -68,7 +68,7 @@ public static String toJson(UnboundPartitionSpec spec, boolean pretty) { } public static PartitionSpec fromJson(Schema schema, JsonNode json) { - return fromJson(json).bind(schema); + return fromJson(json).bindUnchecked(schema); } public static UnboundPartitionSpec fromJson(JsonNode json) { From bd3e5aac8ce8930c3a7b3e4c2263d7614c1fbcc2 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Mon, 30 Jun 2025 22:55:02 +0200 Subject: [PATCH 3/6] Thanks Steven --- .../org/apache/iceberg/TestPartitioning.java | 23 +++++++++++++++++- .../TestAlterTablePartitionFields.java | 24 ++++++++++--------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index da04e67bdd66..1ec8b28e17ce 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -172,7 +172,7 @@ public void testPartitionTypeWithIncompatibleSpecEvolution() { PartitionSpec newSpec = PartitionSpec.builderFor(table.schema()).identity("category").build(); - TableOperations ops = ((HasTableOperations) table).operations(); + TableOperations ops = table.operations(); TableMetadata current = ops.current(); ops.commit(current, current.updatePartitionSpec(newSpec)); @@ -183,6 +183,27 @@ public void testPartitionTypeWithIncompatibleSpecEvolution() { .hasMessageStartingWith("Conflicting partition fields"); } + @Test + public void testPartitionTypeDropInactiveFields() { + TestTables.TestTable table = + TestTables.create(tableDir, "test", SCHEMA, BY_CATEGORY_DATA_SPEC, V2_FORMAT_VERSION); + + StructType actualType = Partitioning.partitionType(table); + assertThat(actualType) + .isEqualTo( + StructType.of( + NestedField.optional(1000, "category", Types.StringType.get()), + NestedField.optional(1001, "data", Types.StringType.get()))); + + // Create a new spec, and drop the field of the old spec + table.updateSpec().removeField("category").commit(); + table.updateSchema().deleteColumn("category").commit(); + + actualType = Partitioning.partitionType(table); + assertThat(actualType) + .isEqualTo(StructType.of(NestedField.optional(1001, "data", Types.StringType.get()))); + } + @Test public void testGroupingKeyTypeWithSpecEvolutionInV1Tables() { TestTables.TestTable table = diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 54d58efb5273..7e5f5454ffa2 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -608,17 +608,7 @@ private void runCreateAndDropPartitionField( } @TestTemplate - public void testDropPartitionAndUnderlyingField() { - String predicateLong = "col_ts >= '2024-04-01 19:25:00'"; - List expectedLong = - Lists.newArrayList( - new Object[] {2000, LocalDateTime.ofEpochSecond(1711999500, 0, ZoneOffset.UTC)}, - new Object[] {3000, LocalDateTime.ofEpochSecond(1714591500, 0, ZoneOffset.UTC)}); - runCreateAndDropPartitionField("col_long", "col_long", expectedLong, predicateLong); - runCreateAndDropPartitionField( - "col_long", "truncate(2, col_long)", expectedLong, predicateLong); - runCreateAndDropPartitionField("col_long", "bucket(16, col_long)", expectedLong, predicateLong); - + public void testDropPartitionAndSourceColumnLong() { String predicateTs = "col_long >= 2200"; List expectedTs = Lists.newArrayList(new Object[] {2000, 2200L}, new Object[] {3000, 2300L}); @@ -627,4 +617,16 @@ public void testDropPartitionAndUnderlyingField() { runCreateAndDropPartitionField("col_ts", "month(col_ts)", expectedTs, predicateTs); runCreateAndDropPartitionField("col_ts", "day(col_ts)", expectedTs, predicateTs); } + + @TestTemplate + public void testDropPartitionAndSourceColumnTimestamp() { + String predicate = "col_ts >= '2024-04-01 19:25:00'"; + List expected = + Lists.newArrayList( + new Object[] {2000, LocalDateTime.ofEpochSecond(1711999500, 0, ZoneOffset.UTC)}, + new Object[] {3000, LocalDateTime.ofEpochSecond(1714591500, 0, ZoneOffset.UTC)}); + runCreateAndDropPartitionField("col_long", "col_long", expected, predicate); + runCreateAndDropPartitionField("col_long", "truncate(2, col_long)", expected, predicate); + runCreateAndDropPartitionField("col_long", "bucket(16, col_long)", expected, predicate); + } } From 429e4f88936e27c8543ba158024ccca9cc917fd1 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 1 Jul 2025 20:33:09 +0200 Subject: [PATCH 4/6] Naming things --- core/src/test/java/org/apache/iceberg/TestPartitioning.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index 1ec8b28e17ce..3bfede545292 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -184,7 +184,7 @@ public void testPartitionTypeWithIncompatibleSpecEvolution() { } @Test - public void testPartitionTypeDropInactiveFields() { + public void testPartitionTypeIgnoreInactiveFields() { TestTables.TestTable table = TestTables.create(tableDir, "test", SCHEMA, BY_CATEGORY_DATA_SPEC, V2_FORMAT_VERSION); From fa5115e9961fc753edc30b75067d787194f84b5a Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 1 Jul 2025 21:23:21 +0200 Subject: [PATCH 5/6] Thanks Steven --- .../java/org/apache/iceberg/PartitionSpec.java | 2 +- .../org/apache/iceberg/TestPartitioning.java | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index f856e251cc30..fc7f5b1a7cfb 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -134,7 +134,7 @@ public StructType partitionType() { Type resultType = field.transform().getResultType(sourceType); // When the source field has been dropped we cannot determine the type - if (resultType == null) { + if (sourceType == null) { resultType = Types.UnknownType.get(); } diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index 3bfede545292..eb77a693c799 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -186,22 +186,29 @@ public void testPartitionTypeWithIncompatibleSpecEvolution() { @Test public void testPartitionTypeIgnoreInactiveFields() { TestTables.TestTable table = - TestTables.create(tableDir, "test", SCHEMA, BY_CATEGORY_DATA_SPEC, V2_FORMAT_VERSION); + TestTables.create( + tableDir, "test", SCHEMA, BY_DATA_CATEGORY_BUCKET_SPEC, V2_FORMAT_VERSION); StructType actualType = Partitioning.partitionType(table); assertThat(actualType) .isEqualTo( StructType.of( - NestedField.optional(1000, "category", Types.StringType.get()), - NestedField.optional(1001, "data", Types.StringType.get()))); + NestedField.optional(1000, "data", Types.StringType.get()), + NestedField.optional(1001, "category_bucket", Types.IntegerType.get()))); // Create a new spec, and drop the field of the old spec - table.updateSpec().removeField("category").commit(); + table.updateSpec().removeField("category_bucket").commit(); table.updateSchema().deleteColumn("category").commit(); actualType = Partitioning.partitionType(table); assertThat(actualType) - .isEqualTo(StructType.of(NestedField.optional(1001, "data", Types.StringType.get()))); + .isEqualTo(StructType.of(NestedField.optional(1000, "data", Types.StringType.get()))); + + table.updateSpec().removeField("data").commit(); + table.updateSchema().deleteColumn("data").commit(); + + actualType = Partitioning.partitionType(table); + assertThat(actualType).isEqualTo(StructType.of()); } @Test From 5a1056c65bc1ca33642eaf107b745679e03d7a19 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Wed, 2 Jul 2025 10:24:24 +0200 Subject: [PATCH 6/6] Add flag --- .../java/org/apache/iceberg/PartitionSpec.java | 14 +++++++++++++- .../org/apache/iceberg/UnboundPartitionSpec.java | 4 ++++ .../org/apache/iceberg/PartitionSpecParser.java | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index fc7f5b1a7cfb..f059c928a967 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -620,8 +620,12 @@ Builder add(int sourceId, int fieldId, String name, Transform transform) { } public PartitionSpec build() { + return build(false); + } + + public PartitionSpec build(boolean allowMissingFields) { PartitionSpec spec = buildUnchecked(); - checkCompatibility(spec, schema); + checkCompatibility(spec, schema, allowMissingFields); return spec; } @@ -631,10 +635,18 @@ PartitionSpec buildUnchecked() { } static void checkCompatibility(PartitionSpec spec, Schema schema) { + checkCompatibility(spec, schema, false); + } + + static void checkCompatibility(PartitionSpec spec, Schema schema, boolean allowMissingFields) { final Map parents = TypeUtil.indexParents(schema.asStruct()); for (PartitionField field : spec.fields) { Type sourceType = schema.findType(field.sourceId()); Transform transform = field.transform(); + // In the case the underlying field is dropped, we cannot check if they are compatible + if (allowMissingFields && sourceType == null) { + continue; + } // In the case of a Version 1 partition-spec field gets deleted, // it is replaced with a void transform, see: // https://iceberg.apache.org/spec/#partition-transforms diff --git a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java index cc8526f9072c..30b3cce35fc8 100644 --- a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java @@ -46,6 +46,10 @@ public PartitionSpec bind(Schema schema) { return copyToBuilder(schema).build(); } + public PartitionSpec bind(Schema schema, boolean ignoreMissingFields) { + return copyToBuilder(schema).build(ignoreMissingFields); + } + PartitionSpec bindUnchecked(Schema schema) { return copyToBuilder(schema).buildUnchecked(); } diff --git a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java index c0653855a0a5..7becf0c62943 100644 --- a/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java +++ b/core/src/main/java/org/apache/iceberg/PartitionSpecParser.java @@ -68,7 +68,7 @@ public static String toJson(UnboundPartitionSpec spec, boolean pretty) { } public static PartitionSpec fromJson(Schema schema, JsonNode json) { - return fromJson(json).bindUnchecked(schema); + return fromJson(json).bind(schema, true); } public static UnboundPartitionSpec fromJson(JsonNode json) {