From 288f170340bacc9e0a5c41e733c9a51622764805 Mon Sep 17 00:00:00 2001 From: Ayush Saxena Date: Thu, 8 Dec 2022 03:50:32 +0530 Subject: [PATCH 1/6] Increase Partition Start Id to 10000. --- .../org/apache/iceberg/PartitionSpec.java | 8 ++-- .../iceberg/TestPartitionSpecValidation.java | 34 +++++++------- .../apache/iceberg/TestManifestReader.java | 6 +-- .../org/apache/iceberg/TestMergeAppend.java | 20 ++++---- .../iceberg/TestMetadataTableScans.java | 10 ++-- ...adataTableScansWithPartitionEvolution.java | 4 +- .../iceberg/TestMetadataUpdateParser.java | 12 ++--- .../iceberg/TestPartitionSpecParser.java | 18 ++++---- .../org/apache/iceberg/TestPartitioning.java | 34 +++++++------- .../org/apache/iceberg/TestTableMetadata.java | 22 +++++---- .../iceberg/TestTableUpdatePartitionSpec.java | 34 +++++++------- .../iceberg/TestUpdatePartitionSpec.java | 28 +++++------ .../rest/requests/TestCreateTableRequest.java | 2 +- .../TableMetadataStatisticsFiles.json | 2 +- .../TableMetadataV2MissingSortOrder.json | 4 +- .../test/resources/TableMetadataV2Valid.json | 2 +- format/spec.md | 6 +-- python/mkdocs/docs/index.md | 6 +-- python/tests/catalog/test_hive.py | 2 +- python/tests/catalog/test_rest.py | 4 +- python/tests/cli/test_console.py | 2 +- python/tests/conftest.py | 6 +-- python/tests/expressions/test_projection.py | 14 +++--- python/tests/table/test_metadata.py | 46 +++++++++++-------- python/tests/table/test_partitioning.py | 32 ++++++------- python_legacy/tests/core/conftest.py | 2 +- python_legacy/tests/hive/conftest.py | 2 +- .../org/apache/iceberg/examples/README.md | 2 +- .../spark/sql/TestCreateTableAsSelect.java | 2 +- ...tMetadataTablesWithPartitionEvolution.java | 6 +-- .../spark/sql/TestCreateTableAsSelect.java | 2 +- ...tMetadataTablesWithPartitionEvolution.java | 6 +-- .../spark/sql/TestCreateTableAsSelect.java | 2 +- 33 files changed, 197 insertions(+), 185 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java index aa6a6051e561..1d3064da0107 100644 --- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java +++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java @@ -49,8 +49,8 @@ * represented by a named {@link PartitionField}. */ public class PartitionSpec implements Serializable { - // IDs for partition fields start at 1000 - private static final int PARTITION_DATA_ID_START = 1000; + // IDs for partition fields start at 10000 + private static final int PARTITION_DATA_ID_START = 10000; private final Schema schema; @@ -600,8 +600,8 @@ static void checkCompatibility(PartitionSpec spec, Schema schema) { } static boolean hasSequentialIds(PartitionSpec spec) { - for (int i = 0; i < spec.fields.length; i += 1) { - if (spec.fields[i].fieldId() != PARTITION_DATA_ID_START + i) { + for (int i = 1; i < spec.fields.length; i += 1) { + if (spec.fields[i].fieldId() != spec.fields[i - 1].fieldId() + 1) { return false; } } diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java index ed1d035cdc4c..08768d399b11 100644 --- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java +++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java @@ -306,26 +306,26 @@ public void testAutoSettingPartitionFieldIds() { .truncate("s", 1, "custom_truncate") .build(); - Assert.assertEquals(1000, spec.fields().get(0).fieldId()); - Assert.assertEquals(1001, spec.fields().get(1).fieldId()); - Assert.assertEquals(1002, spec.fields().get(2).fieldId()); - Assert.assertEquals(1003, spec.fields().get(3).fieldId()); - Assert.assertEquals(1003, spec.lastAssignedFieldId()); + Assert.assertEquals(10000, spec.fields().get(0).fieldId()); + Assert.assertEquals(10001, spec.fields().get(1).fieldId()); + Assert.assertEquals(10002, spec.fields().get(2).fieldId()); + Assert.assertEquals(10003, spec.fields().get(3).fieldId()); + Assert.assertEquals(10003, spec.lastAssignedFieldId()); } @Test public void testAddPartitionFieldsWithFieldIds() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA) - .add(1, 1005, "id_partition1", Transforms.bucket(4)) - .add(1, 1006, "id_partition2", Transforms.bucket(5)) - .add(1, 1002, "id_partition3", Transforms.bucket(6)) + .add(1, 10005, "id_partition1", Transforms.bucket(4)) + .add(1, 10006, "id_partition2", Transforms.bucket(5)) + .add(1, 10002, "id_partition3", Transforms.bucket(6)) .build(); - Assert.assertEquals(1005, spec.fields().get(0).fieldId()); - Assert.assertEquals(1006, spec.fields().get(1).fieldId()); - Assert.assertEquals(1002, spec.fields().get(2).fieldId()); - Assert.assertEquals(1006, spec.lastAssignedFieldId()); + Assert.assertEquals(10005, spec.fields().get(0).fieldId()); + Assert.assertEquals(10006, spec.fields().get(1).fieldId()); + Assert.assertEquals(10002, spec.fields().get(2).fieldId()); + Assert.assertEquals(10006, spec.lastAssignedFieldId()); } @Test @@ -333,13 +333,13 @@ public void testAddPartitionFieldsWithAndWithoutFieldIds() { PartitionSpec spec = PartitionSpec.builderFor(SCHEMA) .add(1, "id_partition2", Transforms.bucket(5)) - .add(1, 1005, "id_partition1", Transforms.bucket(4)) + .add(1, 10005, "id_partition1", Transforms.bucket(4)) .truncate("s", 1, "custom_truncate") .build(); - Assert.assertEquals(1000, spec.fields().get(0).fieldId()); - Assert.assertEquals(1005, spec.fields().get(1).fieldId()); - Assert.assertEquals(1006, spec.fields().get(2).fieldId()); - Assert.assertEquals(1006, spec.lastAssignedFieldId()); + Assert.assertEquals(10000, spec.fields().get(0).fieldId()); + Assert.assertEquals(10005, spec.fields().get(1).fieldId()); + Assert.assertEquals(10006, spec.fields().get(2).fieldId()); + Assert.assertEquals(10006, spec.lastAssignedFieldId()); } } diff --git a/core/src/test/java/org/apache/iceberg/TestManifestReader.java b/core/src/test/java/org/apache/iceberg/TestManifestReader.java index dfc84200fdb2..237786c0b452 100644 --- a/core/src/test/java/org/apache/iceberg/TestManifestReader.java +++ b/core/src/test/java/org/apache/iceberg/TestManifestReader.java @@ -93,7 +93,7 @@ public void testManifestReaderWithPartitionMetadata() throws IOException { List fields = ((PartitionData) entry.file().partition()).getPartitionType().fields(); Assert.assertEquals(1, fields.size()); - Assert.assertEquals(1000, fields.get(0).fieldId()); + Assert.assertEquals(10000, fields.get(0).fieldId()); Assert.assertEquals("data_bucket", fields.get(0).name()); Assert.assertEquals(Types.IntegerType.get(), fields.get(0).type()); } @@ -113,11 +113,11 @@ public void testManifestReaderWithUpdatedPartitionMetadataForV1Table() throws IO List fields = ((PartitionData) entry.file().partition()).getPartitionType().fields(); Assert.assertEquals(2, fields.size()); - Assert.assertEquals(1000, fields.get(0).fieldId()); + Assert.assertEquals(10000, fields.get(0).fieldId()); Assert.assertEquals("id_bucket", fields.get(0).name()); Assert.assertEquals(Types.IntegerType.get(), fields.get(0).type()); - Assert.assertEquals(1001, fields.get(1).fieldId()); + Assert.assertEquals(10001, fields.get(1).fieldId()); Assert.assertEquals("data_bucket", fields.get(1).name()); Assert.assertEquals(Types.IntegerType.get(), fields.get(1).type()); } diff --git a/core/src/test/java/org/apache/iceberg/TestMergeAppend.java b/core/src/test/java/org/apache/iceberg/TestMergeAppend.java index 212bab3c136b..c5178a194084 100644 --- a/core/src/test/java/org/apache/iceberg/TestMergeAppend.java +++ b/core/src/test/java/org/apache/iceberg/TestMergeAppend.java @@ -1240,28 +1240,28 @@ public void testUpdatePartitionSpecFieldIdsForV1Table() { List partitionSpecs = table.ops().current().specs(); PartitionSpec partitionSpec = partitionSpecs.get(0); - Assert.assertEquals(1000, partitionSpec.lastAssignedFieldId()); + Assert.assertEquals(10000, partitionSpec.lastAssignedFieldId()); Types.StructType structType = partitionSpec.partitionType(); List fields = structType.fields(); Assert.assertEquals(1, fields.size()); Assert.assertEquals("data_bucket", fields.get(0).name()); - Assert.assertEquals(1000, fields.get(0).fieldId()); + Assert.assertEquals(10000, fields.get(0).fieldId()); partitionSpec = partitionSpecs.get(1); - Assert.assertEquals(1003, partitionSpec.lastAssignedFieldId()); + Assert.assertEquals(10003, partitionSpec.lastAssignedFieldId()); structType = partitionSpec.partitionType(); fields = structType.fields(); Assert.assertEquals(4, fields.size()); Assert.assertEquals("id_bucket", fields.get(0).name()); - Assert.assertEquals(1000, fields.get(0).fieldId()); + Assert.assertEquals(10000, fields.get(0).fieldId()); Assert.assertEquals("data", fields.get(1).name()); - Assert.assertEquals(1001, fields.get(1).fieldId()); + Assert.assertEquals(10001, fields.get(1).fieldId()); Assert.assertEquals("data_bucket", fields.get(2).name()); - Assert.assertEquals(1002, fields.get(2).fieldId()); + Assert.assertEquals(10002, fields.get(2).fieldId()); Assert.assertEquals("data_partition", fields.get(3).name()); - Assert.assertEquals(1003, fields.get(3).fieldId()); + Assert.assertEquals(10003, fields.get(3).fieldId()); } @Test @@ -1338,10 +1338,10 @@ public void testManifestEntryFieldIdsForChangedPartitionSpecForV1Table() { .next(); Types.NestedField field = ((PartitionData) entry.file().partition()).getPartitionType().fields().get(0); - Assert.assertEquals(1000, field.fieldId()); + Assert.assertEquals(10000, field.fieldId()); Assert.assertEquals("id_bucket", field.name()); field = ((PartitionData) entry.file().partition()).getPartitionType().fields().get(1); - Assert.assertEquals(1001, field.fieldId()); + Assert.assertEquals(10001, field.fieldId()); Assert.assertEquals("data_bucket", field.name()); entry = @@ -1350,7 +1350,7 @@ public void testManifestEntryFieldIdsForChangedPartitionSpecForV1Table() { .iterator() .next(); field = ((PartitionData) entry.file().partition()).getPartitionType().fields().get(0); - Assert.assertEquals(1000, field.fieldId()); + Assert.assertEquals(10000, field.fieldId()); Assert.assertEquals("data_bucket", field.name()); } diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java index a9409322090d..7fc998cfe515 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java @@ -305,7 +305,7 @@ public void testPartitionsTableScanNoFilter() { required( 1, "partition", - Types.StructType.of(optional(1000, "data_bucket", Types.IntegerType.get())))) + Types.StructType.of(optional(10000, "data_bucket", Types.IntegerType.get())))) .asStruct(); TableScan scanNoFilter = partitionsTable.newScan().select("partition.data_bucket"); @@ -481,11 +481,11 @@ public void testFilesTableScanWithDroppedPartition() throws IOException { Types.StructType actualType = schema.findField(DataFile.PARTITION_ID).type().asStructType(); Types.StructType expectedType = Types.StructType.of( - Types.NestedField.optional(1000, "data_bucket", Types.IntegerType.get()), - Types.NestedField.optional(1001, "data_bucket_16", Types.IntegerType.get()), - Types.NestedField.optional(1002, "data_trunc_2", Types.StringType.get())); + Types.NestedField.optional(10000, "data_bucket", Types.IntegerType.get()), + Types.NestedField.optional(10001, "data_bucket_16", Types.IntegerType.get()), + Types.NestedField.optional(10002, "data_trunc_2", Types.StringType.get())); Assert.assertEquals("Partition type must match", expectedType, actualType); - Accessor accessor = schema.accessorForField(1000); + Accessor accessor = schema.accessorForField(10000); try (CloseableIterable tasks = scan.planFiles()) { Set results = diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataTableScansWithPartitionEvolution.java b/core/src/test/java/org/apache/iceberg/TestMetadataTableScansWithPartitionEvolution.java index c825cef876a6..77d6b8c0fe9b 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataTableScansWithPartitionEvolution.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataTableScansWithPartitionEvolution.java @@ -139,8 +139,8 @@ public void testPartitionsTableScanWithAddPartitionOnNestedField() throws IOExce 1, "partition", Types.StructType.of( - optional(1000, "id", Types.IntegerType.get()), - optional(1001, "nested.id", Types.IntegerType.get())))) + optional(10000, "id", Types.IntegerType.get()), + optional(10001, "nested.id", Types.IntegerType.get())))) .asStruct(); TableScan scanNoFilter = partitionsTable.newScan().select("partition"); diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java index 80faccc5f2d6..423f22b60dcb 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataUpdateParser.java @@ -168,12 +168,12 @@ public void testAddPartitionSpecFromJsonWithFieldId() { + "\"name\":\"id_bucket\"," + "\"transform\":\"bucket[8]\"," + "\"source-id\":1," - + "\"field-id\":1000" + + "\"field-id\":10000" + "},{" + "\"name\":\"data_bucket\"," + "\"transform\":\"bucket[16]\"," + "\"source-id\":2," - + "\"field-id\":1001" + + "\"field-id\":10001" + "}]" + "}"; @@ -197,8 +197,8 @@ public void testAddPartitionSpecFromJsonWithFieldId() { @Test public void testAddPartitionSpecFromJsonWithoutFieldId() { - // partition field ids are missing in old PartitionSpec, they always auto-increment from 1000 in - // declared order + // partition field ids are missing in old PartitionSpec, they always auto-increment from 10000 + // in declared order String action = MetadataUpdateParser.ADD_PARTITION_SPEC; String specString = "{" @@ -240,12 +240,12 @@ public void testAddPartitionSpecToJson() { + "\"name\":\"id_bucket\"," + "\"transform\":\"bucket[8]\"," + "\"source-id\":1," - + "\"field-id\":1000" + + "\"field-id\":10000" + "},{" + "\"name\":\"data_bucket\"," + "\"transform\":\"bucket[16]\"," + "\"source-id\":2," - + "\"field-id\":1001" + + "\"field-id\":10001" + "}]" + "}"; diff --git a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java index 1d88e97f9925..308965bf5fc1 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitionSpecParser.java @@ -35,7 +35,7 @@ public void testToJsonForV1Table() { + " \"name\" : \"data_bucket\",\n" + " \"transform\" : \"bucket[16]\",\n" + " \"source-id\" : 2,\n" - + " \"field-id\" : 1000\n" + + " \"field-id\" : 10000\n" + " } ]\n" + "}"; Assert.assertEquals(expected, PartitionSpecParser.toJson(table.spec(), true)); @@ -52,12 +52,12 @@ public void testToJsonForV1Table() { + " \"name\" : \"id_bucket\",\n" + " \"transform\" : \"bucket[8]\",\n" + " \"source-id\" : 1,\n" - + " \"field-id\" : 1000\n" + + " \"field-id\" : 10000\n" + " }, {\n" + " \"name\" : \"data_bucket\",\n" + " \"transform\" : \"bucket[16]\",\n" + " \"source-id\" : 2,\n" - + " \"field-id\" : 1001\n" + + " \"field-id\" : 10001\n" + " } ]\n" + "}"; Assert.assertEquals(expected, PartitionSpecParser.toJson(table.spec(), true)); @@ -72,12 +72,12 @@ public void testFromJsonWithFieldId() { + " \"name\" : \"id_bucket\",\n" + " \"transform\" : \"bucket[8]\",\n" + " \"source-id\" : 1,\n" - + " \"field-id\" : 1001\n" + + " \"field-id\" : 10001\n" + " }, {\n" + " \"name\" : \"data_bucket\",\n" + " \"transform\" : \"bucket[16]\",\n" + " \"source-id\" : 2,\n" - + " \"field-id\" : 1000\n" + + " \"field-id\" : 10000\n" + " } ]\n" + "}"; @@ -85,8 +85,8 @@ public void testFromJsonWithFieldId() { Assert.assertEquals(2, spec.fields().size()); // should be the field ids in the JSON - Assert.assertEquals(1001, spec.fields().get(0).fieldId()); - Assert.assertEquals(1000, spec.fields().get(1).fieldId()); + Assert.assertEquals(10001, spec.fields().get(0).fieldId()); + Assert.assertEquals(10000, spec.fields().get(1).fieldId()); } @Test @@ -109,8 +109,8 @@ public void testFromJsonWithoutFieldId() { Assert.assertEquals(2, spec.fields().size()); // should be the default assignment - Assert.assertEquals(1000, spec.fields().get(0).fieldId()); - Assert.assertEquals(1001, spec.fields().get(1).fieldId()); + Assert.assertEquals(10000, spec.fields().get(0).fieldId()); + Assert.assertEquals(10001, spec.fields().get(1).fieldId()); } @Test diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index 5dfc9ac40ee9..0ce74669ef00 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -74,8 +74,8 @@ public void testPartitionTypeWithSpecEvolutionInV1Tables() { StructType expectedType = StructType.of( - NestedField.optional(1000, "data", Types.StringType.get()), - NestedField.optional(1001, "category_bucket_8", Types.IntegerType.get())); + NestedField.optional(10000, "data", Types.StringType.get()), + NestedField.optional(10001, "category_bucket_8", Types.IntegerType.get())); StructType actualType = Partitioning.partitionType(table); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -91,8 +91,8 @@ public void testPartitionTypeWithSpecEvolutionInV2Tables() { StructType expectedType = StructType.of( - NestedField.optional(1000, "data", Types.StringType.get()), - NestedField.optional(1001, "category", Types.StringType.get())); + NestedField.optional(10000, "data", Types.StringType.get()), + NestedField.optional(10001, "category", Types.StringType.get())); StructType actualType = Partitioning.partitionType(table); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -109,8 +109,8 @@ public void testPartitionTypeWithRenamesInV1Table() { StructType expectedType = StructType.of( - NestedField.optional(1000, "p2", Types.StringType.get()), - NestedField.optional(1001, "category", Types.StringType.get())); + NestedField.optional(10000, "p2", Types.StringType.get()), + NestedField.optional(10001, "category", Types.StringType.get())); StructType actualType = Partitioning.partitionType(table); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -127,8 +127,8 @@ public void testPartitionTypeWithAddingBackSamePartitionFieldInV1Table() { // in v1, we use void transforms instead of dropping partition fields StructType expectedType = StructType.of( - NestedField.optional(1000, "data_1000", Types.StringType.get()), - NestedField.optional(1001, "data", Types.StringType.get())); + NestedField.optional(10000, "data_10000", Types.StringType.get()), + NestedField.optional(10001, "data", Types.StringType.get())); StructType actualType = Partitioning.partitionType(table); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -144,7 +144,7 @@ public void testPartitionTypeWithAddingBackSamePartitionFieldInV2Table() { // in v2, we should be able to reuse the original partition spec StructType expectedType = - StructType.of(NestedField.optional(1000, "data", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "data", Types.StringType.get())); StructType actualType = Partitioning.partitionType(table); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -179,7 +179,7 @@ public void testGroupingKeyTypeWithSpecEvolutionInV1Tables() { Assert.assertEquals("Should have 2 specs", 2, table.specs().size()); StructType expectedType = - StructType.of(NestedField.optional(1000, "data", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "data", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -194,7 +194,7 @@ public void testGroupingKeyTypeWithSpecEvolutionInV2Tables() { Assert.assertEquals("Should have 2 specs", 2, table.specs().size()); StructType expectedType = - StructType.of(NestedField.optional(1000, "data", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "data", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -210,7 +210,7 @@ public void testGroupingKeyTypeWithDroppedPartitionFieldInV1Tables() { Assert.assertEquals("Should have 2 specs", 2, table.specs().size()); StructType expectedType = - StructType.of(NestedField.optional(1000, "data", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "data", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -226,7 +226,7 @@ public void testGroupingKeyTypeWithDroppedPartitionFieldInV2Tables() { Assert.assertEquals("Should have 2 specs", 2, table.specs().size()); StructType expectedType = - StructType.of(NestedField.optional(1000, "data", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "data", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -242,7 +242,7 @@ public void testGroupingKeyTypeWithRenamesInV1Table() { table.updateSpec().renameField("p1", "p2").commit(); StructType expectedType = - StructType.of(NestedField.optional(1000, "p2", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "p2", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -258,7 +258,7 @@ public void testGroupingKeyTypeWithRenamesInV2Table() { table.updateSpec().renameField("p1", "p2").commit(); StructType expectedType = - StructType.of(NestedField.optional(1000, "p2", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "p2", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -301,7 +301,7 @@ public void testGroupingKeyTypeWithAddingBackSamePartitionFieldInV1Table() { table.updateSpec().addField("data").commit(); StructType expectedType = - StructType.of(NestedField.optional(1000, "category", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "category", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } @@ -316,7 +316,7 @@ public void testGroupingKeyTypeWithAddingBackSamePartitionFieldInV2Table() { table.updateSpec().addField("data").commit(); StructType expectedType = - StructType.of(NestedField.optional(1000, "category", Types.StringType.get())); + StructType.of(NestedField.optional(10000, "category", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index e587153d78ba..e3ab9914d59b 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -1072,8 +1072,8 @@ public void testNewTableMetadataReassignmentAllIds() throws Exception { PartitionSpec expected = PartitionSpec.builderFor(metadata.schema()) .withSpecId(0) - .add(1, 1000, "x_partition", Transforms.bucket(4)) - .add(3, 1001, "z_partition", Transforms.bucket(8)) + .add(1, 10000, "x_partition", Transforms.bucket(4)) + .add(3, 10001, "z_partition", Transforms.bucket(8)) .build(); Assert.assertEquals(expected, metadata.spec()); @@ -1081,12 +1081,16 @@ public void testNewTableMetadataReassignmentAllIds() throws Exception { @Test public void testInvalidUpdatePartitionSpecForV1Table() throws Exception { - Schema schema = new Schema(Types.NestedField.required(1, "x", Types.LongType.get())); + Schema schema = + new Schema( + Types.NestedField.required(1, "x", Types.LongType.get()), + Types.NestedField.required(2, "y", Types.LongType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema) .withSpecId(5) - .add(1, 1005, "x_partition", Transforms.bucket(4)) + .add(1, 10005, "x_partition", Transforms.bucket(4)) + .add(2, 10008, "y_partition", Transforms.bucket(4)) .build(); String location = "file://tmp/db/table"; TableMetadata metadata = @@ -1127,9 +1131,9 @@ public void testBuildReplacementForV1Table() { PartitionSpec expected = PartitionSpec.builderFor(updated.schema()) .withSpecId(1) - .add(1, 1000, "x", Transforms.identity()) - .add(2, 1001, "y", Transforms.alwaysNull()) - .add(3, 1002, "z_bucket", Transforms.bucket(8)) + .add(1, 10000, "x", Transforms.identity()) + .add(2, 10001, "y", Transforms.alwaysNull()) + .add(3, 10002, "z_bucket", Transforms.bucket(8)) .build(); Assert.assertEquals( "Should reassign the partition field IDs and reuse any existing IDs for equivalent fields", @@ -1163,8 +1167,8 @@ public void testBuildReplacementForV2Table() { PartitionSpec expected = PartitionSpec.builderFor(updated.schema()) .withSpecId(1) - .add(3, 1002, "z_bucket", Transforms.bucket(8)) - .add(1, 1000, "x", Transforms.identity()) + .add(3, 10002, "z_bucket", Transforms.bucket(8)) + .add(1, 10000, "x", Transforms.identity()) .build(); Assert.assertEquals( "Should reassign the partition field IDs and reuse any existing IDs for equivalent fields", diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java index f770cd279287..329527c8e42f 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java @@ -46,7 +46,7 @@ public TestTableUpdatePartitionSpec(int formatVersion) { public void verifyInitialSpec() { PartitionSpec initialSpec = PartitionSpec.builderFor(table.schema()).bucket("data", 16).build(); Assert.assertEquals("Should use the expected initial spec", initialSpec, table.spec()); - Assert.assertEquals(1000, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10000, table.spec().lastAssignedFieldId()); Assert.assertEquals(0, table.spec().specId()); } @@ -61,7 +61,7 @@ public void testCommitUpdatedSpec() { .bucket("id", 8, "id_bucket_8") .build(); Assert.assertEquals("Should append a partition field to the spec", evolvedSpec, table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); table .updateSpec() @@ -84,11 +84,11 @@ public void testCommitUpdatedSpec() { "Should hard delete id and data buckets", PartitionSpec.builderFor(table.schema()) .withSpecId(2) - .add(2, 1002, "data_trunc_8", Transforms.truncate(8)) + .add(2, 10002, "data_trunc_8", Transforms.truncate(8)) .build(), table.spec()); - Assert.assertEquals(1002, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10002, table.spec().lastAssignedFieldId()); } @Test @@ -129,7 +129,7 @@ public void testRenameField() { .build(); Assert.assertEquals("should match evolved spec", evolvedSpec, table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); table .updateSpec() @@ -146,7 +146,7 @@ public void testRenameField() { .build(); Assert.assertEquals("should match evolved spec", evolvedSpec, table.spec()); - Assert.assertEquals(1002, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10002, table.spec().lastAssignedFieldId()); } @Test @@ -160,7 +160,7 @@ public void testRenameOnlyEvolution() { .build(); Assert.assertEquals("should match evolved spec", evolvedSpec, table.spec()); - Assert.assertEquals(1000, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10000, table.spec().lastAssignedFieldId()); } @Test @@ -180,11 +180,11 @@ public void testRemoveAndAddField() { "Should hard delete data bucket", PartitionSpec.builderFor(table.schema()) .withSpecId(1) - .add(1, 1001, "id_bucket_8", Transforms.bucket(8)) + .add(1, 10001, "id_bucket_8", Transforms.bucket(8)) .build(), table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); } @Test @@ -203,10 +203,10 @@ public void testAddAndRemoveField() { "Should remove and then add a bucket field", PartitionSpec.builderFor(table.schema()) .withSpecId(1) - .add(2, 1001, "data_bucket_6", Transforms.bucket(6)) + .add(2, 10001, "data_bucket_6", Transforms.bucket(6)) .build(), table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); } @Test @@ -221,14 +221,14 @@ public void testAddAfterLastFieldRemoved() { .build(), table.spec()); V1Assert.assertEquals( - "Should match the last assigned field id", 1000, table.spec().lastAssignedFieldId()); + "Should match the last assigned field id", 10000, table.spec().lastAssignedFieldId()); V2Assert.assertEquals( "Should add a new id bucket", PartitionSpec.builderFor(table.schema()).withSpecId(1).build(), table.spec()); V2Assert.assertEquals( - "Should match the last assigned field id", 999, table.spec().lastAssignedFieldId()); - Assert.assertEquals(1000, table.ops().current().lastAssignedPartitionId()); + "Should match the last assigned field id", 9999, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10000, table.ops().current().lastAssignedPartitionId()); table.updateSpec().addField(bucket("id", 8)).commit(); @@ -244,10 +244,10 @@ public void testAddAfterLastFieldRemoved() { "Should add a new id bucket", PartitionSpec.builderFor(table.schema()) .withSpecId(2) - .add(1, 1001, "id_bucket_8", Transforms.bucket(8)) + .add(1, 10001, "id_bucket_8", Transforms.bucket(8)) .build(), table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); - Assert.assertEquals(1001, table.ops().current().lastAssignedPartitionId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.ops().current().lastAssignedPartitionId()); } } diff --git a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java index 97210e51b5ba..400c0197fa12 100644 --- a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java @@ -213,8 +213,8 @@ public void testAddHourToDay() { Assert.assertEquals( "Should have a day and an hour time field", ImmutableList.of( - new PartitionField(2, 1000, "ts_day", Transforms.day()), - new PartitionField(2, 1001, "ts_hour", Transforms.hour())), + new PartitionField(2, 10000, "ts_day", Transforms.day()), + new PartitionField(2, 10001, "ts_hour", Transforms.hour())), byHour.fields()); } @@ -253,8 +253,8 @@ public void testRemoveIdentityByName() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("ts"), 1001, "ts_day", Transforms.day()) - .add(id("id"), 1002, "shard", Transforms.bucket(16)) + .add(id("ts"), 10001, "ts_day", Transforms.day()) + .add(id("id"), 10002, "shard", Transforms.bucket(16)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -276,8 +276,8 @@ public void testRemoveBucketByName() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("category"), 1000, "category", Transforms.identity()) - .add(id("ts"), 1001, "ts_day", Transforms.day()) + .add(id("category"), 10000, "category", Transforms.identity()) + .add(id("ts"), 10001, "ts_day", Transforms.day()) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -301,8 +301,8 @@ public void testRemoveIdentityByEquivalent() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("ts"), 1001, "ts_day", Transforms.day()) - .add(id("id"), 1002, "shard", Transforms.bucket(16)) + .add(id("ts"), 10001, "ts_day", Transforms.day()) + .add(id("id"), 10002, "shard", Transforms.bucket(16)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -324,8 +324,8 @@ public void testRemoveDayByEquivalent() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("category"), 1000, "category", Transforms.identity()) - .add(id("id"), 1002, "shard", Transforms.bucket(16)) + .add(id("category"), 10000, "category", Transforms.identity()) + .add(id("id"), 10002, "shard", Transforms.bucket(16)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -387,9 +387,9 @@ public void testMultipleChanges() { PartitionSpec v2Expected = PartitionSpec.builderFor(SCHEMA) - .add(id("category"), 1000, "category", Transforms.identity()) - .add(id("id"), 1002, "id_bucket", Transforms.bucket(16)) - .add(id("data"), 1003, "prefix", Transforms.truncate(4)) + .add(id("category"), 10000, "category", Transforms.identity()) + .add(id("id"), 10002, "id_bucket", Transforms.bucket(16)) + .add(id("data"), 10003, "prefix", Transforms.truncate(4)) .build(); V2Assert.assertEquals("Should match expected spec", v2Expected, updated); @@ -699,7 +699,7 @@ public void testRemoveAndUpdateWithDifferentTransformation() { Assert.assertEquals("Should match expected spec field size", 2, updated.fields().size()); Assert.assertEquals( "Should match expected field name", - "ts_transformed_1000", + "ts_transformed_10000", updated.fields().get(0).name()); Assert.assertEquals( "Should match expected field name", "ts_transformed", updated.fields().get(1).name()); diff --git a/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateTableRequest.java b/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateTableRequest.java index 9176e8ad4d24..e83e0a8230fc 100644 --- a/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateTableRequest.java +++ b/core/src/test/java/org/apache/iceberg/rest/requests/TestCreateTableRequest.java @@ -61,7 +61,7 @@ public void testRoundTripSerDe() throws JsonProcessingException { "{\"name\":\"test_tbl\",\"location\":\"file://tmp/location/\",\"schema\":{\"type\":\"struct\"," + "\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":true,\"type\":\"int\"}," + "{\"id\":2,\"name\":\"data\",\"required\":false,\"type\":\"string\"}]},\"partition-spec\":{\"spec-id\":0," - + "\"fields\":[{\"name\":\"id_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":1000}]}," + + "\"fields\":[{\"name\":\"id_bucket\",\"transform\":\"bucket[16]\",\"source-id\":1,\"field-id\":10000}]}," + "\"write-order\":{\"order-id\":1,\"fields\":" + "[{\"transform\":\"identity\",\"source-id\":2,\"direction\":\"asc\",\"null-order\":\"nulls-last\"}]}," + "\"properties\":{\"owner\":\"Hank\"},\"stage-create\":false}"; diff --git a/core/src/test/resources/TableMetadataStatisticsFiles.json b/core/src/test/resources/TableMetadataStatisticsFiles.json index 150e2891efc0..f89dc41a2da2 100644 --- a/core/src/test/resources/TableMetadataStatisticsFiles.json +++ b/core/src/test/resources/TableMetadataStatisticsFiles.json @@ -27,7 +27,7 @@ "fields": [] } ], - "last-partition-id": 1000, + "last-partition-id": 10000, "default-sort-order-id": 0, "sort-orders": [ { diff --git a/core/src/test/resources/TableMetadataV2MissingSortOrder.json b/core/src/test/resources/TableMetadataV2MissingSortOrder.json index fbbcf415d264..765e32b1d6f2 100644 --- a/core/src/test/resources/TableMetadataV2MissingSortOrder.json +++ b/core/src/test/resources/TableMetadataV2MissingSortOrder.json @@ -40,12 +40,12 @@ "name": "x", "transform": "identity", "source-id": 1, - "field-id": 1000 + "field-id": 10000 } ] } ], - "last-partition-id": 1000, + "last-partition-id": 10000, "properties": {}, "current-snapshot-id": -1, "snapshots": [], diff --git a/core/src/test/resources/TableMetadataV2Valid.json b/core/src/test/resources/TableMetadataV2Valid.json index 0dc89de58c42..1c7bee4c1aa7 100644 --- a/core/src/test/resources/TableMetadataV2Valid.json +++ b/core/src/test/resources/TableMetadataV2Valid.json @@ -63,7 +63,7 @@ ] } ], - "last-partition-id": 1000, + "last-partition-id": 10000, "default-sort-order-id": 3, "sort-orders": [ { diff --git a/format/spec.md b/format/spec.md index 56abfc6a73b7..148a0a338e7b 100644 --- a/format/spec.md +++ b/format/spec.md @@ -1039,7 +1039,7 @@ Partition specs are serialized as a JSON object with the following fields: |Field|JSON representation|Example| |--- |--- |--- | |**`spec-id`**|`JSON int`|`0`| -|**`fields`**|`JSON list: [`
  `,`
  `...`
`]`|`[ {`
  `"source-id": 4,`
  `"field-id": 1000,`
  `"name": "ts_day",`
  `"transform": "day"`
`}, {`
  `"source-id": 1,`
  `"field-id": 1001,`
  `"name": "id_bucket",`
  `"transform": "bucket[16]"`
`} ]`| +|**`fields`**|`JSON list: [`
  `,`
  `...`
`]`|`[ {`
  `"source-id": 4,`
  `"field-id": 10000,`
  `"name": "ts_day",`
  `"transform": "day"`
`}, {`
  `"source-id": 1,`
  `"field-id": 1001,`
  `"name": "id_bucket",`
  `"transform": "bucket[16]"`
`} ]`| Each partition field in the fields list is stored as an object. See the table for more detail: @@ -1052,7 +1052,7 @@ Each partition field in the fields list is stored as an object. See the table fo |**`month`**|`JSON string: "month"`|`"month"`| |**`day`**|`JSON string: "day"`|`"day"`| |**`hour`**|`JSON string: "hour"`|`"hour"`| -|**`Partition Field`**|`JSON object: {`
  `"source-id": ,`
  `"field-id": ,`
  `"name": ,`
  `"transform": `
`}`|`{`
  `"source-id": 1,`
  `"field-id": 1000,`
  `"name": "id_bucket",`
  `"transform": "bucket[16]"`
`}`| +|**`Partition Field`**|`JSON object: {`
  `"source-id": ,`
  `"field-id": ,`
  `"name": ,`
  `"transform": `
`}`|`{`
  `"source-id": 1,`
  `"field-id": 10000,`
  `"name": "id_bucket",`
  `"transform": "bucket[16]"`
`}`| In some cases partition specs are stored using only the field list instead of the object format that includes the spec ID, like the deprecated `partition-spec` field in table metadata. The object format should be used unless otherwise noted in this spec. @@ -1098,7 +1098,7 @@ Table metadata is serialized as a JSON object according to the following table. |**`partition-spec`**|`JSON partition fields (list)`|`See above, read partition-specs instead`| |**`partition-specs`**|`JSON partition specs (list of objects)`|`See above`| |**`default-spec-id`**|`JSON int`|`0`| -|**`last-partition-id`**|`JSON int`|`1000`| +|**`last-partition-id`**|`JSON int`|`10000`| |**`properties`**|`JSON object: {`
  `"": "",`
  `...`
`}`|`{`
  `"write.format.default": "avro",`
  `"commit.retry.num-retries": "4"`
`}`| |**`current-snapshot-id`**|`JSON long`|`3051729675574597004`| |**`snapshots`**|`JSON list of objects: [ {`
  `"snapshot-id": ,`
  `"timestamp-ms": ,`
  `"summary": {`
    `"operation": ,`
    `... },`
  `"manifest-list": "",`
  `"schema-id": ""`
  `},`
  `...`
`]`|`[ {`
  `"snapshot-id": 3051729675574597004,`
  `"timestamp-ms": 1515100955770,`
  `"summary": {`
    `"operation": "append"`
  `},`
  `"manifest-list": "s3://b/wh/.../s1.avro"`
  `"schema-id": 0`
`} ]`| diff --git a/python/mkdocs/docs/index.md b/python/mkdocs/docs/index.md index 53795a8e07cd..0277ef8ea1e1 100644 --- a/python/mkdocs/docs/index.md +++ b/python/mkdocs/docs/index.md @@ -454,7 +454,7 @@ from pyiceberg.transforms import DayTransform partition_spec = PartitionSpec( PartitionField( - source_id=1, field_id=1000, transform=DayTransform(), name="datetime_day" + source_id=1, field_id=10000, transform=DayTransform(), name="datetime_day" ) ) @@ -499,10 +499,10 @@ Table( current_schema_id=1, partition_specs=[ PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=DayTransform(), name='datetime_day'),)) + PartitionField(source_id=1, field_id=10000, transform=DayTransform(), name='datetime_day'),)) ], default_spec_id=0, - last_partition_id=1000, + last_partition_id=10000, properties={}, current_snapshot_id=None, snapshots=[], diff --git a/python/tests/catalog/test_hive.py b/python/tests/catalog/test_hive.py index f9a47e516ca3..6e2ba1659fbd 100644 --- a/python/tests/catalog/test_hive.py +++ b/python/tests/catalog/test_hive.py @@ -334,7 +334,7 @@ def test_load_table(hive_table: HiveTable) -> None: PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x"), spec_id=0) ], default_spec_id=0, - last_partition_id=1000, + last_partition_id=10000, properties={"read.split.target.size": "134217728"}, current_snapshot_id=3055729675574597004, snapshots=[ diff --git a/python/tests/catalog/test_rest.py b/python/tests/catalog/test_rest.py index 9c1700d97fc4..aeb3c3293ae3 100644 --- a/python/tests/catalog/test_rest.py +++ b/python/tests/catalog/test_rest.py @@ -574,7 +574,7 @@ def test_create_table_200(rest_mock: Mocker, table_schema_simple: Schema) -> Non schema=table_schema_simple, location=None, partition_spec=PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + PartitionField(source_id=1, field_id=10000, transform=TruncateTransform(width=3), name="id"), spec_id=1 ), sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), properties={"owner": "fokko"}, @@ -646,7 +646,7 @@ def test_create_table_409(rest_mock: Mocker, table_schema_simple: Schema) -> Non schema=table_schema_simple, location=None, partition_spec=PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id") + PartitionField(source_id=1, field_id=10000, transform=TruncateTransform(width=3), name="id") ), sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), properties={"owner": "fokko"}, diff --git a/python/tests/cli/test_console.py b/python/tests/cli/test_console.py index 4e0c3e62cd07..dcb25c979ecf 100644 --- a/python/tests/cli/test_console.py +++ b/python/tests/cli/test_console.py @@ -551,7 +551,7 @@ def test_json_describe_table(_: MockCatalog) -> None: assert result.exit_code == 0 assert ( result.output - == """{"identifier": ["default", "foo"], "metadata_location": "s3://tmp/", "metadata": {"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}}\n""" + == """{"identifier": ["default", "foo"], "metadata_location": "s3://tmp/", "metadata": {"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 10000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}}\n""" ) diff --git a/python/tests/conftest.py b/python/tests/conftest.py index bab7b87bcdc5..23ae5c81f4c7 100644 --- a/python/tests/conftest.py +++ b/python/tests/conftest.py @@ -233,7 +233,7 @@ def all_avro_types() -> Dict[str, Any]: "element-id": 1001, }, ], - "field-id": 1000, + "field-id": 10000, }, { "name": "map_with_longs", @@ -244,7 +244,7 @@ def all_avro_types() -> Dict[str, Any]: "key-id": 1101, "value-id": 1102, }, - "field-id": 1000, + "field-id": 10000, }, ], } @@ -273,7 +273,7 @@ def all_avro_types() -> Dict[str, Any]: ], "default-spec-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, + "last-partition-id": 10000, "default-sort-order-id": 3, "sort-orders": [ { diff --git a/python/tests/expressions/test_projection.py b/python/tests/expressions/test_projection.py index 4d0c2c13463a..72ea65ca1822 100644 --- a/python/tests/expressions/test_projection.py +++ b/python/tests/expressions/test_projection.py @@ -70,38 +70,38 @@ def empty_spec() -> PartitionSpec: @pytest.fixture def id_spec() -> PartitionSpec: - return PartitionSpec(PartitionField(1, 1000, IdentityTransform(), "id_part")) + return PartitionSpec(PartitionField(1, 10000, IdentityTransform(), "id_part")) @pytest.fixture def bucket_spec() -> PartitionSpec: - return PartitionSpec(PartitionField(2, 1000, BucketTransform(16), "data_bucket")) + return PartitionSpec(PartitionField(2, 10000, BucketTransform(16), "data_bucket")) @pytest.fixture def day_spec() -> PartitionSpec: - return PartitionSpec(PartitionField(4, 1000, DayTransform(), "date"), PartitionField(3, 1000, DayTransform(), "ddate")) + return PartitionSpec(PartitionField(4, 10000, DayTransform(), "date"), PartitionField(3, 10000, DayTransform(), "ddate")) @pytest.fixture def hour_spec() -> PartitionSpec: - return PartitionSpec(PartitionField(4, 1000, HourTransform(), "hour")) + return PartitionSpec(PartitionField(4, 10000, HourTransform(), "hour")) @pytest.fixture def truncate_str_spec() -> PartitionSpec: - return PartitionSpec(PartitionField(2, 1000, TruncateTransform(2), "data_trunc")) + return PartitionSpec(PartitionField(2, 10000, TruncateTransform(2), "data_trunc")) @pytest.fixture def truncate_int_spec() -> PartitionSpec: - return PartitionSpec(PartitionField(1, 1000, TruncateTransform(10), "id_trunc")) + return PartitionSpec(PartitionField(1, 10000, TruncateTransform(10), "id_trunc")) @pytest.fixture def id_and_bucket_spec() -> PartitionSpec: return PartitionSpec( - PartitionField(1, 1000, IdentityTransform(), "id_part"), PartitionField(2, 1001, BucketTransform(16), "data_bucket") + PartitionField(1, 10000, IdentityTransform(), "id_part"), PartitionField(2, 1001, BucketTransform(16), "data_bucket") ) diff --git a/python/tests/table/test_metadata.py b/python/tests/table/test_metadata.py index 9f46ca673792..1f57333188e8 100644 --- a/python/tests/table/test_metadata.py +++ b/python/tests/table/test_metadata.py @@ -63,7 +63,7 @@ {"id": 3, "name": "z", "required": True, "type": "long"}, ], }, - "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}], "properties": {}, "current-snapshot-id": -1, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], @@ -101,7 +101,7 @@ def test_v2_metadata_parsing(example_table_metadata_v2: Dict[str, Any]) -> None: assert table_metadata.current_schema_id == 1 assert table_metadata.partition_specs[0].spec_id == 0 assert table_metadata.default_spec_id == 0 - assert table_metadata.last_partition_id == 1000 + assert table_metadata.last_partition_id == 10000 assert table_metadata.properties["read.split.target.size"] == "134217728" assert table_metadata.current_snapshot_id == 3055729675574597004 assert table_metadata.snapshots[0].snapshot_id == 3051729675574597004 @@ -134,10 +134,10 @@ def test_v1_metadata_parsing_directly() -> None: assert table_metadata.schemas[0].schema_id == 0 assert table_metadata.current_schema_id == 0 assert table_metadata.partition_specs == [ - PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x")) + PartitionSpec(PartitionField(source_id=1, field_id=10000, transform=IdentityTransform(), name="x")) ] assert table_metadata.default_spec_id == 0 - assert table_metadata.last_partition_id == 1000 + assert table_metadata.last_partition_id == 10000 assert table_metadata.current_snapshot_id is None assert table_metadata.default_sort_order_id == 0 @@ -177,13 +177,13 @@ def test_updating_metadata(example_table_metadata_v2: Dict[str, Any]) -> None: def test_serialize_v1() -> None: table_metadata = TableMetadataV1(**EXAMPLE_TABLE_METADATA_V1) table_metadata_json = table_metadata.json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "d20125c8-7284-442c-9aea-15fee620737c", "last-updated-ms": 1602638573874, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}], "current-schema-id": 0, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 10000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 10000, "properties": {}, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], "snapshot-log": [], "metadata-log": [], "sort-orders": [{"order-id": 0, "fields": []}], "default-sort-order-id": 0, "refs": {}, "format-version": 1, "schema": {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}]}""" assert table_metadata_json == expected def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]) -> None: table_metadata = TableMetadataV2(**example_table_metadata_v2).json() - expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 1000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" + expected = """{"location": "s3://bucket/test/location", "table-uuid": "9c12d441-03fe-4693-9a96-a0705ddf69c1", "last-updated-ms": 1602638573590, "last-column-id": 3, "schemas": [{"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}], "schema-id": 0, "identifier-field-ids": []}, {"type": "struct", "fields": [{"id": 1, "name": "x", "type": "long", "required": true}, {"id": 2, "name": "y", "type": "long", "required": true, "doc": "comment"}, {"id": 3, "name": "z", "type": "long", "required": true}], "schema-id": 1, "identifier-field-ids": [1, 2]}], "current-schema-id": 1, "partition-specs": [{"spec-id": 0, "fields": [{"source-id": 1, "field-id": 1000, "transform": "identity", "name": "x"}]}], "default-spec-id": 0, "last-partition-id": 10000, "properties": {"read.split.target.size": "134217728"}, "current-snapshot-id": 3055729675574597004, "snapshots": [{"snapshot-id": 3051729675574597004, "sequence-number": 0, "timestamp-ms": 1515100955770, "manifest-list": "s3://a/b/1.avro", "summary": {"operation": "append"}}, {"snapshot-id": 3055729675574597004, "parent-snapshot-id": 3051729675574597004, "sequence-number": 1, "timestamp-ms": 1555100955770, "manifest-list": "s3://a/b/2.avro", "summary": {"operation": "append"}, "schema-id": 1}], "snapshot-log": [{"snapshot-id": "3051729675574597004", "timestamp-ms": 1515100955770}, {"snapshot-id": "3055729675574597004", "timestamp-ms": 1555100955770}], "metadata-log": [{"metadata-file": "s3://bucket/.../v1.json", "timestamp-ms": 1515100}], "sort-orders": [{"order-id": 3, "fields": [{"source-id": 2, "transform": "identity", "direction": "asc", "null-order": "nulls-first"}, {"source-id": 3, "transform": "bucket[4]", "direction": "desc", "null-order": "nulls-last"}]}], "default-sort-order-id": 3, "refs": {"test": {"snapshot-id": 3051729675574597004, "type": "tag", "max-ref-age-ms": 10000000}, "main": {"snapshot-id": 3055729675574597004, "type": "branch"}}, "format-version": 2, "last-sequence-number": 34}""" assert table_metadata == expected @@ -202,7 +202,7 @@ def test_migrate_v1_partition_specs() -> None: assert len(table_metadata.partition_specs) == 1 # Spec ID gets added automatically assert table_metadata.partition_specs == [ - PartitionSpec(PartitionField(source_id=1, field_id=1000, transform=IdentityTransform(), name="x")), + PartitionSpec(PartitionField(source_id=1, field_id=10000, transform=IdentityTransform(), name="x")), ] @@ -222,7 +222,7 @@ def test_invalid_format_version() -> None: {"id": 3, "name": "z", "required": True, "type": "long"}, ], }, - "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}], "properties": {}, "current-snapshot-id": -1, "snapshots": [], @@ -258,8 +258,10 @@ def test_current_schema_not_found() -> None: ], "current-schema-id": 2, "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, + "partition-specs": [ + {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}]} + ], + "last-partition-id": 10000, "default-sort-order-id": 0, "properties": {}, "current-snapshot-id": -1, @@ -305,8 +307,10 @@ def test_sort_order_not_found() -> None: ], "current-schema-id": 0, "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, + "partition-specs": [ + {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}]} + ], + "last-partition-id": 10000, "properties": {}, "current-snapshot-id": -1, "snapshots": [], @@ -343,8 +347,10 @@ def test_sort_order_unsorted() -> None: "sort-orders": [], "current-schema-id": 0, "default-spec-id": 0, - "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, + "partition-specs": [ + {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}]} + ], + "last-partition-id": 10000, "properties": {}, "current-snapshot-id": -1, "snapshots": [], @@ -381,8 +387,10 @@ def test_invalid_partition_spec() -> None: "sort-orders": [], "default-sort-order-id": 0, "default-spec-id": 1, - "partition-specs": [{"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}]}], - "last-partition-id": 1000, + "partition-specs": [ + {"spec-id": 0, "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}]} + ], + "last-partition-id": 10000, } with pytest.raises(ValidationError) as exc_info: TableMetadataUtil.parse_obj(table_metadata_spec_not_found) @@ -449,7 +457,7 @@ def test_v1_write_metadata_for_v2() -> None: {"id": 3, "name": "z", "required": True, "type": "long"}, ], }, - "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "partition-spec": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}], "properties": {}, "current-snapshot-id": -1, "snapshots": [{"snapshot-id": 1925, "timestamp-ms": 1602638573822}], @@ -477,11 +485,11 @@ def test_v1_write_metadata_for_v2() -> None: assert metadata_v2["partition-specs"] == [ { "spec-id": 0, - "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 1000}], + "fields": [{"name": "x", "transform": "identity", "source-id": 1, "field-id": 10000}], } ] assert metadata_v2["default-spec-id"] == 0 - assert metadata_v2["last-partition-id"] == 1000 + assert metadata_v2["last-partition-id"] == 10000 assert metadata_v2["sort-orders"] == [{"order-id": 0, "fields": []}] assert metadata_v2["default-sort-order-id"] == 0 # Deprecated fields diff --git a/python/tests/table/test_partitioning.py b/python/tests/table/test_partitioning.py index b2d78c49ba38..1cb2d6f94c62 100644 --- a/python/tests/table/test_partitioning.py +++ b/python/tests/table/test_partitioning.py @@ -27,17 +27,17 @@ def test_partition_field_init() -> None: bucket_transform = BucketTransform(100) # type: ignore - partition_field = PartitionField(3, 1000, bucket_transform, "id") + partition_field = PartitionField(3, 10000, bucket_transform, "id") assert partition_field.source_id == 3 - assert partition_field.field_id == 1000 + assert partition_field.field_id == 10000 assert partition_field.transform == bucket_transform assert partition_field.name == "id" assert partition_field == partition_field - assert str(partition_field) == "1000: id: bucket[100](3)" + assert str(partition_field) == "10000: id: bucket[100](3)" assert ( repr(partition_field) - == "PartitionField(source_id=3, field_id=1000, transform=BucketTransform(num_buckets=100), name='id')" + == "PartitionField(source_id=3, field_id=10000, transform=BucketTransform(num_buckets=100), name='id')" ) @@ -48,7 +48,7 @@ def test_unpartitioned_partition_spec_repr() -> None: def test_partition_spec_init() -> None: bucket_transform: BucketTransform = BucketTransform(4) # type: ignore - id_field1 = PartitionField(3, 1001, bucket_transform, "id") + id_field1 = PartitionField(3, 10001, bucket_transform, "id") partition_spec1 = PartitionSpec(id_field1) assert partition_spec1.spec_id == 0 @@ -57,7 +57,7 @@ def test_partition_spec_init() -> None: assert str(partition_spec1) == f"[\n {str(id_field1)}\n]" assert not partition_spec1.is_unpartitioned() # only differ by PartitionField field_id - id_field2 = PartitionField(3, 1002, bucket_transform, "id") + id_field2 = PartitionField(3, 10002, bucket_transform, "id") partition_spec2 = PartitionSpec(id_field2) assert partition_spec1 != partition_spec2 assert partition_spec1.compatible_with(partition_spec2) @@ -89,36 +89,36 @@ def test_serialize_unpartitioned_spec() -> None: def test_serialize_partition_spec() -> None: partitioned = PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), - PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), + PartitionField(source_id=1, field_id=10000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=10001, transform=BucketTransform(num_buckets=25), name="int_bucket"), spec_id=3, ) assert ( partitioned.json() - == """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 1000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 1001, "transform": "bucket[25]", "name": "int_bucket"}]}""" + == """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 10000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 10001, "transform": "bucket[25]", "name": "int_bucket"}]}""" ) def test_deserialize_partition_spec() -> None: - json_partition_spec = """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 1000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 1001, "transform": "bucket[25]", "name": "int_bucket"}]}""" + json_partition_spec = """{"spec-id": 3, "fields": [{"source-id": 1, "field-id": 10000, "transform": "truncate[19]", "name": "str_truncate"}, {"source-id": 2, "field-id": 10001, "transform": "bucket[25]", "name": "int_bucket"}]}""" spec = PartitionSpec.parse_raw(json_partition_spec) assert spec == PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), - PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), + PartitionField(source_id=1, field_id=10000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=10001, transform=BucketTransform(num_buckets=25), name="int_bucket"), spec_id=3, ) def test_partition_type(table_schema_simple: Schema) -> None: spec = PartitionSpec( - PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=19), name="str_truncate"), - PartitionField(source_id=2, field_id=1001, transform=BucketTransform(num_buckets=25), name="int_bucket"), + PartitionField(source_id=1, field_id=10000, transform=TruncateTransform(width=19), name="str_truncate"), + PartitionField(source_id=2, field_id=10001, transform=BucketTransform(num_buckets=25), name="int_bucket"), spec_id=3, ) assert spec.partition_type(table_schema_simple) == StructType( - NestedField(field_id=1000, name="str_truncate", field_type=StringType(), required=False), - NestedField(field_id=1001, name="int_bucket", field_type=IntegerType(), required=False), + NestedField(field_id=10000, name="str_truncate", field_type=StringType(), required=False), + NestedField(field_id=10001, name="int_bucket", field_type=IntegerType(), required=False), ) diff --git a/python_legacy/tests/core/conftest.py b/python_legacy/tests/core/conftest.py index 933743e76454..2c72de7808a7 100644 --- a/python_legacy/tests/core/conftest.py +++ b/python_legacy/tests/core/conftest.py @@ -289,7 +289,7 @@ def base_scan_partition(base_scan_schema, request): if request.param == "none": spec = PartitionSpec.unpartitioned() else: - spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() + spec = PartitionSpecBuilder(base_scan_schema).add(1, 10000, "id", "identity").build() return spec diff --git a/python_legacy/tests/hive/conftest.py b/python_legacy/tests/hive/conftest.py index 44c60d74d469..a6e2370eba80 100644 --- a/python_legacy/tests/hive/conftest.py +++ b/python_legacy/tests/hive/conftest.py @@ -33,5 +33,5 @@ def base_scan_partition(base_scan_schema, request): if request.param == "none": spec = PartitionSpec.unpartitioned() else: - spec = PartitionSpecBuilder(base_scan_schema).add(1, 1000, "id", "identity").build() + spec = PartitionSpecBuilder(base_scan_schema).add(1, 10000, "id", "identity").build() return spec diff --git a/spark/v2.4/spark/src/test/java/org/apache/iceberg/examples/README.md b/spark/v2.4/spark/src/test/java/org/apache/iceberg/examples/README.md index eca410dfeabf..4d50475f8aad 100644 --- a/spark/v2.4/spark/src/test/java/org/apache/iceberg/examples/README.md +++ b/spark/v2.4/spark/src/test/java/org/apache/iceberg/examples/README.md @@ -111,7 +111,7 @@ The metadata file will contain things like the table location, the schema and th ... } ] } ], - "last-partition-id" : 1000, + "last-partition-id" : 10000, "properties" : { }, "current-snapshot-id" : -1, "snapshots" : [ ], diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java index 2581c0fd3c56..c58d955b7b5d 100644 --- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java +++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java @@ -388,7 +388,7 @@ public void testCreateRTASWithPartitionSpecChanging() { PartitionSpec expectedSpec = PartitionSpec.builderFor(expectedSchema) - .alwaysNull("part", "part_1000") + .alwaysNull("part", "part_10000") .identity("part") .identity("id") .withSpecId(2) // The Spec is new diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 82c9a58e33ea..f9a5970c5346 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -495,7 +495,7 @@ public void testPartitionsTableSwitchFields() throws Exception { row(null, "c3", "d3"), row("d1", "c1", null), row("d2", "c2", null)), - "STRUCT", + "STRUCT", PARTITIONS); } else { // In V2 re-adding a former partition field that was part of an older spec will not change its @@ -606,13 +606,13 @@ public void testPartitionTableFilterSwitchFields() throws Exception { assertPartitions( ImmutableList.of(row(null, "c2", null), row(null, "c2", "d2"), row("d2", "c2", null)), - "STRUCT", + "STRUCT", PARTITIONS, "partition.category = 'c2'"); assertPartitions( ImmutableList.of(row(null, "c1", "d1")), - "STRUCT", + "STRUCT", PARTITIONS, "partition.data = 'd1'"); } diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java index 2581c0fd3c56..c58d955b7b5d 100644 --- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java +++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java @@ -388,7 +388,7 @@ public void testCreateRTASWithPartitionSpecChanging() { PartitionSpec expectedSpec = PartitionSpec.builderFor(expectedSchema) - .alwaysNull("part", "part_1000") + .alwaysNull("part", "part_10000") .identity("part") .identity("id") .withSpecId(2) // The Spec is new diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index 82c9a58e33ea..f9a5970c5346 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -495,7 +495,7 @@ public void testPartitionsTableSwitchFields() throws Exception { row(null, "c3", "d3"), row("d1", "c1", null), row("d2", "c2", null)), - "STRUCT", + "STRUCT", PARTITIONS); } else { // In V2 re-adding a former partition field that was part of an older spec will not change its @@ -606,13 +606,13 @@ public void testPartitionTableFilterSwitchFields() throws Exception { assertPartitions( ImmutableList.of(row(null, "c2", null), row(null, "c2", "d2"), row("d2", "c2", null)), - "STRUCT", + "STRUCT", PARTITIONS, "partition.category = 'c2'"); assertPartitions( ImmutableList.of(row(null, "c1", "d1")), - "STRUCT", + "STRUCT", PARTITIONS, "partition.data = 'd1'"); } diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java index 2581c0fd3c56..c58d955b7b5d 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java @@ -388,7 +388,7 @@ public void testCreateRTASWithPartitionSpecChanging() { PartitionSpec expectedSpec = PartitionSpec.builderFor(expectedSchema) - .alwaysNull("part", "part_1000") + .alwaysNull("part", "part_10000") .identity("part") .identity("id") .withSpecId(2) // The Spec is new From 9233d129e52732d07edea5fd293bd42255a38c00 Mon Sep 17 00:00:00 2001 From: Ilya Cherkasov Date: Sat, 9 Mar 2024 11:00:48 +0200 Subject: [PATCH 2/6] Remove odd checks --- .../org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java | 1 - .../org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java | 1 - .../org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java | 1 - 3 files changed, 3 deletions(-) diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java index c58d955b7b5d..1c08b1fd5ae7 100644 --- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java +++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java @@ -388,7 +388,6 @@ public void testCreateRTASWithPartitionSpecChanging() { PartitionSpec expectedSpec = PartitionSpec.builderFor(expectedSchema) - .alwaysNull("part", "part_10000") .identity("part") .identity("id") .withSpecId(2) // The Spec is new diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java index 1eb5cafb3bee..74906241fa95 100644 --- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java +++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java @@ -402,7 +402,6 @@ public void testCreateRTASWithPartitionSpecChanging() { PartitionSpec expectedSpec = PartitionSpec.builderFor(expectedSchema) - .alwaysNull("part", "part_10000") .identity("part") .identity("id") .withSpecId(2) // The Spec is new diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java index 4a83df80a8d5..4098a155be0d 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java @@ -429,7 +429,6 @@ public void testCreateRTASWithPartitionSpecChanging() { PartitionSpec expectedSpec = PartitionSpec.builderFor(expectedSchema) - .alwaysNull("part", "part_10000") .identity("part") .identity("id") .withSpecId(2) // The Spec is new From d39864719d525e6401e69da12c67d6f68d85625e Mon Sep 17 00:00:00 2001 From: Ilya Cherkasov Date: Sat, 9 Mar 2024 11:16:24 +0200 Subject: [PATCH 3/6] Removes odd line from spec --- format/spec.md | 1 - 1 file changed, 1 deletion(-) diff --git a/format/spec.md b/format/spec.md index 8005065a9e85..400bf7b06170 100644 --- a/format/spec.md +++ b/format/spec.md @@ -1139,7 +1139,6 @@ Supported partition transforms are listed below. |**`month`**|`JSON string: "month"`|`"month"`| |**`day`**|`JSON string: "day"`|`"day"`| |**`hour`**|`JSON string: "hour"`|`"hour"`| -|**`Partition Field`**|`JSON object: {`
  `"source-id": ,`
  `"field-id": ,`
  `"name": ,`
  `"transform": `
`}`|`{`
  `"source-id": 1,`
  `"field-id": 10000,`
  `"name": "id_bucket",`
  `"transform": "bucket[16]"`
`}`| In some cases partition specs are stored using only the field list instead of the object format that includes the spec ID, like the deprecated `partition-spec` field in table metadata. The object format should be used unless otherwise noted in this spec. From 75ef242d7f18eff512743da1ab9784786411b353 Mon Sep 17 00:00:00 2001 From: Ilya Cherkasov Date: Sat, 9 Mar 2024 11:20:18 +0200 Subject: [PATCH 4/6] fixing tests --- .../org/apache/iceberg/TestTableUpdatePartitionSpec.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java index 2a9b681f4e87..5a008bbe9b5a 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java +++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java @@ -202,7 +202,7 @@ public void testRemoveAndAddYearField() { .build(); Assert.assertEquals("should match evolved spec", evolvedSpec, table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); table.updateSpec().removeField("year_field_year").addField(year("year_field")).commit(); @@ -220,11 +220,11 @@ public void testRemoveAndAddYearField() { PartitionSpec.builderFor(table.schema()) .withSpecId(1) .bucket("data", 16) - .add(3, 1001, "year_field_year", Transforms.year()) + .add(3, 10001, "year_field_year", Transforms.year()) .build(), table.spec()); - Assert.assertEquals(1001, table.spec().lastAssignedFieldId()); + Assert.assertEquals(10001, table.spec().lastAssignedFieldId()); } @Test From 3b61f0b4265448206d0e8929cf698374830182e6 Mon Sep 17 00:00:00 2001 From: Ilya Cherkasov Date: Sat, 9 Mar 2024 15:26:21 +0200 Subject: [PATCH 5/6] fixing tests --- .../java/org/apache/iceberg/TestContentFileParser.java | 8 ++++---- .../java/org/apache/iceberg/TestFileScanTaskParser.java | 8 ++++---- .../java/org/apache/iceberg/TestMetadataTableScans.java | 2 +- .../test/java/org/apache/iceberg/TestPartitioning.java | 2 +- .../source/TestMetadataTablesWithPartitionEvolution.java | 6 +++--- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java index 4fda388159ec..e2ecfab02a4e 100644 --- a/core/src/test/java/org/apache/iceberg/TestContentFileParser.java +++ b/core/src/test/java/org/apache/iceberg/TestContentFileParser.java @@ -126,7 +126,7 @@ private static String dataFileJsonWithRequiredOnly(PartitionSpec spec) { + "\"partition\":{},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}"; } else { return "{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\"," - + "\"partition\":{\"1000\":1},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}"; + + "\"partition\":{\"10000\":1},\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}"; } } @@ -144,7 +144,7 @@ private static String dataFileJsonWithAllOptional(PartitionSpec spec) { + "\"split-offsets\":[128,256],\"sort-order-id\":1}"; } else { return "{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":1},\"file-size-in-bytes\":350,\"record-count\":10," + + "\"file-format\":\"PARQUET\",\"partition\":{\"10000\":1},\"file-size-in-bytes\":350,\"record-count\":10," + "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]}," + "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]}," + "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]}," @@ -281,7 +281,7 @@ private static String deleteFileJsonWithRequiredOnly(PartitionSpec spec) { + "\"file-format\":\"PARQUET\",\"partition\":{},\"file-size-in-bytes\":1234,\"record-count\":9}"; } else { return "{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":9},\"file-size-in-bytes\":1234,\"record-count\":9}"; + + "\"file-format\":\"PARQUET\",\"partition\":{\"10000\":9},\"file-size-in-bytes\":1234,\"record-count\":9}"; } } @@ -299,7 +299,7 @@ private static String deleteFileJsonWithAllOptional(PartitionSpec spec) { + "\"split-offsets\":[128],\"equality-ids\":[3],\"sort-order-id\":1}"; } else { return "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":9},\"file-size-in-bytes\":1234,\"record-count\":10," + + "\"file-format\":\"PARQUET\",\"partition\":{\"10000\":9},\"file-size-in-bytes\":1234,\"record-count\":10," + "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]}," + "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]}," + "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]}," diff --git a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java index 42785c7a778e..46d827397e75 100644 --- a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java +++ b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java @@ -70,16 +70,16 @@ private String expectedFileScanTaskJson() { + "{\"id\":3,\"name\":\"id\",\"required\":true,\"type\":\"int\"}," + "{\"id\":4,\"name\":\"data\",\"required\":true,\"type\":\"string\"}]}," + "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\"," - + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]}," + + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":10000}]}," + "\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0}," + + "\"file-format\":\"PARQUET\",\"partition\":{\"10000\":0}," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"start\":0,\"length\":10," + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," - + "\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}," + + "\"partition\":{\"10000\":0},\"file-size-in-bytes\":10,\"record-count\":1}," + "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},\"file-size-in-bytes\":10," + + "\"file-format\":\"PARQUET\",\"partition\":{\"10000\":0},\"file-size-in-bytes\":10," + "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}"; } diff --git a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java index 70944cdac67e..1e4aeb11ed50 100644 --- a/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java +++ b/core/src/test/java/org/apache/iceberg/TestMetadataTableScans.java @@ -577,7 +577,7 @@ public void testFilesTableScanWithDroppedPartition() throws IOException { Types.NestedField.optional(10001, "data_bucket_16", Types.IntegerType.get()), Types.NestedField.optional(10002, "data_trunc_2", Types.StringType.get())); assertThat(actualType).as("Partition type must match").isEqualTo(expectedType); - Accessor accessor = schema.accessorForField(1000); + Accessor accessor = schema.accessorForField(10000); try (CloseableIterable tasks = scan.planFiles()) { Set results = diff --git a/core/src/test/java/org/apache/iceberg/TestPartitioning.java b/core/src/test/java/org/apache/iceberg/TestPartitioning.java index 48b681d55db5..1899c4ba70f7 100644 --- a/core/src/test/java/org/apache/iceberg/TestPartitioning.java +++ b/core/src/test/java/org/apache/iceberg/TestPartitioning.java @@ -361,7 +361,7 @@ public void testGroupingKeyTypeWithProjectedSchema() { Schema projectedSchema = table.schema().select("id", "data"); StructType expectedType = - StructType.of(NestedField.optional(1001, "data", Types.StringType.get())); + StructType.of(NestedField.optional(10001, "data", Types.StringType.get())); StructType actualType = Partitioning.groupingKeyType(projectedSchema, table.specs().values()); Assert.assertEquals("Types must match", expectedType, actualType); } diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java index a417454b45dc..78e7efbd8eeb 100644 --- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java @@ -474,7 +474,7 @@ public void testPartitionsTableSwitchFields() throws Exception { row(null, "c3", "d3"), row("d1", "c1", null), row("d2", "c2", null)), - "STRUCT", + "STRUCT", PARTITIONS); } else { // In V2 re-adding a former partition field that was part of an older spec will not change its @@ -579,13 +579,13 @@ public void testPartitionTableFilterSwitchFields() throws Exception { assertPartitions( ImmutableList.of(row(null, "c2", null), row(null, "c2", "d2"), row("d2", "c2", null)), - "STRUCT", + "STRUCT", PARTITIONS, "partition.category = 'c2'"); assertPartitions( ImmutableList.of(row(null, "c1", "d1")), - "STRUCT", + "STRUCT", PARTITIONS, "partition.data = 'd1'"); } From adb48509075ac1325921dda35dc66094f7faa70f Mon Sep 17 00:00:00 2001 From: Ilya Cherkasov Date: Sat, 9 Mar 2024 16:28:09 +0200 Subject: [PATCH 6/6] Fixing tests --- .../spark/extensions/TestAlterTablePartitionFields.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java index 38e5c942c9ff..c2196eb6712a 100644 --- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java +++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java @@ -394,7 +394,7 @@ public void testReplacePartition() { TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) - .addField("hour", 3, 1001, "ts_hour") + .addField("hour", 3, 10001, "ts_hour") .build(); } assertThat(table.spec()) @@ -428,7 +428,7 @@ public void testReplacePartitionAndRename() { TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) - .addField("hour", 3, 1001, "hour_col") + .addField("hour", 3, 10001, "hour_col") .build(); } assertThat(table.spec()) @@ -462,7 +462,7 @@ public void testReplaceNamedPartition() { TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) - .addField("hour", 3, 1001, "ts_hour") + .addField("hour", 3, 10001, "ts_hour") .build(); } assertThat(table.spec()) @@ -496,7 +496,7 @@ public void testReplaceNamedPartitionAndRenameDifferently() { TestHelpers.newExpectedSpecBuilder() .withSchema(table.schema()) .withSpecId(2) - .addField("hour", 3, 1001, "hour_col") + .addField("hour", 3, 10001, "hour_col") .build(); } assertThat(table.spec())