From b8bdd5b1a3d5eb429ebb174a919f6be8e7e02eb0 Mon Sep 17 00:00:00 2001 From: Eduard Tudenhoefner Date: Fri, 19 Nov 2021 11:09:17 +0100 Subject: [PATCH] Core: Use Immutables for TableMetadata The motivation for this change is * `TableMetadata` is truly immutable * we get a Builder for free, which allows easy creation of new `TableMetadata` instances --- baseline.gradle | 4 +- build.gradle | 2 + .../org/apache/iceberg/TableMetadata.java | 597 +++++++++--------- .../apache/iceberg/TableMetadataParser.java | 27 +- .../org/apache/iceberg/TestTableMetadata.java | 198 ++++-- versions.props | 1 + 6 files changed, 469 insertions(+), 360 deletions(-) diff --git a/baseline.gradle b/baseline.gradle index 9ebc8afff761..501935f1b308 100644 --- a/baseline.gradle +++ b/baseline.gradle @@ -63,8 +63,8 @@ subprojects { pluginManager.withPlugin('com.palantir.baseline-error-prone') { tasks.withType(JavaCompile).configureEach { options.errorprone.errorproneArgs.addAll ( - // error-prone is slow, don't run on tests and generated src - '-XepExcludedPaths:.*/(test|generated-src)/.*', + // error-prone is slow, don't run on tests/generated-src/generated + '-XepExcludedPaths:.*/(test|generated-src|generated)/.*', // specific to Palantir '-Xep:ConsistentLoggerName:OFF', // Uses name `log` but we use name `LOG` '-Xep:FinalClass:OFF', diff --git a/build.gradle b/build.gradle index 85bfe423c846..3bc3d9ca0051 100644 --- a/build.gradle +++ b/build.gradle @@ -204,6 +204,7 @@ project(':iceberg-common') { project(':iceberg-core') { dependencies { + annotationProcessor "org.immutables:value" api project(':iceberg-api') implementation project(':iceberg-common') implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') @@ -212,6 +213,7 @@ project(':iceberg-core') { exclude group: 'org.tukaani' // xz compression is not supported } + implementation "org.immutables:value" implementation "com.fasterxml.jackson.core:jackson-databind" implementation "com.fasterxml.jackson.core:jackson-core" implementation "com.github.ben-manes.caffeine:caffeine" diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java index 59c71d7b55a8..ab79f81ecc02 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadata.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java @@ -30,6 +30,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Predicate; import java.util.stream.Collectors; +import javax.annotation.Nullable; import org.apache.iceberg.exceptions.ValidationException; import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; import org.apache.iceberg.relocated.com.google.common.base.Objects; @@ -44,11 +45,13 @@ import org.apache.iceberg.types.TypeUtil; import org.apache.iceberg.util.Pair; import org.apache.iceberg.util.PropertyUtil; +import org.immutables.value.Value; /** * Metadata for a table. */ -public class TableMetadata implements Serializable { +@Value.Immutable +public abstract class TableMetadata implements Serializable { static final long INITIAL_SEQUENCE_NUMBER = 0; static final long INVALID_SEQUENCE_NUMBER = -1; static final int DEFAULT_TABLE_FORMAT_VERSION = 1; @@ -120,13 +123,22 @@ static TableMetadata newTableMetadata(Schema schema, // break existing tables. MetricsConfig.fromProperties(properties).validateReferencedColumns(schema); - return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - INITIAL_SEQUENCE_NUMBER, System.currentTimeMillis(), - lastColumnId.get(), freshSchema.schemaId(), ImmutableList.of(freshSchema), - freshSpec.specId(), ImmutableList.of(freshSpec), freshSpec.lastAssignedFieldId(), - freshSortOrderId, ImmutableList.of(freshSortOrder), - ImmutableMap.copyOf(properties), -1, ImmutableList.of(), - ImmutableList.of(), ImmutableList.of()); + return ImmutableTableMetadata.builder() + .formatVersion(formatVersion) + .uuid(UUID.randomUUID().toString()) + .location(location) + .lastSequenceNumber(INITIAL_SEQUENCE_NUMBER) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(lastColumnId.get()) + .currentSchemaId(freshSchema.schemaId()) + .schemas(ImmutableList.of(freshSchema)) + .defaultSpecId(freshSpec.specId()) + .specs(ImmutableList.of(freshSpec)) + .lastAssignedPartitionId(freshSpec.lastAssignedFieldId()) + .defaultSortOrderId(freshSortOrderId) + .sortOrders(ImmutableList.of(freshSortOrder)) + .properties(properties) + .build(); } public static class SnapshotLogEntry implements HistoryEntry { @@ -216,87 +228,19 @@ public String toString() { } } - // stored metadata - private final String metadataFileLocation; - private final int formatVersion; - private final String uuid; - private final String location; - private final long lastSequenceNumber; - private final long lastUpdatedMillis; - private final int lastColumnId; - private final int currentSchemaId; - private final List schemas; - private final int defaultSpecId; - private final List specs; - private final int lastAssignedPartitionId; - private final int defaultSortOrderId; - private final List sortOrders; - private final Map properties; - private final long currentSnapshotId; - private final List snapshots; - private final Map snapshotsById; - private final Map schemasById; - private final Map specsById; - private final Map sortOrdersById; - private final List snapshotLog; - private final List previousFiles; - - @SuppressWarnings("checkstyle:CyclomaticComplexity") - TableMetadata(String metadataFileLocation, - int formatVersion, - String uuid, - String location, - long lastSequenceNumber, - long lastUpdatedMillis, - int lastColumnId, - int currentSchemaId, - List schemas, - int defaultSpecId, - List specs, - int lastAssignedPartitionId, - int defaultSortOrderId, - List sortOrders, - Map properties, - long currentSnapshotId, - List snapshots, - List snapshotLog, - List previousFiles) { - Preconditions.checkArgument(specs != null && !specs.isEmpty(), "Partition specs cannot be null or empty"); - Preconditions.checkArgument(sortOrders != null && !sortOrders.isEmpty(), "Sort orders cannot be null or empty"); - Preconditions.checkArgument(formatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, - "Unsupported format version: v%s", formatVersion); - Preconditions.checkArgument(formatVersion == 1 || uuid != null, - "UUID is required in format v%s", formatVersion); - Preconditions.checkArgument(formatVersion > 1 || lastSequenceNumber == 0, - "Sequence number must be 0 in v1: %s", lastSequenceNumber); - - this.metadataFileLocation = metadataFileLocation; - this.formatVersion = formatVersion; - this.uuid = uuid; - this.location = location; - this.lastSequenceNumber = lastSequenceNumber; - this.lastUpdatedMillis = lastUpdatedMillis; - this.lastColumnId = lastColumnId; - this.currentSchemaId = currentSchemaId; - this.schemas = schemas; - this.specs = specs; - this.defaultSpecId = defaultSpecId; - this.lastAssignedPartitionId = lastAssignedPartitionId; - this.defaultSortOrderId = defaultSortOrderId; - this.sortOrders = sortOrders; - this.properties = properties; - this.currentSnapshotId = currentSnapshotId; - this.snapshots = snapshots; - this.snapshotLog = snapshotLog; - this.previousFiles = previousFiles; - - this.snapshotsById = indexAndValidateSnapshots(snapshots, lastSequenceNumber); - this.schemasById = indexSchemas(); - this.specsById = indexSpecs(specs); - this.sortOrdersById = indexSortOrders(sortOrders); + @Value.Check + protected void check() { + Preconditions.checkArgument(specs() != null && !specs().isEmpty(), "Partition specs cannot be null or empty"); + Preconditions.checkArgument(sortOrders() != null && !sortOrders().isEmpty(), "Sort orders cannot be null or empty"); + Preconditions.checkArgument(formatVersion() <= SUPPORTED_TABLE_FORMAT_VERSION, + "Unsupported format version: v%s", formatVersion()); + Preconditions.checkArgument(formatVersion() == 1 || uuid() != null, + "UUID is required in format v%s", formatVersion()); + Preconditions.checkArgument(formatVersion() > 1 || lastSequenceNumber() == 0, + "Sequence number must be 0 in v1: %s", lastSequenceNumber()); HistoryEntry last = null; - for (HistoryEntry logEntry : snapshotLog) { + for (HistoryEntry logEntry : snapshotLog()) { if (last != null) { Preconditions.checkArgument( (logEntry.timestampMillis() - last.timestampMillis()) >= -ONE_MINUTE, @@ -308,13 +252,13 @@ public String toString() { Preconditions.checkArgument( // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew - lastUpdatedMillis - last.timestampMillis() >= -ONE_MINUTE, + lastUpdatedMillis() - last.timestampMillis() >= -ONE_MINUTE, "Invalid update timestamp %s: before last snapshot log entry at %s", - lastUpdatedMillis, last.timestampMillis()); + lastUpdatedMillis(), last.timestampMillis()); } MetadataLogEntry previous = null; - for (MetadataLogEntry metadataEntry : previousFiles) { + for (MetadataLogEntry metadataEntry : previousFiles()) { if (previous != null) { Preconditions.checkArgument( // commits can happen concurrently from different machines. @@ -324,157 +268,150 @@ public String toString() { } previous = metadataEntry; } - // Make sure that this update's lastUpdatedMillis is > max(previousFile's timestamp) + // Make sure that this update's lastUpdatedMillis is > max(previousFile's timestamp) if (previous != null) { Preconditions.checkArgument( // commits can happen concurrently from different machines. // A tolerance helps us avoid failure for small clock skew - lastUpdatedMillis - previous.timestampMillis >= -ONE_MINUTE, + lastUpdatedMillis() - previous.timestampMillis >= -ONE_MINUTE, "Invalid update timestamp %s: before the latest metadata log entry timestamp %s", - lastUpdatedMillis, previous.timestampMillis); + lastUpdatedMillis(), previous.timestampMillis); } Preconditions.checkArgument( - currentSnapshotId < 0 || snapshotsById.containsKey(currentSnapshotId), + currentSnapshotId() < 0 || snapshotsById().containsKey(currentSnapshotId()), "Invalid table metadata: Cannot find current version"); } - public int formatVersion() { - return formatVersion; - } + public abstract int formatVersion(); - public String metadataFileLocation() { - return metadataFileLocation; - } + @Nullable + public abstract String metadataFileLocation(); - public String uuid() { - return uuid; - } + @Nullable + public abstract String uuid(); - public long lastSequenceNumber() { - return lastSequenceNumber; - } + public abstract long lastSequenceNumber(); public long nextSequenceNumber() { - return formatVersion > 1 ? lastSequenceNumber + 1 : INITIAL_SEQUENCE_NUMBER; + return formatVersion() > 1 ? lastSequenceNumber() + 1 : INITIAL_SEQUENCE_NUMBER; } - public long lastUpdatedMillis() { - return lastUpdatedMillis; - } + public abstract long lastUpdatedMillis(); - public int lastColumnId() { - return lastColumnId; - } + public abstract int lastColumnId(); public Schema schema() { - return schemasById.get(currentSchemaId); + return schemasById().get(currentSchemaId()); } - public List schemas() { - return schemas; - } + public abstract List schemas(); + @Value.Derived public Map schemasById() { - return schemasById; + return indexSchemas(); } - public int currentSchemaId() { - return currentSchemaId; - } + public abstract int currentSchemaId(); public PartitionSpec spec() { - return specsById.get(defaultSpecId); + return specsById().get(defaultSpecId()); } public PartitionSpec spec(int id) { - return specsById.get(id); + return specsById().get(id); } - public List specs() { - return specs; - } + public abstract List specs(); + @Value.Derived public Map specsById() { - return specsById; + return indexSpecs(specs()); } - int lastAssignedPartitionId() { - return lastAssignedPartitionId; - } + abstract int lastAssignedPartitionId(); - public int defaultSpecId() { - return defaultSpecId; - } + public abstract int defaultSpecId(); - public int defaultSortOrderId() { - return defaultSortOrderId; - } + public abstract int defaultSortOrderId(); public SortOrder sortOrder() { - return sortOrdersById.get(defaultSortOrderId); + return sortOrdersById().get(defaultSortOrderId()); } - public List sortOrders() { - return sortOrders; - } + public abstract List sortOrders(); + @Value.Derived public Map sortOrdersById() { - return sortOrdersById; + return indexSortOrders(sortOrders()); } - public String location() { - return location; - } + @Nullable + public abstract String location(); + @Value.Default public Map properties() { - return properties; + return ImmutableMap.of(); } public String property(String property, String defaultValue) { - return properties.getOrDefault(property, defaultValue); + return properties().getOrDefault(property, defaultValue); } public boolean propertyAsBoolean(String property, boolean defaultValue) { - return PropertyUtil.propertyAsBoolean(properties, property, defaultValue); + return PropertyUtil.propertyAsBoolean(properties(), property, defaultValue); } public int propertyAsInt(String property, int defaultValue) { - return PropertyUtil.propertyAsInt(properties, property, defaultValue); + return PropertyUtil.propertyAsInt(properties(), property, defaultValue); } public long propertyAsLong(String property, long defaultValue) { - return PropertyUtil.propertyAsLong(properties, property, defaultValue); + return PropertyUtil.propertyAsLong(properties(), property, defaultValue); + } + + @Value.Derived + public Map snapshotsById() { + return indexAndValidateSnapshots(snapshots(), lastSequenceNumber()); } public Snapshot snapshot(long snapshotId) { - return snapshotsById.get(snapshotId); + return snapshotsById().get(snapshotId); + } + + @Value.Default + public long currentSnapshotId() { + return -1L; } public Snapshot currentSnapshot() { - return snapshotsById.get(currentSnapshotId); + return snapshotsById().get(currentSnapshotId()); } + @Value.Default public List snapshots() { - return snapshots; + return ImmutableList.of(); } + @Value.Default public List snapshotLog() { - return snapshotLog; + return ImmutableList.of(); } + @Value.Default public List previousFiles() { - return previousFiles; + return ImmutableList.of(); } public TableMetadata withUUID() { - if (uuid != null) { + if (uuid() != null) { return this; } else { - return new TableMetadata(null, formatVersion, UUID.randomUUID().toString(), location, - lastSequenceNumber, lastUpdatedMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, - currentSnapshotId, snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .uuid(UUID.randomUUID().toString()) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } } @@ -482,25 +419,30 @@ public TableMetadata updateSchema(Schema newSchema, int newLastColumnId) { PartitionSpec.checkCompatibility(spec(), newSchema); SortOrder.checkCompatibility(sortOrder(), newSchema); // rebuild all of the partition specs and sort orders for the new current schema - List updatedSpecs = Lists.transform(specs, spec -> updateSpecSchema(newSchema, spec)); - List updatedSortOrders = Lists.transform(sortOrders, order -> updateSortOrderSchema(newSchema, order)); + List updatedSpecs = Lists.transform(specs(), spec -> updateSpecSchema(newSchema, spec)); + List updatedSortOrders = Lists.transform(sortOrders(), order -> updateSortOrderSchema(newSchema, order)); int newSchemaId = reuseOrCreateNewSchemaId(newSchema); - if (currentSchemaId == newSchemaId && newLastColumnId == lastColumnId) { + if (currentSchemaId() == newSchemaId && newLastColumnId == lastColumnId()) { // the new spec and last column Id is already current and no change is needed return this; } - ImmutableList.Builder builder = ImmutableList.builder().addAll(schemas); - if (!schemasById.containsKey(newSchemaId)) { + ImmutableList.Builder builder = ImmutableList.builder().addAll(schemas()); + if (!schemasById().containsKey(newSchemaId)) { builder.add(new Schema(newSchemaId, newSchema.columns(), newSchema.identifierFieldIds())); } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), newLastColumnId, - newSchemaId, builder.build(), defaultSpecId, updatedSpecs, lastAssignedPartitionId, - defaultSortOrderId, updatedSortOrders, properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(newLastColumnId) + .currentSchemaId(newSchemaId) + .schemas(builder.build()) + .specs(updatedSpecs) + .sortOrders(updatedSortOrders) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } // The caller is responsible to pass a newPartitionSpec with correct partition field IDs @@ -508,12 +450,12 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { Schema schema = schema(); PartitionSpec.checkCompatibility(newPartitionSpec, schema); - ValidationException.check(formatVersion > 1 || PartitionSpec.hasSequentialIds(newPartitionSpec), + ValidationException.check(formatVersion() > 1 || PartitionSpec.hasSequentialIds(newPartitionSpec), "Spec does not use sequential IDs that are required in v1: %s", newPartitionSpec); // if the spec already exists, use the same ID. otherwise, use 1 more than the highest ID. int newDefaultSpecId = INITIAL_SPEC_ID; - for (PartitionSpec spec : specs) { + for (PartitionSpec spec : specs()) { if (newPartitionSpec.compatibleWith(spec)) { newDefaultSpecId = spec.specId(); break; @@ -522,23 +464,26 @@ public TableMetadata updatePartitionSpec(PartitionSpec newPartitionSpec) { } } - if (defaultSpecId == newDefaultSpecId) { + if (defaultSpecId() == newDefaultSpecId) { // the new spec is already current and no change is needed return this; } ImmutableList.Builder builder = ImmutableList.builder() - .addAll(specs); - if (!specsById.containsKey(newDefaultSpecId)) { + .addAll(specs()); + if (!specsById().containsKey(newDefaultSpecId)) { // get a fresh spec to ensure the spec ID is set to the new default builder.add(freshSpec(newDefaultSpecId, schema, newPartitionSpec)); } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, newDefaultSpecId, - builder.build(), Math.max(lastAssignedPartitionId, newPartitionSpec.lastAssignedFieldId()), - defaultSortOrderId, sortOrders, properties, - currentSnapshotId, snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .defaultSpecId(newDefaultSpecId) + .specs(builder.build()) + .lastAssignedPartitionId(Math.max(lastAssignedPartitionId(), newPartitionSpec.lastAssignedFieldId())) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata replaceSortOrder(SortOrder newOrder) { @@ -547,7 +492,7 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { // determine the next order id int newOrderId = INITIAL_SORT_ORDER_ID; - for (SortOrder order : sortOrders) { + for (SortOrder order : sortOrders()) { if (order.sameOrder(newOrder)) { newOrderId = order.orderId(); break; @@ -556,14 +501,14 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { } } - if (newOrderId == defaultSortOrderId) { + if (newOrderId == defaultSortOrderId()) { return this; } ImmutableList.Builder builder = ImmutableList.builder(); - builder.addAll(sortOrders); + builder.addAll(sortOrders()); - if (!sortOrdersById.containsKey(newOrderId)) { + if (!sortOrdersById().containsKey(newOrderId)) { if (newOrder.isUnsorted()) { newOrderId = SortOrder.unsorted().orderId(); builder.add(SortOrder.unsorted()); @@ -573,60 +518,70 @@ public TableMetadata replaceSortOrder(SortOrder newOrder) { } } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, newOrderId, builder.build(), properties, currentSnapshotId, snapshots, snapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .defaultSortOrderId(newOrderId) + .sortOrders(builder.build()) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata addStagedSnapshot(Snapshot snapshot) { - ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() > lastSequenceNumber, + ValidationException.check(formatVersion() == 1 || snapshot.sequenceNumber() > lastSequenceNumber(), "Cannot add snapshot with sequence number %s older than last sequence number %s", - snapshot.sequenceNumber(), lastSequenceNumber); + snapshot.sequenceNumber(), lastSequenceNumber()); List newSnapshots = ImmutableList.builder() - .addAll(snapshots) + .addAll(snapshots()) .add(snapshot) .build(); - return new TableMetadata(null, formatVersion, uuid, location, - snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, - currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, - defaultSortOrderId, sortOrders, properties, currentSnapshotId, newSnapshots, snapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastSequenceNumber(snapshot.sequenceNumber()) + .lastUpdatedMillis(snapshot.timestampMillis()) + .snapshots(newSnapshots) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata replaceCurrentSnapshot(Snapshot snapshot) { // there can be operations (viz. rollback, cherrypick) where an existing snapshot could be replacing current - if (snapshotsById.containsKey(snapshot.snapshotId())) { + if (snapshotsById().containsKey(snapshot.snapshotId())) { return setCurrentSnapshotTo(snapshot); } - ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() > lastSequenceNumber, + ValidationException.check(formatVersion() == 1 || snapshot.sequenceNumber() > lastSequenceNumber(), "Cannot add snapshot with sequence number %s older than last sequence number %s", - snapshot.sequenceNumber(), lastSequenceNumber); + snapshot.sequenceNumber(), lastSequenceNumber()); List newSnapshots = ImmutableList.builder() - .addAll(snapshots) + .addAll(snapshots()) .add(snapshot) .build(); List newSnapshotLog = ImmutableList.builder() - .addAll(snapshotLog) + .addAll(snapshotLog()) .add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshot.snapshotId())) .build(); - return new TableMetadata(null, formatVersion, uuid, location, - snapshot.sequenceNumber(), snapshot.timestampMillis(), lastColumnId, - currentSchemaId, schemas, defaultSpecId, specs, lastAssignedPartitionId, - defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), newSnapshots, newSnapshotLog, - addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastSequenceNumber(snapshot.sequenceNumber()) + .lastUpdatedMillis(snapshot.timestampMillis()) + .snapshots(newSnapshots) + .currentSnapshotId(snapshot.snapshotId()) + .snapshots(newSnapshots) + .snapshotLog(newSnapshotLog) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata removeSnapshotsIf(Predicate removeIf) { - List filtered = Lists.newArrayListWithExpectedSize(snapshots.size()); - for (Snapshot snapshot : snapshots) { + List filtered = Lists.newArrayListWithExpectedSize(snapshots().size()); + for (Snapshot snapshot : snapshots()) { // keep the current snapshot and any snapshots that do not match the removeIf condition - if (snapshot.snapshotId() == currentSnapshotId || !removeIf.test(snapshot)) { + if (snapshot.snapshotId() == currentSnapshotId() || !removeIf.test(snapshot)) { filtered.add(snapshot); } } @@ -634,7 +589,7 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { // update the snapshot log Set validIds = Sets.newHashSet(Iterables.transform(filtered, Snapshot::snapshotId)); List newSnapshotLog = Lists.newArrayList(); - for (HistoryEntry logEntry : snapshotLog) { + for (HistoryEntry logEntry : snapshotLog()) { if (validIds.contains(logEntry.snapshotId())) { // copy the log entries that are still valid newSnapshotLog.add(logEntry); @@ -648,46 +603,53 @@ public TableMetadata removeSnapshotsIf(Predicate removeIf) { } } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, filtered, - ImmutableList.copyOf(newSnapshotLog), addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .snapshots(filtered) + .snapshotLog(newSnapshotLog) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } private TableMetadata setCurrentSnapshotTo(Snapshot snapshot) { - ValidationException.check(snapshotsById.containsKey(snapshot.snapshotId()), + ValidationException.check(snapshotsById().containsKey(snapshot.snapshotId()), "Cannot set current snapshot to unknown: %s", snapshot.snapshotId()); - ValidationException.check(formatVersion == 1 || snapshot.sequenceNumber() <= lastSequenceNumber, + ValidationException.check(formatVersion() == 1 || snapshot.sequenceNumber() <= lastSequenceNumber(), "Last sequence number %s is less than existing snapshot sequence number %s", - lastSequenceNumber, snapshot.sequenceNumber()); + lastSequenceNumber(), snapshot.sequenceNumber()); - if (currentSnapshotId == snapshot.snapshotId()) { + if (currentSnapshotId() == snapshot.snapshotId()) { // change is a noop return this; } long nowMillis = System.currentTimeMillis(); List newSnapshotLog = ImmutableList.builder() - .addAll(snapshotLog) + .addAll(snapshotLog()) .add(new SnapshotLogEntry(nowMillis, snapshot.snapshotId())) .build(); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, nowMillis, lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshot.snapshotId(), snapshots, - newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder().from(this) + .lastUpdatedMillis(nowMillis) + .currentSnapshotId(snapshot.snapshotId()) + .snapshotLog(newSnapshotLog) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata replaceProperties(Map rawProperties) { ValidationException.check(rawProperties != null, "Cannot set properties to null"); Map newProperties = unreservedProperties(rawProperties); - TableMetadata metadata = new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, newProperties, currentSnapshotId, snapshots, - snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis, newProperties)); - int newFormatVersion = PropertyUtil.propertyAsInt(rawProperties, TableProperties.FORMAT_VERSION, formatVersion); - if (formatVersion != newFormatVersion) { + TableMetadata metadata = ImmutableTableMetadata.builder().from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .properties(newProperties) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis(), newProperties)) + .build(); + + int newFormatVersion = PropertyUtil.propertyAsInt(rawProperties, TableProperties.FORMAT_VERSION, formatVersion()); + if (formatVersion() != newFormatVersion) { metadata = metadata.upgradeToFormatVersion(newFormatVersion); } @@ -696,21 +658,23 @@ public TableMetadata replaceProperties(Map rawProperties) { public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { List newSnapshotLog = Lists.newArrayList(); - for (HistoryEntry logEntry : snapshotLog) { + for (HistoryEntry logEntry : snapshotLog()) { if (!snapshotIds.contains(logEntry.snapshotId())) { // copy the log entries that are still valid newSnapshotLog.add(logEntry); } } - ValidationException.check(currentSnapshotId < 0 || // not set - Iterables.getLast(newSnapshotLog).snapshotId() == currentSnapshotId, + ValidationException.check(currentSnapshotId() < 0 || // not set + Iterables.getLast(newSnapshotLog).snapshotId() == currentSnapshotId(), "Cannot set invalid snapshot log: latest entry is not the current snapshot"); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .snapshotLog(newSnapshotLog) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } /** @@ -722,62 +686,71 @@ public TableMetadata removeSnapshotLogEntries(Set snapshotIds) { * @return {@link TableMetadata} with updated {@link #currentSnapshotId} and {@link #snapshotLog} */ public TableMetadata withCurrentSnapshotOnly(long snapshotId) { - if ((currentSnapshotId == -1L && snapshotId == -1L && snapshots.isEmpty()) || - (currentSnapshotId == snapshotId && snapshots.size() == 1)) { + if ((currentSnapshotId() == -1L && snapshotId == -1L && snapshots().isEmpty()) || + (currentSnapshotId() == snapshotId && snapshots().size() == 1)) { return this; } List newSnapshotLog = Lists.newArrayList(); if (snapshotId != -1L) { - Snapshot snapshot = snapshotsById.get(snapshotId); + Snapshot snapshot = snapshotsById().get(snapshotId); Preconditions.checkArgument(snapshot != null, "Non-existent snapshot"); newSnapshotLog.add(new SnapshotLogEntry(snapshot.timestampMillis(), snapshotId)); } - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, snapshotId, - snapshots, newSnapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .currentSnapshotId(snapshotId) + .snapshotLog(newSnapshotLog) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata withCurrentSchema(int schemaId) { - if (currentSchemaId == schemaId) { + if (currentSchemaId() == schemaId) { return this; } - Preconditions.checkArgument(schemasById.containsKey(schemaId), "Non-existent schema"); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, schemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + Preconditions.checkArgument(schemasById().containsKey(schemaId), "Non-existent schema"); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .currentSchemaId(schemaId) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata withDefaultSortOrder(int sortOrderId) { - if (defaultSortOrderId == sortOrderId) { + if (defaultSortOrderId() == sortOrderId) { return this; } - Preconditions.checkArgument(sortOrdersById.containsKey(sortOrderId), "Non-existent sort-order"); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, sortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + Preconditions.checkArgument(sortOrdersById().containsKey(sortOrderId), "Non-existent sort-order"); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .defaultSortOrderId(sortOrderId) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata withDefaultSpec(int specId) { - if (defaultSpecId == specId) { + if (defaultSpecId() == specId) { return this; } - Preconditions.checkArgument(specsById.containsKey(specId), "Non-existent partition spec"); - return new TableMetadata(null, formatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, specId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + Preconditions.checkArgument(specsById().containsKey(specId), "Non-existent partition spec"); + return ImmutableTableMetadata.builder() + .from(this) + .lastUpdatedMillis(System.currentTimeMillis()) + .defaultSpecId(specId) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil.NextID nextID) { PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(partitionSpec.schema()) .withSpecId(partitionSpec.specId()); - if (formatVersion > 1) { + if (formatVersion() > 1) { // for v2 and later, reuse any existing field IDs, but reproduce the same spec - Map, Integer> transformToFieldId = specs.stream() + Map, Integer> transformToFieldId = specs().stream() .flatMap(spec -> spec.fields().stream()) .collect(Collectors.toMap( field -> Pair.of(field.sourceId(), field.transform().toString()), @@ -829,37 +802,37 @@ private PartitionSpec reassignPartitionIds(PartitionSpec partitionSpec, TypeUtil public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec updatedPartitionSpec, SortOrder updatedSortOrder, String newLocation, Map updatedProperties) { - ValidationException.check(formatVersion > 1 || PartitionSpec.hasSequentialIds(updatedPartitionSpec), + ValidationException.check(formatVersion() > 1 || PartitionSpec.hasSequentialIds(updatedPartitionSpec), "Spec does not use sequential IDs that are required in v1: %s", updatedPartitionSpec); - AtomicInteger newLastColumnId = new AtomicInteger(lastColumnId); + AtomicInteger newLastColumnId = new AtomicInteger(lastColumnId()); Schema freshSchema = TypeUtil.assignFreshIds(updatedSchema, schema(), newLastColumnId::incrementAndGet); // determine the next spec id - OptionalInt maxSpecId = specs.stream().mapToInt(PartitionSpec::specId).max(); + OptionalInt maxSpecId = specs().stream().mapToInt(PartitionSpec::specId).max(); int nextSpecId = maxSpecId.orElse(TableMetadata.INITIAL_SPEC_ID) + 1; // rebuild the partition spec using the new column ids PartitionSpec freshSpec = freshSpec(nextSpecId, freshSchema, updatedPartitionSpec); // reassign partition field ids with existing partition specs in the table - AtomicInteger lastPartitionId = new AtomicInteger(lastAssignedPartitionId); + AtomicInteger lastPartitionId = new AtomicInteger(lastAssignedPartitionId()); PartitionSpec newSpec = reassignPartitionIds(freshSpec, lastPartitionId::incrementAndGet); // if the spec already exists, use the same ID. otherwise, use 1 more than the highest ID. - int specId = specs.stream() + int specId = specs().stream() .filter(newSpec::compatibleWith) .findFirst() .map(PartitionSpec::specId) .orElse(nextSpecId); - ImmutableList.Builder specListBuilder = ImmutableList.builder().addAll(specs); - if (!specsById.containsKey(specId)) { + ImmutableList.Builder specListBuilder = ImmutableList.builder().addAll(specs()); + if (!specsById().containsKey(specId)) { specListBuilder.add(newSpec); } // determine the next order id - OptionalInt maxOrderId = sortOrders.stream().mapToInt(SortOrder::orderId).max(); + OptionalInt maxOrderId = sortOrders().stream().mapToInt(SortOrder::orderId).max(); int nextOrderId = maxOrderId.isPresent() ? maxOrderId.getAsInt() + 1 : INITIAL_SORT_ORDER_ID; // rebuild the sort order using new column ids @@ -867,38 +840,51 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update SortOrder freshSortOrder = freshSortOrder(freshSortOrderId, freshSchema, updatedSortOrder); // if the order already exists, use the same ID. otherwise, use the fresh order ID - Optional sameSortOrder = sortOrders.stream() + Optional sameSortOrder = sortOrders().stream() .filter(sortOrder -> sortOrder.sameOrder(freshSortOrder)) .findAny(); int orderId = sameSortOrder.map(SortOrder::orderId).orElse(freshSortOrderId); - ImmutableList.Builder sortOrdersBuilder = ImmutableList.builder().addAll(sortOrders); - if (!sortOrdersById.containsKey(orderId)) { + ImmutableList.Builder sortOrdersBuilder = ImmutableList.builder().addAll(sortOrders()); + if (!sortOrdersById().containsKey(orderId)) { sortOrdersBuilder.add(freshSortOrder); } Map newProperties = Maps.newHashMap(); - newProperties.putAll(this.properties); + newProperties.putAll(this.properties()); newProperties.putAll(unreservedProperties(updatedProperties)); // check if there is format version override - int newFormatVersion = PropertyUtil.propertyAsInt(updatedProperties, TableProperties.FORMAT_VERSION, formatVersion); + int newFormatVersion = + PropertyUtil.propertyAsInt(updatedProperties, TableProperties.FORMAT_VERSION, formatVersion()); // determine the next schema id int freshSchemaId = reuseOrCreateNewSchemaId(freshSchema); - ImmutableList.Builder schemasBuilder = ImmutableList.builder().addAll(schemas); + ImmutableList.Builder schemasBuilder = ImmutableList.builder().addAll(schemas()); - if (!schemasById.containsKey(freshSchemaId)) { + if (!schemasById().containsKey(freshSchemaId)) { schemasBuilder.add(new Schema(freshSchemaId, freshSchema.columns(), freshSchema.identifierFieldIds())); } - TableMetadata metadata = new TableMetadata(null, formatVersion, uuid, newLocation, - lastSequenceNumber, System.currentTimeMillis(), newLastColumnId.get(), freshSchemaId, schemasBuilder.build(), - specId, specListBuilder.build(), Math.max(lastAssignedPartitionId, newSpec.lastAssignedFieldId()), - orderId, sortOrdersBuilder.build(), ImmutableMap.copyOf(newProperties), - -1, snapshots, ImmutableList.of(), addPreviousFile(metadataFileLocation, lastUpdatedMillis, newProperties)); + TableMetadata metadata = ImmutableTableMetadata.builder() + .from(this) + .location(newLocation) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(newLastColumnId.get()) + .currentSchemaId(freshSchemaId) + .schemas(schemasBuilder.build()) + .defaultSpecId(specId) + .specs(specListBuilder.build()) + .lastAssignedPartitionId(Math.max(lastAssignedPartitionId(), newSpec.lastAssignedFieldId())) + .defaultSortOrderId(orderId) + .sortOrders(sortOrdersBuilder.build()) + .properties(newProperties) + .currentSnapshotId(-1L) + .snapshotLog(ImmutableList.of()) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis(), newProperties)) + .build(); - if (formatVersion != newFormatVersion) { + if (formatVersion() != newFormatVersion) { metadata = metadata.upgradeToFormatVersion(newFormatVersion); } @@ -906,48 +892,51 @@ public TableMetadata buildReplacement(Schema updatedSchema, PartitionSpec update } public TableMetadata updateLocation(String newLocation) { - return new TableMetadata(null, formatVersion, uuid, newLocation, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .location(newLocation) + .lastUpdatedMillis(System.currentTimeMillis()) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } public TableMetadata upgradeToFormatVersion(int newFormatVersion) { Preconditions.checkArgument(newFormatVersion <= SUPPORTED_TABLE_FORMAT_VERSION, "Cannot upgrade table to unsupported format version: v%s (supported: v%s)", newFormatVersion, SUPPORTED_TABLE_FORMAT_VERSION); - Preconditions.checkArgument(newFormatVersion >= formatVersion, - "Cannot downgrade v%s table to v%s", formatVersion, newFormatVersion); + Preconditions.checkArgument(newFormatVersion >= formatVersion(), + "Cannot downgrade v%s table to v%s", formatVersion(), newFormatVersion); - if (newFormatVersion == formatVersion) { + if (newFormatVersion == formatVersion()) { return this; } - - return new TableMetadata(null, newFormatVersion, uuid, location, - lastSequenceNumber, System.currentTimeMillis(), lastColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentSnapshotId, - snapshots, snapshotLog, addPreviousFile(metadataFileLocation, lastUpdatedMillis)); + return ImmutableTableMetadata.builder() + .from(this) + .formatVersion(newFormatVersion) + .lastUpdatedMillis(System.currentTimeMillis()) + .previousFiles(addPreviousFile(metadataFileLocation(), lastUpdatedMillis())) + .build(); } private List addPreviousFile(String previousFileLocation, long timestampMillis) { - return addPreviousFile(previousFileLocation, timestampMillis, properties); + return addPreviousFile(previousFileLocation, timestampMillis, properties()); } private List addPreviousFile(String previousFileLocation, long timestampMillis, Map updatedProperties) { if (previousFileLocation == null) { - return previousFiles; + return previousFiles(); } int maxSize = Math.max(1, PropertyUtil.propertyAsInt(updatedProperties, TableProperties.METADATA_PREVIOUS_VERSIONS_MAX, TableProperties.METADATA_PREVIOUS_VERSIONS_MAX_DEFAULT)); List newMetadataLog; - if (previousFiles.size() >= maxSize) { - int removeIndex = previousFiles.size() - maxSize + 1; - newMetadataLog = Lists.newArrayList(previousFiles.subList(removeIndex, previousFiles.size())); + if (previousFiles().size() >= maxSize) { + int removeIndex = previousFiles().size() - maxSize + 1; + newMetadataLog = Lists.newArrayList(previousFiles().subList(removeIndex, previousFiles().size())); } else { - newMetadataLog = Lists.newArrayList(previousFiles); + newMetadataLog = Lists.newArrayList(previousFiles()); } newMetadataLog.add(new MetadataLogEntry(timestampMillis, previousFileLocation)); @@ -1025,7 +1014,7 @@ private static Map indexAndValidateSnapshots(List snap private Map indexSchemas() { ImmutableMap.Builder builder = ImmutableMap.builder(); - for (Schema schema : schemas) { + for (Schema schema : schemas()) { builder.put(schema.schemaId(), schema); } return builder.build(); @@ -1049,8 +1038,8 @@ private static Map indexSortOrders(List sortOrder private int reuseOrCreateNewSchemaId(Schema newSchema) { // if the schema already exists, use its id; otherwise use the highest id + 1 - int newSchemaId = currentSchemaId; - for (Schema schema : schemas) { + int newSchemaId = currentSchemaId(); + for (Schema schema : schemas()) { if (schema.sameSchema(newSchema)) { newSchemaId = schema.schemaId(); break; diff --git a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java index 1e26fbb9b11b..95233a95c11b 100644 --- a/core/src/main/java/org/apache/iceberg/TableMetadataParser.java +++ b/core/src/main/java/org/apache/iceberg/TableMetadataParser.java @@ -288,7 +288,7 @@ static TableMetadata fromJson(FileIO io, InputFile file, JsonNode node) { return fromJson(io, file.location(), node); } - @SuppressWarnings("checkstyle:CyclomaticComplexity") + @SuppressWarnings({"checkstyle:CyclomaticComplexity", "checkstyle:MethodLength"}) static TableMetadata fromJson(FileIO io, String metadataLocation, JsonNode node) { Preconditions.checkArgument(node.isObject(), "Cannot parse metadata from a non-object: %s", node); @@ -430,9 +430,26 @@ static TableMetadata fromJson(FileIO io, String metadataLocation, JsonNode node) } } - return new TableMetadata(metadataLocation, formatVersion, uuid, location, - lastSequenceNumber, lastUpdatedMillis, lastAssignedColumnId, currentSchemaId, schemas, defaultSpecId, specs, - lastAssignedPartitionId, defaultSortOrderId, sortOrders, properties, currentVersionId, - snapshots, entries.build(), metadataEntries.build()); + return ImmutableTableMetadata.builder() + .metadataFileLocation(metadataLocation) + .formatVersion(formatVersion) + .uuid(uuid) + .location(location) + .lastSequenceNumber(lastSequenceNumber) + .lastUpdatedMillis(lastUpdatedMillis) + .lastColumnId(lastAssignedColumnId) + .currentSchemaId(currentSchemaId) + .schemas(schemas) + .defaultSpecId(defaultSpecId) + .specs(specs) + .lastAssignedPartitionId(lastAssignedPartitionId) + .defaultSortOrderId(defaultSortOrderId) + .sortOrders(sortOrders) + .properties(properties) + .currentSnapshotId(currentVersionId) + .snapshots(snapshots) + .snapshotLog(entries.build()) + .previousFiles(metadataEntries.build()) + .build(); } } diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java index 6ddbeab1d1d4..d1574f313d6f 100644 --- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java +++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java @@ -103,12 +103,26 @@ public void testJsonConversion() throws Exception { Schema schema = new Schema(6, Types.NestedField.required(10, "x", Types.StringType.get())); - TableMetadata expected = new TableMetadata(null, 2, UUID.randomUUID().toString(), TEST_LOCATION, - SEQ_NO, System.currentTimeMillis(), 3, - 7, ImmutableList.of(TEST_SCHEMA, schema), - 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), - 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, - Arrays.asList(previousSnapshot, currentSnapshot), snapshotLog, ImmutableList.of()); + TableMetadata expected = ImmutableTableMetadata.builder() + .formatVersion(2) + .uuid(UUID.randomUUID().toString()) + .location(TEST_LOCATION) + .lastSequenceNumber(SEQ_NO) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(3) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA, schema)) + .defaultSpecId(5) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(3) + .sortOrders(ImmutableList.of(SORT_ORDER_3)) + .properties(ImmutableMap.of("property", "value")) + .currentSnapshotId(currentSnapshotId) + .snapshots(Arrays.asList(previousSnapshot, currentSnapshot)) + .snapshotLog(snapshotLog) + .previousFiles(ImmutableList.of()) + .build(); String asJson = TableMetadataParser.toJson(expected); TableMetadata metadata = TableMetadataParser.fromJson(ops.io(), asJson); @@ -176,11 +190,24 @@ public void testBackwardCompat() throws Exception { ops.io(), currentSnapshotId, previousSnapshotId, currentSnapshotId, null, null, null, ImmutableList.of( new GenericManifestFile(localInput("file:/tmp/manfiest.2.avro"), spec.specId()))); - TableMetadata expected = new TableMetadata(null, 1, null, TEST_LOCATION, - 0, System.currentTimeMillis(), 3, TableMetadata.INITIAL_SCHEMA_ID, - ImmutableList.of(schema), 6, ImmutableList.of(spec), spec.lastAssignedFieldId(), - TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(sortOrder), ImmutableMap.of("property", "value"), - currentSnapshotId, Arrays.asList(previousSnapshot, currentSnapshot), ImmutableList.of(), ImmutableList.of()); + TableMetadata expected = ImmutableTableMetadata.builder() + .formatVersion(1) + .uuid(null) + .location(TEST_LOCATION) + .lastSequenceNumber(0) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(3) + .currentSchemaId(TableMetadata.INITIAL_SCHEMA_ID) + .schemas(ImmutableList.of(schema)) + .defaultSpecId(6) + .specs(ImmutableList.of(spec)) + .lastAssignedPartitionId(spec.lastAssignedFieldId()) + .defaultSortOrderId(TableMetadata.INITIAL_SORT_ORDER_ID) + .sortOrders(ImmutableList.of(sortOrder)) + .properties(ImmutableMap.of("property", "value")) + .currentSnapshotId(currentSnapshotId) + .snapshots(Arrays.asList(previousSnapshot, currentSnapshot)) + .build(); String asJson = toJsonWithoutSpecAndSchemaList(expected); TableMetadata metadata = TableMetadataParser.fromJson(ops.io(), asJson); @@ -297,12 +324,26 @@ public void testJsonWithPreviousMetadataLog() throws Exception { previousMetadataLog.add(new MetadataLogEntry(currentTimestamp, "/tmp/000001-" + UUID.randomUUID().toString() + ".metadata.json")); - TableMetadata base = new TableMetadata(null, 1, UUID.randomUUID().toString(), TEST_LOCATION, - 0, System.currentTimeMillis(), 3, - 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), - 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, - Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + TableMetadata base = ImmutableTableMetadata.builder() + .formatVersion(1) + .uuid(UUID.randomUUID().toString()) + .location(TEST_LOCATION) + .lastSequenceNumber(0) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(3) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA)) + .defaultSpecId(5) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(3) + .sortOrders(ImmutableList.of(SORT_ORDER_3)) + .properties(ImmutableMap.of("property", "value")) + .currentSnapshotId(currentSnapshotId) + .snapshots(Arrays.asList(previousSnapshot, currentSnapshot)) + .snapshotLog(reversedSnapshotLog) + .previousFiles(ImmutableList.copyOf(previousMetadataLog)) + .build(); String asJson = TableMetadataParser.toJson(base); TableMetadata metadataFromJson = TableMetadataParser.fromJson(ops.io(), asJson); @@ -332,12 +373,27 @@ public void testAddPreviousMetadataRemoveNone() { MetadataLogEntry latestPreviousMetadata = new MetadataLogEntry(currentTimestamp - 80, "/tmp/000003-" + UUID.randomUUID().toString() + ".metadata.json"); - TableMetadata base = new TableMetadata(latestPreviousMetadata.file(), 1, UUID.randomUUID().toString(), - TEST_LOCATION, 0, currentTimestamp - 80, 3, - 7, ImmutableList.of(TEST_SCHEMA), 5, ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), - 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of("property", "value"), currentSnapshotId, - Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + TableMetadata base = ImmutableTableMetadata.builder() + .metadataFileLocation(localInput(latestPreviousMetadata.file()).location()) + .formatVersion(1) + .uuid(UUID.randomUUID().toString()) + .location(TEST_LOCATION) + .lastSequenceNumber(0) + .lastUpdatedMillis(currentTimestamp - 80) + .lastColumnId(3) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA)) + .defaultSpecId(5) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(3) + .sortOrders(ImmutableList.of(SORT_ORDER_3)) + .properties(ImmutableMap.of("property", "value")) + .currentSnapshotId(currentSnapshotId) + .snapshots(Arrays.asList(previousSnapshot, currentSnapshot)) + .snapshotLog(reversedSnapshotLog) + .previousFiles(ImmutableList.copyOf(previousMetadataLog)) + .build(); previousMetadataLog.add(latestPreviousMetadata); @@ -378,13 +434,27 @@ public void testAddPreviousMetadataRemoveOne() { MetadataLogEntry latestPreviousMetadata = new MetadataLogEntry(currentTimestamp - 50, "/tmp/000006-" + UUID.randomUUID().toString() + ".metadata.json"); - TableMetadata base = new TableMetadata(latestPreviousMetadata.file(), 1, UUID.randomUUID().toString(), - TEST_LOCATION, 0, currentTimestamp - 50, 3, - 7, ImmutableList.of(TEST_SCHEMA), 5, - ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), - ImmutableMap.of("property", "value"), currentSnapshotId, - Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + TableMetadata base = ImmutableTableMetadata.builder() + .metadataFileLocation(localInput(latestPreviousMetadata.file()).location()) + .formatVersion(1) + .uuid(UUID.randomUUID().toString()) + .location(TEST_LOCATION) + .lastSequenceNumber(0) + .lastUpdatedMillis(currentTimestamp - 50) + .lastColumnId(3) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA)) + .defaultSpecId(5) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(3) + .sortOrders(ImmutableList.of(SORT_ORDER_3)) + .properties(ImmutableMap.of("property", "value")) + .currentSnapshotId(currentSnapshotId) + .snapshots(Arrays.asList(previousSnapshot, currentSnapshot)) + .snapshotLog(reversedSnapshotLog) + .previousFiles(ImmutableList.copyOf(previousMetadataLog)) + .build(); previousMetadataLog.add(latestPreviousMetadata); @@ -430,13 +500,27 @@ public void testAddPreviousMetadataRemoveMultiple() { MetadataLogEntry latestPreviousMetadata = new MetadataLogEntry(currentTimestamp - 50, "/tmp/000006-" + UUID.randomUUID().toString() + ".metadata.json"); - TableMetadata base = new TableMetadata(latestPreviousMetadata.file(), 1, UUID.randomUUID().toString(), - TEST_LOCATION, 0, currentTimestamp - 50, 3, 7, ImmutableList.of(TEST_SCHEMA), 2, - ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), - TableMetadata.INITIAL_SORT_ORDER_ID, ImmutableList.of(SortOrder.unsorted()), - ImmutableMap.of("property", "value"), currentSnapshotId, - Arrays.asList(previousSnapshot, currentSnapshot), reversedSnapshotLog, - ImmutableList.copyOf(previousMetadataLog)); + TableMetadata base = ImmutableTableMetadata.builder() + .metadataFileLocation(localInput(latestPreviousMetadata.file()).location()) + .formatVersion(1) + .uuid(UUID.randomUUID().toString()) + .location(TEST_LOCATION) + .lastSequenceNumber(0) + .lastUpdatedMillis(currentTimestamp - 50) + .lastColumnId(3) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA)) + .defaultSpecId(2) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(TableMetadata.INITIAL_SORT_ORDER_ID) + .sortOrders(ImmutableList.of(SortOrder.unsorted())) + .properties(ImmutableMap.of("property", "value")) + .currentSnapshotId(currentSnapshotId) + .snapshots(Arrays.asList(previousSnapshot, currentSnapshot)) + .snapshotLog(reversedSnapshotLog) + .previousFiles(ImmutableList.copyOf(previousMetadataLog)) + .build(); previousMetadataLog.add(latestPreviousMetadata); @@ -458,12 +542,20 @@ public void testAddPreviousMetadataRemoveMultiple() { public void testV2UUIDValidation() { AssertHelpers.assertThrows("Should reject v2 metadata without a UUID", IllegalArgumentException.class, "UUID is required in format v2", - () -> new TableMetadata(null, 2, null, TEST_LOCATION, SEQ_NO, System.currentTimeMillis(), - LAST_ASSIGNED_COLUMN_ID, 7, ImmutableList.of(TEST_SCHEMA), - SPEC_5.specId(), ImmutableList.of(SPEC_5), SPEC_5.lastAssignedFieldId(), - 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, - ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) - ); + () -> ImmutableTableMetadata.builder() + .formatVersion(2) + .location(TEST_LOCATION) + .lastSequenceNumber(SEQ_NO) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(LAST_ASSIGNED_COLUMN_ID) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA)) + .defaultSpecId(SPEC_5.specId()) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(3) + .sortOrders(ImmutableList.of(SORT_ORDER_3)) + .build()); } @Test @@ -471,12 +563,20 @@ public void testVersionValidation() { int unsupportedVersion = TableMetadata.SUPPORTED_TABLE_FORMAT_VERSION + 1; AssertHelpers.assertThrows("Should reject unsupported metadata", IllegalArgumentException.class, "Unsupported format version: v" + unsupportedVersion, - () -> new TableMetadata(null, unsupportedVersion, null, TEST_LOCATION, SEQ_NO, - System.currentTimeMillis(), LAST_ASSIGNED_COLUMN_ID, - 7, ImmutableList.of(TEST_SCHEMA), SPEC_5.specId(), ImmutableList.of(SPEC_5), - SPEC_5.lastAssignedFieldId(), 3, ImmutableList.of(SORT_ORDER_3), ImmutableMap.of(), -1L, - ImmutableList.of(), ImmutableList.of(), ImmutableList.of()) - ); + () -> ImmutableTableMetadata.builder() + .formatVersion(unsupportedVersion) + .location(TEST_LOCATION) + .lastSequenceNumber(SEQ_NO) + .lastUpdatedMillis(System.currentTimeMillis()) + .lastColumnId(LAST_ASSIGNED_COLUMN_ID) + .currentSchemaId(7) + .schemas(ImmutableList.of(TEST_SCHEMA)) + .defaultSpecId(SPEC_5.specId()) + .specs(ImmutableList.of(SPEC_5)) + .lastAssignedPartitionId(SPEC_5.lastAssignedFieldId()) + .defaultSortOrderId(3) + .sortOrders(ImmutableList.of(SORT_ORDER_3)) + .build()); } @Test diff --git a/versions.props b/versions.props index bf19bc9ee8c6..838fe73d104b 100644 --- a/versions.props +++ b/versions.props @@ -25,6 +25,7 @@ org.glassfish.jaxb:jaxb-runtime = 2.3.3 software.amazon.awssdk:* = 2.15.7 org.scala-lang:scala-library = 2.12.10 org.projectnessie:* = 0.15.1 +org.immutables:value = 2.8.8 # test deps org.junit.vintage:junit-vintage-engine = 5.7.2